@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,828 @@
|
|
|
1
|
+
"""The World Host's disk for its request ledger: what the flusher drained from a session's outbox waits
|
|
2
|
+
here, in the session's order, until the platform has stored it, beside the session's account on this
|
|
3
|
+
host (what it lost, and whether everything it held was accounted for).
|
|
4
|
+
|
|
5
|
+
Each account is a directory named by its random ``source``, holding its record (``account.jsonl``: a
|
|
6
|
+
snapshot of the account and the segments it holds, then a line per change, compacted as it grows) and its
|
|
7
|
+
entries as numbered segments, one entry a line. Every line is sealed (:class:`~.spill_files.SpillSeal`) and
|
|
8
|
+
bound to its account and place: a segment line to its segment and ordinal, a record line to the one before
|
|
9
|
+
it. So a line a version worker wrote, copied, moved or changed is never sent; what a segment the record
|
|
10
|
+
lists no longer holds in place, whether it is gone, unreadable, short or reordered, is counted lost when the
|
|
11
|
+
platform stores the rest of it, or when a restarted host takes the account; and a restarted host never takes
|
|
12
|
+
an account as complete whose record is broken or whose directory holds a segment the record does not list.
|
|
13
|
+
An account belongs to one outbox (``epoch``), so whatever reaches the spill after a session's account closed
|
|
14
|
+
joins that account while the host holds it; a page that lands after the account was settled opens another,
|
|
15
|
+
never final. Past ``max_bytes`` the host's oldest segments go, counted lost. Every file goes through
|
|
16
|
+
:class:`~.spill_files.SpillDirectory`, keeping only entries the platform parses (:func:`ledger_line`)."""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import collections
|
|
21
|
+
import errno
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import stat
|
|
26
|
+
import threading
|
|
27
|
+
import uuid
|
|
28
|
+
from collections.abc import Iterator
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from functools import partial
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Callable, Optional, TypeVar
|
|
33
|
+
|
|
34
|
+
from .cache_layout import SESSIONS_DIR, VERSIONS_DIR, WORKERS_DIR
|
|
35
|
+
from .call_log import _children
|
|
36
|
+
from .handler_errors import log_line
|
|
37
|
+
from .spill_files import TOMBSTONE, SpillDirectory, SpillSeal
|
|
38
|
+
|
|
39
|
+
SPILL_DIR = "ledger"
|
|
40
|
+
DEFAULT_SPILL_MB = 4096
|
|
41
|
+
HOST_DIRS = (SESSIONS_DIR, VERSIONS_DIR, WORKERS_DIR)
|
|
42
|
+
LEDGER_BATCH_BYTES = 4 << 20
|
|
43
|
+
SEGMENT_READ_BYTES = LEDGER_BATCH_BYTES + (1 << 20)
|
|
44
|
+
SEGMENT_TARGET_BYTES = 256 << 10
|
|
45
|
+
SEGMENTS_IN_CAP = 64
|
|
46
|
+
SEGMENT_MISSES = 3
|
|
47
|
+
ACCOUNT_BYTES = 256
|
|
48
|
+
ACCOUNT_FILE = "account.jsonl"
|
|
49
|
+
ACCOUNT_READ_BYTES = 8 << 20
|
|
50
|
+
RECORD_LINES_MIN = 64
|
|
51
|
+
SESSION_ID_CHARS = 512
|
|
52
|
+
OUTBOXES_MAX = 64
|
|
53
|
+
COUNT_MAX = (1 << 53) - 1
|
|
54
|
+
ENTRY_MAX_DEPTH = 64
|
|
55
|
+
HEX32 = re.compile(r"[0-9a-f]{32}")
|
|
56
|
+
SEGMENT = re.compile(r"[0-9]{12}\.jsonl")
|
|
57
|
+
WHOLE = re.compile(r"[0-9]{1,9}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def counted(value: object) -> bool:
|
|
61
|
+
return type(value) is int and 0 <= value <= COUNT_MAX
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _deeper_than(entry: dict, levels: int) -> bool:
|
|
65
|
+
"""Whether ``entry`` holds a value more than ``levels`` below itself, found without recursing:
|
|
66
|
+
the encoder's own depth limit moves with the frame it runs in, so an entry near it encodes in
|
|
67
|
+
one place and raises in the next."""
|
|
68
|
+
layer = [entry]
|
|
69
|
+
for _ in range(levels + 1):
|
|
70
|
+
layer = [child for node in layer for child in _children(node)]
|
|
71
|
+
return bool(layer)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def ledger_line(entry: object) -> bytes | None:
|
|
75
|
+
"""``entry`` as the JSON the spill keeps, or None when the platform would not parse it: one it cannot
|
|
76
|
+
parse (a non-finite number from a worker's tenant code) refuses every session sent beside it, and one
|
|
77
|
+
nested past :data:`ENTRY_MAX_DEPTH` would fail the flush that encodes it."""
|
|
78
|
+
if not (isinstance(entry, dict) and isinstance(entry.get("id"), str) and HEX32.fullmatch(entry["id"])):
|
|
79
|
+
return None
|
|
80
|
+
if _deeper_than(entry, ENTRY_MAX_DEPTH):
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
text = json.dumps(entry, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
|
84
|
+
except (TypeError, ValueError, RecursionError):
|
|
85
|
+
return None
|
|
86
|
+
data = text.encode("utf-8", "replace")
|
|
87
|
+
return data if len(data) < LEDGER_BATCH_BYTES else None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
Line = TypeVar("Line", bytes, Optional[bytes])
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def runs(lines: list[Line], max_bytes: int, per_line: int = 0) -> Iterator[list[Line]]:
|
|
94
|
+
"""``lines`` in order, in runs of at most ``max_bytes``, each line ``per_line`` more (always one)."""
|
|
95
|
+
run: list[Line] = []
|
|
96
|
+
spent = 0
|
|
97
|
+
for line in lines:
|
|
98
|
+
size = len(line or b"") + per_line
|
|
99
|
+
if run and spent + size > max_bytes:
|
|
100
|
+
yield run
|
|
101
|
+
run, spent = [], 0
|
|
102
|
+
run.append(line)
|
|
103
|
+
spent += size
|
|
104
|
+
if run:
|
|
105
|
+
yield run
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _not_json(constant: str) -> None:
|
|
109
|
+
raise ValueError(constant)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _json(raw: bytes | None) -> object:
|
|
113
|
+
try:
|
|
114
|
+
return json.loads(raw or b"", parse_constant=_not_json)
|
|
115
|
+
except (ValueError, RecursionError):
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _entry(data: bytes | None) -> dict | None:
|
|
120
|
+
entry = _json(data)
|
|
121
|
+
return entry if ledger_line(entry) is not None else None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _lines(raw: bytes | None) -> list[bytes]:
|
|
125
|
+
return [line for line in (raw or b"").split(b"\n") if line]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _compact(value: object) -> bytes:
|
|
129
|
+
return json.dumps(value, separators=(",", ":")).encode("utf-8")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class SpillRoot:
|
|
133
|
+
"""Where a host spills: a directory of its own under the cache root, the path read as written (never
|
|
134
|
+
the root itself, nor a directory the host clears or fills on its own) and opened from the cache root
|
|
135
|
+
held open, each step through no link and made the host's own; a link, a file or a directory it cannot
|
|
136
|
+
open or make its own on the way is set aside and made again."""
|
|
137
|
+
|
|
138
|
+
def __init__(self, cache: str, parts: tuple[str, ...]) -> None:
|
|
139
|
+
self.cache = cache
|
|
140
|
+
self.parts = parts
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def at(cls, cache_root: Path | str, given: str | None) -> SpillRoot:
|
|
144
|
+
cache = os.path.normpath(os.path.abspath(cache_root))
|
|
145
|
+
relative = os.path.relpath(os.path.normpath(os.path.join(cache, given or SPILL_DIR)), cache)
|
|
146
|
+
parts = Path(relative).parts
|
|
147
|
+
if relative == "." or parts[0] == ".." or parts[0] in HOST_DIRS:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
f"WORLDS_HOST_LEDGER_SPILL_DIR must be a directory of its own under the cache root {cache}, "
|
|
150
|
+
f"not {given}"
|
|
151
|
+
)
|
|
152
|
+
return cls(cache, parts)
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def path(self) -> Path:
|
|
156
|
+
return Path(self.cache).joinpath(*self.parts)
|
|
157
|
+
|
|
158
|
+
def open(self) -> SpillDirectory:
|
|
159
|
+
os.makedirs(self.cache, exist_ok=True)
|
|
160
|
+
current = SpillDirectory(os.open(self.cache, os.O_RDONLY | os.O_DIRECTORY))
|
|
161
|
+
for part in self.parts:
|
|
162
|
+
current = self._step(current, part)
|
|
163
|
+
return current
|
|
164
|
+
|
|
165
|
+
@staticmethod
|
|
166
|
+
def _step(parent: SpillDirectory, part: str) -> SpillDirectory:
|
|
167
|
+
try:
|
|
168
|
+
parent.ensure(part)
|
|
169
|
+
return SpillRoot._into(parent, part)
|
|
170
|
+
finally:
|
|
171
|
+
parent.close()
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
def _into(parent: SpillDirectory, part: str) -> SpillDirectory:
|
|
175
|
+
try:
|
|
176
|
+
return SpillDirectory.owned(part, parent.descriptor)
|
|
177
|
+
except OSError:
|
|
178
|
+
parent.discard(part)
|
|
179
|
+
parent.make(part)
|
|
180
|
+
parent.sweep()
|
|
181
|
+
return SpillDirectory.owned(part, parent.descriptor)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass(eq=False)
|
|
185
|
+
class Segment:
|
|
186
|
+
seq: int
|
|
187
|
+
name: str
|
|
188
|
+
size: int
|
|
189
|
+
entries: int
|
|
190
|
+
sending: bool = False
|
|
191
|
+
misses: int = 0
|
|
192
|
+
full: bool = False
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass(eq=False)
|
|
196
|
+
class Account:
|
|
197
|
+
"""One host's account of one session's outbox (``epoch``; None for one the host only heard of from a
|
|
198
|
+
dead worker or closed before it drained anything): its spilled entries, oldest first, what it lost,
|
|
199
|
+
under the random ``source`` that names its directory, and, once ``closed``, whether everything the
|
|
200
|
+
host held of it was moved or counted (``complete``); final once complete with nothing left on disk.
|
|
201
|
+
``home`` says its directory and record exist; one the disk refused keeps its counts in memory."""
|
|
202
|
+
|
|
203
|
+
source: str
|
|
204
|
+
session_id: str
|
|
205
|
+
epoch: str | None = None
|
|
206
|
+
shed: int = 0
|
|
207
|
+
lost: int = 0
|
|
208
|
+
closed: bool = False
|
|
209
|
+
complete: bool = False
|
|
210
|
+
home: bool = False
|
|
211
|
+
refused: bool = False
|
|
212
|
+
told: tuple[int, int, bool] | None = None
|
|
213
|
+
outboxes: dict[str, tuple[int, int]] = field(default_factory=dict)
|
|
214
|
+
segments: collections.deque[Segment] = field(default_factory=collections.deque)
|
|
215
|
+
sending: bool = False
|
|
216
|
+
records: int = 0
|
|
217
|
+
chain: bytes = b""
|
|
218
|
+
|
|
219
|
+
def account(self) -> tuple[int, int, bool]:
|
|
220
|
+
return (self.shed, self.lost, self.closed and self.complete and not self.segments)
|
|
221
|
+
|
|
222
|
+
def counts(self) -> dict:
|
|
223
|
+
return {"shed": self.shed, "lost": self.lost, "closed": self.closed, "complete": self.complete}
|
|
224
|
+
|
|
225
|
+
def snapshot(self) -> dict:
|
|
226
|
+
return {"session": self.session_id, **self.counts(), "segments": {s.name: s.entries for s in self.segments}}
|
|
227
|
+
|
|
228
|
+
def lose(self, entries: int) -> None:
|
|
229
|
+
self.lost = min(COUNT_MAX, self.lost + entries)
|
|
230
|
+
|
|
231
|
+
def outbox(self, epoch: str, shed: int, lost: int) -> bool:
|
|
232
|
+
"""An outbox's own running counts (``epoch`` names the outbox): what they grew by since they were last
|
|
233
|
+
seen joins the account. True when the account grew."""
|
|
234
|
+
was_shed, was_lost = self.outboxes.pop(epoch, (0, 0))
|
|
235
|
+
self.outboxes[epoch] = (max(was_shed, shed), max(was_lost, lost))
|
|
236
|
+
if len(self.outboxes) > OUTBOXES_MAX:
|
|
237
|
+
self.outboxes.pop(next(iter(self.outboxes)))
|
|
238
|
+
grew_shed, grew_lost = max(0, shed - was_shed), max(0, lost - was_lost)
|
|
239
|
+
self.shed = min(COUNT_MAX, self.shed + grew_shed)
|
|
240
|
+
self.lost = min(COUNT_MAX, self.lost + grew_lost)
|
|
241
|
+
return bool(grew_shed or grew_lost)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class AccountRecord:
|
|
245
|
+
"""An account's record as its sealed lines replay it: the newest snapshot (``session``, the counts and
|
|
246
|
+
the segments it held), then each change after it (``segment``: a segment's entries, ``gone``: a segment
|
|
247
|
+
stored or let go, ``counts``); a line before the snapshot, or off its shape, changes nothing. Each line
|
|
248
|
+
is sealed over the one before it, so a record missing, repeating or reordering a line is ``broken``."""
|
|
249
|
+
|
|
250
|
+
def __init__(self) -> None:
|
|
251
|
+
self.broken = False
|
|
252
|
+
self.session_id: str | None = None
|
|
253
|
+
self.counts = {"shed": 0, "lost": 0, "closed": False, "complete": False}
|
|
254
|
+
self.segments: dict[str, int] = {}
|
|
255
|
+
self._kinds: dict[str, Callable[[dict], None]] = {
|
|
256
|
+
"session": self._snapshot,
|
|
257
|
+
"segment": self._segment,
|
|
258
|
+
"gone": self._gone,
|
|
259
|
+
"counts": self._counted,
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
def apply(self, value: object) -> None:
|
|
263
|
+
line = value if isinstance(value, dict) else {}
|
|
264
|
+
kind = next((name for name in self._kinds if name in line), None)
|
|
265
|
+
known = kind == "session" or self.session_id is not None
|
|
266
|
+
if kind is not None and known:
|
|
267
|
+
self._kinds[kind](line)
|
|
268
|
+
|
|
269
|
+
def account(self, source: str, sizes: dict[str, int | None], unlisted: bool) -> Account:
|
|
270
|
+
"""The account a restarted host takes: closed, complete only if its host closed it complete and its
|
|
271
|
+
record and segments are as that host left them, and what a listed segment no longer holds on disk
|
|
272
|
+
counted lost."""
|
|
273
|
+
assert self.session_id is not None
|
|
274
|
+
missing = sum(entries for name, entries in self.segments.items() if sizes.get(name) is None)
|
|
275
|
+
intact = not (self.broken or unlisted)
|
|
276
|
+
account = Account(
|
|
277
|
+
source,
|
|
278
|
+
self.session_id,
|
|
279
|
+
shed=self.counts["shed"],
|
|
280
|
+
lost=self.counts["lost"],
|
|
281
|
+
closed=True,
|
|
282
|
+
complete=self.counts["closed"] and self.counts["complete"] and intact,
|
|
283
|
+
home=True,
|
|
284
|
+
)
|
|
285
|
+
account.lose(missing)
|
|
286
|
+
account.segments.extend(
|
|
287
|
+
Segment(int(name[:12]), name, sizes[name] or 0, entries)
|
|
288
|
+
for name, entries in sorted(self.segments.items())
|
|
289
|
+
if sizes.get(name) is not None
|
|
290
|
+
)
|
|
291
|
+
return account
|
|
292
|
+
|
|
293
|
+
def _snapshot(self, line: dict) -> None:
|
|
294
|
+
session_id, segments = line.get("session"), line.get("segments")
|
|
295
|
+
named = isinstance(session_id, str) and 0 < len(session_id) <= SESSION_ID_CHARS
|
|
296
|
+
if named and self._valid_counts(line) and isinstance(segments, dict):
|
|
297
|
+
self.session_id = session_id
|
|
298
|
+
self.counts = {key: line[key] for key in self.counts}
|
|
299
|
+
self.segments = {name: n for name, n in segments.items() if SEGMENT.fullmatch(name) and counted(n)}
|
|
300
|
+
|
|
301
|
+
def _segment(self, line: dict) -> None:
|
|
302
|
+
name, entries = line.get("segment"), line.get("entries")
|
|
303
|
+
if isinstance(name, str) and SEGMENT.fullmatch(name) and counted(entries):
|
|
304
|
+
self.segments[name] = entries
|
|
305
|
+
|
|
306
|
+
def _gone(self, line: dict) -> None:
|
|
307
|
+
self.segments.pop(str(line.get("gone")), None)
|
|
308
|
+
|
|
309
|
+
def _counted(self, line: dict) -> None:
|
|
310
|
+
counts = line.get("counts")
|
|
311
|
+
if isinstance(counts, dict) and self._valid_counts(counts):
|
|
312
|
+
self.counts = {key: counts[key] for key in self.counts}
|
|
313
|
+
|
|
314
|
+
@staticmethod
|
|
315
|
+
def _valid_counts(value: dict) -> bool:
|
|
316
|
+
whole = counted(value.get("shed")) and counted(value.get("lost"))
|
|
317
|
+
return whole and isinstance(value.get("closed"), bool) and isinstance(value.get("complete"), bool)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@dataclass(eq=False)
|
|
321
|
+
class Piece:
|
|
322
|
+
"""What one round sends of an account: its oldest segment, or its account alone, what of the segment did
|
|
323
|
+
not come back (``loss``, counted when the platform stores the piece) and the account as it was sent."""
|
|
324
|
+
|
|
325
|
+
account: Account
|
|
326
|
+
segment: Segment | None
|
|
327
|
+
loss: int = 0
|
|
328
|
+
sent: tuple[int, int, bool] = (0, 0, False)
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def size(self) -> int:
|
|
332
|
+
return ACCOUNT_BYTES + (self.segment.size if self.segment is not None else 0)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class LedgerSpill:
|
|
336
|
+
"""The entries the host drained and the platform has not stored, on disk at ``root``, at most ``max_bytes``
|
|
337
|
+
of segments: past it the host's oldest go, counted lost in their accounts. A session's entries go to the
|
|
338
|
+
account of the outbox that answered them; :meth:`finished` closes the session's newest account, and only a
|
|
339
|
+
new outbox, or a page that lands after its account was settled, opens another. :meth:`round` hands out one
|
|
340
|
+
piece of every session at a time, oldest account first, so a session's entries go in order and no session
|
|
341
|
+
waits behind another; a piece is :meth:`stored` or :meth:`released`."""
|
|
342
|
+
|
|
343
|
+
def __init__(self, root: SpillRoot, max_bytes: int, seal: SpillSeal) -> None:
|
|
344
|
+
if max_bytes <= 0:
|
|
345
|
+
raise ValueError(f"the ledger spill's cap (WORLDS_HOST_LEDGER_SPILL_MB) must be positive, not {max_bytes}")
|
|
346
|
+
self.root = root
|
|
347
|
+
self.max_bytes = max_bytes
|
|
348
|
+
self.segment_bytes = min(SEGMENT_TARGET_BYTES, max_bytes // SEGMENTS_IN_CAP)
|
|
349
|
+
self.seal = seal
|
|
350
|
+
self.bytes = 0
|
|
351
|
+
self.accounts: dict[str, Account] = {}
|
|
352
|
+
self.latest: dict[str, Account] = {}
|
|
353
|
+
self.epochs: dict[tuple[str, str | None], Account] = {}
|
|
354
|
+
self._oldest: dict[Segment, Account] = {}
|
|
355
|
+
self._next = 1
|
|
356
|
+
self._directory: SpillDirectory | None = None
|
|
357
|
+
self._lock = threading.Lock()
|
|
358
|
+
self._sweeping = threading.Lock()
|
|
359
|
+
self._sweep_asked = threading.Event()
|
|
360
|
+
self._incomplete: set[str] = set()
|
|
361
|
+
|
|
362
|
+
@classmethod
|
|
363
|
+
def at(cls, cache_root: Path | str, given_dir: str | None, given_mb: str | None, secret: str) -> LedgerSpill:
|
|
364
|
+
"""The spill a deployed host keeps (``WORLDS_HOST_LEDGER_SPILL_DIR``, ``..._MB``), not opened yet."""
|
|
365
|
+
megabytes = given_mb or str(DEFAULT_SPILL_MB)
|
|
366
|
+
if not WHOLE.fullmatch(megabytes):
|
|
367
|
+
raise ValueError(f"WORLDS_HOST_LEDGER_SPILL_MB must be a whole number of MiB, not {megabytes!r}")
|
|
368
|
+
return cls(SpillRoot.at(cache_root, given_dir), int(megabytes) << 20, SpillSeal(secret))
|
|
369
|
+
|
|
370
|
+
def open(self) -> LedgerSpill:
|
|
371
|
+
"""Open the spill's directory (made if missing) and take every account a previous host left in it:
|
|
372
|
+
each is closed, final only if that host had closed it complete, and what a segment its record lists
|
|
373
|
+
no longer holds is counted lost; anything else in the directory is set aside."""
|
|
374
|
+
self._directory = self.root.open()
|
|
375
|
+
found = [self._adopt(name) for name in self.directory.names() if not name.startswith(TOMBSTONE)]
|
|
376
|
+
adopted = [account for account in found if account is not None]
|
|
377
|
+
segments = sorted(((segment, account) for account in adopted for segment in account.segments), key=_seq)
|
|
378
|
+
with self._lock:
|
|
379
|
+
self._oldest = dict(segments)
|
|
380
|
+
self.bytes = sum(segment.size for segment in self._oldest)
|
|
381
|
+
self._next = 1 + max((segment.seq for segment in self._oldest), default=0)
|
|
382
|
+
dropped = self._bound()
|
|
383
|
+
self._dropped(dropped)
|
|
384
|
+
self._sweep_later()
|
|
385
|
+
return self
|
|
386
|
+
|
|
387
|
+
@property
|
|
388
|
+
def directory(self) -> SpillDirectory:
|
|
389
|
+
if self._directory is None:
|
|
390
|
+
raise RuntimeError("the ledger spill is not open")
|
|
391
|
+
return self._directory
|
|
392
|
+
|
|
393
|
+
def write(
|
|
394
|
+
self, session_id: str, lines: list[bytes | None], epoch: str | None = None, source: str | None = None
|
|
395
|
+
) -> None:
|
|
396
|
+
"""``lines`` into the account of the outbox ``epoch`` (the session's newest when None; a new one is
|
|
397
|
+
kept under ``source``, the platform's account of the life, when given), an entry the platform would
|
|
398
|
+
not parse counted lost; past the cap the host's oldest segments go. Raises OSError, writing nothing
|
|
399
|
+
it can be sure of, when the disk refuses it: the outbox still holds the entries."""
|
|
400
|
+
kept = [line for line in lines if line is not None]
|
|
401
|
+
with self._lock:
|
|
402
|
+
account = self._account(session_id, epoch, source)
|
|
403
|
+
self._housed(account)
|
|
404
|
+
self._append(account, kept)
|
|
405
|
+
self._lose(account, len(lines) - len(kept))
|
|
406
|
+
dropped = self._bound()
|
|
407
|
+
self._dropped(dropped)
|
|
408
|
+
self._unsendable(session_id, len(lines) - len(kept))
|
|
409
|
+
|
|
410
|
+
def outbox(self, session_id: str, epoch: str, shed: int, lost: int, source: str | None = None) -> bool:
|
|
411
|
+
"""The counts the outbox ``epoch`` reported in a drain answer; True when its account grew."""
|
|
412
|
+
with self._lock:
|
|
413
|
+
account = self._account(session_id, epoch, source)
|
|
414
|
+
grew = account.outbox(epoch, shed, lost)
|
|
415
|
+
if grew:
|
|
416
|
+
self._remember(account)
|
|
417
|
+
return grew
|
|
418
|
+
|
|
419
|
+
def lose(self, session_id: str, entries: int) -> None:
|
|
420
|
+
with self._lock:
|
|
421
|
+
self._lose(self._account(session_id, None), entries)
|
|
422
|
+
|
|
423
|
+
def finished(self, session_id: str, complete: bool) -> None:
|
|
424
|
+
"""The host is done with the session: its newest account closes, complete when everything the host
|
|
425
|
+
held of it was moved here or counted; an account closes once, the first verdict standing, except that
|
|
426
|
+
an incomplete verdict after a complete one closes a new account, so it is never lost. The session
|
|
427
|
+
stays :meth:`incomplete` for as long as the host serves it."""
|
|
428
|
+
with self._lock:
|
|
429
|
+
if not complete:
|
|
430
|
+
self._incomplete.add(session_id)
|
|
431
|
+
account = self._account(session_id, None)
|
|
432
|
+
if account.closed and (complete or not account.complete):
|
|
433
|
+
return
|
|
434
|
+
if account.closed:
|
|
435
|
+
account = self._open_account(session_id, None)
|
|
436
|
+
account.closed, account.complete = True, complete
|
|
437
|
+
self._remember(account)
|
|
438
|
+
|
|
439
|
+
def round(self, held: set[str], only: str | None = None) -> list[Piece]:
|
|
440
|
+
"""The next piece of every session not ``held`` (or of ``only``): its oldest account's oldest
|
|
441
|
+
segment, or that account alone when it has only its counts left to tell."""
|
|
442
|
+
with self._lock:
|
|
443
|
+
taken: set[str] = set(held)
|
|
444
|
+
accounts = [account for account in self.accounts.values() if only in (None, account.session_id)]
|
|
445
|
+
pieces = [self._piece(account, taken) for account in accounts]
|
|
446
|
+
return [piece for piece in pieces if piece is not None]
|
|
447
|
+
|
|
448
|
+
def pending(self, session_id: str) -> bool:
|
|
449
|
+
"""Whether entries of the session wait here unsent, or are on their way in a send."""
|
|
450
|
+
with self._lock:
|
|
451
|
+
return any(account.segments or account.sending for account in self._of(session_id))
|
|
452
|
+
|
|
453
|
+
def counts(self, session_id: str) -> tuple[int, int]:
|
|
454
|
+
"""What the session's accounts here shed and lost."""
|
|
455
|
+
with self._lock:
|
|
456
|
+
accounts = self._of(session_id)
|
|
457
|
+
return sum(a.shed for a in accounts), sum(a.lost for a in accounts)
|
|
458
|
+
|
|
459
|
+
def incomplete(self, session_id: str) -> bool:
|
|
460
|
+
"""Whether an account of the session closed incomplete while the host served it, or one here did."""
|
|
461
|
+
with self._lock:
|
|
462
|
+
closed = (account.closed and not account.complete for account in self._of(session_id))
|
|
463
|
+
return session_id in self._incomplete or any(closed)
|
|
464
|
+
|
|
465
|
+
def _of(self, session_id: str) -> list[Account]:
|
|
466
|
+
return [account for account in self.accounts.values() if account.session_id == session_id]
|
|
467
|
+
|
|
468
|
+
def item(self, piece: Piece) -> dict | None:
|
|
469
|
+
"""The flush item a piece sends: its segment's entries the platform parses, with its account as it
|
|
470
|
+
stands once what the segment did not give back is counted, final when the account is complete and
|
|
471
|
+
this is all it holds. None while its segment could not be read and has reads left; one read in none
|
|
472
|
+
of ``SEGMENT_MISSES`` tries is sent empty. Nothing is counted until :meth:`stored`."""
|
|
473
|
+
requests = self._read(piece)
|
|
474
|
+
if requests is None:
|
|
475
|
+
return None
|
|
476
|
+
with self._lock:
|
|
477
|
+
account = piece.account
|
|
478
|
+
expected = piece.segment.entries if piece.segment is not None else 0
|
|
479
|
+
piece.loss = max(0, expected - len(requests))
|
|
480
|
+
last = all(segment is piece.segment for segment in account.segments)
|
|
481
|
+
piece.sent = (
|
|
482
|
+
account.shed,
|
|
483
|
+
min(COUNT_MAX, account.lost + piece.loss),
|
|
484
|
+
account.closed and account.complete and last,
|
|
485
|
+
)
|
|
486
|
+
shed, lost, final = piece.sent
|
|
487
|
+
return {
|
|
488
|
+
"sessionId": account.session_id,
|
|
489
|
+
"requests": requests,
|
|
490
|
+
"source": account.source,
|
|
491
|
+
"shed": shed,
|
|
492
|
+
"lost": lost,
|
|
493
|
+
"final": final,
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
def stored(self, pieces: list[Piece]) -> None:
|
|
497
|
+
"""Pieces the platform stored: their segments go, what they did not give back is counted, and their
|
|
498
|
+
accounts were told."""
|
|
499
|
+
with self._lock:
|
|
500
|
+
for piece in pieces:
|
|
501
|
+
self._drop(piece.segment)
|
|
502
|
+
self._lose(piece.account, piece.loss)
|
|
503
|
+
piece.account.told = piece.sent
|
|
504
|
+
|
|
505
|
+
def released(self, pieces: list[Piece]) -> None:
|
|
506
|
+
"""Pieces sent or not: each may be handed out again."""
|
|
507
|
+
with self._lock:
|
|
508
|
+
for piece in pieces:
|
|
509
|
+
piece.account.sending = False
|
|
510
|
+
_mark(piece.segment, False)
|
|
511
|
+
|
|
512
|
+
def settle(self, served: set[str]) -> None:
|
|
513
|
+
"""Forget every account whose entries were stored and whose counts were told, unless it is still open
|
|
514
|
+
for a session the host serves; their directories are set aside at once and deleted after, off the
|
|
515
|
+
caller's thread."""
|
|
516
|
+
with self._lock:
|
|
517
|
+
self._incomplete &= served
|
|
518
|
+
done = [account for account in self.accounts.values() if self._settled(account, served)]
|
|
519
|
+
for account in done:
|
|
520
|
+
self._forget(account)
|
|
521
|
+
if done:
|
|
522
|
+
self._sweep_later()
|
|
523
|
+
|
|
524
|
+
def _settled(self, account: Account, served: set[str]) -> bool:
|
|
525
|
+
idle = not (account.segments or account.sending)
|
|
526
|
+
open_here = not account.closed and account.session_id in served
|
|
527
|
+
return idle and not open_here and account.told == account.account()
|
|
528
|
+
|
|
529
|
+
def _forget(self, account: Account) -> None:
|
|
530
|
+
del self.accounts[account.source]
|
|
531
|
+
if self.latest.get(account.session_id) is account:
|
|
532
|
+
del self.latest[account.session_id]
|
|
533
|
+
self.epochs.pop((account.session_id, account.epoch), None)
|
|
534
|
+
if account.home:
|
|
535
|
+
self._discard(account.source)
|
|
536
|
+
|
|
537
|
+
def _discard(self, name: str) -> None:
|
|
538
|
+
try:
|
|
539
|
+
self.directory.discard(name)
|
|
540
|
+
except OSError as error:
|
|
541
|
+
log_line(f"request ledger spill: {name} not set aside: {type(error).__name__}")
|
|
542
|
+
|
|
543
|
+
def _account(self, session_id: str, epoch: str | None, source: str | None = None) -> Account:
|
|
544
|
+
found = self.epochs.get((session_id, epoch)) if epoch is not None else self.latest.get(session_id)
|
|
545
|
+
return found if found is not None else self._open_account(session_id, epoch, source)
|
|
546
|
+
|
|
547
|
+
def _open_account(self, session_id: str, epoch: str | None, source: str | None = None) -> Account:
|
|
548
|
+
fresh = source is None or source in self.accounts
|
|
549
|
+
account = Account(uuid.uuid4().hex if fresh else source, session_id, epoch)
|
|
550
|
+
self.accounts[account.source] = self.latest[session_id] = self.epochs[(session_id, epoch)] = account
|
|
551
|
+
self._house(account)
|
|
552
|
+
return account
|
|
553
|
+
|
|
554
|
+
def _house(self, account: Account) -> bool:
|
|
555
|
+
"""The account's directory and record made, when they are not yet; False while the disk refuses them,
|
|
556
|
+
the account kept in memory meanwhile."""
|
|
557
|
+
if account.home:
|
|
558
|
+
return True
|
|
559
|
+
try:
|
|
560
|
+
self.directory.reowned(partial(self.directory.ensure, account.source))
|
|
561
|
+
self._compact(account)
|
|
562
|
+
except OSError as error:
|
|
563
|
+
self._refused(account, error)
|
|
564
|
+
return False
|
|
565
|
+
account.home = True
|
|
566
|
+
return True
|
|
567
|
+
|
|
568
|
+
def _housed(self, account: Account) -> None:
|
|
569
|
+
if not self._house(account):
|
|
570
|
+
raise OSError(errno.EIO, f"the spill cannot hold session {account.session_id}'s account")
|
|
571
|
+
|
|
572
|
+
def _refused(self, account: Account, error: OSError) -> None:
|
|
573
|
+
if not account.refused:
|
|
574
|
+
log_line(
|
|
575
|
+
f"request ledger spill: session {account.session_id} has no account on disk: {type(error).__name__}"
|
|
576
|
+
)
|
|
577
|
+
account.refused = True
|
|
578
|
+
|
|
579
|
+
def _adopt(self, name: str) -> Account | None:
|
|
580
|
+
try:
|
|
581
|
+
return self._found(name)
|
|
582
|
+
except Exception as error: # noqa: BLE001 - one account a previous host left never stops the others
|
|
583
|
+
log_line(f"request ledger spill: {name} not taken: {type(error).__name__}")
|
|
584
|
+
return None
|
|
585
|
+
|
|
586
|
+
def _found(self, name: str) -> Account | None:
|
|
587
|
+
mode = self.directory.mode(name)
|
|
588
|
+
if not (HEX32.fullmatch(name) and mode is not None and stat.S_ISDIR(mode)):
|
|
589
|
+
return self._unknown(name)
|
|
590
|
+
with self.directory.child(name) as directory:
|
|
591
|
+
record = self._replayed(name, directory.tail(ACCOUNT_FILE, ACCOUNT_READ_BYTES))
|
|
592
|
+
sizes = {segment: directory.size(segment) for segment in record.segments}
|
|
593
|
+
unlisted = any(SEGMENT.fullmatch(file) and file not in sizes for file in directory.names())
|
|
594
|
+
if record.session_id is None:
|
|
595
|
+
return self._unknown(name)
|
|
596
|
+
account = record.account(name, sizes, unlisted)
|
|
597
|
+
self.accounts[name] = account
|
|
598
|
+
self._rewrite(account)
|
|
599
|
+
return account
|
|
600
|
+
|
|
601
|
+
def _replayed(self, source: str, raw: bytes | None) -> AccountRecord:
|
|
602
|
+
record = AccountRecord()
|
|
603
|
+
place = b""
|
|
604
|
+
for line in _lines(raw):
|
|
605
|
+
data = self.seal.opened(source, line, place)
|
|
606
|
+
record.broken = data is None
|
|
607
|
+
if record.broken:
|
|
608
|
+
break
|
|
609
|
+
record.apply(_json(data))
|
|
610
|
+
place = SpillSeal.tag(line)
|
|
611
|
+
return record
|
|
612
|
+
|
|
613
|
+
def _unknown(self, name: str) -> None:
|
|
614
|
+
log_line(f"request ledger spill: {name} holds no account this host sealed; removed")
|
|
615
|
+
self._discard(name)
|
|
616
|
+
|
|
617
|
+
@staticmethod
|
|
618
|
+
def _piece(account: Account, taken: set[str]) -> Piece | None:
|
|
619
|
+
if account.session_id in taken:
|
|
620
|
+
return None
|
|
621
|
+
if account.sending:
|
|
622
|
+
taken.add(account.session_id)
|
|
623
|
+
return None
|
|
624
|
+
segment = account.segments[0] if account.segments else None
|
|
625
|
+
if segment is None and account.told == account.account():
|
|
626
|
+
return None
|
|
627
|
+
taken.add(account.session_id)
|
|
628
|
+
account.sending = True
|
|
629
|
+
return Piece(account, _mark(segment, True))
|
|
630
|
+
|
|
631
|
+
def _read(self, piece: Piece) -> list[dict] | None:
|
|
632
|
+
segment = piece.segment
|
|
633
|
+
if segment is None:
|
|
634
|
+
return []
|
|
635
|
+
raw = self._raw(piece.account, segment)
|
|
636
|
+
if raw is None and self._missed(segment):
|
|
637
|
+
return None
|
|
638
|
+
source = piece.account.source
|
|
639
|
+
opened = (self.seal.opened(source, line, _place(segment, n)) for n, line in enumerate(_lines(raw)))
|
|
640
|
+
return [entry for entry in map(_entry, opened) if entry is not None]
|
|
641
|
+
|
|
642
|
+
def _missed(self, segment: Segment) -> bool:
|
|
643
|
+
with self._lock:
|
|
644
|
+
segment.misses += 1
|
|
645
|
+
return segment.misses < SEGMENT_MISSES
|
|
646
|
+
|
|
647
|
+
def _raw(self, account: Account, segment: Segment) -> bytes | None:
|
|
648
|
+
try:
|
|
649
|
+
with self.directory.child(account.source) as directory:
|
|
650
|
+
return directory.tail(segment.name, SEGMENT_READ_BYTES)
|
|
651
|
+
except OSError:
|
|
652
|
+
return None
|
|
653
|
+
|
|
654
|
+
def _append(self, account: Account, lines: list[bytes]) -> None:
|
|
655
|
+
for run in runs(lines, LEDGER_BATCH_BYTES, SpillSeal.OVERHEAD):
|
|
656
|
+
self._add(account, run)
|
|
657
|
+
|
|
658
|
+
def _add(self, account: Account, lines: list[bytes]) -> None:
|
|
659
|
+
"""A run of lines on disk, each sealed to its segment and place in it, the record told first: a
|
|
660
|
+
segment the record lists and the disk does not hold is counted lost by a restarted host, never one
|
|
661
|
+
it holds and the record misses."""
|
|
662
|
+
size = sum(len(line) + SpillSeal.OVERHEAD for line in lines)
|
|
663
|
+
tail = account.segments[-1] if account.segments else None
|
|
664
|
+
if tail is not None and self._extends(tail, size):
|
|
665
|
+
return self._extend(account, tail, self._sealed(account, tail.name, tail.entries, lines), len(lines))
|
|
666
|
+
seq, self._next = self._next, self._next + 1
|
|
667
|
+
name = f"{seq:012d}.jsonl"
|
|
668
|
+
data = self._sealed(account, name, 0, lines)
|
|
669
|
+
entries = len(lines)
|
|
670
|
+
segment = Segment(seq, name, len(data), entries)
|
|
671
|
+
account.segments.append(segment)
|
|
672
|
+
try:
|
|
673
|
+
self._write_record(account, {"segment": segment.name, "entries": entries})
|
|
674
|
+
with self.directory.child(account.source) as directory:
|
|
675
|
+
directory.create(segment.name, data)
|
|
676
|
+
except OSError:
|
|
677
|
+
account.segments.remove(segment)
|
|
678
|
+
self._record(account, {"gone": segment.name})
|
|
679
|
+
raise
|
|
680
|
+
self._oldest[segment] = account
|
|
681
|
+
self.bytes += segment.size
|
|
682
|
+
|
|
683
|
+
def _sealed(self, account: Account, segment: str, first: int, lines: list[bytes]) -> bytes:
|
|
684
|
+
places = (_place_of(segment, first + n) for n in range(len(lines)))
|
|
685
|
+
return b"".join(self.seal.seal(account.source, line, place) for line, place in zip(lines, places))
|
|
686
|
+
|
|
687
|
+
def _extends(self, tail: Segment, size: int) -> bool:
|
|
688
|
+
return not (tail.sending or tail.full) and tail.size + size <= self.segment_bytes
|
|
689
|
+
|
|
690
|
+
def _extend(self, account: Account, segment: Segment, data: bytes, entries: int) -> None:
|
|
691
|
+
segment.entries += entries
|
|
692
|
+
try:
|
|
693
|
+
self._write_record(account, {"segment": segment.name, "entries": segment.entries})
|
|
694
|
+
except OSError:
|
|
695
|
+
segment.entries -= entries
|
|
696
|
+
raise
|
|
697
|
+
try:
|
|
698
|
+
with self.directory.child(account.source) as directory:
|
|
699
|
+
directory.add(segment.name, data)
|
|
700
|
+
except OSError:
|
|
701
|
+
segment.entries -= entries
|
|
702
|
+
segment.full = True
|
|
703
|
+
self._record(account, {"segment": segment.name, "entries": segment.entries})
|
|
704
|
+
raise
|
|
705
|
+
segment.size += len(data)
|
|
706
|
+
self.bytes += len(data)
|
|
707
|
+
|
|
708
|
+
def _bound(self) -> int:
|
|
709
|
+
dropped = 0
|
|
710
|
+
while self.bytes > self.max_bytes:
|
|
711
|
+
victim = next((segment for segment in self._oldest if not segment.sending), None)
|
|
712
|
+
if victim is None:
|
|
713
|
+
break
|
|
714
|
+
account = self._oldest[victim]
|
|
715
|
+
self._lose(account, victim.entries)
|
|
716
|
+
self._drop(victim)
|
|
717
|
+
dropped += victim.entries
|
|
718
|
+
return dropped
|
|
719
|
+
|
|
720
|
+
def _drop(self, segment: Segment | None) -> None:
|
|
721
|
+
"""A segment stored or let go: deleted only once the record says so, so a restarted host never counts
|
|
722
|
+
it lost for having been deleted (one the record could not drop stays until its account goes)."""
|
|
723
|
+
account = self._oldest.pop(segment, None) if segment is not None else None
|
|
724
|
+
if account is None:
|
|
725
|
+
return
|
|
726
|
+
account.segments.remove(segment)
|
|
727
|
+
self.bytes -= segment.size
|
|
728
|
+
try:
|
|
729
|
+
self._write_record(account, {"gone": segment.name})
|
|
730
|
+
with self.directory.child(account.source) as directory:
|
|
731
|
+
directory.unlink(segment.name)
|
|
732
|
+
except OSError as error:
|
|
733
|
+
log_line(f"request ledger spill: segment {segment.seq} not removed: {type(error).__name__}")
|
|
734
|
+
|
|
735
|
+
def _lose(self, account: Account, entries: int) -> None:
|
|
736
|
+
if entries <= 0:
|
|
737
|
+
return
|
|
738
|
+
account.lose(entries)
|
|
739
|
+
self._remember(account)
|
|
740
|
+
|
|
741
|
+
def _remember(self, account: Account) -> None:
|
|
742
|
+
self._record(account, {"counts": account.counts()})
|
|
743
|
+
|
|
744
|
+
def _record(self, account: Account, change: dict) -> None:
|
|
745
|
+
"""One change to the account's record, or the record rewritten as a snapshot once its changes
|
|
746
|
+
outnumber its segments; a record the disk refuses is logged, its counts kept in memory."""
|
|
747
|
+
if not account.home:
|
|
748
|
+
return
|
|
749
|
+
try:
|
|
750
|
+
self._write_record(account, change)
|
|
751
|
+
except OSError as error:
|
|
752
|
+
log_line(f"request ledger spill: session {account.session_id} record not written: {type(error).__name__}")
|
|
753
|
+
|
|
754
|
+
def _write_record(self, account: Account, change: dict) -> None:
|
|
755
|
+
if account.records >= max(RECORD_LINES_MIN, 4 * len(account.segments)):
|
|
756
|
+
return self._compact(account)
|
|
757
|
+
line = self.seal.seal(account.source, _compact(change), account.chain)
|
|
758
|
+
with self.directory.child(account.source) as directory:
|
|
759
|
+
directory.add(ACCOUNT_FILE, line)
|
|
760
|
+
account.records += 1
|
|
761
|
+
account.chain = SpillSeal.tag(line)
|
|
762
|
+
|
|
763
|
+
def _compact(self, account: Account) -> None:
|
|
764
|
+
line = self.seal.seal(account.source, _compact(account.snapshot()))
|
|
765
|
+
with self.directory.child(account.source) as directory:
|
|
766
|
+
directory.replace(ACCOUNT_FILE, line)
|
|
767
|
+
account.records = 1
|
|
768
|
+
account.chain = SpillSeal.tag(line)
|
|
769
|
+
|
|
770
|
+
def _rewrite(self, account: Account) -> None:
|
|
771
|
+
try:
|
|
772
|
+
self._compact(account)
|
|
773
|
+
except OSError as error:
|
|
774
|
+
log_line(f"request ledger spill: session {account.session_id} record not rewritten: {type(error).__name__}")
|
|
775
|
+
|
|
776
|
+
def _sweep_later(self) -> None:
|
|
777
|
+
"""Delete what was set aside on a thread of its own, once more after any sweep asked for while one
|
|
778
|
+
ran: a tree tenant code planted never holds up a pass."""
|
|
779
|
+
self._sweep_asked.set()
|
|
780
|
+
if self._sweeping.acquire(blocking=False):
|
|
781
|
+
threading.Thread(target=self._sweep, name="world-host-ledger-sweep", daemon=True).start()
|
|
782
|
+
|
|
783
|
+
def _sweep(self) -> None:
|
|
784
|
+
try:
|
|
785
|
+
self._sweep_while_asked()
|
|
786
|
+
finally:
|
|
787
|
+
self._sweeping.release()
|
|
788
|
+
if self._sweep_asked.is_set():
|
|
789
|
+
self._sweep_later()
|
|
790
|
+
|
|
791
|
+
def _sweep_while_asked(self) -> None:
|
|
792
|
+
while self._sweep_asked.is_set():
|
|
793
|
+
self._sweep_asked.clear()
|
|
794
|
+
self._sweep_once()
|
|
795
|
+
|
|
796
|
+
def _sweep_once(self) -> None:
|
|
797
|
+
try:
|
|
798
|
+
self.directory.sweep()
|
|
799
|
+
except OSError as error:
|
|
800
|
+
log_line(f"request ledger spill: sweep failed: {type(error).__name__}")
|
|
801
|
+
|
|
802
|
+
@staticmethod
|
|
803
|
+
def _dropped(entries: int) -> None:
|
|
804
|
+
if entries:
|
|
805
|
+
log_line(f"request ledger spill: let go of {entries} unsent entries past its cap")
|
|
806
|
+
|
|
807
|
+
@staticmethod
|
|
808
|
+
def _unsendable(session_id: str, entries: int) -> None:
|
|
809
|
+
if entries:
|
|
810
|
+
log_line(f"request ledger spill: session {session_id} let go of {entries} unsendable entries, counted lost")
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _mark(segment: Segment | None, sending: bool) -> Segment | None:
|
|
814
|
+
if segment is not None:
|
|
815
|
+
segment.sending = sending
|
|
816
|
+
return segment
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _seq(pair: tuple[Segment, Account]) -> int:
|
|
820
|
+
return pair[0].seq
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _place_of(segment: str, ordinal: int) -> bytes:
|
|
824
|
+
return f"{segment}:{ordinal}".encode("ascii")
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
def _place(segment: Segment, ordinal: int) -> bytes:
|
|
828
|
+
return _place_of(segment.name, ordinal)
|