@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,958 @@
|
|
|
1
|
+
"""The World Host's half of the world request ledger: every request a session's world receives,
|
|
2
|
+
through every door, waits in the session's :class:`RequestOutbox`; the :class:`LedgerFlusher` moves
|
|
3
|
+
it to the host's :class:`~.ledger_spill.LedgerSpill` on disk, and from there to the platform.
|
|
4
|
+
|
|
5
|
+
Only the platform's acknowledgement lets an entry go from the disk. The grader's call log
|
|
6
|
+
(:class:`~.call_log.CallLog`) keeps its own bounded view; no bound of it reaches the ledger. An outbox
|
|
7
|
+
is moved to disk every few seconds and as soon as it passes half its ceiling, so it reaches its
|
|
8
|
+
ceilings (bodies shed past :data:`OUTBOX_MAX_CHARS`, entries dropped past :data:`OUTBOX_MAX_ENTRIES`)
|
|
9
|
+
only while that move is held up. What an outbox sheds or drops, and what the disk's cap lets go, is
|
|
10
|
+
counted in the session's account and stored with it. Bodies are never logged."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import collections
|
|
15
|
+
import enum
|
|
16
|
+
import http.client
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
import urllib.error
|
|
22
|
+
import urllib.request
|
|
23
|
+
import uuid
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from functools import partial
|
|
27
|
+
from typing import IO, TYPE_CHECKING, Callable, Iterator, Mapping, Protocol
|
|
28
|
+
|
|
29
|
+
from ..world_session_contract import REQUEST_LOG_PAGE_MAX
|
|
30
|
+
from .call_capture import Exchange
|
|
31
|
+
from .call_log import CallJournal
|
|
32
|
+
from .handler_errors import log_line
|
|
33
|
+
from .ledger_spill import HEX32, LEDGER_BATCH_BYTES, LedgerSpill, Piece, counted, ledger_line, runs
|
|
34
|
+
from .validation import fail
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from .host_session import Session
|
|
38
|
+
from .world_host import WorldHost
|
|
39
|
+
|
|
40
|
+
LEDGER_FLUSH_S = 2.0
|
|
41
|
+
LEDGER_STOP_S = 10.0
|
|
42
|
+
LEDGER_BARRIER_S = 20.0
|
|
43
|
+
BARRIER_POLL_S = 0.05
|
|
44
|
+
NO_PLATFORM = "this World Host sends no request ledger: it has no platform to send to"
|
|
45
|
+
INHERITED_REQUESTS = (
|
|
46
|
+
"this session was reopened from a checkpoint or recovered after its worker or host stopped, "
|
|
47
|
+
"so the requests its previous holder had not stored cannot be proven"
|
|
48
|
+
)
|
|
49
|
+
INHERITED_MAX_CHARS = 500
|
|
50
|
+
INGEST_ANSWER_BYTES = 64 * 1024
|
|
51
|
+
LEDGER_ROUNDS = 10
|
|
52
|
+
LEDGER_BACKOFF_MAX_S = 60.0
|
|
53
|
+
LEDGER_ROUTE = "/api/public/world-hosts/requests"
|
|
54
|
+
OUTBOX_MAX_ENTRIES = 20_000
|
|
55
|
+
OUTBOX_MAX_CHARS = 8 << 20
|
|
56
|
+
TAKE_MAX_PAGES = 2 * OUTBOX_MAX_ENTRIES // REQUEST_LOG_PAGE_MAX
|
|
57
|
+
TAKE_DEADLINE_S = 5.0
|
|
58
|
+
LEDGER_PASS_S = 10.0
|
|
59
|
+
LEFT_JOURNAL_BYTES = 64 << 20
|
|
60
|
+
DRAIN_BYTES = 4 << 20
|
|
61
|
+
DRAIN_ANSWER_BYTES = DRAIN_BYTES + (1 << 20)
|
|
62
|
+
OUTBOX_CURSOR = re.compile(r"[0-9a-f]{32}:[0-9]{1,18}")
|
|
63
|
+
RETRIED_STATUSES = frozenset({401, 403, 408, 429})
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _now() -> str:
|
|
67
|
+
return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _size(value: object) -> int:
|
|
71
|
+
return len(json.dumps(value, ensure_ascii=False).encode("utf-8", "replace"))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _late(until: float | None) -> bool:
|
|
75
|
+
return until is not None and time.monotonic() >= until
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _timeout(until: float | None) -> float | None:
|
|
79
|
+
return None if until is None else max(0.05, until - time.monotonic())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class MalformedPage(ValueError):
|
|
83
|
+
"""A drain answer the flusher cannot use: a version worker runs tenant code."""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass
|
|
87
|
+
class _Budget:
|
|
88
|
+
"""What a take still may move (``bytes``) and the pages it moved."""
|
|
89
|
+
|
|
90
|
+
bytes: int
|
|
91
|
+
pages: int = 0
|
|
92
|
+
|
|
93
|
+
def spend(self, size: int) -> None:
|
|
94
|
+
self.bytes -= size
|
|
95
|
+
self.pages += 1
|
|
96
|
+
|
|
97
|
+
def spent(self) -> bool:
|
|
98
|
+
return self.bytes <= 0
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True)
|
|
102
|
+
class OutboxPage:
|
|
103
|
+
"""One drain answer as the flusher may trust it (:meth:`of`): each entry as the line the spill
|
|
104
|
+
keeps, None for one the platform would not parse; ``epoch`` names the outbox that answered,
|
|
105
|
+
``pending`` how many entries it holds after this page and ``inherited`` why it cannot say it holds
|
|
106
|
+
every request it was given."""
|
|
107
|
+
|
|
108
|
+
lines: list[bytes | None]
|
|
109
|
+
cursor: str | None
|
|
110
|
+
more: bool
|
|
111
|
+
epoch: str
|
|
112
|
+
shed: int
|
|
113
|
+
lost: int
|
|
114
|
+
pending: int
|
|
115
|
+
inherited: str | None = None
|
|
116
|
+
source: str | None = None
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def of(cls, answer: object) -> OutboxPage:
|
|
120
|
+
page = answer if isinstance(answer, dict) else {}
|
|
121
|
+
requests, cursor, epoch, inherited, source = (
|
|
122
|
+
page.get(key) for key in ("requests", "cursor", "epoch", "inherited", "source")
|
|
123
|
+
)
|
|
124
|
+
valid = (
|
|
125
|
+
isinstance(requests, list)
|
|
126
|
+
and isinstance(epoch, str)
|
|
127
|
+
and bool(HEX32.fullmatch(epoch))
|
|
128
|
+
and counted(page.get("shed"))
|
|
129
|
+
and counted(page.get("lost"))
|
|
130
|
+
and counted(page.get("pending"))
|
|
131
|
+
and isinstance(page.get("more"), bool)
|
|
132
|
+
and (cursor is None if not requests else isinstance(cursor, str) and bool(OUTBOX_CURSOR.fullmatch(cursor)))
|
|
133
|
+
and (inherited is None or (isinstance(inherited, str) and len(inherited) <= INHERITED_MAX_CHARS))
|
|
134
|
+
and (source is None or LedgerLife.sourced(source))
|
|
135
|
+
)
|
|
136
|
+
if not valid:
|
|
137
|
+
raise MalformedPage("a drain answer off the outbox's contract")
|
|
138
|
+
lines = [ledger_line(entry) for entry in requests]
|
|
139
|
+
return cls(lines, cursor, page["more"], epoch, page["shed"], page["lost"], page["pending"], inherited, source)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def size(self) -> int:
|
|
143
|
+
return sum(len(line or b"") for line in self.lines)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class MoveFailed(Exception):
|
|
147
|
+
"""A move of a session's outbox that failed once the outbox had answered ``page``: ``left`` is what it
|
|
148
|
+
still holds, the page's own entries included unless they were ``kept`` on the spill."""
|
|
149
|
+
|
|
150
|
+
def __init__(self, page: OutboxPage, kept: bool) -> None:
|
|
151
|
+
super().__init__("a move of a session's outbox to the spill failed")
|
|
152
|
+
self.left = page.pending + (0 if kept else len(page.lines))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class RequestOutbox:
|
|
156
|
+
"""One session's requests on their way to the ledger, oldest first, each under the id
|
|
157
|
+
the ledger keeps it by and the outbox's own ``seq``. :meth:`drain` is the only way out:
|
|
158
|
+
handed the ``cursor`` of the last page the host took, it lets go of every entry up to it
|
|
159
|
+
and answers the next page. A cursor names the outbox it came from (``epoch``), so one a
|
|
160
|
+
replaced outbox answered lets go of nothing here. Past half of either ceiling it calls
|
|
161
|
+
``on_pressure``, which wakes the host's flusher. A worker's session journals its outbox
|
|
162
|
+
beside its store (:data:`~.session_files.LEDGER_JOURNAL`), rewritten from memory once
|
|
163
|
+
twice the character ceiling was appended, so a worker that dies leaves its unsent
|
|
164
|
+
entries to the next one (:meth:`resume`)."""
|
|
165
|
+
|
|
166
|
+
def __init__(
|
|
167
|
+
self,
|
|
168
|
+
journal: CallJournal | None = None,
|
|
169
|
+
*,
|
|
170
|
+
max_entries: int = OUTBOX_MAX_ENTRIES,
|
|
171
|
+
max_chars: int = OUTBOX_MAX_CHARS,
|
|
172
|
+
on_pressure: Callable[[], None] | None = None,
|
|
173
|
+
) -> None:
|
|
174
|
+
self.journal = journal
|
|
175
|
+
self.max_entries = max_entries
|
|
176
|
+
self.max_chars = max_chars
|
|
177
|
+
self.on_pressure = on_pressure
|
|
178
|
+
self.epoch = uuid.uuid4().hex
|
|
179
|
+
self.entries: collections.deque[dict] = collections.deque()
|
|
180
|
+
self._charges: collections.deque[int] = collections.deque()
|
|
181
|
+
self._unshed = 0
|
|
182
|
+
self.chars = 0
|
|
183
|
+
self.seq = 0
|
|
184
|
+
self.shed = 0
|
|
185
|
+
self.lost = 0
|
|
186
|
+
self.inherited: str | None = None
|
|
187
|
+
self._lock = threading.Lock()
|
|
188
|
+
|
|
189
|
+
def append(self, line: dict, exchange: Exchange) -> None:
|
|
190
|
+
at = _now()
|
|
191
|
+
with self._lock:
|
|
192
|
+
self.seq += 1
|
|
193
|
+
entry = {"id": uuid.uuid4().hex, "seq": self.seq, "at": at, **line, **exchange.as_json()}
|
|
194
|
+
self._hold(entry, exchange.charge)
|
|
195
|
+
self._journal(entry)
|
|
196
|
+
pressed = 2 * self.chars > self.max_chars or 2 * len(self.entries) > self.max_entries
|
|
197
|
+
self._bound()
|
|
198
|
+
if pressed and self.on_pressure is not None:
|
|
199
|
+
self.on_pressure()
|
|
200
|
+
|
|
201
|
+
def drain(self, cursor: str | None, limit: int = REQUEST_LOG_PAGE_MAX, max_bytes: int = DRAIN_BYTES) -> dict:
|
|
202
|
+
"""Let go of every entry up to ``cursor`` and answer the entries after it, at most
|
|
203
|
+
``limit`` and as many as fit ``max_bytes`` of JSON (always one), with the
|
|
204
|
+
``cursor`` that lets go of them, whether ``more`` wait and how many (``pending``), and the
|
|
205
|
+
outbox's running counts."""
|
|
206
|
+
with self._lock:
|
|
207
|
+
if self._ack(cursor):
|
|
208
|
+
self._rewrite()
|
|
209
|
+
page, spent = [], 0
|
|
210
|
+
for entry in self.entries:
|
|
211
|
+
spent += _size(entry)
|
|
212
|
+
if len(page) == limit or (page and spent > max_bytes):
|
|
213
|
+
break
|
|
214
|
+
page.append(entry)
|
|
215
|
+
return {
|
|
216
|
+
"requests": page,
|
|
217
|
+
"cursor": f"{self.epoch}:{page[-1]['seq']}" if page else None,
|
|
218
|
+
"more": len(self.entries) > len(page),
|
|
219
|
+
"pending": len(self.entries) - len(page),
|
|
220
|
+
"epoch": self.epoch,
|
|
221
|
+
"shed": self.shed,
|
|
222
|
+
"lost": self.lost,
|
|
223
|
+
**({"inherited": self.inherited} if self.inherited else {}),
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
def resume(self, entries: list[dict], inherited: str = INHERITED_REQUESTS) -> None:
|
|
227
|
+
"""Carry on with the entries a previous holder kept, numbered on from the last; what it
|
|
228
|
+
had not kept is unknown, so the outbox can no longer say it holds every request."""
|
|
229
|
+
with self._lock:
|
|
230
|
+
self.inherited = inherited
|
|
231
|
+
for entry in entries:
|
|
232
|
+
self._hold(entry, Exchange.from_json(entry).charge)
|
|
233
|
+
self.seq = max([self.seq, *(int(entry["seq"]) for entry in entries)])
|
|
234
|
+
self._unshed = 0
|
|
235
|
+
self._bound()
|
|
236
|
+
self._rewrite()
|
|
237
|
+
|
|
238
|
+
def _hold(self, entry: dict, charge: int) -> None:
|
|
239
|
+
self.entries.append(entry)
|
|
240
|
+
self._charges.append(charge)
|
|
241
|
+
self.chars += charge
|
|
242
|
+
|
|
243
|
+
def _pop(self) -> None:
|
|
244
|
+
self.entries.popleft()
|
|
245
|
+
self.chars -= self._charges.popleft()
|
|
246
|
+
self._unshed = max(0, self._unshed - 1)
|
|
247
|
+
|
|
248
|
+
def _ack(self, cursor: str | None) -> bool:
|
|
249
|
+
epoch, _, seq = (cursor or "").partition(":")
|
|
250
|
+
if epoch != self.epoch:
|
|
251
|
+
return False
|
|
252
|
+
acked = False
|
|
253
|
+
while self.entries and int(self.entries[0]["seq"]) <= int(seq):
|
|
254
|
+
self._pop()
|
|
255
|
+
acked = True
|
|
256
|
+
return acked
|
|
257
|
+
|
|
258
|
+
def _bound(self) -> None:
|
|
259
|
+
while self.chars > self.max_chars and self._unshed < len(self.entries):
|
|
260
|
+
self._shed(self._unshed)
|
|
261
|
+
self._unshed += 1
|
|
262
|
+
while len(self.entries) > self.max_entries:
|
|
263
|
+
self._pop()
|
|
264
|
+
self.lost += 1
|
|
265
|
+
|
|
266
|
+
def _shed(self, index: int) -> None:
|
|
267
|
+
charge = self._charges[index]
|
|
268
|
+
if charge == 0:
|
|
269
|
+
return
|
|
270
|
+
entry = self.entries[index]
|
|
271
|
+
self.entries[index] = {**entry, **Exchange.from_json(entry).shed().as_json()}
|
|
272
|
+
self._charges[index] = 0
|
|
273
|
+
self.chars -= charge
|
|
274
|
+
self.shed += 1
|
|
275
|
+
|
|
276
|
+
def _journal(self, entry: dict) -> None:
|
|
277
|
+
if self.journal is None:
|
|
278
|
+
return
|
|
279
|
+
self.journal.append(CallJournal.line(entry))
|
|
280
|
+
if self.journal.appended > 2 * self.max_chars:
|
|
281
|
+
self._rewrite()
|
|
282
|
+
|
|
283
|
+
def _rewrite(self) -> None:
|
|
284
|
+
if self.journal is not None:
|
|
285
|
+
self.journal.rewrite(list(self.entries), 8 * self.max_chars)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
@dataclass
|
|
289
|
+
class LedgerLife:
|
|
290
|
+
"""The platform's account of one life of a session (``source``, from its open): the host keeps the
|
|
291
|
+
life's first outbox (``epoch``, the first a drain answers) under it, so that account alone settles
|
|
292
|
+
it, and every later outbox, a dead worker's included, under an account of its own."""
|
|
293
|
+
|
|
294
|
+
source: str | None = None
|
|
295
|
+
epoch: str | None = None
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def sourced(source: object) -> bool:
|
|
299
|
+
return isinstance(source, str) and bool(HEX32.fullmatch(source)) and source != "0" * 32
|
|
300
|
+
|
|
301
|
+
@classmethod
|
|
302
|
+
def of(cls, source: object) -> LedgerLife:
|
|
303
|
+
if source is not None and not cls.sourced(source):
|
|
304
|
+
fail("host", "ledgerSource is the 32-hex request ledger account the platform opened for this life")
|
|
305
|
+
return cls(source if isinstance(source, str) else None)
|
|
306
|
+
|
|
307
|
+
def stamp(self, page: dict) -> dict:
|
|
308
|
+
epoch = page.get("epoch") if isinstance(page, dict) else None
|
|
309
|
+
if not isinstance(epoch, str):
|
|
310
|
+
return page
|
|
311
|
+
self.epoch = self.epoch or epoch
|
|
312
|
+
return {**page, "source": self.source if epoch == self.epoch else None}
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass(frozen=True)
|
|
316
|
+
class LedgerAccount:
|
|
317
|
+
"""This host's account of one session's request ledger as the session leaves it: ``final``
|
|
318
|
+
once everything the host held of it is stored, with what its outbox shed and lost."""
|
|
319
|
+
|
|
320
|
+
final: bool
|
|
321
|
+
shed: int = 0
|
|
322
|
+
lost: int = 0
|
|
323
|
+
|
|
324
|
+
def as_json(self) -> dict:
|
|
325
|
+
return {"final": self.final, "shed": self.shed, "lost": self.lost}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
class HostLedger(Protocol):
|
|
329
|
+
"""What a host tells its ledger: a session's outbox is filling (``wake``), a session is leaving
|
|
330
|
+
(``leave``), a session is about to move to another host and its requests must be stored first
|
|
331
|
+
(``settle``), a dead worker left a session's unsent entries (``adopt``), the host is stopping
|
|
332
|
+
(``stop``, which gives ``settle`` half its time first), a grade waits until a session's requests
|
|
333
|
+
are stored (``barrier``)."""
|
|
334
|
+
|
|
335
|
+
def wake(self, session_id: str) -> None: ...
|
|
336
|
+
|
|
337
|
+
def leave(self, session: Session) -> None: ...
|
|
338
|
+
|
|
339
|
+
def settle(self, session: Session, until: float) -> LedgerAccount: ...
|
|
340
|
+
|
|
341
|
+
def adopt(self, session_id: str, entries: list[dict]) -> None: ...
|
|
342
|
+
|
|
343
|
+
def stop(self, deadline_s: float = ..., *, settle: Callable[[float], object] = ...) -> None: ...
|
|
344
|
+
|
|
345
|
+
def barrier(self, session: Session) -> dict: ...
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _settled(deadline_s: float) -> bool:
|
|
349
|
+
return True
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
class NoLedger:
|
|
353
|
+
"""The ledger of a host with no platform to send to: a version worker, whose supervisor
|
|
354
|
+
drains its outboxes, or a dev host, whose outboxes keep to their ceilings and whose
|
|
355
|
+
accounts are never final."""
|
|
356
|
+
|
|
357
|
+
def wake(self, session_id: str) -> None:
|
|
358
|
+
return None
|
|
359
|
+
|
|
360
|
+
def leave(self, session: Session) -> None:
|
|
361
|
+
return None
|
|
362
|
+
|
|
363
|
+
def settle(self, session: Session, until: float) -> LedgerAccount:
|
|
364
|
+
return LedgerAccount(final=False)
|
|
365
|
+
|
|
366
|
+
def adopt(self, session_id: str, entries: list[dict]) -> None:
|
|
367
|
+
return None
|
|
368
|
+
|
|
369
|
+
def stop(self, deadline_s: float = LEDGER_STOP_S, *, settle: Callable[[float], object] = _settled) -> None:
|
|
370
|
+
settle(deadline_s)
|
|
371
|
+
|
|
372
|
+
def barrier(self, session: Session) -> dict:
|
|
373
|
+
return _unstored(NO_PLATFORM)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _unstored(reason: str, shed: int = 0, lost: int = 0) -> dict:
|
|
377
|
+
return {"stored": False, "shed": shed, "lost": lost, "reason": reason}
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
class Delivery(enum.Enum):
|
|
381
|
+
"""What came of one send: the platform stored it, refused it as sent, or did not answer
|
|
382
|
+
(no answer, a server error, a throttle, or a refusal of the host itself)."""
|
|
383
|
+
|
|
384
|
+
STORED = "stored"
|
|
385
|
+
REFUSED = "refused"
|
|
386
|
+
UNAVAILABLE = "unavailable"
|
|
387
|
+
|
|
388
|
+
@classmethod
|
|
389
|
+
def of(cls, status: int) -> Delivery:
|
|
390
|
+
if 200 <= status < 300:
|
|
391
|
+
return cls.STORED
|
|
392
|
+
refused = 400 <= status < 500 and status not in RETRIED_STATUSES
|
|
393
|
+
return cls.REFUSED if refused else cls.UNAVAILABLE
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
@dataclass(frozen=True)
|
|
397
|
+
class Answer:
|
|
398
|
+
"""What came of one send, with how many of its requests the platform stored as refused (off its
|
|
399
|
+
contract), each session's when the platform said (``by_session``); a sink may answer a bare
|
|
400
|
+
:class:`Delivery`, a bool (stored or not) or the refused count."""
|
|
401
|
+
|
|
402
|
+
delivery: Delivery
|
|
403
|
+
refused: int = 0
|
|
404
|
+
by_session: Mapping[str, int] | None = None
|
|
405
|
+
|
|
406
|
+
@classmethod
|
|
407
|
+
def of(cls, answer: object) -> Answer:
|
|
408
|
+
if isinstance(answer, Answer):
|
|
409
|
+
return answer
|
|
410
|
+
if isinstance(answer, Delivery):
|
|
411
|
+
return cls(answer)
|
|
412
|
+
if answer is True:
|
|
413
|
+
return cls(Delivery.STORED)
|
|
414
|
+
stored = counted(answer) and answer is not False
|
|
415
|
+
return cls(Delivery.STORED, int(answer)) if stored else cls(Delivery.UNAVAILABLE)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
class LedgerSink(Protocol):
|
|
419
|
+
"""Where a flush goes, sent by ``until`` (a monotonic time) when one is given."""
|
|
420
|
+
|
|
421
|
+
def send(self, sessions: list[dict], until: float | None = None) -> Answer | Delivery | int | bool | None: ...
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
class _Unredirected(urllib.request.HTTPRedirectHandler):
|
|
425
|
+
def redirect_request(
|
|
426
|
+
self,
|
|
427
|
+
req: urllib.request.Request,
|
|
428
|
+
fp: IO[bytes],
|
|
429
|
+
code: int,
|
|
430
|
+
msg: str,
|
|
431
|
+
headers: http.client.HTTPMessage,
|
|
432
|
+
newurl: str,
|
|
433
|
+
) -> None:
|
|
434
|
+
return None
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
class PlatformLedger:
|
|
438
|
+
"""The platform's ledger door (:data:`LEDGER_ROUTE`), behind the control-plane secret; an answer that
|
|
439
|
+
redirects the flush is not followed, so it reads as unavailable, never as stored."""
|
|
440
|
+
|
|
441
|
+
_opener = urllib.request.build_opener(_Unredirected)
|
|
442
|
+
|
|
443
|
+
def __init__(self, platform_url: str, secret: str, timeout_s: float = 30.0, host_url: str | None = None) -> None:
|
|
444
|
+
self.url = f"{platform_url.rstrip('/')}{LEDGER_ROUTE}"
|
|
445
|
+
self.secret = secret
|
|
446
|
+
self.timeout_s = timeout_s
|
|
447
|
+
self.host = {"host": host_url} if host_url else {}
|
|
448
|
+
|
|
449
|
+
def send(self, sessions: list[dict], until: float | None = None) -> Answer:
|
|
450
|
+
flush = json.dumps(
|
|
451
|
+
{**self.host, "sessions": sessions}, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
|
452
|
+
)
|
|
453
|
+
body = flush.encode("utf-8", "replace")
|
|
454
|
+
headers = {"content-type": "application/json", "x-host-secret": self.secret}
|
|
455
|
+
request = urllib.request.Request(self.url, data=body, method="POST", headers=headers)
|
|
456
|
+
timeout = self.timeout_s if until is None else min(self.timeout_s, until - time.monotonic())
|
|
457
|
+
if timeout <= 0:
|
|
458
|
+
return Answer(Delivery.UNAVAILABLE)
|
|
459
|
+
try:
|
|
460
|
+
with self._opener.open(request, timeout=timeout) as answer:
|
|
461
|
+
delivery = Delivery.of(answer.status)
|
|
462
|
+
said = _said(answer.read(INGEST_ANSWER_BYTES)) if delivery is Delivery.STORED else {}
|
|
463
|
+
return Answer(delivery, _refused(said), _refused_sessions(said))
|
|
464
|
+
except urllib.error.HTTPError as error:
|
|
465
|
+
error.close()
|
|
466
|
+
log_line(f"request ledger: the platform answered a flush {error.code}")
|
|
467
|
+
return Answer(Delivery.of(error.code))
|
|
468
|
+
except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
|
|
469
|
+
log_line(f"request ledger: the platform did not take a flush: {type(error).__name__}")
|
|
470
|
+
return Answer(Delivery.UNAVAILABLE)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _said(answer: bytes) -> dict:
|
|
474
|
+
try:
|
|
475
|
+
said = json.loads(answer or b"{}")
|
|
476
|
+
except ValueError:
|
|
477
|
+
return {}
|
|
478
|
+
return said if isinstance(said, dict) else {}
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _refused(said: dict) -> int:
|
|
482
|
+
refused = said.get("refused", 0)
|
|
483
|
+
return refused if counted(refused) else 0
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _refused_sessions(said: dict) -> dict[str, int] | None:
|
|
487
|
+
sessions = said.get("refusedSessions")
|
|
488
|
+
if not isinstance(sessions, dict):
|
|
489
|
+
return None
|
|
490
|
+
return {sid: n for sid, n in sessions.items() if isinstance(sid, str) and counted(n) and n > 0}
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
@dataclass
|
|
494
|
+
class SendBackoff:
|
|
495
|
+
"""How long the sender waits after a round the platform took none of: ``first_s``,
|
|
496
|
+
doubling to ``max_s``; a round it stores ends the wait. Nothing else cuts it short."""
|
|
497
|
+
|
|
498
|
+
first_s: float = LEDGER_FLUSH_S
|
|
499
|
+
max_s: float = LEDGER_BACKOFF_MAX_S
|
|
500
|
+
delay_s: float = 0.0
|
|
501
|
+
until: float = 0.0
|
|
502
|
+
|
|
503
|
+
def ready(self) -> bool:
|
|
504
|
+
return time.monotonic() >= self.until
|
|
505
|
+
|
|
506
|
+
def failed(self) -> None:
|
|
507
|
+
self.delay_s = min(self.max_s, max(self.first_s, 2 * self.delay_s))
|
|
508
|
+
self.until = time.monotonic() + self.delay_s
|
|
509
|
+
|
|
510
|
+
def cleared(self) -> None:
|
|
511
|
+
self.delay_s, self.until = 0.0, 0.0
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
class LedgerFlusher:
|
|
515
|
+
"""Moves the host's outboxes to its spill and the spill to a :class:`LedgerSink`, on two
|
|
516
|
+
threads. The spiller moves every live session's outbox to disk every ``interval_s``, and an
|
|
517
|
+
outbox past half its ceiling at once; it never waits on the platform. The sender sends the
|
|
518
|
+
spill oldest first, one piece of every session a round, batched across sessions up to
|
|
519
|
+
``batch_bytes`` a POST, each piece let go of once stored. A round the platform takes none of
|
|
520
|
+
starts a :class:`SendBackoff`; a batch it refuses as sent is resent a session at a time, and
|
|
521
|
+
a session refused alone is held for ``backoff.max_s`` while the rest go on. A session
|
|
522
|
+
leaving the host (:meth:`leave`), a dead worker's journal (:meth:`adopt`) and every session
|
|
523
|
+
when the host stops (:meth:`stop`) go to the spill too, their accounts closing once stored.
|
|
524
|
+
A worker runs tenant code, so a leaving session's take moves at most ``take_pages`` pages and
|
|
525
|
+
``take_bytes`` bytes, within ``take_s`` of the worker's first answer, and counts the rest lost
|
|
526
|
+
(``takes_cut`` counts the cuts). A drain answer off the contract, or any failure, is that
|
|
527
|
+
session's alone."""
|
|
528
|
+
|
|
529
|
+
def __init__(
|
|
530
|
+
self,
|
|
531
|
+
host: WorldHost,
|
|
532
|
+
sink: LedgerSink,
|
|
533
|
+
spill: LedgerSpill,
|
|
534
|
+
*,
|
|
535
|
+
interval_s: float = LEDGER_FLUSH_S,
|
|
536
|
+
batch_bytes: int = LEDGER_BATCH_BYTES,
|
|
537
|
+
rounds: int = LEDGER_ROUNDS,
|
|
538
|
+
backoff: SendBackoff | None = None,
|
|
539
|
+
take_pages: int = TAKE_MAX_PAGES,
|
|
540
|
+
take_s: float = TAKE_DEADLINE_S,
|
|
541
|
+
take_bytes: int | None = None,
|
|
542
|
+
) -> None:
|
|
543
|
+
self.host = host
|
|
544
|
+
self.sink = sink
|
|
545
|
+
self.spill = spill
|
|
546
|
+
self.interval_s = interval_s
|
|
547
|
+
self.batch_bytes = batch_bytes
|
|
548
|
+
self.rounds = rounds
|
|
549
|
+
self.backoff = backoff or SendBackoff(interval_s)
|
|
550
|
+
self.take_pages = take_pages
|
|
551
|
+
self.take_s = take_s
|
|
552
|
+
self.take_bytes = spill.max_bytes // 2 if take_bytes is None else take_bytes
|
|
553
|
+
self.takes_cut = 0
|
|
554
|
+
self.acked: dict[str, str] = {}
|
|
555
|
+
self.refused: collections.Counter[str] = collections.Counter()
|
|
556
|
+
self._moving: collections.Counter[str] = collections.Counter()
|
|
557
|
+
self._held: dict[str, float] = {}
|
|
558
|
+
self._pressed: set[str] = set()
|
|
559
|
+
self._next_pass = 0.0
|
|
560
|
+
self._turn = 0
|
|
561
|
+
self._lock = threading.Lock()
|
|
562
|
+
self._stop = threading.Event()
|
|
563
|
+
self._spill_due = threading.Event()
|
|
564
|
+
self._send_due = threading.Event()
|
|
565
|
+
self._threads: list[threading.Thread] = []
|
|
566
|
+
|
|
567
|
+
def start(self) -> LedgerFlusher:
|
|
568
|
+
self._threads = [
|
|
569
|
+
threading.Thread(target=self._spilling, name="world-host-ledger-spill", daemon=True),
|
|
570
|
+
threading.Thread(target=self._sending, name="world-host-ledger-send", daemon=True),
|
|
571
|
+
]
|
|
572
|
+
for thread in self._threads:
|
|
573
|
+
thread.start()
|
|
574
|
+
return self
|
|
575
|
+
|
|
576
|
+
def wake(self, session_id: str) -> None:
|
|
577
|
+
with self._lock:
|
|
578
|
+
self._pressed.add(session_id)
|
|
579
|
+
self._spill_due.set()
|
|
580
|
+
|
|
581
|
+
def stop(self, deadline_s: float = LEDGER_STOP_S, *, settle: Callable[[float], object] = _settled) -> None:
|
|
582
|
+
"""Within ``deadline_s`` in all, the time the host's grace leaves after its drain: let the last
|
|
583
|
+
finishes run (``settle``, half of it, both threads still running), then stop both threads, send
|
|
584
|
+
what the spill holds, move every live outbox to it and close each account (complete when
|
|
585
|
+
everything was moved or counted), and send once more: no drain or send outlives it, and what is
|
|
586
|
+
not stored by then stays on disk for the next host on this cache."""
|
|
587
|
+
until = time.monotonic() + deadline_s
|
|
588
|
+
settle(deadline_s / 2)
|
|
589
|
+
self._stop.set()
|
|
590
|
+
self._spill_due.set()
|
|
591
|
+
self._send_due.set()
|
|
592
|
+
for thread in self._threads:
|
|
593
|
+
thread.join(max(0.0, until - time.monotonic()))
|
|
594
|
+
self._guarded(partial(self._last, until))
|
|
595
|
+
|
|
596
|
+
def flush(self, until: float | None = None) -> None:
|
|
597
|
+
self.spill_live(until)
|
|
598
|
+
self.replay(until)
|
|
599
|
+
|
|
600
|
+
def leave(self, session: Session) -> None:
|
|
601
|
+
"""A take for a session whose state goes next: what the take could not reach is counted lost
|
|
602
|
+
and the account closes, complete unless the move failed."""
|
|
603
|
+
self._leave(session.session_id, partial(self.host.ledger_page, session))
|
|
604
|
+
with self._lock:
|
|
605
|
+
self.acked.pop(session.session_id, None)
|
|
606
|
+
|
|
607
|
+
def settle(self, session: Session, until: float) -> LedgerAccount:
|
|
608
|
+
"""Send everything this host holds of a session about to move to another host, by ``until``:
|
|
609
|
+
its outbox moved to the spill and every piece of it stored. Final when all of it is stored and
|
|
610
|
+
no account of it closed incomplete; what the platform refused of it is counted lost, so an
|
|
611
|
+
account is never final over a refusal it does not count. The next host keeps its own."""
|
|
612
|
+
session_id = session.session_id
|
|
613
|
+
page, sent = self._durable(session_id, partial(self.host.ledger_page, session), until)
|
|
614
|
+
shed, lost = self.spill.counts(session_id)
|
|
615
|
+
with self._lock:
|
|
616
|
+
refused = self.refused[session_id]
|
|
617
|
+
final = page is not None and not page.more and sent and not self.spill.incomplete(session_id)
|
|
618
|
+
return LedgerAccount(final, shed, lost + refused)
|
|
619
|
+
|
|
620
|
+
def barrier(self, session: Session, deadline_s: float = LEDGER_BARRIER_S) -> dict:
|
|
621
|
+
"""A grade's wait: the session's outbox moved to the spill and every piece of it sent, by
|
|
622
|
+
``deadline_s``; stored only when nothing of it waits here, the platform refused none of it and
|
|
623
|
+
the outbox can say it holds every request it was given. What was shed or lost is counted."""
|
|
624
|
+
session_id = session.session_id
|
|
625
|
+
until = time.monotonic() + deadline_s
|
|
626
|
+
page, sent = self._durable(session_id, partial(self.host.requests, session_id), until)
|
|
627
|
+
shed, lost = self.spill.counts(session_id)
|
|
628
|
+
held = session_id in self._holding()
|
|
629
|
+
with self._lock:
|
|
630
|
+
refused = self.refused[session_id]
|
|
631
|
+
reasons = (
|
|
632
|
+
(page is None, "the session's outbox could not be read"),
|
|
633
|
+
(page is not None and page.more, "the session's outbox was not moved before the deadline"),
|
|
634
|
+
(held, "the platform refused the session's requests as sent; they wait on this host"),
|
|
635
|
+
(not sent, "the platform did not take the session's requests before the deadline"),
|
|
636
|
+
(refused > 0, f"the platform refused {refused} requests of a flush that carried this session's"),
|
|
637
|
+
(page is not None and page.inherited is not None, page.inherited if page is not None else None),
|
|
638
|
+
)
|
|
639
|
+
reason = next((said for found, said in reasons if found), None)
|
|
640
|
+
return _unstored(reason, shed, lost + refused) if reason else {"stored": True, "shed": shed, "lost": lost}
|
|
641
|
+
|
|
642
|
+
def _durable(self, session_id: str, drain: Callable[..., object], until: float) -> tuple[OutboxPage | None, bool]:
|
|
643
|
+
"""The session's outbox moved to the spill and its pieces sent by ``until``: the last page (None
|
|
644
|
+
when it could not be read) and whether nothing of the session waits here any more."""
|
|
645
|
+
with self._lock:
|
|
646
|
+
self._moving[session_id] += 1
|
|
647
|
+
try:
|
|
648
|
+
page = self._each(session_id, partial(self._drained, session_id, drain, until))
|
|
649
|
+
finally:
|
|
650
|
+
with self._lock:
|
|
651
|
+
self._moving[session_id] -= 1
|
|
652
|
+
self._moving += collections.Counter()
|
|
653
|
+
moved = isinstance(page, OutboxPage)
|
|
654
|
+
return (page if moved else None), moved and self._sent(session_id, until)
|
|
655
|
+
|
|
656
|
+
def _drained(self, session_id: str, drain: Callable[..., object], until: float) -> OutboxPage | None:
|
|
657
|
+
"""Until a drain answers nothing, so the outbox has let go of everything the spill holds."""
|
|
658
|
+
page = self._move(session_id, drain, 1, until)
|
|
659
|
+
while page is not None and (page.more or page.lines) and not _late(until):
|
|
660
|
+
page = self._move(session_id, drain, TAKE_MAX_PAGES, until)
|
|
661
|
+
return page
|
|
662
|
+
|
|
663
|
+
def _sent(self, session_id: str, until: float) -> bool:
|
|
664
|
+
while self.spill.pending(session_id):
|
|
665
|
+
held = session_id in self._holding()
|
|
666
|
+
if held or _late(until):
|
|
667
|
+
return False
|
|
668
|
+
pieces = self.spill.round(self._holding(), only=session_id)
|
|
669
|
+
stored = self._send_round(pieces, until) if pieces else None
|
|
670
|
+
if stored is False:
|
|
671
|
+
return False
|
|
672
|
+
time.sleep(0 if pieces else BARRIER_POLL_S)
|
|
673
|
+
return True
|
|
674
|
+
|
|
675
|
+
def adopt(self, session_id: str, entries: list[dict]) -> None:
|
|
676
|
+
"""A session's entries a dead worker's journal kept, moved to the spill; its account closes
|
|
677
|
+
incomplete, since a dead worker's last entries are not known to be in its journal."""
|
|
678
|
+
for run in runs([ledger_line(entry) for entry in entries], self.batch_bytes):
|
|
679
|
+
self._each(session_id, partial(self._adopted, session_id, run))
|
|
680
|
+
self._each(session_id, partial(self.spill.finished, session_id, False))
|
|
681
|
+
|
|
682
|
+
def _adopted(self, session_id: str, run: list[bytes | None]) -> None:
|
|
683
|
+
"""A run of a dead worker's entries into the spill; one the disk refuses is counted lost, since
|
|
684
|
+
nothing else holds it any more."""
|
|
685
|
+
try:
|
|
686
|
+
self.spill.write(session_id, run)
|
|
687
|
+
except OSError:
|
|
688
|
+
self.spill.lose(session_id, len(run))
|
|
689
|
+
raise
|
|
690
|
+
|
|
691
|
+
def spill_live(self, until: float | None = None, only: set[str] | None = None) -> None:
|
|
692
|
+
"""Move each live session's outbox (or ``only`` those) to the spill, at most ``rounds`` pages each
|
|
693
|
+
and none past ``until``, starting one session further along each pass so a slow one holds back
|
|
694
|
+
no other for long, and forget what the host no longer serves and the platform has."""
|
|
695
|
+
self.spill.settle(set(self.host.session_ids(draining=True)))
|
|
696
|
+
live = self.host.session_ids()
|
|
697
|
+
with self._lock:
|
|
698
|
+
self.acked = {sid: cursor for sid, cursor in self.acked.items() if sid in live}
|
|
699
|
+
self.refused = collections.Counter({sid: n for sid, n in self.refused.items() if sid in live})
|
|
700
|
+
for session_id in self._rotated([sid for sid in live if only is None or sid in only]):
|
|
701
|
+
if _late(until):
|
|
702
|
+
break
|
|
703
|
+
self._each(session_id, partial(self._keep, session_id, until))
|
|
704
|
+
|
|
705
|
+
def replay(self, until: float | None = None) -> None:
|
|
706
|
+
"""Send the spill until it is empty, a round finds the platform taking none of it (a backoff
|
|
707
|
+
starts), or ``until`` (unset: until the flusher stops)."""
|
|
708
|
+
while not self._over(until):
|
|
709
|
+
pieces = self.spill.round(self._holding())
|
|
710
|
+
if not pieces:
|
|
711
|
+
return
|
|
712
|
+
if not self._send_round(pieces, until):
|
|
713
|
+
return self.backoff.failed()
|
|
714
|
+
self.backoff.cleared()
|
|
715
|
+
|
|
716
|
+
def _spilling(self) -> None:
|
|
717
|
+
while not self._stop.is_set():
|
|
718
|
+
self._spill_due.wait(max(0.0, self._next_pass - time.monotonic()))
|
|
719
|
+
self._spill_due.clear()
|
|
720
|
+
if self._stop.is_set():
|
|
721
|
+
return
|
|
722
|
+
self._guarded(self._spill_due_outboxes)
|
|
723
|
+
self._send_due.set()
|
|
724
|
+
|
|
725
|
+
def _sending(self) -> None:
|
|
726
|
+
while not self._stop.is_set():
|
|
727
|
+
self._send_due.wait(self.interval_s)
|
|
728
|
+
self._send_due.clear()
|
|
729
|
+
if not self._stop.is_set() and self.backoff.ready():
|
|
730
|
+
self._guarded(self.replay)
|
|
731
|
+
|
|
732
|
+
def _spill_due_outboxes(self) -> None:
|
|
733
|
+
with self._lock:
|
|
734
|
+
pressed, self._pressed = self._pressed, set()
|
|
735
|
+
whole = time.monotonic() >= self._next_pass
|
|
736
|
+
self._next_pass = time.monotonic() + self.interval_s if whole else self._next_pass
|
|
737
|
+
self.spill_live(time.monotonic() + LEDGER_PASS_S, None if whole else pressed)
|
|
738
|
+
|
|
739
|
+
def _last(self, until: float) -> None:
|
|
740
|
+
self.replay(until)
|
|
741
|
+
for session_id in self._rotated(self.host.session_ids()):
|
|
742
|
+
if _late(until):
|
|
743
|
+
break
|
|
744
|
+
self._leave(session_id, partial(self.host.requests, session_id), until)
|
|
745
|
+
self.replay(until)
|
|
746
|
+
|
|
747
|
+
def _take(self, session_id: str, drain: Callable[..., object], until: float | None = None) -> OutboxPage | None:
|
|
748
|
+
"""The outbox to the spill within a take's bounds and ``until``: ``take_s`` start at the worker's
|
|
749
|
+
first answer; answers the last page."""
|
|
750
|
+
first = self._move(session_id, drain, 1, until)
|
|
751
|
+
bounded = time.monotonic() + self.take_s
|
|
752
|
+
if first is None or not first.more:
|
|
753
|
+
return first
|
|
754
|
+
budget = _Budget(self.take_bytes - first.size, 1)
|
|
755
|
+
cut = self._move(session_id, drain, self.take_pages - 1, min(until or bounded, bounded), budget, first)
|
|
756
|
+
assert cut is not None
|
|
757
|
+
if cut.more:
|
|
758
|
+
self._cut(session_id, cut, budget.pages)
|
|
759
|
+
return cut
|
|
760
|
+
|
|
761
|
+
def _cut(self, session_id: str, page: OutboxPage, pages: int) -> None:
|
|
762
|
+
with self._lock:
|
|
763
|
+
self.takes_cut += 1
|
|
764
|
+
log_line(
|
|
765
|
+
f"request ledger: session {session_id} stopped after {pages} pages with {page.pending} entries"
|
|
766
|
+
" still on its outbox"
|
|
767
|
+
)
|
|
768
|
+
|
|
769
|
+
def _rotated(self, session_ids: list[str]) -> list[str]:
|
|
770
|
+
start = self._turn % len(session_ids) if session_ids else 0
|
|
771
|
+
self._turn += 1
|
|
772
|
+
return session_ids[start:] + session_ids[:start]
|
|
773
|
+
|
|
774
|
+
def _keep(self, session_id: str, until: float | None) -> None:
|
|
775
|
+
"""A spiller pass's move of one session, left to a barrier or a settle moving it now."""
|
|
776
|
+
with self._lock:
|
|
777
|
+
moving = self._moving[session_id] > 0
|
|
778
|
+
if not moving:
|
|
779
|
+
self._move(session_id, partial(self.host.requests, session_id), self.rounds, until)
|
|
780
|
+
|
|
781
|
+
def _leave(self, session_id: str, drain: Callable[..., object], until: float | None = None) -> None:
|
|
782
|
+
"""Move what a session leaving the host still holds, count what the move could not reach, and
|
|
783
|
+
close its account: complete unless the move failed. A move that failed once its outbox answered
|
|
784
|
+
a page counts what that page said was left; one that failed before has nothing to count."""
|
|
785
|
+
moved, left = self._each(session_id, partial(self._taken, session_id, drain, until)) or (False, 0)
|
|
786
|
+
self._each(session_id, partial(self._close, session_id, moved, left))
|
|
787
|
+
|
|
788
|
+
def _taken(self, session_id: str, drain: Callable[..., object], until: float | None) -> tuple[bool, int]:
|
|
789
|
+
try:
|
|
790
|
+
last = self._take(session_id, drain, until)
|
|
791
|
+
except MoveFailed as failed:
|
|
792
|
+
self.spill.lose(session_id, failed.left)
|
|
793
|
+
raise
|
|
794
|
+
moved = last is not None
|
|
795
|
+
return moved, last.pending if moved and last.more else 0
|
|
796
|
+
|
|
797
|
+
def _close(self, session_id: str, moved: bool, left: int) -> None:
|
|
798
|
+
self.spill.lose(session_id, left)
|
|
799
|
+
self.spill.finished(session_id, moved)
|
|
800
|
+
|
|
801
|
+
def _move(
|
|
802
|
+
self,
|
|
803
|
+
session_id: str,
|
|
804
|
+
drain: Callable[..., object],
|
|
805
|
+
pages: int,
|
|
806
|
+
until: float | None,
|
|
807
|
+
budget: _Budget | None = None,
|
|
808
|
+
page: OutboxPage | None = None,
|
|
809
|
+
) -> OutboxPage | None:
|
|
810
|
+
"""The session's outbox to the spill a page at a time, at most ``pages``, none past ``until`` and
|
|
811
|
+
none past ``budget``'s bytes, each page let go of by the outbox's next drain once it is on disk;
|
|
812
|
+
answers the last page the outbox answered, ``page`` (the one before this move) when it moved none.
|
|
813
|
+
A move that fails once the outbox answered a page raises :class:`MoveFailed`."""
|
|
814
|
+
for _ in range(pages):
|
|
815
|
+
if _late(until) or (budget is not None and budget.spent()):
|
|
816
|
+
return page
|
|
817
|
+
page = self._moved(session_id, drain, page, until)
|
|
818
|
+
if budget is not None:
|
|
819
|
+
budget.spend(page.size)
|
|
820
|
+
if not page.more:
|
|
821
|
+
return page
|
|
822
|
+
return page
|
|
823
|
+
|
|
824
|
+
def _moved(
|
|
825
|
+
self, session_id: str, drain: Callable[..., object], last: OutboxPage | None, until: float | None
|
|
826
|
+
) -> OutboxPage:
|
|
827
|
+
try:
|
|
828
|
+
page = OutboxPage.of(drain(self.acked.get(session_id), timeout=_timeout(until)))
|
|
829
|
+
except Exception as error:
|
|
830
|
+
if last is None:
|
|
831
|
+
raise
|
|
832
|
+
raise MoveFailed(last, kept=True) from error
|
|
833
|
+
self._note(session_id, page)
|
|
834
|
+
try:
|
|
835
|
+
self._keep_page(session_id, page)
|
|
836
|
+
except OSError as error:
|
|
837
|
+
raise MoveFailed(page, kept=False) from error
|
|
838
|
+
return page
|
|
839
|
+
|
|
840
|
+
def _keep_page(self, session_id: str, page: OutboxPage) -> None:
|
|
841
|
+
if not page.lines:
|
|
842
|
+
return
|
|
843
|
+
self.spill.write(session_id, page.lines, page.epoch, page.source)
|
|
844
|
+
with self._lock:
|
|
845
|
+
self.acked[session_id] = page.cursor
|
|
846
|
+
|
|
847
|
+
def _note(self, session_id: str, page: OutboxPage) -> None:
|
|
848
|
+
if self.spill.outbox(session_id, page.epoch, page.shed, page.lost, page.source):
|
|
849
|
+
log_line(
|
|
850
|
+
f"request ledger: session {session_id} outbox shed the headers and bodies of {page.shed} requests,"
|
|
851
|
+
f" lost {page.lost} entries"
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
def _send_round(self, pieces: list[Piece], until: float | None) -> bool:
|
|
855
|
+
"""One piece of every session, in batches read from disk one at a time, until a batch finds
|
|
856
|
+
the platform unavailable; True when the platform stored any of it."""
|
|
857
|
+
outcomes: list[Delivery] = []
|
|
858
|
+
try:
|
|
859
|
+
for ready in self._ready(pieces):
|
|
860
|
+
outcome = Delivery.UNAVAILABLE if self._over(until) else self._deliver(ready, until)
|
|
861
|
+
outcomes.append(outcome)
|
|
862
|
+
if outcome is Delivery.UNAVAILABLE:
|
|
863
|
+
break
|
|
864
|
+
finally:
|
|
865
|
+
self.spill.released(pieces)
|
|
866
|
+
return Delivery.STORED in outcomes
|
|
867
|
+
|
|
868
|
+
def _deliver(self, batch: list[tuple[Piece, dict]], until: float | None) -> Delivery:
|
|
869
|
+
"""One batch; STORED when any of it was stored, what the platform refused of it recorded before
|
|
870
|
+
its pieces read as stored, so a barrier never answers stored over a refusal. A batch of several
|
|
871
|
+
the platform refuses as sent goes again a session at a time; a session it refuses alone is held."""
|
|
872
|
+
answer = self._send([item for _, item in batch], until)
|
|
873
|
+
outcome = answer.delivery
|
|
874
|
+
if outcome is Delivery.STORED:
|
|
875
|
+
self._refused(batch, answer)
|
|
876
|
+
self.spill.stored([piece for piece, _ in batch])
|
|
877
|
+
if outcome is Delivery.REFUSED and len(batch) == 1:
|
|
878
|
+
self._hold(batch[0][0])
|
|
879
|
+
if outcome is not Delivery.REFUSED or len(batch) == 1:
|
|
880
|
+
return outcome
|
|
881
|
+
return self._one_by_one(batch, until)
|
|
882
|
+
|
|
883
|
+
def _one_by_one(self, batch: list[tuple[Piece, dict]], until: float | None) -> Delivery:
|
|
884
|
+
outcomes: list[Delivery] = []
|
|
885
|
+
for one in batch:
|
|
886
|
+
outcomes.append(self._deliver([one], until))
|
|
887
|
+
if outcomes[-1] is Delivery.UNAVAILABLE:
|
|
888
|
+
return Delivery.UNAVAILABLE
|
|
889
|
+
return Delivery.STORED if Delivery.STORED in outcomes else Delivery.REFUSED
|
|
890
|
+
|
|
891
|
+
def _refused(self, batch: list[tuple[Piece, dict]], answer: Answer) -> None:
|
|
892
|
+
"""What the platform refused of a stored batch, each session's own when it said, else the batch's
|
|
893
|
+
count against every session in it."""
|
|
894
|
+
if not answer.refused:
|
|
895
|
+
return
|
|
896
|
+
log_line(f"request ledger: the platform refused {answer.refused} requests of a flush")
|
|
897
|
+
sessions = {piece.account.session_id for piece, _ in batch}
|
|
898
|
+
each = answer.by_session or {sid: answer.refused for sid in sessions}
|
|
899
|
+
with self._lock:
|
|
900
|
+
for session_id in sessions:
|
|
901
|
+
self.refused[session_id] += each.get(session_id, 0)
|
|
902
|
+
|
|
903
|
+
def _hold(self, piece: Piece) -> None:
|
|
904
|
+
session_id = piece.account.session_id
|
|
905
|
+
log_line(f"request ledger: the platform refused session {session_id}'s entries; held {self.backoff.max_s:g} s")
|
|
906
|
+
with self._lock:
|
|
907
|
+
self._held[session_id] = time.monotonic() + self.backoff.max_s
|
|
908
|
+
|
|
909
|
+
def _holding(self) -> set[str]:
|
|
910
|
+
now = time.monotonic()
|
|
911
|
+
with self._lock:
|
|
912
|
+
self._held = {sid: until for sid, until in self._held.items() if until > now}
|
|
913
|
+
return set(self._held)
|
|
914
|
+
|
|
915
|
+
def _ready(self, pieces: list[Piece]) -> Iterator[list[tuple[Piece, dict]]]:
|
|
916
|
+
"""The pieces in batches of at most ``batch_bytes``, each read from disk as it is reached; a piece
|
|
917
|
+
whose segment could not be read this round is left out."""
|
|
918
|
+
for batch in self._batches(pieces):
|
|
919
|
+
items = [(piece, self.spill.item(piece)) for piece in batch]
|
|
920
|
+
ready = [(piece, item) for piece, item in items if item is not None]
|
|
921
|
+
if ready:
|
|
922
|
+
yield ready
|
|
923
|
+
|
|
924
|
+
def _batches(self, pieces: list[Piece]) -> list[list[Piece]]:
|
|
925
|
+
batches: list[list[Piece]] = []
|
|
926
|
+
spent = self.batch_bytes
|
|
927
|
+
for piece in pieces:
|
|
928
|
+
if spent + piece.size > self.batch_bytes:
|
|
929
|
+
batches.append([])
|
|
930
|
+
spent = 0
|
|
931
|
+
batches[-1].append(piece)
|
|
932
|
+
spent += piece.size
|
|
933
|
+
return batches
|
|
934
|
+
|
|
935
|
+
def _send(self, items: list[dict], until: float | None) -> Answer:
|
|
936
|
+
try:
|
|
937
|
+
return Answer.of(self.sink.send(items, until))
|
|
938
|
+
except Exception as error: # noqa: BLE001 - a sink that fails is a send not taken, retried
|
|
939
|
+
log_line(f"request ledger: a flush failed: {type(error).__name__}")
|
|
940
|
+
return Answer(Delivery.UNAVAILABLE)
|
|
941
|
+
|
|
942
|
+
def _over(self, until: float | None) -> bool:
|
|
943
|
+
return _late(until) if until is not None else self._stop.is_set()
|
|
944
|
+
|
|
945
|
+
@staticmethod
|
|
946
|
+
def _each(session_id: str, work: Callable[[], object]) -> object:
|
|
947
|
+
try:
|
|
948
|
+
return work()
|
|
949
|
+
except Exception as error: # noqa: BLE001 - one session's ledger never stops another's
|
|
950
|
+
log_line(f"request ledger: session {session_id} not spilled: {type(error.__cause__ or error).__name__}")
|
|
951
|
+
return None
|
|
952
|
+
|
|
953
|
+
@staticmethod
|
|
954
|
+
def _guarded(work: Callable[[], object]) -> None:
|
|
955
|
+
try:
|
|
956
|
+
work()
|
|
957
|
+
except Exception as error: # noqa: BLE001 - the flusher outlives any one pass
|
|
958
|
+
log_line(f"request ledger: a flush failed: {type(error).__name__}")
|