@withgateway/sdk 0.6.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +573 -0
- package/dist/agents/index.d.ts +573 -0
- package/dist/agents/index.js +1443 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1394 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +5019 -1599
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +5003 -1584
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +2689 -401
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2650 -394
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1983 -1745
- package/dist/worlds/index.d.ts +1983 -1745
- package/dist/worlds/index.js +1295 -226
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1290 -222
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/host.py +74 -11
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +23 -267
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +43 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +367 -217
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +7 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
- package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
- package/runtime/gatewaysdk/environment/schema/world_host.py +44 -15
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -2
- package/skills/connector-schema-authoring/connector-worlds.md +2 -2
- package/skills/connector-schema-authoring/world-schema.md +29 -23
- package/skills/world-data-ingestion/SKILL.md +4 -2
- package/skills/worlds-getting-started/SKILL.md +7 -7
- package/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"""A UI world session's page as rrweb records it (``ui_replay_record.js``, MIT, vendored unmodified),
|
|
2
|
+
for people to rewatch on the session page. It is never evidence: verifiers read the page's
|
|
3
|
+
``via: page`` interactions (:mod:`.ui_interactions`).
|
|
4
|
+
|
|
5
|
+
The recorder loads :data:`REPLAY_SCRIPT_PATH` only where the host serves it and sends each report
|
|
6
|
+
to :data:`REPLAY_PATH` on the host that holds the session, a worker's session too, since a
|
|
7
|
+
recording is no world state. Reports are merged into chunks per page load (:class:`ReplayChunk`)
|
|
8
|
+
and :class:`ReplayShipper` sends each one on to the platform on a channel of its own, so a page's
|
|
9
|
+
recording never crowds out the request ledger. Its contract is pinned for both sides by
|
|
10
|
+
``conformance/world-replay-reports.json``."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import enum
|
|
15
|
+
import gzip
|
|
16
|
+
import http.client
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import re
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.parse
|
|
24
|
+
import urllib.request
|
|
25
|
+
from collections import OrderedDict, deque
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from .handler_errors import log_line
|
|
31
|
+
from .request_ledger import UNREDIRECTED, SendBackoff
|
|
32
|
+
from .ui_interactions import RECORDER_PREFIX, REPLAY_SOURCE
|
|
33
|
+
|
|
34
|
+
REPLAY_PATH = f"{RECORDER_PREFIX}replay"
|
|
35
|
+
REPLAY_SCRIPT_PATH = f"{RECORDER_PREFIX}replay.js"
|
|
36
|
+
REPLAY_ROUTE = "/api/public/world-hosts/replays"
|
|
37
|
+
MAX_REPORT_BYTES = 8 << 20
|
|
38
|
+
MAX_EVENTS = 5000
|
|
39
|
+
MAX_FIRST = 10_000_000
|
|
40
|
+
MAX_TIMESTAMP = 253_402_300_799_999
|
|
41
|
+
EVENT_TYPES = frozenset(range(7))
|
|
42
|
+
LOAD = re.compile(r"[A-Za-z0-9_-]{1,32}")
|
|
43
|
+
REPORT_KEYS = frozenset({"load", "first", "events"})
|
|
44
|
+
CHUNK_BYTES = 1 << 20
|
|
45
|
+
CHUNK_S = 10.0
|
|
46
|
+
MAX_SESSION_BYTES = 64 << 20
|
|
47
|
+
MAX_QUEUED_BYTES = 64 << 20
|
|
48
|
+
MAX_SESSIONS = 10_000
|
|
49
|
+
MAX_TRIES = 5
|
|
50
|
+
SHIP_TIMEOUT_S = 30.0
|
|
51
|
+
REPLAY_STOP_S = 2.0
|
|
52
|
+
TICK_S = 1.0
|
|
53
|
+
BUSY_RETRY_S = 5
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ReplayRefused(Exception):
|
|
57
|
+
def __init__(self, status: int, error: str) -> None:
|
|
58
|
+
super().__init__(error)
|
|
59
|
+
self.status = status
|
|
60
|
+
self.error = error
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _stamp(value: object) -> bool:
|
|
64
|
+
plain = isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
65
|
+
return plain and math.isfinite(value) and 0 < value <= MAX_TIMESTAMP
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _event(value: object) -> bool:
|
|
69
|
+
kind = value.get("type") if isinstance(value, dict) else None
|
|
70
|
+
shaped = isinstance(kind, int) and not isinstance(kind, bool) and kind in EVENT_TYPES
|
|
71
|
+
return shaped and _stamp(value.get("timestamp")) and isinstance(value.get("data"), dict)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _report(value: object) -> bool:
|
|
75
|
+
if not isinstance(value, dict) or set(value) != REPORT_KEYS:
|
|
76
|
+
return False
|
|
77
|
+
load, first, events = value["load"], value["first"], value["events"]
|
|
78
|
+
named = isinstance(load, str) and bool(LOAD.fullmatch(load))
|
|
79
|
+
counted = isinstance(first, int) and not isinstance(first, bool) and 0 <= first <= MAX_FIRST
|
|
80
|
+
listed = isinstance(events, list) and 0 < len(events) <= MAX_EVENTS
|
|
81
|
+
return named and counted and listed and all(_event(event) for event in events)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _encoded(value: Any) -> bytes:
|
|
85
|
+
"""UTF-8 JSON the size of what the page sent; a lone surrogate (text a page cut mid-character)
|
|
86
|
+
is replaced, never an error, and a `NaN` or `Infinity` is refused (``ValueError``)."""
|
|
87
|
+
return json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8", "replace")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class ReplayReport:
|
|
92
|
+
"""One report a page's recorder sent: its load, the index of its first event, its events."""
|
|
93
|
+
|
|
94
|
+
load: str
|
|
95
|
+
first: int
|
|
96
|
+
events: list[dict]
|
|
97
|
+
raw_bytes: int
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def read(cls, raw: bytes, scrub: Callable[[str], str]) -> ReplayReport:
|
|
101
|
+
"""``scrub`` (the session's :class:`~.call_redaction.CallRedaction`) runs over the report
|
|
102
|
+
re-encoded, where no JSON escape hides a spelling, and the shape is checked on what it
|
|
103
|
+
leaves."""
|
|
104
|
+
if len(raw) > MAX_REPORT_BYTES:
|
|
105
|
+
raise ReplayRefused(413, f"a replay report is at most {MAX_REPORT_BYTES} bytes")
|
|
106
|
+
try:
|
|
107
|
+
report: Any = json.loads(scrub(_encoded(json.loads(raw.decode("utf-8", "replace"))).decode("utf-8")))
|
|
108
|
+
except (ValueError, UnicodeError, OverflowError, RecursionError):
|
|
109
|
+
raise ReplayRefused(400, "a replay report is JSON") from None
|
|
110
|
+
if not _report(report):
|
|
111
|
+
raise ReplayRefused(
|
|
112
|
+
400,
|
|
113
|
+
f"a replay report is {{load, first, events}}: at most {MAX_EVENTS} rrweb events, each "
|
|
114
|
+
"{type 0-6, timestamp, data}",
|
|
115
|
+
)
|
|
116
|
+
return cls(report["load"], report["first"], report["events"], len(raw))
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def next(self) -> int:
|
|
120
|
+
return self.first + len(self.events)
|
|
121
|
+
|
|
122
|
+
def after(self, index: int) -> ReplayReport:
|
|
123
|
+
"""The report from ``index`` on, what a resend adds past what is held."""
|
|
124
|
+
kept = self.events[max(0, index - self.first) :]
|
|
125
|
+
return ReplayReport(self.load, max(index, self.first), kept, len(_encoded(kept)))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass
|
|
129
|
+
class ReplayChunk:
|
|
130
|
+
"""A session page load's reports merged in index order, closed at :data:`CHUNK_BYTES` or
|
|
131
|
+
:data:`CHUNK_S`; a resent report adds only the events past what the chunk holds, and a report
|
|
132
|
+
that leaves a hole starts the next chunk."""
|
|
133
|
+
|
|
134
|
+
session_id: str
|
|
135
|
+
load: str
|
|
136
|
+
first: int
|
|
137
|
+
events: list[dict] = field(default_factory=list)
|
|
138
|
+
raw_bytes: int = 0
|
|
139
|
+
opened: float = field(default_factory=time.monotonic)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def next(self) -> int:
|
|
143
|
+
return self.first + len(self.events)
|
|
144
|
+
|
|
145
|
+
def takes(self, report: ReplayReport) -> bool:
|
|
146
|
+
return report.load == self.load and report.first <= self.next < report.next
|
|
147
|
+
|
|
148
|
+
def holds(self, report: ReplayReport) -> bool:
|
|
149
|
+
return report.load == self.load and self.first <= report.first and report.next <= self.next
|
|
150
|
+
|
|
151
|
+
def fits(self, report: ReplayReport) -> bool:
|
|
152
|
+
within = len(self.events) + len(report.events) <= MAX_EVENTS
|
|
153
|
+
return within and self.raw_bytes + report.raw_bytes <= CHUNK_BYTES
|
|
154
|
+
|
|
155
|
+
def add(self, report: ReplayReport) -> None:
|
|
156
|
+
self.events.extend(report.events)
|
|
157
|
+
self.raw_bytes += report.raw_bytes
|
|
158
|
+
|
|
159
|
+
def due(self, now: float) -> bool:
|
|
160
|
+
return self.raw_bytes >= CHUNK_BYTES or now - self.opened >= CHUNK_S
|
|
161
|
+
|
|
162
|
+
def body(self) -> bytes:
|
|
163
|
+
return gzip.compress(_encoded({"load": self.load, "first": self.first, "events": self.events}), mtime=0)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@dataclass(eq=False)
|
|
167
|
+
class ShippedChunk:
|
|
168
|
+
session_id: str
|
|
169
|
+
body: bytes
|
|
170
|
+
tries: int = 0
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class Taken(enum.Enum):
|
|
174
|
+
KEPT = "kept"
|
|
175
|
+
FULL = "full"
|
|
176
|
+
BUSY = "busy"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class ReplayShipper:
|
|
180
|
+
"""Sends a host's replay chunks on to the platform's replay door (:data:`REPLAY_ROUTE`, behind
|
|
181
|
+
the control-plane secret), oldest first, from a queue of its own: at most
|
|
182
|
+
:data:`MAX_SESSION_BYTES` a session (past it the session's recording is :attr:`Taken.FULL`) and
|
|
183
|
+
:data:`MAX_QUEUED_BYTES` waiting on the host (past it a report waits: :attr:`Taken.BUSY`). A
|
|
184
|
+
failed send holds back that session's chunks alone (a :class:`SendBackoff` each) and is tried
|
|
185
|
+
again up to :data:`MAX_TRIES` times; a chunk the platform refuses, or one that failed every try,
|
|
186
|
+
is let go, so no chunk holds up another session's."""
|
|
187
|
+
|
|
188
|
+
def __init__(
|
|
189
|
+
self, platform_url: str, secret: str, timeout_s: float = SHIP_TIMEOUT_S, backoff: SendBackoff | None = None
|
|
190
|
+
) -> None:
|
|
191
|
+
self.url = f"{platform_url.rstrip('/')}{REPLAY_ROUTE}"
|
|
192
|
+
self.secret = secret
|
|
193
|
+
self.timeout_s = timeout_s
|
|
194
|
+
self._paced = backoff or SendBackoff()
|
|
195
|
+
self._open: dict[tuple[str, str], ReplayChunk] = {}
|
|
196
|
+
self._queue: deque[ShippedChunk] = deque()
|
|
197
|
+
self._queued = 0
|
|
198
|
+
self._held: dict[str, SendBackoff] = {}
|
|
199
|
+
self._taken: OrderedDict[str, int] = OrderedDict()
|
|
200
|
+
self._ready = threading.Condition()
|
|
201
|
+
self._until: float | None = None
|
|
202
|
+
self._thread = threading.Thread(target=self._run, name="replay-shipper", daemon=True)
|
|
203
|
+
|
|
204
|
+
def start(self) -> ReplayShipper:
|
|
205
|
+
self._thread.start()
|
|
206
|
+
return self
|
|
207
|
+
|
|
208
|
+
def take(self, session_id: str, report: ReplayReport) -> Taken:
|
|
209
|
+
with self._ready:
|
|
210
|
+
waiting = self._queued + sum(chunk.raw_bytes for chunk in self._open.values())
|
|
211
|
+
if waiting >= MAX_QUEUED_BYTES:
|
|
212
|
+
return Taken.BUSY
|
|
213
|
+
spent = self._taken.pop(session_id, 0) + report.raw_bytes
|
|
214
|
+
self._taken[session_id] = spent
|
|
215
|
+
while len(self._taken) > MAX_SESSIONS:
|
|
216
|
+
self._taken.popitem(last=False)
|
|
217
|
+
if spent > MAX_SESSION_BYTES:
|
|
218
|
+
return Taken.FULL
|
|
219
|
+
self._merge(session_id, report)
|
|
220
|
+
self._ready.notify()
|
|
221
|
+
return Taken.KEPT
|
|
222
|
+
|
|
223
|
+
def stop(self, deadline_s: float = REPLAY_STOP_S) -> None:
|
|
224
|
+
"""Ships what is left until ``deadline_s`` has passed, then lets the rest go."""
|
|
225
|
+
with self._ready:
|
|
226
|
+
self._until = time.monotonic() + deadline_s
|
|
227
|
+
self._close(lambda chunk: True)
|
|
228
|
+
self._ready.notify()
|
|
229
|
+
self._thread.join(max(0.0, self._until - time.monotonic()))
|
|
230
|
+
|
|
231
|
+
def waiting(self) -> int:
|
|
232
|
+
with self._ready:
|
|
233
|
+
return len(self._queue) + len(self._open)
|
|
234
|
+
|
|
235
|
+
def _merge(self, session_id: str, report: ReplayReport) -> None:
|
|
236
|
+
key = (session_id, report.load)
|
|
237
|
+
held = self._open.get(key)
|
|
238
|
+
if held is not None and held.holds(report):
|
|
239
|
+
return
|
|
240
|
+
joined = held is not None and held.takes(report)
|
|
241
|
+
fresh = report.after(held.next) if joined else report
|
|
242
|
+
if joined and held.fits(fresh):
|
|
243
|
+
held.add(fresh)
|
|
244
|
+
return
|
|
245
|
+
self._close(lambda chunk: (chunk.session_id, chunk.load) == key)
|
|
246
|
+
chunk = ReplayChunk(session_id, fresh.load, fresh.first)
|
|
247
|
+
chunk.add(fresh)
|
|
248
|
+
self._open[key] = chunk
|
|
249
|
+
|
|
250
|
+
def _close(self, which: Callable[[ReplayChunk], bool]) -> None:
|
|
251
|
+
for key, chunk in list(self._open.items()):
|
|
252
|
+
if not which(chunk):
|
|
253
|
+
continue
|
|
254
|
+
del self._open[key]
|
|
255
|
+
try:
|
|
256
|
+
shipped = ShippedChunk(chunk.session_id, chunk.body())
|
|
257
|
+
except (ValueError, RecursionError) as error:
|
|
258
|
+
log_line(f"page replay: a chunk of session {chunk.session_id} could not be packed: {error}; let go")
|
|
259
|
+
continue
|
|
260
|
+
self._queue.append(shipped)
|
|
261
|
+
self._queued += len(shipped.body)
|
|
262
|
+
|
|
263
|
+
def _done(self) -> bool:
|
|
264
|
+
stopping = self._until is not None
|
|
265
|
+
return stopping and (not self._queue or time.monotonic() >= self._until)
|
|
266
|
+
|
|
267
|
+
def _due(self, chunk: ShippedChunk) -> bool:
|
|
268
|
+
held = self._held.get(chunk.session_id)
|
|
269
|
+
return held is None or held.ready()
|
|
270
|
+
|
|
271
|
+
def _next(self) -> ShippedChunk | None:
|
|
272
|
+
with self._ready:
|
|
273
|
+
while not self._done():
|
|
274
|
+
now = time.monotonic()
|
|
275
|
+
self._close(lambda chunk: chunk.due(now))
|
|
276
|
+
ready = next((chunk for chunk in self._queue if self._due(chunk)), None)
|
|
277
|
+
if ready is not None:
|
|
278
|
+
return ready
|
|
279
|
+
soonest = min((held.until for held in self._held.values()), default=now + TICK_S)
|
|
280
|
+
self._ready.wait(timeout=min(TICK_S, max(0.05, soonest - now)))
|
|
281
|
+
return None
|
|
282
|
+
|
|
283
|
+
def _run(self) -> None:
|
|
284
|
+
while (chunk := self._next()) is not None:
|
|
285
|
+
try:
|
|
286
|
+
answered = self._send(chunk)
|
|
287
|
+
except Exception as error: # noqa: BLE001 - one chunk never ends the shipper
|
|
288
|
+
log_line(f"page replay: sending a chunk failed: {type(error).__name__}")
|
|
289
|
+
answered = False
|
|
290
|
+
with self._ready:
|
|
291
|
+
self._settle(chunk, answered)
|
|
292
|
+
|
|
293
|
+
def _settle(self, chunk: ShippedChunk, answered: bool) -> None:
|
|
294
|
+
chunk.tries += 1
|
|
295
|
+
if not answered and chunk.tries < MAX_TRIES:
|
|
296
|
+
held = self._held.setdefault(chunk.session_id, SendBackoff(self._paced.first_s, self._paced.max_s))
|
|
297
|
+
held.failed()
|
|
298
|
+
return
|
|
299
|
+
if not answered:
|
|
300
|
+
log_line(f"page replay: a chunk of session {chunk.session_id} failed {MAX_TRIES} sends; let go")
|
|
301
|
+
self._held.pop(chunk.session_id, None)
|
|
302
|
+
self._queue.remove(chunk)
|
|
303
|
+
self._queued -= len(chunk.body)
|
|
304
|
+
|
|
305
|
+
def _send(self, chunk: ShippedChunk) -> bool:
|
|
306
|
+
"""True once the platform answered for the chunk, kept or refused; False to try again."""
|
|
307
|
+
headers = {"content-type": "application/gzip", "x-host-secret": self.secret}
|
|
308
|
+
query = urllib.parse.urlencode({"session": chunk.session_id})
|
|
309
|
+
request = urllib.request.Request(f"{self.url}?{query}", data=chunk.body, method="POST", headers=headers)
|
|
310
|
+
timeout = self.timeout_s if self._until is None else min(self.timeout_s, self._until - time.monotonic())
|
|
311
|
+
try:
|
|
312
|
+
with UNREDIRECTED.open(request, timeout=max(0.1, timeout)) as answer:
|
|
313
|
+
return 200 <= answer.status < 300
|
|
314
|
+
except urllib.error.HTTPError as error:
|
|
315
|
+
error.close()
|
|
316
|
+
log_line(f"page replay: the platform answered a chunk {error.code}")
|
|
317
|
+
return 400 <= error.code < 500 and error.code not in (408, 429)
|
|
318
|
+
except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
|
|
319
|
+
log_line(f"page replay: the platform did not take a chunk: {type(error).__name__}")
|
|
320
|
+
return False
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class UiReplay:
|
|
324
|
+
"""The ui origin's half of page replay: the script, served only where reports are shipped, and
|
|
325
|
+
the replay door. Neither takes the session's admission turn: a recording is no world state."""
|
|
326
|
+
|
|
327
|
+
@staticmethod
|
|
328
|
+
def script(shipper: ReplayShipper | None) -> tuple[int, dict[str, str], bytes]:
|
|
329
|
+
if shipper is None:
|
|
330
|
+
return 404, {"content-type": "application/json"}, b'{"error":"this host keeps no page replay"}'
|
|
331
|
+
headers = {
|
|
332
|
+
"content-type": "text/javascript; charset=utf-8",
|
|
333
|
+
"cache-control": "public, max-age=31536000, immutable",
|
|
334
|
+
}
|
|
335
|
+
return 200, headers, REPLAY_SOURCE.read_bytes()
|
|
336
|
+
|
|
337
|
+
@staticmethod
|
|
338
|
+
def take(
|
|
339
|
+
method: str,
|
|
340
|
+
raw: bytes,
|
|
341
|
+
session_id: str,
|
|
342
|
+
scrub: Callable[[str], str],
|
|
343
|
+
shipper: ReplayShipper | None,
|
|
344
|
+
) -> tuple:
|
|
345
|
+
if shipper is None:
|
|
346
|
+
return 404, {"error": "this host keeps no page replay"}
|
|
347
|
+
if method != "POST":
|
|
348
|
+
return 405, {"error": f"{REPLAY_PATH} takes POST"}
|
|
349
|
+
try:
|
|
350
|
+
report = ReplayReport.read(raw, scrub)
|
|
351
|
+
except ReplayRefused as refused:
|
|
352
|
+
return refused.status, {"error": refused.error}
|
|
353
|
+
taken = shipper.take(session_id, report)
|
|
354
|
+
answers: dict[Taken, tuple] = {
|
|
355
|
+
Taken.KEPT: (200, {"kept": True}),
|
|
356
|
+
Taken.FULL: (
|
|
357
|
+
410,
|
|
358
|
+
{"kept": False, "error": "this session's page recording is full; the page stops recording"},
|
|
359
|
+
),
|
|
360
|
+
Taken.BUSY: (
|
|
361
|
+
503,
|
|
362
|
+
{"kept": False, "error": "page replays wait on this host"},
|
|
363
|
+
{"Retry-After": str(BUSY_RETRY_S)},
|
|
364
|
+
),
|
|
365
|
+
}
|
|
366
|
+
return answers[taken]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
ui_replay_record.js is @rrweb/record 2.1.7 (umd/record.min.js), unmodified, from the npm registry
|
|
2
|
+
(tarball integrity sha512-lIAbWdH8JwRuujURxTBtgrlsPvyBUGXUa9aB5hK8s5DdHgagyLFkIaAWhtthuOwT3o4o4+w38yUVPLc3W2oOrw==).
|
|
3
|
+
Source: https://github.com/rrweb-io/rrweb (packages/record). Its licence follows.
|
|
4
|
+
|
|
5
|
+
MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2018 Contributors (https://github.com/rrweb-io/rrweb/graphs/contributors)
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|