@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -40,6 +40,7 @@ STATIC_SOURCE = Path(__file__).with_name("ui_static")
|
|
|
40
40
|
#: overwrites an index.html only when it carries it.
|
|
41
41
|
GENERATED_MARK = "<!-- gateway-world-ui: generated -->"
|
|
42
42
|
DEFAULT_ENTRY = "/"
|
|
43
|
+
UI_RESERVED = "/__world-ui"
|
|
43
44
|
GENERATED_FILES = ("index.html", "app.js", "contract.json")
|
|
44
45
|
#: What ``ui init`` adds to a connector.toml that declares no ui.
|
|
45
46
|
UI_SECTION = """
|
|
@@ -54,6 +55,17 @@ generated = true
|
|
|
54
55
|
"""
|
|
55
56
|
|
|
56
57
|
|
|
58
|
+
def checked_api_path(api_path: Any) -> str:
|
|
59
|
+
"""``[ui] api_path`` as both readers of ``[ui]`` keep it: a same-origin prefix, never the
|
|
60
|
+
root and never under :data:`UI_RESERVED`."""
|
|
61
|
+
if not isinstance(api_path, str) or not api_path.startswith("/") or api_path == "/":
|
|
62
|
+
fail("spec", "ui.api_path is the same-origin prefix the ui calls the api at", "/ui/api_path")
|
|
63
|
+
kept = api_path.rstrip("/")
|
|
64
|
+
if kept == UI_RESERVED or kept.startswith(f"{UI_RESERVED}/"):
|
|
65
|
+
fail("spec", f"ui.api_path is under {UI_RESERVED}, the prefix the ui origin keeps for itself", "/ui/api_path")
|
|
66
|
+
return kept
|
|
67
|
+
|
|
68
|
+
|
|
57
69
|
@dataclass(frozen=True)
|
|
58
70
|
class UiDeclaration:
|
|
59
71
|
static: str
|
|
@@ -72,6 +84,44 @@ class UiDeclaration:
|
|
|
72
84
|
contract at compile time). What the connector pages badge and label."""
|
|
73
85
|
return "product" if self.generated is False else "generated"
|
|
74
86
|
|
|
87
|
+
@staticmethod
|
|
88
|
+
def kind_of(spec: bytes | None) -> str:
|
|
89
|
+
"""The ui a connector.toml's bytes declare, as the connector pages show it:
|
|
90
|
+
``product``, ``generated``, or ``none`` without the file or its ``[ui]``."""
|
|
91
|
+
declaration = UiDeclaration.parse(spec) if spec is not None else None
|
|
92
|
+
return declaration.kind if declaration is not None else "none"
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def parse(spec: bytes) -> UiDeclaration | None:
|
|
96
|
+
"""``[ui]`` of a connector.toml, or None when it declares none. Read
|
|
97
|
+
without the api runtime's index.html check, which is what compile is
|
|
98
|
+
about to satisfy."""
|
|
99
|
+
try:
|
|
100
|
+
raw = tomllib.loads(spec.decode("utf-8"))
|
|
101
|
+
except ValueError as error:
|
|
102
|
+
fail("spec", f"Cannot parse connector.toml: {error}", "/connector.toml")
|
|
103
|
+
ui = raw.get("ui")
|
|
104
|
+
if ui is None:
|
|
105
|
+
return None
|
|
106
|
+
if not isinstance(ui, dict):
|
|
107
|
+
fail("spec", "[ui] must be a table", "/ui")
|
|
108
|
+
# No default here: api.py (the runtime that answers `[ui] api_path` requests)
|
|
109
|
+
# fails hard on a missing `static`, so this reader agrees rather than quietly
|
|
110
|
+
# picking a directory the api side would never have looked in.
|
|
111
|
+
static = ui.get("static")
|
|
112
|
+
if not isinstance(static, str) or not static or static.startswith("/") or ".." in static.split("/"):
|
|
113
|
+
fail("spec", "ui.static is a directory inside the world holding the ui", "/ui/static")
|
|
114
|
+
entry = ui.get("entry", DEFAULT_ENTRY)
|
|
115
|
+
# `//host` and `/\host` are paths only in form: a browser resolves them to
|
|
116
|
+
# another origin, which would carry the session token off the ui's own.
|
|
117
|
+
if not isinstance(entry, str) or not entry.startswith("/") or entry.startswith("//") or "\\" in entry:
|
|
118
|
+
fail("spec", "ui.entry is the path the agent lands on, starting with / on the ui's own origin", "/ui/entry")
|
|
119
|
+
api_path = checked_api_path(ui.get("api_path", "/__world-api"))
|
|
120
|
+
generated = ui.get("generated")
|
|
121
|
+
if generated is not None and not isinstance(generated, bool):
|
|
122
|
+
fail("spec", "ui.generated is true (compile writes the ui) or false (hand-written)", "/ui/generated")
|
|
123
|
+
return UiDeclaration(static=static.rstrip("/"), entry=entry, api_path=api_path, generated=generated)
|
|
124
|
+
|
|
75
125
|
|
|
76
126
|
def ui_kind(world: Path) -> str:
|
|
77
127
|
"""The world's ui as the connector pages show it: ``product``,
|
|
@@ -82,38 +132,15 @@ def ui_kind(world: Path) -> str:
|
|
|
82
132
|
|
|
83
133
|
def read_ui_declaration(world: Path) -> UiDeclaration | None:
|
|
84
134
|
"""``[ui]`` of the world's connector.toml, or None when the world declares
|
|
85
|
-
none (no connector.toml, or no section).
|
|
86
|
-
runtime's index.html check, which is what compile is about to satisfy."""
|
|
135
|
+
none (no connector.toml, or no section)."""
|
|
87
136
|
path = world / "connector.toml"
|
|
88
137
|
if not path.is_file():
|
|
89
138
|
return None
|
|
90
139
|
try:
|
|
91
|
-
|
|
92
|
-
except
|
|
140
|
+
spec = path.read_bytes()
|
|
141
|
+
except OSError as error:
|
|
93
142
|
fail("spec", f"Cannot parse connector.toml: {error}", "/connector.toml")
|
|
94
|
-
|
|
95
|
-
if ui is None:
|
|
96
|
-
return None
|
|
97
|
-
if not isinstance(ui, dict):
|
|
98
|
-
fail("spec", "[ui] must be a table", "/ui")
|
|
99
|
-
# No default here: api.py (the runtime that answers `[ui] api_path` requests)
|
|
100
|
-
# fails hard on a missing `static`, so this reader agrees rather than quietly
|
|
101
|
-
# picking a directory the api side would never have looked in.
|
|
102
|
-
static = ui.get("static")
|
|
103
|
-
if not isinstance(static, str) or not static or static.startswith("/") or ".." in static.split("/"):
|
|
104
|
-
fail("spec", "ui.static is a directory inside the world holding the ui", "/ui/static")
|
|
105
|
-
entry = ui.get("entry", DEFAULT_ENTRY)
|
|
106
|
-
# `//host` and `/\host` are paths only in form: a browser resolves them to
|
|
107
|
-
# another origin, which would carry the session token off the ui's own.
|
|
108
|
-
if not isinstance(entry, str) or not entry.startswith("/") or entry.startswith("//") or "\\" in entry:
|
|
109
|
-
fail("spec", "ui.entry is the path the agent lands on, starting with / on the ui's own origin", "/ui/entry")
|
|
110
|
-
api_path = ui.get("api_path", "/__world-api")
|
|
111
|
-
if not isinstance(api_path, str) or not api_path.startswith("/") or api_path == "/":
|
|
112
|
-
fail("spec", "ui.api_path is the same-origin prefix the ui calls the api at", "/ui/api_path")
|
|
113
|
-
generated = ui.get("generated")
|
|
114
|
-
if generated is not None and not isinstance(generated, bool):
|
|
115
|
-
fail("spec", "ui.generated is true (compile writes the ui) or false (hand-written)", "/ui/generated")
|
|
116
|
-
return UiDeclaration(static=static.rstrip("/"), entry=entry, api_path=api_path.rstrip("/"), generated=generated)
|
|
143
|
+
return UiDeclaration.parse(spec)
|
|
117
144
|
|
|
118
145
|
|
|
119
146
|
def is_generated(index: Path) -> bool:
|
|
@@ -187,8 +214,6 @@ def init_ui(world: str | Path) -> dict:
|
|
|
187
214
|
if not declared:
|
|
188
215
|
text = spec_path.read_text(encoding="utf-8")
|
|
189
216
|
first_route = re.search(r"^\[\[operations\]\]", text, re.M)
|
|
190
|
-
# Before the first route so the section reads with the other tables,
|
|
191
|
-
# at the end when the file declares no routes yet.
|
|
192
217
|
text = (
|
|
193
218
|
text[: first_route.start()] + UI_SECTION.lstrip("\n") + "\n" + text[first_route.start() :]
|
|
194
219
|
if first_route
|
|
@@ -208,10 +233,6 @@ def init_ui(world: str | Path) -> dict:
|
|
|
208
233
|
}
|
|
209
234
|
|
|
210
235
|
|
|
211
|
-
# --------------------------------------------------------------------------- #
|
|
212
|
-
# contract.json: the world as the page renders it.
|
|
213
|
-
# --------------------------------------------------------------------------- #
|
|
214
|
-
|
|
215
236
|
VERBS = ("list", "get", "create", "update", "delete")
|
|
216
237
|
|
|
217
238
|
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"""What a UI world's page does that sends no request of its own: a click, a committed value,
|
|
2
|
+
a submit, a key, a scroll, a route change. A host that records notes so when it opens a UI
|
|
3
|
+
session (``ui.recording``, :meth:`UiRecorder.recording`), puts :data:`RECORDER_TAG` on every
|
|
4
|
+
page it serves (:meth:`UiRecorder.page`) and notes the page as ``ui.served``; the recorder
|
|
5
|
+
reports each interaction at :data:`EVENTS_PATH`, and :meth:`UiRecorder.record` keeps each one as
|
|
6
|
+
a call of the session (:meth:`~.call_log.CallLog.record_interaction`): a ``via: page`` entry
|
|
7
|
+
named ``ui.<kind>`` in the request ledger and in the grader's call log.
|
|
8
|
+
|
|
9
|
+
An interaction is what the page reported, admitted as the page's own api prefix is (the session
|
|
10
|
+
cookie, never another site): the session's token holder can report one without a browser."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, ClassVar
|
|
20
|
+
|
|
21
|
+
from .call_log import CallLog
|
|
22
|
+
from .call_redaction import REDACTED, ROOT, _card_digits
|
|
23
|
+
from .tasks import PAGE_RECORDING as RECORDING_OPERATION
|
|
24
|
+
from .tasks import PAGE_SERVED as SERVED_OPERATION
|
|
25
|
+
from .tasks import PAGE_UNRECORDED_LIFE as UNRECORDED_OPERATION
|
|
26
|
+
from .ui import UI_RESERVED
|
|
27
|
+
|
|
28
|
+
RECORDER_PREFIX = f"{UI_RESERVED}/"
|
|
29
|
+
RECORDER_PATH = f"{RECORDER_PREFIX}recorder.js"
|
|
30
|
+
EVENTS_PATH = f"{RECORDER_PREFIX}events"
|
|
31
|
+
RECORDER_SOURCE = Path(__file__).with_name("ui_recorder.js")
|
|
32
|
+
REPLAY_SOURCE = Path(__file__).with_name("ui_replay_record.js")
|
|
33
|
+
RECORDER_VERSION = hashlib.sha256(RECORDER_SOURCE.read_bytes() + REPLAY_SOURCE.read_bytes()).hexdigest()[:12]
|
|
34
|
+
RECORDER_ANSWER = {"recorder": RECORDER_VERSION}
|
|
35
|
+
RECORDER_TAG = f'<script defer src="{RECORDER_PATH}?v={RECORDER_VERSION}"></script>'.encode()
|
|
36
|
+
MAX_EVENTS = 32
|
|
37
|
+
MAX_BATCH_BYTES = 32 * 1024
|
|
38
|
+
NAME_CHARS = 120
|
|
39
|
+
TEXT_CHARS = 200
|
|
40
|
+
VALUE_CHARS = 512
|
|
41
|
+
PAGE_CHARS = 2048
|
|
42
|
+
SELECTOR_CHARS = 256
|
|
43
|
+
OPTION_CHARS = 60
|
|
44
|
+
MAX_OPTIONS = 50
|
|
45
|
+
MAX_FILES = 20
|
|
46
|
+
LOAD_CHARS = 32
|
|
47
|
+
TARGET_FIELDS = {
|
|
48
|
+
"tag": NAME_CHARS,
|
|
49
|
+
"role": NAME_CHARS,
|
|
50
|
+
"name": NAME_CHARS,
|
|
51
|
+
"id": NAME_CHARS,
|
|
52
|
+
"testid": NAME_CHARS,
|
|
53
|
+
"href": NAME_CHARS,
|
|
54
|
+
"opens": NAME_CHARS,
|
|
55
|
+
"type": NAME_CHARS,
|
|
56
|
+
"field": NAME_CHARS,
|
|
57
|
+
"autocomplete": NAME_CHARS,
|
|
58
|
+
"checked": NAME_CHARS,
|
|
59
|
+
"expanded": NAME_CHARS,
|
|
60
|
+
"pressed": NAME_CHARS,
|
|
61
|
+
"selected": NAME_CHARS,
|
|
62
|
+
"disabled": NAME_CHARS,
|
|
63
|
+
"group": NAME_CHARS,
|
|
64
|
+
"within": NAME_CHARS,
|
|
65
|
+
"text": TEXT_CHARS,
|
|
66
|
+
"context": TEXT_CHARS,
|
|
67
|
+
"selector": SELECTOR_CHARS,
|
|
68
|
+
"xpath": SELECTOR_CHARS,
|
|
69
|
+
}
|
|
70
|
+
TARGET_LISTS = {"options": (MAX_OPTIONS, OPTION_CHARS)}
|
|
71
|
+
PAGE_SUFFIXES = frozenset({"", ".html", ".htm"})
|
|
72
|
+
SECRET_INPUT_TYPES = frozenset({"password", "hidden"})
|
|
73
|
+
SECRET_AUTOCOMPLETE = re.compile(r"(^|\s)(current-password|new-password|one-time-code|cc-[a-z-]+)(\s|$)")
|
|
74
|
+
SECRET_WORDS = frozenset(
|
|
75
|
+
{"password", "passwd", "passcode", "pwd", "pass", "secret", "token", "otp", "totp", "pin", "cvc", "cvv", "cvn"}
|
|
76
|
+
| {"ssn", "iban", "apikey", "jwt", "2fa", "mfa"}
|
|
77
|
+
)
|
|
78
|
+
SECRET_PAIRS = frozenset(
|
|
79
|
+
{"api key", "card number", "card no", "card num", "cc number", "credit card", "security code"}
|
|
80
|
+
| {"verification code", "access key", "private key", "client secret", "backup code", "recovery code"}
|
|
81
|
+
| {"authenticator code", "auth code", "account number", "routing number", "sort code"}
|
|
82
|
+
)
|
|
83
|
+
_BODY = re.compile(rb"<body[\s>]", re.IGNORECASE)
|
|
84
|
+
_OPENINGS = tuple(
|
|
85
|
+
re.compile(pattern, re.IGNORECASE) for pattern in (rb"<head(\s[^>]*)?>", rb"<html(\s[^>]*)?>", rb"<!doctype[^>]*>")
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class InteractionRefused(Exception):
|
|
90
|
+
"""A report the events door will not keep, with the status it answers."""
|
|
91
|
+
|
|
92
|
+
def __init__(self, status: int, error: str):
|
|
93
|
+
super().__init__(error)
|
|
94
|
+
self.status = status
|
|
95
|
+
self.error = error
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _text(value: Any, chars: int, what: str) -> str:
|
|
99
|
+
if not isinstance(value, str):
|
|
100
|
+
raise InteractionRefused(400, f"{what} is a string")
|
|
101
|
+
return value[:chars]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _words(text: str) -> list[str]:
|
|
105
|
+
spaced = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", text).lower()
|
|
106
|
+
return [word for word in re.split(r"[^a-z0-9]+", spaced) if word]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def credential_named(text: str) -> bool:
|
|
110
|
+
"""Whether a field's name, id or label says it holds a secret: one of :data:`SECRET_WORDS`, or
|
|
111
|
+
two words in a row that are one of :data:`SECRET_PAIRS`; the recorder applies the same lists."""
|
|
112
|
+
words = _words(text)
|
|
113
|
+
return any(
|
|
114
|
+
word in SECRET_WORDS or f"{word} {after}" in SECRET_PAIRS for word, after in zip(words, words[1:] + [""])
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _count(value: Any) -> int | None:
|
|
119
|
+
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _strings(value: Any, limit: int, chars: int, what: str) -> list[str]:
|
|
123
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
124
|
+
raise InteractionRefused(400, f"{what} is a list of strings")
|
|
125
|
+
return [item[:chars] for item in value[:limit]]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _target(raw: Any, what: str) -> dict[str, Any]:
|
|
129
|
+
if not isinstance(raw, dict):
|
|
130
|
+
raise InteractionRefused(400, f"an event's {what} is an object")
|
|
131
|
+
kept: dict[str, Any] = {
|
|
132
|
+
name: _text(raw[name], chars, f"{what}.{name}") for name, chars in TARGET_FIELDS.items() if name in raw
|
|
133
|
+
}
|
|
134
|
+
lists = TARGET_LISTS.items()
|
|
135
|
+
kept.update({name: _strings(raw[name], *bounds, f"{what}.{name}") for name, bounds in lists if name in raw})
|
|
136
|
+
return kept
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _point(raw: Any) -> dict[str, int] | None:
|
|
140
|
+
x, y = (_count(raw.get(axis)) for axis in ("x", "y")) if isinstance(raw, dict) else (None, None)
|
|
141
|
+
return None if x is None or y is None else {"x": x, "y": y}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _files(raw: Any) -> list[dict[str, Any]]:
|
|
145
|
+
if not isinstance(raw, list) or not all(isinstance(item, dict) for item in raw):
|
|
146
|
+
raise InteractionRefused(400, "an event's files is a list of {name, size, type}")
|
|
147
|
+
return [
|
|
148
|
+
{
|
|
149
|
+
"name": _text(item.get("name"), NAME_CHARS, "files.name"),
|
|
150
|
+
"size": _count(item.get("size")),
|
|
151
|
+
"type": _text(item.get("type") or "", NAME_CHARS, "files.type"),
|
|
152
|
+
}
|
|
153
|
+
for item in raw[:MAX_FILES]
|
|
154
|
+
]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass(frozen=True)
|
|
158
|
+
class UiInteraction:
|
|
159
|
+
"""One interaction as the page reported it. The kinds are subclasses, chosen by
|
|
160
|
+
:meth:`of` from ``kind``; each says what it carries and what of it the ledger lists
|
|
161
|
+
(:meth:`summary`)."""
|
|
162
|
+
|
|
163
|
+
page: str
|
|
164
|
+
target: dict[str, Any]
|
|
165
|
+
value: str | None = None
|
|
166
|
+
at: int | None = None
|
|
167
|
+
seq: int | None = None
|
|
168
|
+
point: dict[str, int] | None = None
|
|
169
|
+
files: list[dict[str, Any]] | None = None
|
|
170
|
+
source: dict[str, Any] | None = None
|
|
171
|
+
load: str | None = None
|
|
172
|
+
kind: ClassVar[str]
|
|
173
|
+
carries_value: ClassVar[bool] = False
|
|
174
|
+
carries_files: ClassVar[bool] = False
|
|
175
|
+
carries_source: ClassVar[bool] = False
|
|
176
|
+
pointer: ClassVar[bool] = False
|
|
177
|
+
states_a_fact: ClassVar[bool] = False
|
|
178
|
+
kinds: ClassVar[dict[str, type[UiInteraction]]] = {}
|
|
179
|
+
|
|
180
|
+
def __init_subclass__(cls, **options: Any) -> None:
|
|
181
|
+
super().__init_subclass__(**options)
|
|
182
|
+
UiInteraction.kinds[cls.kind] = cls
|
|
183
|
+
|
|
184
|
+
@classmethod
|
|
185
|
+
def of(cls, raw: Any) -> UiInteraction:
|
|
186
|
+
if not isinstance(raw, dict):
|
|
187
|
+
raise InteractionRefused(400, "each event is an object")
|
|
188
|
+
kind = cls.kinds.get(raw.get("kind")) if isinstance(raw.get("kind"), str) else None
|
|
189
|
+
if kind is None:
|
|
190
|
+
raise InteractionRefused(400, f"an event's kind is one of {', '.join(cls.kinds)}")
|
|
191
|
+
value = raw.get("value")
|
|
192
|
+
return kind(
|
|
193
|
+
page=_text(raw.get("page"), PAGE_CHARS, "page"),
|
|
194
|
+
target=_target(raw.get("target") or {}, "target"),
|
|
195
|
+
value=_text(value, VALUE_CHARS, "value") if kind.carries_value and value is not None else None,
|
|
196
|
+
at=_count(raw.get("at")),
|
|
197
|
+
seq=_count(raw.get("seq")),
|
|
198
|
+
point=_point(raw.get("point")) if kind.pointer else None,
|
|
199
|
+
files=_files(raw.get("files") or []) if kind.carries_files else None,
|
|
200
|
+
source=_target(raw.get("source") or {}, "source") if kind.carries_source else None,
|
|
201
|
+
load=_text(raw["load"], LOAD_CHARS, "load") if raw.get("load") is not None else None,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
@property
|
|
205
|
+
def operation(self) -> str:
|
|
206
|
+
return f"ui.{self.kind}"
|
|
207
|
+
|
|
208
|
+
def summary(self) -> dict[str, str]:
|
|
209
|
+
name = self.target.get("name")
|
|
210
|
+
return {"target": name} if name else {}
|
|
211
|
+
|
|
212
|
+
def query(self) -> dict[str, Any]:
|
|
213
|
+
"""The summary with the page load's numbering, read by a grade whose journal carries no bodies."""
|
|
214
|
+
numbered = {key: value for key, value in (("load", self.load), ("seq", self.seq)) if value is not None}
|
|
215
|
+
return {**self.summary(), **numbered}
|
|
216
|
+
|
|
217
|
+
def secret_value(self) -> bool:
|
|
218
|
+
target = self.target
|
|
219
|
+
named = (target.get(key) for key in ("field", "id", "name"))
|
|
220
|
+
return not self.states_a_fact and (
|
|
221
|
+
target.get("type", "").lower() in SECRET_INPUT_TYPES
|
|
222
|
+
or bool(SECRET_AUTOCOMPLETE.search(target.get("autocomplete", "").lower()))
|
|
223
|
+
or any(credential_named(name) or ROOT.child(name).credential for name in named if name)
|
|
224
|
+
or _card_digits(self.value or "")
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
def detail(self) -> dict[str, Any]:
|
|
228
|
+
value = {} if self.value is None else {"value": REDACTED if self.secret_value() else self.value}
|
|
229
|
+
carried = {"point": self.point, "files": self.files, "source": self.source, "load": self.load}
|
|
230
|
+
kept = {key: item for key, item in carried.items() if item is not None}
|
|
231
|
+
return {"page": self.page, "target": self.target, **value, **kept, "at": self.at, "seq": self.seq}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class UiClick(UiInteraction):
|
|
235
|
+
kind = "click"
|
|
236
|
+
pointer = True
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class UiDoubleClick(UiInteraction):
|
|
240
|
+
kind = "dblclick"
|
|
241
|
+
pointer = True
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class UiContextMenu(UiInteraction):
|
|
245
|
+
kind = "contextmenu"
|
|
246
|
+
pointer = True
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class UiHover(UiInteraction):
|
|
250
|
+
kind = "hover"
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class UiChange(UiInteraction):
|
|
254
|
+
kind = "change"
|
|
255
|
+
carries_value = True
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class UiSubmit(UiInteraction):
|
|
259
|
+
kind = "submit"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class UiKey(UiInteraction):
|
|
263
|
+
kind = "key"
|
|
264
|
+
carries_value = True
|
|
265
|
+
states_a_fact = True
|
|
266
|
+
|
|
267
|
+
def summary(self) -> dict[str, str]:
|
|
268
|
+
return {"key": self.value or "", **super().summary()}
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class UiScroll(UiInteraction):
|
|
272
|
+
kind = "scroll"
|
|
273
|
+
carries_value = True
|
|
274
|
+
states_a_fact = True
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class UiCopy(UiInteraction):
|
|
278
|
+
kind = "copy"
|
|
279
|
+
carries_value = True
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class UiCut(UiInteraction):
|
|
283
|
+
kind = "cut"
|
|
284
|
+
carries_value = True
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class UiPaste(UiInteraction):
|
|
288
|
+
kind = "paste"
|
|
289
|
+
carries_value = True
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
class UiDrag(UiInteraction):
|
|
293
|
+
kind = "drag"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class UiDrop(UiInteraction):
|
|
297
|
+
kind = "drop"
|
|
298
|
+
carries_source = True
|
|
299
|
+
|
|
300
|
+
def summary(self) -> dict[str, str]:
|
|
301
|
+
moved = (self.source or {}).get("name")
|
|
302
|
+
return {**super().summary(), **({"source": moved} if moved else {})}
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class UiFile(UiInteraction):
|
|
306
|
+
kind = "file"
|
|
307
|
+
carries_files = True
|
|
308
|
+
|
|
309
|
+
def summary(self) -> dict[str, str]:
|
|
310
|
+
return {**super().summary(), "files": str(len(self.files or []))}
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class UiVisibility(UiInteraction):
|
|
314
|
+
kind = "visibility"
|
|
315
|
+
carries_value = True
|
|
316
|
+
states_a_fact = True
|
|
317
|
+
|
|
318
|
+
def summary(self) -> dict[str, str]:
|
|
319
|
+
return {"state": self.value or ""}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class UiOpen(UiInteraction):
|
|
323
|
+
kind = "open"
|
|
324
|
+
carries_value = True
|
|
325
|
+
states_a_fact = True
|
|
326
|
+
|
|
327
|
+
def summary(self) -> dict[str, str]:
|
|
328
|
+
return {"to": self.value or ""}
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
class UiNavigate(UiInteraction):
|
|
332
|
+
kind = "navigate"
|
|
333
|
+
|
|
334
|
+
def summary(self) -> dict[str, str]:
|
|
335
|
+
return {"to": self.page}
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@dataclass(frozen=True)
|
|
339
|
+
class UiBatch:
|
|
340
|
+
"""One report of the recorder: at most :data:`MAX_EVENTS` interactions in
|
|
341
|
+
:data:`MAX_BATCH_BYTES` of JSON, kept whole or not at all, in the order the page sent them."""
|
|
342
|
+
|
|
343
|
+
interactions: list[UiInteraction] = field(default_factory=list)
|
|
344
|
+
|
|
345
|
+
@classmethod
|
|
346
|
+
def read(cls, raw: bytes) -> UiBatch:
|
|
347
|
+
if len(raw) > MAX_BATCH_BYTES:
|
|
348
|
+
raise InteractionRefused(413, f"a report is at most {MAX_BATCH_BYTES} bytes")
|
|
349
|
+
try:
|
|
350
|
+
body = json.loads(raw or b"null")
|
|
351
|
+
except ValueError as error:
|
|
352
|
+
raise InteractionRefused(400, f"malformed json: {error}") from error
|
|
353
|
+
events = body.get("events") if isinstance(body, dict) else None
|
|
354
|
+
if not isinstance(events, list):
|
|
355
|
+
raise InteractionRefused(400, "a report is {events: [...]}")
|
|
356
|
+
if len(events) > MAX_EVENTS:
|
|
357
|
+
raise InteractionRefused(400, f"a report holds at most {MAX_EVENTS} events")
|
|
358
|
+
return cls([UiInteraction.of(event) for event in events])
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class UiRecorder:
|
|
362
|
+
"""The ui origin's half of interaction recording: the note that the session's pages are
|
|
363
|
+
recorded, the tag on each page and the note that it was served, the script behind the tag, and
|
|
364
|
+
the events door."""
|
|
365
|
+
|
|
366
|
+
@staticmethod
|
|
367
|
+
def serves(path: str) -> bool:
|
|
368
|
+
return path.startswith(RECORDER_PREFIX)
|
|
369
|
+
|
|
370
|
+
@staticmethod
|
|
371
|
+
def is_page(path: str) -> bool:
|
|
372
|
+
return Path(path).suffix.lower() in PAGE_SUFFIXES
|
|
373
|
+
|
|
374
|
+
@staticmethod
|
|
375
|
+
def page(status: int, headers: dict[str, str], payload: bytes) -> bytes | None:
|
|
376
|
+
"""``payload`` with the recorder after its opening tag, or None when it is not a page:
|
|
377
|
+
not a 200, not HTML, no ``<head>``, ``<html>`` or doctype to follow, or already tagged."""
|
|
378
|
+
html = headers.get("content-type", "").startswith("text/html")
|
|
379
|
+
body = _BODY.search(payload)
|
|
380
|
+
head = payload[: body.start()] if body else payload
|
|
381
|
+
opening = next(filter(None, (pattern.search(head) for pattern in _OPENINGS)), None)
|
|
382
|
+
if status != 200 or not html or opening is None or RECORDER_TAG in payload:
|
|
383
|
+
return None
|
|
384
|
+
return payload[: opening.end()] + RECORDER_TAG + payload[opening.end() :]
|
|
385
|
+
|
|
386
|
+
@staticmethod
|
|
387
|
+
def recording(calls: CallLog) -> None:
|
|
388
|
+
version = {"recorder": RECORDER_VERSION}
|
|
389
|
+
calls.record_interaction(RECORDING_OPERATION, "/", version, version, graded=False)
|
|
390
|
+
|
|
391
|
+
@staticmethod
|
|
392
|
+
def unrecorded(calls: CallLog) -> None:
|
|
393
|
+
calls.record_interaction(UNRECORDED_OPERATION, "/", {}, {}, graded=False)
|
|
394
|
+
|
|
395
|
+
@staticmethod
|
|
396
|
+
def served(page: str, calls: CallLog) -> None:
|
|
397
|
+
calls.record_interaction(SERVED_OPERATION, page, {"to": page}, {"page": page}, graded=False)
|
|
398
|
+
|
|
399
|
+
@staticmethod
|
|
400
|
+
def script() -> tuple[int, dict[str, str], bytes]:
|
|
401
|
+
headers = {
|
|
402
|
+
"content-type": "text/javascript; charset=utf-8",
|
|
403
|
+
"cache-control": "public, max-age=31536000, immutable",
|
|
404
|
+
}
|
|
405
|
+
return 200, headers, RECORDER_SOURCE.read_bytes()
|
|
406
|
+
|
|
407
|
+
@staticmethod
|
|
408
|
+
def record(method: str, path: str, raw: bytes, calls: CallLog) -> tuple[int, dict]:
|
|
409
|
+
if path != EVENTS_PATH:
|
|
410
|
+
return 404, {"error": f"the recorder answers {RECORDER_PATH} and {EVENTS_PATH}"}
|
|
411
|
+
if method != "POST":
|
|
412
|
+
return 405, {"error": f"{EVENTS_PATH} takes POST"}
|
|
413
|
+
try:
|
|
414
|
+
batch = UiBatch.read(raw)
|
|
415
|
+
except InteractionRefused as refused:
|
|
416
|
+
return refused.status, {"error": refused.error}
|
|
417
|
+
for interaction in batch.interactions:
|
|
418
|
+
calls.record_interaction(interaction.operation, interaction.page, interaction.query(), interaction.detail())
|
|
419
|
+
return 200, {"recorded": len(batch.interactions)}
|