@withgateway/sdk 0.6.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +573 -0
- package/dist/agents/index.d.ts +573 -0
- package/dist/agents/index.js +1443 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1394 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +5019 -1599
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +5003 -1584
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +2689 -401
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2650 -394
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1983 -1745
- package/dist/worlds/index.d.ts +1983 -1745
- package/dist/worlds/index.js +1295 -226
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1290 -222
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/host.py +74 -11
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +23 -267
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +43 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +367 -217
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +7 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
- package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
- package/runtime/gatewaysdk/environment/schema/world_host.py +44 -15
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -2
- package/skills/connector-schema-authoring/connector-worlds.md +2 -2
- package/skills/connector-schema-authoring/world-schema.md +29 -23
- package/skills/world-data-ingestion/SKILL.md +4 -2
- package/skills/worlds-getting-started/SKILL.md +7 -7
- package/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Binds a leased pool browser to the one session it serves: the relay's egress proxy then
|
|
2
|
+
loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from .host_requests import BrowserRelay
|
|
16
|
+
from .validation import fail
|
|
17
|
+
|
|
18
|
+
GUARD_REQUIRED_FROM = 1794787200.0
|
|
19
|
+
BIND_TIMEOUT_S = 5.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class RelayNavigation:
|
|
24
|
+
opener: Callable[..., Any] = field(default=urllib.request.urlopen)
|
|
25
|
+
clock: Callable[[], float] = field(default=time.time)
|
|
26
|
+
|
|
27
|
+
def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
|
|
28
|
+
request = urllib.request.Request(
|
|
29
|
+
f"{relay.http_url}/navigation",
|
|
30
|
+
data=json.dumps({"hosts": hosts}).encode(),
|
|
31
|
+
method="PUT",
|
|
32
|
+
headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
|
|
33
|
+
)
|
|
34
|
+
try:
|
|
35
|
+
with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
|
|
36
|
+
return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
|
|
37
|
+
except urllib.error.HTTPError as error:
|
|
38
|
+
if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
|
|
39
|
+
log_relay("browser relay predates the navigation guard", relay)
|
|
40
|
+
return "unguarded"
|
|
41
|
+
fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
|
|
42
|
+
except (OSError, ValueError) as error:
|
|
43
|
+
log_relay("browser relay could not be bound", relay, error=str(error))
|
|
44
|
+
fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
|
|
48
|
+
print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
|
|
@@ -15,6 +15,8 @@ from email.message import Message
|
|
|
15
15
|
from typing import TYPE_CHECKING, Any
|
|
16
16
|
|
|
17
17
|
from .handler_errors import carried
|
|
18
|
+
from .host_browser import RelayNavigation
|
|
19
|
+
from .host_requests import BrowserRelay
|
|
18
20
|
from .host_surfaces import public_host
|
|
19
21
|
from .host_workers import WORKER_EXITED, refusal
|
|
20
22
|
from .validation import ContractError, fail
|
|
@@ -35,10 +37,18 @@ class ControlPlane:
|
|
|
35
37
|
"""One host's control plane. A host with no secret has no control plane at all rather than an
|
|
36
38
|
open one, and one with no worlds domain could not tell its own address from a session's."""
|
|
37
39
|
|
|
38
|
-
def __init__(
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
host: WorldHost,
|
|
43
|
+
secret: str,
|
|
44
|
+
operations: Mapping[str, Callable[..., dict]],
|
|
45
|
+
worlds_domain: str,
|
|
46
|
+
navigation: RelayNavigation | None = None,
|
|
47
|
+
):
|
|
39
48
|
if not secret:
|
|
40
49
|
raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
|
|
41
50
|
self.host = host
|
|
51
|
+
self.navigation = navigation or RelayNavigation()
|
|
42
52
|
self.operations = operations
|
|
43
53
|
self.domain = self.domain_of(worlds_domain)
|
|
44
54
|
self._secret = secret.encode()
|
|
@@ -109,6 +119,20 @@ class ControlPlane:
|
|
|
109
119
|
return 404, {"error": "unknown control route"}
|
|
110
120
|
|
|
111
121
|
def _open(self, body: dict) -> dict:
|
|
122
|
+
answer = self._open_session(body)
|
|
123
|
+
relay = BrowserRelay.parse(body.get("browser"))
|
|
124
|
+
if relay is not None:
|
|
125
|
+
self._bind_or_close(body["sessionId"], relay)
|
|
126
|
+
return answer
|
|
127
|
+
|
|
128
|
+
def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
|
|
129
|
+
try:
|
|
130
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
131
|
+
except Exception:
|
|
132
|
+
self.host.close(session_id, "browser_unbound")
|
|
133
|
+
raise
|
|
134
|
+
|
|
135
|
+
def _open_session(self, body: dict) -> dict:
|
|
112
136
|
return self.host.open(
|
|
113
137
|
session_id=body["sessionId"],
|
|
114
138
|
version_id=body["versionId"],
|
|
@@ -123,14 +147,21 @@ class ControlPlane:
|
|
|
123
147
|
pending=body.get("pending") is True,
|
|
124
148
|
auth=body.get("auth"),
|
|
125
149
|
ledger_source=body.get("ledgerSource"),
|
|
150
|
+
parameters=body.get("parameters"),
|
|
126
151
|
)
|
|
127
152
|
|
|
153
|
+
def _swap_browser(self, session_id: str, browser: Any) -> dict:
|
|
154
|
+
relay = BrowserRelay.parse(browser)
|
|
155
|
+
if relay is not None:
|
|
156
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
157
|
+
return self.host.set_browser(session_id, browser)
|
|
158
|
+
|
|
128
159
|
def _session(self, method: str, parts: list[str], body: dict) -> Answer:
|
|
129
160
|
sid = parts[2]
|
|
130
161
|
self.host.forwarding.refuse(sid)
|
|
131
162
|
verb = parts[3] if len(parts) > 3 else None
|
|
132
163
|
if method == "PUT" and verb == "browser" and len(parts) == 4:
|
|
133
|
-
return 200, self.
|
|
164
|
+
return 200, self._swap_browser(sid, body.get("browser"))
|
|
134
165
|
if method == "POST" and verb == "export" and len(parts) == 5:
|
|
135
166
|
return self._upload(sid, parts[4], body.get("url"))
|
|
136
167
|
row = session_verb(method, verb)
|
|
@@ -31,6 +31,7 @@ from .handler_errors import log_line
|
|
|
31
31
|
from .host_owners import PROXY_TIMEOUT_S
|
|
32
32
|
from .request_ledger import LEDGER_STOP_S
|
|
33
33
|
from .session_move import same_host
|
|
34
|
+
from .ui_replay import REPLAY_STOP_S
|
|
34
35
|
|
|
35
36
|
if TYPE_CHECKING:
|
|
36
37
|
from .host_session import Session
|
|
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
|
|
|
47
48
|
LINGER_S = 5.0
|
|
48
49
|
IN_FLIGHT_S = PROXY_TIMEOUT_S
|
|
49
50
|
EXIT_S = 110.0
|
|
50
|
-
assert
|
|
51
|
+
assert (
|
|
52
|
+
LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
|
|
53
|
+
), "a stopping host's exit fits the grace past its drain"
|
|
51
54
|
|
|
52
55
|
MOVED = "moved"
|
|
53
56
|
STAYED = "stayed"
|
|
@@ -28,6 +28,16 @@ class DependencyRef:
|
|
|
28
28
|
snapshot: dict[str, str] | None
|
|
29
29
|
raw: dict[str, Any]
|
|
30
30
|
|
|
31
|
+
@property
|
|
32
|
+
def for_worker(self) -> dict[str, Any]:
|
|
33
|
+
"""The entry a version worker is handed: its snapshot without the url, a tree the supervisor built."""
|
|
34
|
+
return {**self.raw, "snapshot": worker_snapshot(self.snapshot)}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
|
|
38
|
+
"""A snapshot as a version worker sees it: its sha256 and digest, never the presigned url."""
|
|
39
|
+
return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
|
|
40
|
+
|
|
31
41
|
|
|
32
42
|
def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
|
|
33
43
|
if dependencies is None:
|
|
@@ -77,6 +87,10 @@ class BrowserRelay:
|
|
|
77
87
|
raise AssertionError # fail() never returns
|
|
78
88
|
return cls(url.rstrip("/"), token)
|
|
79
89
|
|
|
90
|
+
@property
|
|
91
|
+
def http_url(self) -> str:
|
|
92
|
+
return "http://" + self.url.removeprefix("ws://")
|
|
93
|
+
|
|
80
94
|
|
|
81
95
|
def restore_sha256(restore: object) -> str | None:
|
|
82
96
|
"""The sha256 of the export an open restores from, or None for an open that restores nothing."""
|
|
@@ -10,7 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
from typing import TYPE_CHECKING
|
|
11
11
|
|
|
12
12
|
from .door import OPEN_MODE, VENDOR_MODE, Door, same
|
|
13
|
-
from .host_surfaces import SessionHost, capability_host
|
|
13
|
+
from .host_surfaces import SessionHost, capability_host, session_host
|
|
14
14
|
from .request_ledger import LedgerLife
|
|
15
15
|
from .session_move import SessionMove
|
|
16
16
|
|
|
@@ -75,6 +75,8 @@ class Session:
|
|
|
75
75
|
placement: dict | None = None
|
|
76
76
|
settings: ApiSettings | None = None
|
|
77
77
|
restore_sha256: str | None = None
|
|
78
|
+
snapshot_sha256: str | None = None
|
|
79
|
+
parameters: dict[str, str] = field(default_factory=dict)
|
|
78
80
|
pending: InitVar[bool] = False
|
|
79
81
|
move: SessionMove = field(init=False, repr=False, compare=False)
|
|
80
82
|
auth: str = VENDOR_MODE
|
|
@@ -119,8 +121,17 @@ class Session:
|
|
|
119
121
|
capability = self.door.capability if self.door is not None else None
|
|
120
122
|
return self.auth == OPEN_MODE and capability is not None and same(named.capability, capability)
|
|
121
123
|
|
|
124
|
+
def browser_hosts(self, domain: str) -> list[str]:
|
|
125
|
+
"""The only hosts this session's browser may load: its ui and api, and their capability
|
|
126
|
+
pair when the session is open."""
|
|
127
|
+
hosts = [session_host(self.session_id, surface, domain) for surface in ("ui", "api")]
|
|
128
|
+
capability = self.door.capability if self.door is not None else None
|
|
129
|
+
if self.auth == OPEN_MODE and capability is not None:
|
|
130
|
+
hosts += [capability_host(self.session_id, capability, surface, domain) for surface in ("ui", "api")]
|
|
131
|
+
return hosts
|
|
132
|
+
|
|
122
133
|
def api_host(self, domain: str, opened: bool) -> str:
|
|
123
134
|
"""The host a link the world writes points at when a request names none."""
|
|
124
135
|
if opened and self.door is not None and self.door.capability is not None:
|
|
125
136
|
return capability_host(self.session_id, self.door.capability, "api", domain)
|
|
126
|
-
return
|
|
137
|
+
return session_host(self.session_id, "api", domain)
|
|
@@ -260,6 +260,12 @@ def _capability_host(rest: str, surface: str) -> SessionHost | None:
|
|
|
260
260
|
return SessionHost(session_id, surface, capability)
|
|
261
261
|
|
|
262
262
|
|
|
263
|
+
def session_host(session_id: str, surface: str, domain: str) -> str:
|
|
264
|
+
"""The session's own hostname for ``surface`` (``api`` or ``ui``)."""
|
|
265
|
+
suffix = "-api" if surface == "api" else ""
|
|
266
|
+
return f"w-{session_id}{suffix}.{domain}"
|
|
267
|
+
|
|
268
|
+
|
|
263
269
|
def capability_host(session_id: str, capability: str, surface: str, domain: str) -> str:
|
|
264
270
|
"""The capability hostname of an open session's ``surface`` (``api`` or ``ui``)."""
|
|
265
271
|
suffix = "-api" if surface == "api" else ""
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
|
+
import re
|
|
7
8
|
import shutil
|
|
8
9
|
import tarfile
|
|
9
10
|
import threading
|
|
@@ -21,6 +22,7 @@ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
|
|
|
21
22
|
from .compiler import check_bundle
|
|
22
23
|
from .connector import Spec, load_spec
|
|
23
24
|
from .database import WorldDatabase
|
|
25
|
+
from .database.base import reflink
|
|
24
26
|
from .dependencies import Dependency
|
|
25
27
|
from .door import Door
|
|
26
28
|
from .host_surfaces import UiSurface, load_ui_surface
|
|
@@ -33,6 +35,8 @@ from .validation import fail
|
|
|
33
35
|
|
|
34
36
|
BUNDLED_WORLD = "server/world"
|
|
35
37
|
WORLD_SPEC = "schema/world.json"
|
|
38
|
+
SNAPSHOTS_DIR = "snapshots"
|
|
39
|
+
_SHA256 = re.compile(r"[0-9a-f]{64}")
|
|
36
40
|
#: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
|
|
37
41
|
DEFAULT_MAX_BUNDLE_MB = 2048
|
|
38
42
|
MAX_BUNDLE_MEMBERS = 200_000
|
|
@@ -53,6 +57,41 @@ def world_tree(bundle: Path) -> Path:
|
|
|
53
57
|
)
|
|
54
58
|
|
|
55
59
|
|
|
60
|
+
def clone_mode(directory: Path) -> str:
|
|
61
|
+
"""``reflink`` when session copies on ``directory`` share blocks until written, ``copy`` when each one
|
|
62
|
+
copies its file: the disk decides, so the host probes it once and reports it in its stats."""
|
|
63
|
+
probe = directory / f".clone-probe-{uuid.uuid4().hex}"
|
|
64
|
+
clone = probe.with_name(f"{probe.name}.clone")
|
|
65
|
+
try:
|
|
66
|
+
probe.write_bytes(b"probe")
|
|
67
|
+
return "reflink" if reflink(probe, clone) else "copy"
|
|
68
|
+
finally:
|
|
69
|
+
probe.unlink(missing_ok=True)
|
|
70
|
+
clone.unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def data_tree(bundle: Path) -> Path:
|
|
74
|
+
return bundle if is_tools_bundle(bundle) else world_tree(bundle)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class DataKey:
|
|
79
|
+
"""A version as a session opens it: on the version's own data, or on one snapshot of it, named by the
|
|
80
|
+
sha256 of the snapshot file. Two snapshots of one version never share a tree, a template or a session."""
|
|
81
|
+
|
|
82
|
+
version_id: str
|
|
83
|
+
sha256: str | None
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def of(cls, version_id: str, snapshot: dict[str, str] | None) -> DataKey:
|
|
87
|
+
if not snapshot:
|
|
88
|
+
return cls(version_id, None)
|
|
89
|
+
sha256 = str(snapshot.get("sha256") or "").lower()
|
|
90
|
+
if not _SHA256.fullmatch(sha256):
|
|
91
|
+
fail("host", "a snapshot names the sha256 of its file (64 hex digits)")
|
|
92
|
+
return cls(version_id, sha256)
|
|
93
|
+
|
|
94
|
+
|
|
56
95
|
@dataclass
|
|
57
96
|
class CachedVersion:
|
|
58
97
|
"""One cached version, of either shape the host serves: an api bundle (a
|
|
@@ -124,8 +163,9 @@ class VersionCache:
|
|
|
124
163
|
#: Off in the supervisor: only a version worker imports world-authored code.
|
|
125
164
|
self.run_handlers = run_handlers
|
|
126
165
|
self.max_bundle_bytes = max_bundle_mb * (1 << 20)
|
|
166
|
+
self.clone_mode = clone_mode(self.root)
|
|
127
167
|
self._lock = threading.Lock()
|
|
128
|
-
self._versions: dict[
|
|
168
|
+
self._versions: dict[DataKey, CachedVersion] = {}
|
|
129
169
|
self._builds = Lanes()
|
|
130
170
|
|
|
131
171
|
def ensure(
|
|
@@ -134,37 +174,76 @@ class VersionCache:
|
|
|
134
174
|
bundle_url: str,
|
|
135
175
|
snapshot: dict[str, str] | None,
|
|
136
176
|
) -> CachedVersion:
|
|
137
|
-
|
|
177
|
+
key = DataKey.of(version_id, snapshot)
|
|
178
|
+
hit = self._cached(key)
|
|
138
179
|
if hit is not None:
|
|
139
180
|
return hit
|
|
140
181
|
with self._builds.turn(version_id):
|
|
141
|
-
return self._cached(
|
|
182
|
+
return self._cached(key) or self._store(key, self._build(key, bundle_url, snapshot))
|
|
142
183
|
|
|
143
|
-
def _cached(self,
|
|
184
|
+
def _cached(self, key: DataKey) -> CachedVersion | None:
|
|
144
185
|
with self._lock:
|
|
145
|
-
return self._versions.get(
|
|
186
|
+
return self._versions.get(key)
|
|
146
187
|
|
|
147
|
-
def _store(self, cached: CachedVersion) -> CachedVersion:
|
|
188
|
+
def _store(self, key: DataKey, cached: CachedVersion) -> CachedVersion:
|
|
148
189
|
with self._lock:
|
|
149
|
-
self._versions[
|
|
190
|
+
self._versions[key] = cached
|
|
150
191
|
return cached
|
|
151
192
|
|
|
152
|
-
def has(self, version_id: str) -> bool:
|
|
193
|
+
def has(self, version_id: str, snapshot: dict[str, str] | None = None) -> bool:
|
|
194
|
+
key = DataKey.of(version_id, snapshot)
|
|
153
195
|
with self._lock:
|
|
154
|
-
return
|
|
196
|
+
return key in self._versions
|
|
155
197
|
|
|
156
198
|
def version_ids(self) -> list[str]:
|
|
157
199
|
with self._lock:
|
|
158
|
-
return sorted(self._versions)
|
|
200
|
+
return sorted({key.version_id for key in self._versions})
|
|
159
201
|
|
|
160
|
-
def _build(self,
|
|
161
|
-
|
|
162
|
-
if not
|
|
163
|
-
self._download(bundle_url,
|
|
164
|
-
if
|
|
165
|
-
|
|
166
|
-
|
|
202
|
+
def _build(self, key: DataKey, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
|
|
203
|
+
base = self.root / key.version_id / "bundle"
|
|
204
|
+
if not base.is_dir():
|
|
205
|
+
self._download(bundle_url, base)
|
|
206
|
+
if key.sha256 is None or snapshot is None:
|
|
207
|
+
return self._load(key.version_id, base, self.root / key.version_id / "template.db", fetched=False)
|
|
208
|
+
bundle = self._with_snapshot(key, base, snapshot)
|
|
209
|
+
return self._load(key.version_id, bundle, bundle.parent / "template.db", fetched=True)
|
|
210
|
+
|
|
211
|
+
def _with_snapshot(self, key: DataKey, base: Path, snapshot: dict[str, str]) -> Path:
|
|
212
|
+
"""The version's bundle with the snapshot's rows as its data, under ``snapshots/<sha256>``: cloned
|
|
213
|
+
from the downloaded bundle (never linked, so nothing written here reaches it) without the data the
|
|
214
|
+
snapshot replaces, then fetched and verified. Renamed into place only once complete; one built
|
|
215
|
+
before is reused without a url, which is how a version worker finds it."""
|
|
216
|
+
home = self.root / key.version_id / SNAPSHOTS_DIR / str(key.sha256)
|
|
217
|
+
if (home / "bundle").is_dir():
|
|
218
|
+
return home / "bundle"
|
|
219
|
+
url, digest = snapshot.get("url"), snapshot.get("digest")
|
|
220
|
+
if not url or not isinstance(digest, str):
|
|
221
|
+
fail(
|
|
222
|
+
"host",
|
|
223
|
+
f"snapshot {key.sha256} of version {key.version_id} is not cached on this host, "
|
|
224
|
+
"and the open names no url and digest to fetch it",
|
|
167
225
|
)
|
|
226
|
+
tree = data_tree(base)
|
|
227
|
+
contract = check_bundle(tree)
|
|
228
|
+
replaced = snapshots.default_snapshot(tree, contract)
|
|
229
|
+
staging = home.parent / f".{key.sha256}.{uuid.uuid4().hex}"
|
|
230
|
+
try:
|
|
231
|
+
for path in sorted(base.rglob("*")):
|
|
232
|
+
target = staging / "bundle" / path.relative_to(base)
|
|
233
|
+
if path.is_dir():
|
|
234
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
235
|
+
elif path != replaced:
|
|
236
|
+
WorldDatabase.clone_file(path, target)
|
|
237
|
+
staged = staging / "bundle" / tree.relative_to(base)
|
|
238
|
+
snapshots.fetch(staged, contract, url, str(key.sha256), digest)
|
|
239
|
+
os.replace(staging, home)
|
|
240
|
+
finally:
|
|
241
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
242
|
+
return home / "bundle"
|
|
243
|
+
|
|
244
|
+
def _load(self, version_id: str, bundle: Path, template: Path, *, fetched: bool) -> CachedVersion:
|
|
245
|
+
if is_tools_bundle(bundle):
|
|
246
|
+
tools = load_tools_version(bundle, template, fetched=fetched, run_handlers=self.run_handlers)
|
|
168
247
|
return CachedVersion(
|
|
169
248
|
version_id=version_id,
|
|
170
249
|
bundle=bundle,
|
|
@@ -179,8 +258,6 @@ class VersionCache:
|
|
|
179
258
|
)
|
|
180
259
|
tree = world_tree(bundle)
|
|
181
260
|
contract = check_bundle(tree)
|
|
182
|
-
if snapshot:
|
|
183
|
-
snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
184
261
|
served = snapshots.ServedSnapshot.of(tree)
|
|
185
262
|
spec = load_spec(tree)
|
|
186
263
|
settings = load_api_settings(tree, spec)
|
|
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
|
|
|
13
13
|
from .._toml import tomllib
|
|
14
14
|
from .call_redaction import REDACTED, CallRedaction
|
|
15
15
|
from .database.base import SideTable
|
|
16
|
-
from .tasks import JOURNAL, JOURNAL_META, WITHHELD_FIELD
|
|
16
|
+
from .tasks import JOURNAL, JOURNAL_META, PAGE_NOTES, WITHHELD_FIELD
|
|
17
17
|
from .validation import fail
|
|
18
18
|
|
|
19
19
|
if TYPE_CHECKING:
|
|
@@ -26,7 +26,7 @@ JOURNAL_TABLES = frozenset(JOURNAL_KEYS)
|
|
|
26
26
|
MAX_JOURNAL_REQUESTS = 20_000
|
|
27
27
|
NOT_RECORDED = "not recorded: only a World Host session records the requests its world receives"
|
|
28
28
|
LINKED_NOT_RECORDED = "not recorded for a linked world: the root world's verifiers and the task's checks read it"
|
|
29
|
-
VIAS = ("api", "ui", "tool", "webhook")
|
|
29
|
+
VIAS = ("api", "ui", "tool", "webhook", "page")
|
|
30
30
|
JSON_HINTS = ("json",)
|
|
31
31
|
WITHHELD_ROUTE_BODY = ["body", "selectors"]
|
|
32
32
|
WITHHELD_TOOL_ARGS = ["args", "selectors"]
|
|
@@ -260,8 +260,13 @@ class CallsView:
|
|
|
260
260
|
return None
|
|
261
261
|
if entry.raw["via"] == "tool" and entry.raw["method"] is None:
|
|
262
262
|
return self._tool(entry) if self.admitted(entry.operation) else None
|
|
263
|
+
if entry.raw["via"] == "page":
|
|
264
|
+
return None if entry.operation in PAGE_NOTES else self._interaction(entry)
|
|
263
265
|
return self._route(entry)
|
|
264
266
|
|
|
267
|
+
def _interaction(self, entry: JournalEntry) -> dict:
|
|
268
|
+
return {**self._tool(entry), "path": entry.raw["path"], "query": entry.raw["query"]}
|
|
269
|
+
|
|
265
270
|
def _tool(self, entry: JournalEntry) -> dict:
|
|
266
271
|
body = entry.request
|
|
267
272
|
args = body.value if body.whole else REDACTED
|
|
@@ -10,6 +10,7 @@ import tempfile
|
|
|
10
10
|
import threading
|
|
11
11
|
import urllib.parse
|
|
12
12
|
from dataclasses import dataclass, field
|
|
13
|
+
from functools import partial
|
|
13
14
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
14
15
|
from pathlib import Path
|
|
15
16
|
from typing import Any, Callable
|
|
@@ -33,6 +34,7 @@ from .tools_world import (
|
|
|
33
34
|
ToolsWorldState,
|
|
34
35
|
load_tools_version,
|
|
35
36
|
)
|
|
37
|
+
from .ui_interactions import RECORDER_PATH, UiRecorder
|
|
36
38
|
from .validation import ContractError
|
|
37
39
|
from .verbs import refusal, session_verb, unknown_verb
|
|
38
40
|
|
|
@@ -51,7 +53,7 @@ def load_local_version(root: Path, scratch: Path) -> ToolsVersion:
|
|
|
51
53
|
f"{root} is not a schema world (no gateway-env.toml with a [schema] table); "
|
|
52
54
|
"`gateway worlds schema init` starts one",
|
|
53
55
|
)
|
|
54
|
-
return load_tools_version(root, scratch / "template.db",
|
|
56
|
+
return load_tools_version(root, scratch / "template.db", fetched=False, run_handlers=True)
|
|
55
57
|
|
|
56
58
|
|
|
57
59
|
def load_local_dependencies(root: Path, version: ToolsVersion, scratch: Path) -> dict[str, DependencyWorld]:
|
|
@@ -317,10 +319,7 @@ def make_session_handler(
|
|
|
317
319
|
raw = self.rfile.read(length) if length else b""
|
|
318
320
|
path = self.path.split("?", 1)[0]
|
|
319
321
|
if ui_origin:
|
|
320
|
-
|
|
321
|
-
if app is not None and api_target(ui, path) is not None:
|
|
322
|
-
return self._ui_api(method, path, raw, Via.UI)
|
|
323
|
-
return self._ui(method, path)
|
|
322
|
+
return self._ui_origin(method, path, raw)
|
|
324
323
|
origin = self.headers.get("Origin")
|
|
325
324
|
if origin and not _local_origin(origin):
|
|
326
325
|
return self._reply(403, {"error": "this session answers this machine's pages, not another site's"})
|
|
@@ -344,11 +343,32 @@ def make_session_handler(
|
|
|
344
343
|
)
|
|
345
344
|
send_response(self, status, body, extra, app.settings.error_content_type)
|
|
346
345
|
|
|
346
|
+
def _ui_origin(self, method: str, path: str, raw: bytes) -> None:
|
|
347
|
+
assert ui is not None
|
|
348
|
+
if app is not None and api_target(ui, path) is not None:
|
|
349
|
+
return self._ui_api(method, path, raw, Via.UI)
|
|
350
|
+
if path == RECORDER_PATH:
|
|
351
|
+
return self._page(method, UiRecorder.script)
|
|
352
|
+
if UiRecorder.serves(path):
|
|
353
|
+
return self._recorded(method, path, raw)
|
|
354
|
+
return self._page(method, partial(self._served, path))
|
|
355
|
+
|
|
356
|
+
def _served(self, path: str) -> tuple[int, dict[str, str], bytes]:
|
|
357
|
+
assert ui is not None
|
|
358
|
+
status, headers, payload = static_file(ui, path)
|
|
359
|
+
page = UiRecorder.page(status, headers, payload)
|
|
360
|
+
if page is not None:
|
|
361
|
+
UiRecorder.served(path, calls)
|
|
362
|
+
return status, headers, payload if page is None else page
|
|
363
|
+
|
|
347
364
|
def _ui(self, method: str, path: str) -> None:
|
|
365
|
+
assert ui is not None
|
|
366
|
+
return self._page(method, partial(static_file, ui, path))
|
|
367
|
+
|
|
368
|
+
def _page(self, method: str, answer: Callable[[], tuple[int, dict[str, str], bytes]]) -> None:
|
|
348
369
|
if method != "GET":
|
|
349
370
|
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
350
|
-
|
|
351
|
-
status, headers, payload = static_file(ui, path)
|
|
371
|
+
status, headers, payload = answer()
|
|
352
372
|
self.send_response(status)
|
|
353
373
|
for name, value in headers.items():
|
|
354
374
|
self.send_header(name, value)
|
|
@@ -356,6 +376,11 @@ def make_session_handler(
|
|
|
356
376
|
self.end_headers()
|
|
357
377
|
write_body(self, payload)
|
|
358
378
|
|
|
379
|
+
def _recorded(self, method: str, path: str, raw: bytes) -> None:
|
|
380
|
+
if cross_site({k.lower(): v for k, v in self.headers.items()}, _local_origin):
|
|
381
|
+
return self._reply(403, {"error": "the recorder answers the page on this server, not another site"})
|
|
382
|
+
return self._reply(*UiRecorder.record(method, path, raw, calls))
|
|
383
|
+
|
|
359
384
|
def _ui_api(self, method: str, path: str, raw: bytes, via: Via) -> None:
|
|
360
385
|
"""Answers with the world's own credential for the pin, as the host does for the session cookie."""
|
|
361
386
|
assert app is not None and ui is not None
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""The grader's view of a UI world's page (``via: page``): which checks read it, and whether what
|
|
2
|
+
the page reported can decide them. A check that does not name the page reads what it read before
|
|
3
|
+
pages reported anything; the host's notes (``ui.recording`` at open, ``ui.served`` per page) are
|
|
4
|
+
evidence for :class:`PageReport`, never a row a check reads."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
from collections.abc import Iterable, Mapping
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .tasks import (
|
|
14
|
+
DOOR_FIELD,
|
|
15
|
+
JOURNAL,
|
|
16
|
+
ORDER_KEYS,
|
|
17
|
+
PAGE_DOOR,
|
|
18
|
+
PAGE_GAP,
|
|
19
|
+
PAGE_NOTES,
|
|
20
|
+
PAGE_RECORDING,
|
|
21
|
+
PAGE_SERVED,
|
|
22
|
+
PAGE_UNHEARD,
|
|
23
|
+
PAGE_UNJOURNALED,
|
|
24
|
+
PAGE_UNRECORDED,
|
|
25
|
+
PAGE_UNRECORDED_LIFE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
PAGE_OPERATIONS = "ui."
|
|
29
|
+
PAGE_PATHS = ("args.target.", "request.body.target.")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _listed(value: Any) -> list[Any]:
|
|
33
|
+
return list(value) if isinstance(value, (list, tuple)) else [value]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _names_door(value: Any) -> bool:
|
|
37
|
+
return any(item == PAGE_DOOR for item in _listed(value))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _names_operation(value: Any) -> bool:
|
|
41
|
+
return any(isinstance(item, str) and item.startswith(PAGE_OPERATIONS) for item in _listed(value))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class PageCheck:
|
|
46
|
+
"""One check as the page sees it: it names the page when a clause or its ``field`` pins the
|
|
47
|
+
page door or a ``ui.*`` operation, or a clause reads what a page interaction acted on."""
|
|
48
|
+
|
|
49
|
+
check: Mapping[str, Any]
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def names_the_page(self) -> bool:
|
|
53
|
+
clauses = [self.check.get("where") or {}, *(self.check.get(key) or {} for key in ORDER_KEYS)]
|
|
54
|
+
return any(self._clause(clause) for clause in clauses) or self._field()
|
|
55
|
+
|
|
56
|
+
def _clause(self, clause: Mapping[str, Any]) -> bool:
|
|
57
|
+
pinned = _names_door(clause.get(DOOR_FIELD)) or _names_operation(clause.get("operation"))
|
|
58
|
+
return pinned or any(path.startswith(PAGE_PATHS) for path in clause)
|
|
59
|
+
|
|
60
|
+
def _field(self) -> bool:
|
|
61
|
+
value = self.check.get("value")
|
|
62
|
+
named = {DOOR_FIELD: _names_door, "operation": _names_operation}.get(self.check.get("field") or "")
|
|
63
|
+
return named is not None and named(value)
|
|
64
|
+
|
|
65
|
+
def keeps(self, values: Mapping[str, Any]) -> bool:
|
|
66
|
+
if values.get(DOOR_FIELD) != PAGE_DOOR:
|
|
67
|
+
return True
|
|
68
|
+
return self.names_the_page and values.get("operation") not in PAGE_NOTES
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class PageReport:
|
|
73
|
+
"""What the session's journal says about its pages: whether a host noted it recorded them,
|
|
74
|
+
whether it served any, whether any interaction came back, and each page load's numbered
|
|
75
|
+
interactions. A grade with no journal has none of this to read."""
|
|
76
|
+
|
|
77
|
+
journaled: bool = False
|
|
78
|
+
recording: bool = False
|
|
79
|
+
unrecorded: bool = False
|
|
80
|
+
served: bool = False
|
|
81
|
+
reported: bool = False
|
|
82
|
+
loads: dict[str, set[int]] = field(default_factory=lambda: defaultdict(set))
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def of(cls, state: Mapping, entity: str) -> PageReport:
|
|
86
|
+
prefix = entity[: entity.rfind(".") + 1]
|
|
87
|
+
rows = state.get(f"{prefix}{JOURNAL}")
|
|
88
|
+
report = cls(journaled=isinstance(rows, list))
|
|
89
|
+
for row in rows if isinstance(rows, list) else ():
|
|
90
|
+
if isinstance(row, Mapping) and row.get(DOOR_FIELD) == PAGE_DOOR:
|
|
91
|
+
report._note(row)
|
|
92
|
+
return report
|
|
93
|
+
|
|
94
|
+
def _note(self, row: Mapping[str, Any]) -> None:
|
|
95
|
+
operation = row.get("operation")
|
|
96
|
+
self.recording = self.recording or operation == PAGE_RECORDING
|
|
97
|
+
self.unrecorded = self.unrecorded or operation == PAGE_UNRECORDED_LIFE
|
|
98
|
+
self.served = self.served or operation == PAGE_SERVED
|
|
99
|
+
if operation in PAGE_NOTES:
|
|
100
|
+
return
|
|
101
|
+
self.reported = True
|
|
102
|
+
query = row.get("query")
|
|
103
|
+
load, seq = (query.get("load"), query.get("seq")) if isinstance(query, Mapping) else (None, None)
|
|
104
|
+
if isinstance(load, str) and isinstance(seq, int) and not isinstance(seq, bool):
|
|
105
|
+
self.loads[load].add(seq)
|
|
106
|
+
|
|
107
|
+
def missing(self) -> int:
|
|
108
|
+
return sum(max(seqs) - len(seqs) for seqs in self.loads.values() if seqs)
|
|
109
|
+
|
|
110
|
+
def refusal(self) -> str | None:
|
|
111
|
+
if not self.journaled:
|
|
112
|
+
return PAGE_UNJOURNALED
|
|
113
|
+
if self.unrecorded or not self.recording:
|
|
114
|
+
return PAGE_UNRECORDED
|
|
115
|
+
if self.served and not self.reported:
|
|
116
|
+
return PAGE_UNHEARD
|
|
117
|
+
gaps = self.missing()
|
|
118
|
+
return PAGE_GAP.format(missing=gaps) if gaps else None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def page_rows(check: Mapping[str, Any], rows: Iterable[Any]) -> list[Any]:
|
|
122
|
+
view = PageCheck(check)
|
|
123
|
+
return [row for row in rows if view.keeps(row.values)]
|