@withgateway/sdk 0.6.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/agents/index.d.mts +573 -0
  2. package/dist/agents/index.d.ts +573 -0
  3. package/dist/agents/index.js +1443 -0
  4. package/dist/agents/index.js.map +1 -0
  5. package/dist/agents/index.mjs +1394 -0
  6. package/dist/agents/index.mjs.map +1 -0
  7. package/dist/apiError-DHpfbmdx.d.mts +17 -0
  8. package/dist/apiError-DHpfbmdx.d.ts +17 -0
  9. package/dist/cli.js +5019 -1599
  10. package/dist/cli.js.map +1 -1
  11. package/dist/cli.mjs +5003 -1584
  12. package/dist/cli.mjs.map +1 -1
  13. package/dist/files/index.d.mts +3 -1
  14. package/dist/files/index.d.ts +3 -1
  15. package/dist/files/index.js +371 -102
  16. package/dist/files/index.js.map +1 -1
  17. package/dist/files/index.mjs +371 -102
  18. package/dist/files/index.mjs.map +1 -1
  19. package/dist/hooks/index.js +346 -136
  20. package/dist/hooks/index.js.map +1 -1
  21. package/dist/hooks/index.mjs +346 -136
  22. package/dist/hooks/index.mjs.map +1 -1
  23. package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
  24. package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
  25. package/dist/index.d.mts +7 -4
  26. package/dist/index.d.ts +7 -4
  27. package/dist/index.js +2689 -401
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.mjs +2650 -394
  30. package/dist/index.mjs.map +1 -1
  31. package/dist/media-CQUC7TkS.d.mts +39 -0
  32. package/dist/media-DQnBloaX.d.ts +39 -0
  33. package/dist/traces/index.d.mts +3 -1
  34. package/dist/traces/index.d.ts +3 -1
  35. package/dist/traces/index.js +371 -102
  36. package/dist/traces/index.js.map +1 -1
  37. package/dist/traces/index.mjs +371 -102
  38. package/dist/traces/index.mjs.map +1 -1
  39. package/dist/tracing/index.d.mts +3 -2
  40. package/dist/tracing/index.d.ts +3 -2
  41. package/dist/tracing/index.js +257 -16
  42. package/dist/tracing/index.js.map +1 -1
  43. package/dist/tracing/index.mjs +257 -16
  44. package/dist/tracing/index.mjs.map +1 -1
  45. package/dist/transport-CpRe7zgd.d.mts +123 -0
  46. package/dist/transport-CpRe7zgd.d.ts +123 -0
  47. package/dist/worlds/index.d.mts +1983 -1745
  48. package/dist/worlds/index.d.ts +1983 -1745
  49. package/dist/worlds/index.js +1295 -226
  50. package/dist/worlds/index.js.map +1 -1
  51. package/dist/worlds/index.mjs +1290 -222
  52. package/dist/worlds/index.mjs.map +1 -1
  53. package/package.json +6 -1
  54. package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
  55. package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
  56. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
  57. package/runtime/gatewaysdk/environment/schema/api.py +3 -4
  58. package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
  59. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  60. package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
  62. package/runtime/gatewaysdk/environment/schema/host.py +74 -11
  63. package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
  64. package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
  65. package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
  66. package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
  67. package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
  68. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
  69. package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
  70. package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
  71. package/runtime/gatewaysdk/environment/schema/local.py +32 -7
  72. package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +23 -267
  74. package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
  75. package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
  76. package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +43 -5
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +367 -217
  79. package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
  80. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
  81. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -2
  83. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +7 -7
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
  89. package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
  90. package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
  92. package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
  93. package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
  94. package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
  95. package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
  96. package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
  97. package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
  98. package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
  99. package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
  100. package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
  101. package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
  102. package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
  103. package/runtime/gatewaysdk/environment/schema/world_host.py +44 -15
  104. package/runtime/gatewaysdk/environment/secret_values.py +1 -1
  105. package/runtime/gatewaysdk/environment/world.py +34 -43
  106. package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
  107. package/skills/benchmarks-getting-started/SKILL.md +1 -1
  108. package/skills/connector-schema-authoring/SKILL.md +1 -2
  109. package/skills/connector-schema-authoring/connector-worlds.md +2 -2
  110. package/skills/connector-schema-authoring/world-schema.md +29 -23
  111. package/skills/world-data-ingestion/SKILL.md +4 -2
  112. package/skills/worlds-getting-started/SKILL.md +7 -7
  113. package/skills/worlds-getting-started/references/cli-mcp-map.md +32 -2
  114. package/skills/worlds-getting-started/references/gotchas.md +6 -6
  115. package/skills/worlds-getting-started/references/journal-checks.md +17 -2
  116. package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  117. package/skills/worlds-getting-started/references/ui-map.md +3 -2
  118. package/dist/media-BIApiPzS.d.mts +0 -102
  119. package/dist/media-BIApiPzS.d.ts +0 -102
@@ -0,0 +1,48 @@
1
+ """Binds a leased pool browser to the one session it serves: the relay's egress proxy then
2
+ loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import sys
8
+ import time
9
+ import urllib.error
10
+ import urllib.request
11
+ from collections.abc import Callable
12
+ from dataclasses import dataclass, field
13
+ from typing import Any
14
+
15
+ from .host_requests import BrowserRelay
16
+ from .validation import fail
17
+
18
+ GUARD_REQUIRED_FROM = 1794787200.0
19
+ BIND_TIMEOUT_S = 5.0
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class RelayNavigation:
24
+ opener: Callable[..., Any] = field(default=urllib.request.urlopen)
25
+ clock: Callable[[], float] = field(default=time.time)
26
+
27
+ def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
28
+ request = urllib.request.Request(
29
+ f"{relay.http_url}/navigation",
30
+ data=json.dumps({"hosts": hosts}).encode(),
31
+ method="PUT",
32
+ headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
33
+ )
34
+ try:
35
+ with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
36
+ return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
37
+ except urllib.error.HTTPError as error:
38
+ if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
39
+ log_relay("browser relay predates the navigation guard", relay)
40
+ return "unguarded"
41
+ fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
42
+ except (OSError, ValueError) as error:
43
+ log_relay("browser relay could not be bound", relay, error=str(error))
44
+ fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
45
+
46
+
47
+ def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
48
+ print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
@@ -15,6 +15,8 @@ from email.message import Message
15
15
  from typing import TYPE_CHECKING, Any
16
16
 
17
17
  from .handler_errors import carried
18
+ from .host_browser import RelayNavigation
19
+ from .host_requests import BrowserRelay
18
20
  from .host_surfaces import public_host
19
21
  from .host_workers import WORKER_EXITED, refusal
20
22
  from .validation import ContractError, fail
@@ -35,10 +37,18 @@ class ControlPlane:
35
37
  """One host's control plane. A host with no secret has no control plane at all rather than an
36
38
  open one, and one with no worlds domain could not tell its own address from a session's."""
37
39
 
38
- def __init__(self, host: WorldHost, secret: str, operations: Mapping[str, Callable[..., dict]], worlds_domain: str):
40
+ def __init__(
41
+ self,
42
+ host: WorldHost,
43
+ secret: str,
44
+ operations: Mapping[str, Callable[..., dict]],
45
+ worlds_domain: str,
46
+ navigation: RelayNavigation | None = None,
47
+ ):
39
48
  if not secret:
40
49
  raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
41
50
  self.host = host
51
+ self.navigation = navigation or RelayNavigation()
42
52
  self.operations = operations
43
53
  self.domain = self.domain_of(worlds_domain)
44
54
  self._secret = secret.encode()
@@ -109,6 +119,20 @@ class ControlPlane:
109
119
  return 404, {"error": "unknown control route"}
110
120
 
111
121
  def _open(self, body: dict) -> dict:
122
+ answer = self._open_session(body)
123
+ relay = BrowserRelay.parse(body.get("browser"))
124
+ if relay is not None:
125
+ self._bind_or_close(body["sessionId"], relay)
126
+ return answer
127
+
128
+ def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
129
+ try:
130
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
131
+ except Exception:
132
+ self.host.close(session_id, "browser_unbound")
133
+ raise
134
+
135
+ def _open_session(self, body: dict) -> dict:
112
136
  return self.host.open(
113
137
  session_id=body["sessionId"],
114
138
  version_id=body["versionId"],
@@ -123,14 +147,21 @@ class ControlPlane:
123
147
  pending=body.get("pending") is True,
124
148
  auth=body.get("auth"),
125
149
  ledger_source=body.get("ledgerSource"),
150
+ parameters=body.get("parameters"),
126
151
  )
127
152
 
153
+ def _swap_browser(self, session_id: str, browser: Any) -> dict:
154
+ relay = BrowserRelay.parse(browser)
155
+ if relay is not None:
156
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
157
+ return self.host.set_browser(session_id, browser)
158
+
128
159
  def _session(self, method: str, parts: list[str], body: dict) -> Answer:
129
160
  sid = parts[2]
130
161
  self.host.forwarding.refuse(sid)
131
162
  verb = parts[3] if len(parts) > 3 else None
132
163
  if method == "PUT" and verb == "browser" and len(parts) == 4:
133
- return 200, self.host.set_browser(sid, body.get("browser"))
164
+ return 200, self._swap_browser(sid, body.get("browser"))
134
165
  if method == "POST" and verb == "export" and len(parts) == 5:
135
166
  return self._upload(sid, parts[4], body.get("url"))
136
167
  row = session_verb(method, verb)
@@ -31,6 +31,7 @@ from .handler_errors import log_line
31
31
  from .host_owners import PROXY_TIMEOUT_S
32
32
  from .request_ledger import LEDGER_STOP_S
33
33
  from .session_move import same_host
34
+ from .ui_replay import REPLAY_STOP_S
34
35
 
35
36
  if TYPE_CHECKING:
36
37
  from .host_session import Session
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
47
48
  LINGER_S = 5.0
48
49
  IN_FLIGHT_S = PROXY_TIMEOUT_S
49
50
  EXIT_S = 110.0
50
- assert LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + 5.0 <= EXIT_S, "a stopping host's exit fits the grace past its drain"
51
+ assert (
52
+ LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
53
+ ), "a stopping host's exit fits the grace past its drain"
51
54
 
52
55
  MOVED = "moved"
53
56
  STAYED = "stayed"
@@ -28,6 +28,16 @@ class DependencyRef:
28
28
  snapshot: dict[str, str] | None
29
29
  raw: dict[str, Any]
30
30
 
31
+ @property
32
+ def for_worker(self) -> dict[str, Any]:
33
+ """The entry a version worker is handed: its snapshot without the url, a tree the supervisor built."""
34
+ return {**self.raw, "snapshot": worker_snapshot(self.snapshot)}
35
+
36
+
37
+ def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
38
+ """A snapshot as a version worker sees it: its sha256 and digest, never the presigned url."""
39
+ return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
40
+
31
41
 
32
42
  def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
33
43
  if dependencies is None:
@@ -77,6 +87,10 @@ class BrowserRelay:
77
87
  raise AssertionError # fail() never returns
78
88
  return cls(url.rstrip("/"), token)
79
89
 
90
+ @property
91
+ def http_url(self) -> str:
92
+ return "http://" + self.url.removeprefix("ws://")
93
+
80
94
 
81
95
  def restore_sha256(restore: object) -> str | None:
82
96
  """The sha256 of the export an open restores from, or None for an open that restores nothing."""
@@ -10,7 +10,7 @@ from pathlib import Path
10
10
  from typing import TYPE_CHECKING
11
11
 
12
12
  from .door import OPEN_MODE, VENDOR_MODE, Door, same
13
- from .host_surfaces import SessionHost, capability_host
13
+ from .host_surfaces import SessionHost, capability_host, session_host
14
14
  from .request_ledger import LedgerLife
15
15
  from .session_move import SessionMove
16
16
 
@@ -75,6 +75,8 @@ class Session:
75
75
  placement: dict | None = None
76
76
  settings: ApiSettings | None = None
77
77
  restore_sha256: str | None = None
78
+ snapshot_sha256: str | None = None
79
+ parameters: dict[str, str] = field(default_factory=dict)
78
80
  pending: InitVar[bool] = False
79
81
  move: SessionMove = field(init=False, repr=False, compare=False)
80
82
  auth: str = VENDOR_MODE
@@ -119,8 +121,17 @@ class Session:
119
121
  capability = self.door.capability if self.door is not None else None
120
122
  return self.auth == OPEN_MODE and capability is not None and same(named.capability, capability)
121
123
 
124
+ def browser_hosts(self, domain: str) -> list[str]:
125
+ """The only hosts this session's browser may load: its ui and api, and their capability
126
+ pair when the session is open."""
127
+ hosts = [session_host(self.session_id, surface, domain) for surface in ("ui", "api")]
128
+ capability = self.door.capability if self.door is not None else None
129
+ if self.auth == OPEN_MODE and capability is not None:
130
+ hosts += [capability_host(self.session_id, capability, surface, domain) for surface in ("ui", "api")]
131
+ return hosts
132
+
122
133
  def api_host(self, domain: str, opened: bool) -> str:
123
134
  """The host a link the world writes points at when a request names none."""
124
135
  if opened and self.door is not None and self.door.capability is not None:
125
136
  return capability_host(self.session_id, self.door.capability, "api", domain)
126
- return f"w-{self.session_id}-api.{domain}"
137
+ return session_host(self.session_id, "api", domain)
@@ -260,6 +260,12 @@ def _capability_host(rest: str, surface: str) -> SessionHost | None:
260
260
  return SessionHost(session_id, surface, capability)
261
261
 
262
262
 
263
+ def session_host(session_id: str, surface: str, domain: str) -> str:
264
+ """The session's own hostname for ``surface`` (``api`` or ``ui``)."""
265
+ suffix = "-api" if surface == "api" else ""
266
+ return f"w-{session_id}{suffix}.{domain}"
267
+
268
+
263
269
  def capability_host(session_id: str, capability: str, surface: str, domain: str) -> str:
264
270
  """The capability hostname of an open session's ``surface`` (``api`` or ``ui``)."""
265
271
  suffix = "-api" if surface == "api" else ""
@@ -4,6 +4,7 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  import os
7
+ import re
7
8
  import shutil
8
9
  import tarfile
9
10
  import threading
@@ -21,6 +22,7 @@ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
21
22
  from .compiler import check_bundle
22
23
  from .connector import Spec, load_spec
23
24
  from .database import WorldDatabase
25
+ from .database.base import reflink
24
26
  from .dependencies import Dependency
25
27
  from .door import Door
26
28
  from .host_surfaces import UiSurface, load_ui_surface
@@ -33,6 +35,8 @@ from .validation import fail
33
35
 
34
36
  BUNDLED_WORLD = "server/world"
35
37
  WORLD_SPEC = "schema/world.json"
38
+ SNAPSHOTS_DIR = "snapshots"
39
+ _SHA256 = re.compile(r"[0-9a-f]{64}")
36
40
  #: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
37
41
  DEFAULT_MAX_BUNDLE_MB = 2048
38
42
  MAX_BUNDLE_MEMBERS = 200_000
@@ -53,6 +57,41 @@ def world_tree(bundle: Path) -> Path:
53
57
  )
54
58
 
55
59
 
60
+ def clone_mode(directory: Path) -> str:
61
+ """``reflink`` when session copies on ``directory`` share blocks until written, ``copy`` when each one
62
+ copies its file: the disk decides, so the host probes it once and reports it in its stats."""
63
+ probe = directory / f".clone-probe-{uuid.uuid4().hex}"
64
+ clone = probe.with_name(f"{probe.name}.clone")
65
+ try:
66
+ probe.write_bytes(b"probe")
67
+ return "reflink" if reflink(probe, clone) else "copy"
68
+ finally:
69
+ probe.unlink(missing_ok=True)
70
+ clone.unlink(missing_ok=True)
71
+
72
+
73
+ def data_tree(bundle: Path) -> Path:
74
+ return bundle if is_tools_bundle(bundle) else world_tree(bundle)
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class DataKey:
79
+ """A version as a session opens it: on the version's own data, or on one snapshot of it, named by the
80
+ sha256 of the snapshot file. Two snapshots of one version never share a tree, a template or a session."""
81
+
82
+ version_id: str
83
+ sha256: str | None
84
+
85
+ @classmethod
86
+ def of(cls, version_id: str, snapshot: dict[str, str] | None) -> DataKey:
87
+ if not snapshot:
88
+ return cls(version_id, None)
89
+ sha256 = str(snapshot.get("sha256") or "").lower()
90
+ if not _SHA256.fullmatch(sha256):
91
+ fail("host", "a snapshot names the sha256 of its file (64 hex digits)")
92
+ return cls(version_id, sha256)
93
+
94
+
56
95
  @dataclass
57
96
  class CachedVersion:
58
97
  """One cached version, of either shape the host serves: an api bundle (a
@@ -124,8 +163,9 @@ class VersionCache:
124
163
  #: Off in the supervisor: only a version worker imports world-authored code.
125
164
  self.run_handlers = run_handlers
126
165
  self.max_bundle_bytes = max_bundle_mb * (1 << 20)
166
+ self.clone_mode = clone_mode(self.root)
127
167
  self._lock = threading.Lock()
128
- self._versions: dict[str, CachedVersion] = {}
168
+ self._versions: dict[DataKey, CachedVersion] = {}
129
169
  self._builds = Lanes()
130
170
 
131
171
  def ensure(
@@ -134,37 +174,76 @@ class VersionCache:
134
174
  bundle_url: str,
135
175
  snapshot: dict[str, str] | None,
136
176
  ) -> CachedVersion:
137
- hit = self._cached(version_id)
177
+ key = DataKey.of(version_id, snapshot)
178
+ hit = self._cached(key)
138
179
  if hit is not None:
139
180
  return hit
140
181
  with self._builds.turn(version_id):
141
- return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
182
+ return self._cached(key) or self._store(key, self._build(key, bundle_url, snapshot))
142
183
 
143
- def _cached(self, version_id: str) -> CachedVersion | None:
184
+ def _cached(self, key: DataKey) -> CachedVersion | None:
144
185
  with self._lock:
145
- return self._versions.get(version_id)
186
+ return self._versions.get(key)
146
187
 
147
- def _store(self, cached: CachedVersion) -> CachedVersion:
188
+ def _store(self, key: DataKey, cached: CachedVersion) -> CachedVersion:
148
189
  with self._lock:
149
- self._versions[cached.version_id] = cached
190
+ self._versions[key] = cached
150
191
  return cached
151
192
 
152
- def has(self, version_id: str) -> bool:
193
+ def has(self, version_id: str, snapshot: dict[str, str] | None = None) -> bool:
194
+ key = DataKey.of(version_id, snapshot)
153
195
  with self._lock:
154
- return version_id in self._versions
196
+ return key in self._versions
155
197
 
156
198
  def version_ids(self) -> list[str]:
157
199
  with self._lock:
158
- return sorted(self._versions)
200
+ return sorted({key.version_id for key in self._versions})
159
201
 
160
- def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
161
- bundle = self.root / version_id / "bundle"
162
- if not bundle.is_dir():
163
- self._download(bundle_url, bundle)
164
- if is_tools_bundle(bundle):
165
- tools = load_tools_version(
166
- bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
202
+ def _build(self, key: DataKey, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
203
+ base = self.root / key.version_id / "bundle"
204
+ if not base.is_dir():
205
+ self._download(bundle_url, base)
206
+ if key.sha256 is None or snapshot is None:
207
+ return self._load(key.version_id, base, self.root / key.version_id / "template.db", fetched=False)
208
+ bundle = self._with_snapshot(key, base, snapshot)
209
+ return self._load(key.version_id, bundle, bundle.parent / "template.db", fetched=True)
210
+
211
+ def _with_snapshot(self, key: DataKey, base: Path, snapshot: dict[str, str]) -> Path:
212
+ """The version's bundle with the snapshot's rows as its data, under ``snapshots/<sha256>``: cloned
213
+ from the downloaded bundle (never linked, so nothing written here reaches it) without the data the
214
+ snapshot replaces, then fetched and verified. Renamed into place only once complete; one built
215
+ before is reused without a url, which is how a version worker finds it."""
216
+ home = self.root / key.version_id / SNAPSHOTS_DIR / str(key.sha256)
217
+ if (home / "bundle").is_dir():
218
+ return home / "bundle"
219
+ url, digest = snapshot.get("url"), snapshot.get("digest")
220
+ if not url or not isinstance(digest, str):
221
+ fail(
222
+ "host",
223
+ f"snapshot {key.sha256} of version {key.version_id} is not cached on this host, "
224
+ "and the open names no url and digest to fetch it",
167
225
  )
226
+ tree = data_tree(base)
227
+ contract = check_bundle(tree)
228
+ replaced = snapshots.default_snapshot(tree, contract)
229
+ staging = home.parent / f".{key.sha256}.{uuid.uuid4().hex}"
230
+ try:
231
+ for path in sorted(base.rglob("*")):
232
+ target = staging / "bundle" / path.relative_to(base)
233
+ if path.is_dir():
234
+ target.mkdir(parents=True, exist_ok=True)
235
+ elif path != replaced:
236
+ WorldDatabase.clone_file(path, target)
237
+ staged = staging / "bundle" / tree.relative_to(base)
238
+ snapshots.fetch(staged, contract, url, str(key.sha256), digest)
239
+ os.replace(staging, home)
240
+ finally:
241
+ shutil.rmtree(staging, ignore_errors=True)
242
+ return home / "bundle"
243
+
244
+ def _load(self, version_id: str, bundle: Path, template: Path, *, fetched: bool) -> CachedVersion:
245
+ if is_tools_bundle(bundle):
246
+ tools = load_tools_version(bundle, template, fetched=fetched, run_handlers=self.run_handlers)
168
247
  return CachedVersion(
169
248
  version_id=version_id,
170
249
  bundle=bundle,
@@ -179,8 +258,6 @@ class VersionCache:
179
258
  )
180
259
  tree = world_tree(bundle)
181
260
  contract = check_bundle(tree)
182
- if snapshot:
183
- snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
184
261
  served = snapshots.ServedSnapshot.of(tree)
185
262
  spec = load_spec(tree)
186
263
  settings = load_api_settings(tree, spec)
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
13
13
  from .._toml import tomllib
14
14
  from .call_redaction import REDACTED, CallRedaction
15
15
  from .database.base import SideTable
16
- from .tasks import JOURNAL, JOURNAL_META, WITHHELD_FIELD
16
+ from .tasks import JOURNAL, JOURNAL_META, PAGE_NOTES, WITHHELD_FIELD
17
17
  from .validation import fail
18
18
 
19
19
  if TYPE_CHECKING:
@@ -26,7 +26,7 @@ JOURNAL_TABLES = frozenset(JOURNAL_KEYS)
26
26
  MAX_JOURNAL_REQUESTS = 20_000
27
27
  NOT_RECORDED = "not recorded: only a World Host session records the requests its world receives"
28
28
  LINKED_NOT_RECORDED = "not recorded for a linked world: the root world's verifiers and the task's checks read it"
29
- VIAS = ("api", "ui", "tool", "webhook")
29
+ VIAS = ("api", "ui", "tool", "webhook", "page")
30
30
  JSON_HINTS = ("json",)
31
31
  WITHHELD_ROUTE_BODY = ["body", "selectors"]
32
32
  WITHHELD_TOOL_ARGS = ["args", "selectors"]
@@ -260,8 +260,13 @@ class CallsView:
260
260
  return None
261
261
  if entry.raw["via"] == "tool" and entry.raw["method"] is None:
262
262
  return self._tool(entry) if self.admitted(entry.operation) else None
263
+ if entry.raw["via"] == "page":
264
+ return None if entry.operation in PAGE_NOTES else self._interaction(entry)
263
265
  return self._route(entry)
264
266
 
267
+ def _interaction(self, entry: JournalEntry) -> dict:
268
+ return {**self._tool(entry), "path": entry.raw["path"], "query": entry.raw["query"]}
269
+
265
270
  def _tool(self, entry: JournalEntry) -> dict:
266
271
  body = entry.request
267
272
  args = body.value if body.whole else REDACTED
@@ -10,6 +10,7 @@ import tempfile
10
10
  import threading
11
11
  import urllib.parse
12
12
  from dataclasses import dataclass, field
13
+ from functools import partial
13
14
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
14
15
  from pathlib import Path
15
16
  from typing import Any, Callable
@@ -33,6 +34,7 @@ from .tools_world import (
33
34
  ToolsWorldState,
34
35
  load_tools_version,
35
36
  )
37
+ from .ui_interactions import RECORDER_PATH, UiRecorder
36
38
  from .validation import ContractError
37
39
  from .verbs import refusal, session_verb, unknown_verb
38
40
 
@@ -51,7 +53,7 @@ def load_local_version(root: Path, scratch: Path) -> ToolsVersion:
51
53
  f"{root} is not a schema world (no gateway-env.toml with a [schema] table); "
52
54
  "`gateway worlds schema init` starts one",
53
55
  )
54
- return load_tools_version(root, scratch / "template.db", None, run_handlers=True)
56
+ return load_tools_version(root, scratch / "template.db", fetched=False, run_handlers=True)
55
57
 
56
58
 
57
59
  def load_local_dependencies(root: Path, version: ToolsVersion, scratch: Path) -> dict[str, DependencyWorld]:
@@ -317,10 +319,7 @@ def make_session_handler(
317
319
  raw = self.rfile.read(length) if length else b""
318
320
  path = self.path.split("?", 1)[0]
319
321
  if ui_origin:
320
- assert ui is not None
321
- if app is not None and api_target(ui, path) is not None:
322
- return self._ui_api(method, path, raw, Via.UI)
323
- return self._ui(method, path)
322
+ return self._ui_origin(method, path, raw)
324
323
  origin = self.headers.get("Origin")
325
324
  if origin and not _local_origin(origin):
326
325
  return self._reply(403, {"error": "this session answers this machine's pages, not another site's"})
@@ -344,11 +343,32 @@ def make_session_handler(
344
343
  )
345
344
  send_response(self, status, body, extra, app.settings.error_content_type)
346
345
 
346
+ def _ui_origin(self, method: str, path: str, raw: bytes) -> None:
347
+ assert ui is not None
348
+ if app is not None and api_target(ui, path) is not None:
349
+ return self._ui_api(method, path, raw, Via.UI)
350
+ if path == RECORDER_PATH:
351
+ return self._page(method, UiRecorder.script)
352
+ if UiRecorder.serves(path):
353
+ return self._recorded(method, path, raw)
354
+ return self._page(method, partial(self._served, path))
355
+
356
+ def _served(self, path: str) -> tuple[int, dict[str, str], bytes]:
357
+ assert ui is not None
358
+ status, headers, payload = static_file(ui, path)
359
+ page = UiRecorder.page(status, headers, payload)
360
+ if page is not None:
361
+ UiRecorder.served(path, calls)
362
+ return status, headers, payload if page is None else page
363
+
347
364
  def _ui(self, method: str, path: str) -> None:
365
+ assert ui is not None
366
+ return self._page(method, partial(static_file, ui, path))
367
+
368
+ def _page(self, method: str, answer: Callable[[], tuple[int, dict[str, str], bytes]]) -> None:
348
369
  if method != "GET":
349
370
  return self._reply(405, {"error": "the static ui answers GET only"})
350
- assert ui is not None
351
- status, headers, payload = static_file(ui, path)
371
+ status, headers, payload = answer()
352
372
  self.send_response(status)
353
373
  for name, value in headers.items():
354
374
  self.send_header(name, value)
@@ -356,6 +376,11 @@ def make_session_handler(
356
376
  self.end_headers()
357
377
  write_body(self, payload)
358
378
 
379
+ def _recorded(self, method: str, path: str, raw: bytes) -> None:
380
+ if cross_site({k.lower(): v for k, v in self.headers.items()}, _local_origin):
381
+ return self._reply(403, {"error": "the recorder answers the page on this server, not another site"})
382
+ return self._reply(*UiRecorder.record(method, path, raw, calls))
383
+
359
384
  def _ui_api(self, method: str, path: str, raw: bytes, via: Via) -> None:
360
385
  """Answers with the world's own credential for the pin, as the host does for the session cookie."""
361
386
  assert app is not None and ui is not None
@@ -0,0 +1,123 @@
1
+ """The grader's view of a UI world's page (``via: page``): which checks read it, and whether what
2
+ the page reported can decide them. A check that does not name the page reads what it read before
3
+ pages reported anything; the host's notes (``ui.recording`` at open, ``ui.served`` per page) are
4
+ evidence for :class:`PageReport`, never a row a check reads."""
5
+
6
+ from __future__ import annotations
7
+
8
+ from collections import defaultdict
9
+ from collections.abc import Iterable, Mapping
10
+ from dataclasses import dataclass, field
11
+ from typing import Any
12
+
13
+ from .tasks import (
14
+ DOOR_FIELD,
15
+ JOURNAL,
16
+ ORDER_KEYS,
17
+ PAGE_DOOR,
18
+ PAGE_GAP,
19
+ PAGE_NOTES,
20
+ PAGE_RECORDING,
21
+ PAGE_SERVED,
22
+ PAGE_UNHEARD,
23
+ PAGE_UNJOURNALED,
24
+ PAGE_UNRECORDED,
25
+ PAGE_UNRECORDED_LIFE,
26
+ )
27
+
28
+ PAGE_OPERATIONS = "ui."
29
+ PAGE_PATHS = ("args.target.", "request.body.target.")
30
+
31
+
32
+ def _listed(value: Any) -> list[Any]:
33
+ return list(value) if isinstance(value, (list, tuple)) else [value]
34
+
35
+
36
+ def _names_door(value: Any) -> bool:
37
+ return any(item == PAGE_DOOR for item in _listed(value))
38
+
39
+
40
+ def _names_operation(value: Any) -> bool:
41
+ return any(isinstance(item, str) and item.startswith(PAGE_OPERATIONS) for item in _listed(value))
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class PageCheck:
46
+ """One check as the page sees it: it names the page when a clause or its ``field`` pins the
47
+ page door or a ``ui.*`` operation, or a clause reads what a page interaction acted on."""
48
+
49
+ check: Mapping[str, Any]
50
+
51
+ @property
52
+ def names_the_page(self) -> bool:
53
+ clauses = [self.check.get("where") or {}, *(self.check.get(key) or {} for key in ORDER_KEYS)]
54
+ return any(self._clause(clause) for clause in clauses) or self._field()
55
+
56
+ def _clause(self, clause: Mapping[str, Any]) -> bool:
57
+ pinned = _names_door(clause.get(DOOR_FIELD)) or _names_operation(clause.get("operation"))
58
+ return pinned or any(path.startswith(PAGE_PATHS) for path in clause)
59
+
60
+ def _field(self) -> bool:
61
+ value = self.check.get("value")
62
+ named = {DOOR_FIELD: _names_door, "operation": _names_operation}.get(self.check.get("field") or "")
63
+ return named is not None and named(value)
64
+
65
+ def keeps(self, values: Mapping[str, Any]) -> bool:
66
+ if values.get(DOOR_FIELD) != PAGE_DOOR:
67
+ return True
68
+ return self.names_the_page and values.get("operation") not in PAGE_NOTES
69
+
70
+
71
+ @dataclass
72
+ class PageReport:
73
+ """What the session's journal says about its pages: whether a host noted it recorded them,
74
+ whether it served any, whether any interaction came back, and each page load's numbered
75
+ interactions. A grade with no journal has none of this to read."""
76
+
77
+ journaled: bool = False
78
+ recording: bool = False
79
+ unrecorded: bool = False
80
+ served: bool = False
81
+ reported: bool = False
82
+ loads: dict[str, set[int]] = field(default_factory=lambda: defaultdict(set))
83
+
84
+ @classmethod
85
+ def of(cls, state: Mapping, entity: str) -> PageReport:
86
+ prefix = entity[: entity.rfind(".") + 1]
87
+ rows = state.get(f"{prefix}{JOURNAL}")
88
+ report = cls(journaled=isinstance(rows, list))
89
+ for row in rows if isinstance(rows, list) else ():
90
+ if isinstance(row, Mapping) and row.get(DOOR_FIELD) == PAGE_DOOR:
91
+ report._note(row)
92
+ return report
93
+
94
+ def _note(self, row: Mapping[str, Any]) -> None:
95
+ operation = row.get("operation")
96
+ self.recording = self.recording or operation == PAGE_RECORDING
97
+ self.unrecorded = self.unrecorded or operation == PAGE_UNRECORDED_LIFE
98
+ self.served = self.served or operation == PAGE_SERVED
99
+ if operation in PAGE_NOTES:
100
+ return
101
+ self.reported = True
102
+ query = row.get("query")
103
+ load, seq = (query.get("load"), query.get("seq")) if isinstance(query, Mapping) else (None, None)
104
+ if isinstance(load, str) and isinstance(seq, int) and not isinstance(seq, bool):
105
+ self.loads[load].add(seq)
106
+
107
+ def missing(self) -> int:
108
+ return sum(max(seqs) - len(seqs) for seqs in self.loads.values() if seqs)
109
+
110
+ def refusal(self) -> str | None:
111
+ if not self.journaled:
112
+ return PAGE_UNJOURNALED
113
+ if self.unrecorded or not self.recording:
114
+ return PAGE_UNRECORDED
115
+ if self.served and not self.reported:
116
+ return PAGE_UNHEARD
117
+ gaps = self.missing()
118
+ return PAGE_GAP.format(missing=gaps) if gaps else None
119
+
120
+
121
+ def page_rows(check: Mapping[str, Any], rows: Iterable[Any]) -> list[Any]:
122
+ view = PageCheck(check)
123
+ return [row for row in rows if view.keeps(row.values)]