@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,144 @@
1
+ """The platform's half of a World Host: ``/__host/*``, behind ``X-Host-Secret``.
2
+
3
+ Only a request that reached the pod from inside the network gets that far (:meth:`ControlPlane.serves`),
4
+ and the secret is checked before any body is read. An admitted request is answered as ``(status, body)``
5
+ or, for a refusal, ``(status, body, headers)``: the host's own verbs (stats, turns, drain, versions,
6
+ opens) and every session verb, which the host binds to its own operations and hands in."""
7
+
8
+ from __future__ import annotations
9
+
10
+ import hmac
11
+ import json
12
+ import re
13
+ from collections.abc import Callable, Mapping
14
+ from email.message import Message
15
+ from typing import TYPE_CHECKING, Any
16
+
17
+ from .handler_errors import carried
18
+ from .host_surfaces import public_host
19
+ from .host_workers import WORKER_EXITED, refusal
20
+ from .validation import ContractError, fail
21
+ from .verbs import session_verb, unknown_verb
22
+ from .workers import WorkerGone
23
+
24
+ if TYPE_CHECKING:
25
+ from .world_host import WorldHost
26
+
27
+ Answer = tuple[int, Any] | tuple[int, Any, dict[str, str]]
28
+ HOST_HEADER = re.compile(r"(?:[A-Za-z0-9._-]+|\[[0-9A-Fa-f:.]+\])(?::[0-9]{1,5})?")
29
+ LABEL = r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?"
30
+ DOMAIN = re.compile(rf"(?P<name>(?:{LABEL}\.)+(?=[a-z0-9-]*[a-z]){LABEL})\.?(?::[0-9]{{1,5}})?")
31
+ FORWARDED_BY = ("X-Forwarded-For", "Forwarded")
32
+
33
+
34
+ class ControlPlane:
35
+ """One host's control plane. A host with no secret has no control plane at all rather than an
36
+ open one, and one with no worlds domain could not tell its own address from a session's."""
37
+
38
+ def __init__(self, host: WorldHost, secret: str, operations: Mapping[str, Callable[..., dict]], worlds_domain: str):
39
+ if not secret:
40
+ raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
41
+ self.host = host
42
+ self.operations = operations
43
+ self.domain = self.domain_of(worlds_domain)
44
+ self._secret = secret.encode()
45
+
46
+ @staticmethod
47
+ def domain_of(value: str | None) -> str:
48
+ """``WORLDS_INGRESS_DOMAIN`` as the gate compares it, without surrounding space, a port or a trailing dot.
49
+ Anything but a dotted name under a lettered top label is refused: against a malformed one every Host would
50
+ read as the pod's own, and against ``local`` or an address the platform's own calls would read as public."""
51
+ named = DOMAIN.fullmatch((value or "").strip().lower())
52
+ if named is None:
53
+ fail("host", f"WORLDS_INGRESS_DOMAIN must be the domain session hostnames live under, not {value!r}")
54
+ return named["name"]
55
+
56
+ def internal(self, target: str, headers: Message) -> bool:
57
+ """Whether a request names the pod itself: an origin-form target and exactly one Host, a well-formed
58
+ name or address outside the worlds domain. ``/healthz`` answers any such request, one an edge
59
+ forwarded included, which learns only that the host is up."""
60
+ hosts = headers.get_all("Host") or []
61
+ if not target.startswith("/") or len(hosts) != 1 or not HOST_HEADER.fullmatch(hosts[0]):
62
+ return False
63
+ return not public_host(hosts[0], self.domain)
64
+
65
+ def serves(self, target: str, headers: Message) -> bool:
66
+ """``/__host/*`` and ``/metrics`` also refuse what a proxy forwarded: the ALB and ingress-nginx add
67
+ ``X-Forwarded-For`` whatever Host they pass on; the platform's calls and a scrape carry neither header."""
68
+ return self.internal(target, headers) and not any(headers.get(name) is not None for name in FORWARDED_BY)
69
+
70
+ def admits(self, presented: str | None) -> bool:
71
+ """Compared as bytes in constant time, so a header no ASCII string can hold is refused, not raised."""
72
+ return hmac.compare_digest((presented or "").encode(), self._secret)
73
+
74
+ def answer(self, method: str, path: str, raw: bytes) -> Answer:
75
+ try:
76
+ body = json.loads(raw) if raw else {}
77
+ except ValueError as error:
78
+ return 400, {"error": "malformed json", "detail": str(error)}
79
+ if not isinstance(body, dict):
80
+ return 400, {"error": "malformed json", "detail": "the body must be a json object"}
81
+ parts = path.strip("/").split("/")
82
+ named = parts[2] if len(parts) >= 3 and parts[:2] == ["__host", "sessions"] else None
83
+ try:
84
+ return self._route(method, parts, body)
85
+ except ContractError as error:
86
+ return refusal(self.host.forwarding.instead_of(error, named))
87
+ except KeyError as error:
88
+ return 400, {"error": "missing field", "field": str(error)}
89
+ except WorkerGone as error:
90
+ return 502, {"error": WORKER_EXITED, "detail": str(error)}
91
+
92
+ def _route(self, method: str, parts: list[str], body: dict) -> Answer:
93
+ host = self.host
94
+ if method == "GET" and parts == ["__host", "stats"]:
95
+ return 200, host.stats()
96
+ if method == "GET" and parts == ["__host", "turns"]:
97
+ return 200, {"turns": host.admission.held_ages()}
98
+ if method == "POST" and parts == ["__host", "drain"]:
99
+ host.drain()
100
+ return 200, {"draining": True, "sessions": host.session_count()}
101
+ if method == "GET" and parts == ["__host", "versions"]:
102
+ return 200, {"versions": host.cache.version_ids()}
103
+ if method == "POST" and parts == ["__host", "versions"]:
104
+ return 200, host.warm(body["versionId"], body["bundleUrl"], body.get("snapshot"))
105
+ if method == "POST" and parts == ["__host", "sessions"]:
106
+ return 200, self._open(body)
107
+ if len(parts) >= 3 and parts[:2] == ["__host", "sessions"]:
108
+ return self._session(method, parts, body)
109
+ return 404, {"error": "unknown control route"}
110
+
111
+ def _open(self, body: dict) -> dict:
112
+ return self.host.open(
113
+ session_id=body["sessionId"],
114
+ version_id=body["versionId"],
115
+ bundle_url=body["bundleUrl"],
116
+ task=body.get("task"),
117
+ token=body.get("token"),
118
+ snapshot=body.get("snapshot"),
119
+ browser=body.get("browser"),
120
+ restore=body.get("restore"),
121
+ task_spec=body.get("taskSpec"),
122
+ dependencies=body.get("dependencies"),
123
+ pending=body.get("pending") is True,
124
+ auth=body.get("auth"),
125
+ ledger_source=body.get("ledgerSource"),
126
+ )
127
+
128
+ def _session(self, method: str, parts: list[str], body: dict) -> Answer:
129
+ sid = parts[2]
130
+ self.host.forwarding.refuse(sid)
131
+ verb = parts[3] if len(parts) > 3 else None
132
+ if method == "PUT" and verb == "browser" and len(parts) == 4:
133
+ return 200, self.host.set_browser(sid, body.get("browser"))
134
+ if method == "POST" and verb == "export" and len(parts) == 5:
135
+ return self._upload(sid, parts[4], body.get("url"))
136
+ row = session_verb(method, verb)
137
+ if row is None or len(parts) > 4:
138
+ return 404, unknown_verb(method, verb, hosted=True, prefix=f"/__host/sessions/{sid}/")
139
+ return 200, carried(self.operations[row.route](self.host, sid, **row.read(body)))
140
+
141
+ def _upload(self, sid: str, export_id: str, url: object) -> Answer:
142
+ if not isinstance(url, str) or not url.startswith(("https://", "http://")):
143
+ return 400, {"error": "url must be the presigned upload url"}
144
+ return 200, self.host.upload_export(sid, export_id, url)
@@ -0,0 +1,370 @@
1
+ """A World Host told to stop hands every session it holds to another host through the platform.
2
+
3
+ :class:`SessionHandoff` moves one session a step at a time: it fences it (:class:`SessionMove`),
4
+ waits for the requests already inside and for a call of it still running on its worker, settles
5
+ its request ledger, and asks the platform (:class:`PlatformHandoff`) to take its final export from
6
+ this host and reopen it elsewhere. The platform answers ``moved``, ``stayed``, ``closed`` or
7
+ ``in_progress``. :class:`DrainHandoff` steps every session the host holds, :data:`MOVES_AT_ONCE`
8
+ at a time, until all have gone or the drain ends; a session that stayed serves here and is asked
9
+ again after a backoff without holding a step's place. What is left when the drain ends is shut
10
+ (its requests told to come back) and logged and counted; the fleet reopens it from its last
11
+ export once the host is gone, the path a host loss takes."""
12
+
13
+ from __future__ import annotations
14
+
15
+ import heapq
16
+ import http.client
17
+ import json
18
+ import math
19
+ import queue
20
+ import threading
21
+ import time
22
+ import urllib.error
23
+ import urllib.parse
24
+ import urllib.request
25
+ from collections import Counter
26
+ from collections.abc import Callable
27
+ from dataclasses import dataclass
28
+ from typing import TYPE_CHECKING, Protocol
29
+
30
+ from .handler_errors import log_line
31
+ from .host_owners import PROXY_TIMEOUT_S
32
+ from .request_ledger import LEDGER_STOP_S
33
+ from .session_move import same_host
34
+
35
+ if TYPE_CHECKING:
36
+ from .host_session import Session
37
+ from .world_host import WorldHost
38
+
39
+ HANDOFF_ROUTE = "/api/public/world-hosts/handoff"
40
+ QUIET_S = 50.0
41
+ MOVES_AT_ONCE = 8
42
+ ASK_TIMEOUT_S = 120.0
43
+ ASK_AGAIN_S = 2.0
44
+ BACKOFF_S = (2.0, 4.0, 8.0, 15.0)
45
+ TURN_POLL_S = 0.2
46
+ ENLIST_S = 1.0
47
+ LINGER_S = 5.0
48
+ IN_FLIGHT_S = PROXY_TIMEOUT_S
49
+ EXIT_S = 110.0
50
+ assert LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + 5.0 <= EXIT_S, "a stopping host's exit fits the grace past its drain"
51
+
52
+ MOVED = "moved"
53
+ STAYED = "stayed"
54
+ CLOSED = "closed"
55
+ IN_PROGRESS = "in_progress"
56
+ REFUSED = "refused"
57
+ UNANSWERED = "unanswered"
58
+ GONE = "gone"
59
+ FALLBACK = "fallback"
60
+ UNDECIDED = "undecided"
61
+ OUTCOMES = (MOVED, CLOSED, GONE, FALLBACK, UNDECIDED)
62
+ NOT_HANDED_OFF = {
63
+ FALLBACK: "it was not handed off (fallback); it goes with this host",
64
+ UNDECIDED: "it was not handed off (undecided); it goes with this host",
65
+ GONE: "it left the host another way during the drain",
66
+ }
67
+
68
+
69
+ def bare_origin(value: object) -> str | None:
70
+ """``value`` as an ``http(s)://host[:port]`` origin, or None for anything else: a moved
71
+ session's requests go only to a host the platform named."""
72
+ if not isinstance(value, str):
73
+ return None
74
+ parts = urllib.parse.urlsplit(value)
75
+ bare = parts.scheme in ("http", "https") and parts.hostname and not (parts.username or parts.password)
76
+ if not bare or parts.path not in ("", "/") or parts.query or parts.fragment:
77
+ return None
78
+ return f"{parts.scheme}://{parts.netloc}"
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class HandoffAnswer:
83
+ """What the platform answered a handoff: ``moved`` (to ``owner``), ``stayed``, ``closed``,
84
+ ``in_progress``; ``refused`` when it turned the ask away or was never reached, so this ask moved
85
+ nothing; or ``unanswered`` when no answer it could read came back."""
86
+
87
+ outcome: str
88
+ owner: str | None = None
89
+ reason: str = ""
90
+
91
+ @classmethod
92
+ def of(cls, answer: object) -> HandoffAnswer:
93
+ body = answer if isinstance(answer, dict) else {}
94
+ outcome, reason = body.get("outcome"), str(body.get("reason") or "")[:500]
95
+ owner = bare_origin(body.get("hostUrl"))
96
+ if outcome == MOVED and owner is not None:
97
+ return cls(MOVED, owner, reason)
98
+ if outcome in (STAYED, CLOSED, IN_PROGRESS):
99
+ return cls(outcome, None, reason)
100
+ return cls.unanswered("an answer off the handoff contract")
101
+
102
+ @classmethod
103
+ def unanswered(cls, reason: str) -> HandoffAnswer:
104
+ return cls(UNANSWERED, None, reason)
105
+
106
+
107
+ class HandoffDoor(Protocol):
108
+ """Where a host asks to have a session moved off it."""
109
+
110
+ def ask(self, request: dict, timeout_s: float) -> HandoffAnswer: ...
111
+
112
+
113
+ class PlatformHandoff:
114
+ """The platform's handoff door (:data:`HANDOFF_ROUTE`), behind the control-plane secret. A 4xx,
115
+ or a request that never reached it, is refused: this ask moved nothing, though an earlier one
116
+ may still. No answer to a request it took, or a 5xx, may have moved the session."""
117
+
118
+ def __init__(self, platform_url: str, secret: str) -> None:
119
+ self.url = f"{platform_url.rstrip('/')}{HANDOFF_ROUTE}"
120
+ self.secret = secret
121
+
122
+ def ask(self, request: dict, timeout_s: float) -> HandoffAnswer:
123
+ headers = {"content-type": "application/json", "x-host-secret": self.secret}
124
+ try:
125
+ body = json.dumps(request, allow_nan=False).encode("utf-8")
126
+ sent = urllib.request.Request(self.url, data=body, method="POST", headers=headers)
127
+ with urllib.request.urlopen(sent, timeout=timeout_s) as answer:
128
+ return HandoffAnswer.of(json.loads(answer.read() or b"null"))
129
+ except urllib.error.HTTPError as error:
130
+ return self._refused(error.code)
131
+ except urllib.error.URLError as error:
132
+ return HandoffAnswer(REFUSED, None, f"the platform was not reached: {type(error.reason).__name__}")
133
+ except (OSError, ValueError, http.client.HTTPException) as error:
134
+ return HandoffAnswer.unanswered(type(error).__name__)
135
+
136
+ @staticmethod
137
+ def _refused(status: int) -> HandoffAnswer:
138
+ if status < 500:
139
+ return HandoffAnswer(REFUSED, None, f"the platform refused the handoff ({status})")
140
+ return HandoffAnswer.unanswered(f"the platform answered {status}")
141
+
142
+
143
+ @dataclass(frozen=True)
144
+ class Step:
145
+ """Where one step left a session's handoff: its final ``outcome``, or None and when it is due again."""
146
+
147
+ outcome: str | None
148
+ again_at: float = 0.0
149
+
150
+
151
+ @dataclass
152
+ class SessionHandoff:
153
+ """One session's handoff, a step at a time until ``until`` (monotonic). After a stay the session
154
+ serves here until its next step; from its ask until the platform answers, it is held, so no
155
+ write lands on this copy after the export the platform reopens from."""
156
+
157
+ host: WorldHost
158
+ door: HandoffDoor
159
+ session_id: str
160
+ until: float
161
+ sleep: Callable[[float], None] = time.sleep
162
+ request: dict | None = None
163
+ stays: int = 0
164
+
165
+ def step(self) -> Step:
166
+ session = self.host.held(self.session_id)
167
+ if session is None:
168
+ return Step(GONE)
169
+ try:
170
+ return self._step(session)
171
+ except Exception as error: # noqa: BLE001 - one session's failed step never stops the drain
172
+ log_line(f"handoff: session {self.session_id} step failed: {type(error).__name__}")
173
+ return self._after_failure(session)
174
+
175
+ def final(self) -> str:
176
+ return UNDECIDED if self.request is not None else FALLBACK
177
+
178
+ def _left(self) -> float:
179
+ return max(0.0, self.until - time.monotonic())
180
+
181
+ def _step(self, session: Session) -> Step:
182
+ if self.request is None:
183
+ self.request = self._prepare(session)
184
+ if self.request is None:
185
+ return self._unfence(session, "it did not go quiet in time")
186
+ answer = self.door.ask(self.request, min(ASK_TIMEOUT_S, self._left()))
187
+ return self._settle(session, answer)
188
+
189
+ def _prepare(self, session: Session) -> dict | None:
190
+ """The platform's question, once the session is fenced and quiet and its ledger settled."""
191
+ if not self._quiet(session):
192
+ return None
193
+ account = self.host.ledger.settle(session, time.monotonic() + min(ASK_TIMEOUT_S, self._left()))
194
+ session.move.ask()
195
+ return {
196
+ "hostUrl": self.host.advertise_url,
197
+ "sessionId": session.session_id,
198
+ "ledger": account.as_json(),
199
+ "usage": {"openedAt": session.opened_at, "seconds": session.seconds},
200
+ }
201
+
202
+ def _quiet(self, session: Session) -> bool:
203
+ """Fenced, with no request inside the session and no call of it still running on its worker,
204
+ within :data:`QUIET_S`; a worker that does not say counts as running one."""
205
+ quiet_by = time.monotonic() + min(QUIET_S, self._left())
206
+ if not session.move.fence(quiet_by - time.monotonic()):
207
+ return False
208
+ while self.host.turn_held(session):
209
+ if time.monotonic() >= quiet_by:
210
+ return False
211
+ self.sleep(TURN_POLL_S)
212
+ return True
213
+
214
+ def _settle(self, session: Session, answer: HandoffAnswer) -> Step:
215
+ settle: dict[str, Callable[[Session, HandoffAnswer], Step]] = {
216
+ MOVED: self._moved,
217
+ STAYED: lambda held, said: self._stay(held, said.reason),
218
+ REFUSED: lambda held, said: self._unfence(held, said.reason),
219
+ CLOSED: self._closed,
220
+ }
221
+ return settle.get(answer.outcome, self._ask_again)(session, answer)
222
+
223
+ def _moved(self, session: Session, answer: HandoffAnswer) -> Step:
224
+ assert answer.owner is not None
225
+ if same_host(answer.owner, self.host.advertise_url):
226
+ return self._stay(session, "the platform named this host")
227
+ self.host.forwarding.record(session.session_id, answer.owner)
228
+ session.move.moved(answer.owner)
229
+ self.host.close(session.session_id, reason="moved")
230
+ return Step(MOVED)
231
+
232
+ def _closed(self, session: Session, _answer: HandoffAnswer) -> Step:
233
+ session.move.closed()
234
+ self.host.close(session.session_id)
235
+ return Step(CLOSED)
236
+
237
+ def _ask_again(self, _session: Session, _answer: HandoffAnswer) -> Step:
238
+ return Step(None, time.monotonic() + ASK_AGAIN_S)
239
+
240
+ def _unfence(self, session: Session, reason: str) -> Step:
241
+ """Back as it was on this host's own word, a copy the platform never named still pending; held and
242
+ asked again instead while the platform holds its final export."""
243
+ if not session.move.unfence():
244
+ return self._ask_again(session, HandoffAnswer.unanswered(reason))
245
+ return self._served_here(session, reason)
246
+
247
+ def _stay(self, session: Session, reason: str) -> Step:
248
+ session.move.stay()
249
+ return self._served_here(session, reason)
250
+
251
+ def _served_here(self, session: Session, reason: str) -> Step:
252
+ self.request = None
253
+ pause = BACKOFF_S[min(self.stays, len(BACKOFF_S) - 1)]
254
+ self.stays += 1
255
+ log_line(f"handoff: session {session.session_id} stayed: {reason}")
256
+ return Step(None, time.monotonic() + pause)
257
+
258
+ def _after_failure(self, session: Session) -> Step:
259
+ if self.request is not None:
260
+ return self._ask_again(session, HandoffAnswer.unanswered("a step failed"))
261
+ return self._unfence(session, "a step failed")
262
+
263
+
264
+ class _Schedule:
265
+ """The drain's sessions, each due for its next step at a time, at most ``at_once`` stepping at
266
+ once on daemon threads, so a step stuck past the drain never holds the host up."""
267
+
268
+ def __init__(self, drain: DrainHandoff, door: HandoffDoor, until: float) -> None:
269
+ self.drain = drain
270
+ self.door = door
271
+ self.until = until
272
+ self.handoffs: dict[str, SessionHandoff] = {}
273
+ self.due: list[tuple[float, str]] = []
274
+ self.stepping: set[str] = set()
275
+ self.finished: dict[str, str] = {}
276
+ self.done: queue.Queue[tuple[str, Step]] = queue.Queue()
277
+
278
+ def run(self) -> dict[str, str]:
279
+ while time.monotonic() < self.until:
280
+ self._enlist()
281
+ if not (self.due or self.stepping):
282
+ break
283
+ self._start_due()
284
+ self._collect()
285
+ for sid, handoff in self.handoffs.items():
286
+ self.finished.setdefault(sid, handoff.final())
287
+ return self.finished
288
+
289
+ def _enlist(self) -> None:
290
+ for sid in self.drain.host.session_ids():
291
+ if sid in self.handoffs:
292
+ continue
293
+ self.handoffs[sid] = SessionHandoff(self.drain.host, self.door, sid, self.until, self.drain.sleep)
294
+ heapq.heappush(self.due, (time.monotonic(), sid))
295
+
296
+ def _start_due(self) -> None:
297
+ while self.due and self.due[0][0] <= time.monotonic() and len(self.stepping) < self.drain.at_once:
298
+ _, sid = heapq.heappop(self.due)
299
+ self.stepping.add(sid)
300
+ threading.Thread(target=self._step, args=(sid,), name="world-host-handoff", daemon=True).start()
301
+
302
+ def _step(self, sid: str) -> None:
303
+ self.done.put((sid, self.handoffs[sid].step()))
304
+
305
+ def _collect(self) -> None:
306
+ next_due = self.due[0][0] if self.due and len(self.stepping) < self.drain.at_once else math.inf
307
+ wake_at = min(next_due, self.until, time.monotonic() + ENLIST_S)
308
+ try:
309
+ sid, step = self.done.get(timeout=max(0.0, wake_at - time.monotonic()))
310
+ except queue.Empty:
311
+ return
312
+ self.stepping.discard(sid)
313
+ if step.outcome is None:
314
+ heapq.heappush(self.due, (step.again_at, sid))
315
+ return
316
+ self.finished[sid] = step.outcome
317
+
318
+
319
+ class DrainHandoff:
320
+ """A stopping host's drain: every session it holds handed off within ``drain_s``; then what is
321
+ left is shut, each logged and counted, and the host lingers so requests still on their way, and
322
+ those it sent on, are answered before it stops."""
323
+
324
+ def __init__(
325
+ self,
326
+ host: WorldHost,
327
+ door: HandoffDoor | None,
328
+ drain_s: float,
329
+ *,
330
+ at_once: int = MOVES_AT_ONCE,
331
+ linger_s: float = LINGER_S,
332
+ in_flight_s: float = IN_FLIGHT_S,
333
+ sleep: Callable[[float], None] = time.sleep,
334
+ ) -> None:
335
+ self.host = host
336
+ self.door = door
337
+ self.drain_s = drain_s
338
+ self.at_once = at_once
339
+ self.linger_s = linger_s
340
+ self.in_flight_s = in_flight_s
341
+ self.sleep = sleep
342
+
343
+ def run(self, idle: Callable[[], bool]) -> Counter[str]:
344
+ until = time.monotonic() + self.drain_s
345
+ finished = _Schedule(self, self.door, until).run() if self.door is not None else self._serve_out(until)
346
+ self._report(finished)
347
+ self._shut()
348
+ self._linger(idle)
349
+ return Counter(finished.values())
350
+
351
+ def _serve_out(self, until: float) -> dict[str, str]:
352
+ while self.host.session_count() > 0 and time.monotonic() < until:
353
+ self.sleep(0.5)
354
+ return {}
355
+
356
+ def _report(self, finished: dict[str, str]) -> None:
357
+ for session_id, outcome in finished.items():
358
+ self.host.tally(f"handoffs.{outcome}")
359
+ if outcome in NOT_HANDED_OFF:
360
+ log_line(f"handoff: session {session_id}: {NOT_HANDED_OFF[outcome]}")
361
+
362
+ def _shut(self) -> None:
363
+ for session in filter(None, map(self.host.held, self.host.session_ids())):
364
+ session.move.shut()
365
+
366
+ def _linger(self, idle: Callable[[], bool]) -> None:
367
+ self.sleep(self.linger_s)
368
+ quiet_by = time.monotonic() + self.in_flight_s
369
+ while not idle() and time.monotonic() < quiet_by:
370
+ self.sleep(0.1)
@@ -0,0 +1,97 @@
1
+ """What a World Host does on its own clock: the lease sweep closes the sessions whose lease ran
2
+ out and sweeps the version workers (:meth:`WorldHost.reap_workers`), and the heartbeat tells the
3
+ platform what the host is and holds. A step that fails is logged and the next one goes on."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import threading
9
+ import urllib.error
10
+ import urllib.request
11
+ from collections.abc import Callable
12
+ from typing import TYPE_CHECKING
13
+
14
+ from .handler_errors import log_line
15
+
16
+ if TYPE_CHECKING:
17
+ from .world_host import WorldHost
18
+
19
+ LEASE_SWEEP_S = 30.0
20
+ HEARTBEAT_S = 10.0
21
+
22
+
23
+ def swept(step: Callable[[], list[str]], done: str) -> list[str]:
24
+ """One step of the lease sweep and what it did, each logged; a step that raises did nothing."""
25
+ try:
26
+ names = step()
27
+ except Exception as error: # noqa: BLE001 - one failing step must not stop the steps or the sweeps after it
28
+ log_line(f"sweep step failed: {type(error).__name__}")
29
+ return []
30
+ for name in names:
31
+ log_line(f"{done} {name}")
32
+ return names
33
+
34
+
35
+ def start_lease_sweeper(world_host: WorldHost, interval_s: float = LEASE_SWEEP_S) -> threading.Event:
36
+ """Expire leases and sweep the version workers every ``interval_s`` until the returned event is set."""
37
+ stop = threading.Event()
38
+
39
+ def sweep() -> None:
40
+ while not stop.wait(interval_s):
41
+ swept(world_host.expire_leases, "lease expired, closed session")
42
+ swept(world_host.reap_workers, "stopped idle worker for version")
43
+
44
+ threading.Thread(target=sweep, name="world-host-leases", daemon=True).start()
45
+ return stop
46
+
47
+
48
+ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeout_s: float = 5.0) -> bool:
49
+ """Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
50
+ The platform's registry is what places new sessions: a host it has not
51
+ heard from lately gets none. Returns whether the platform took it."""
52
+ body = json.dumps(world_host.stats()).encode("utf-8")
53
+ req = urllib.request.Request(
54
+ f"{platform_url.rstrip('/')}/api/public/world-hosts",
55
+ data=body,
56
+ method="POST",
57
+ headers={"content-type": "application/json", "x-host-secret": secret},
58
+ )
59
+ try:
60
+ with urllib.request.urlopen(req, timeout=timeout_s) as r:
61
+ return 200 <= r.status < 300
62
+ except (urllib.error.URLError, OSError, ValueError) as error:
63
+ log_line(f"heartbeat failed: {error}")
64
+ return False
65
+
66
+
67
+ class Heartbeat:
68
+ """Tells the platform what this host is and holds every ``interval_s``: at once on
69
+ :meth:`start`, so a fresh replica is placeable immediately; at once on :meth:`wake`,
70
+ so a drain is seen as soon as it begins while the host, still alive, keeps beating
71
+ through its handoff; and once more on :meth:`stop`."""
72
+
73
+ def __init__(self, world_host: WorldHost, platform_url: str, secret: str, interval_s: float = HEARTBEAT_S):
74
+ self.world_host = world_host
75
+ self.platform_url = platform_url
76
+ self.secret = secret
77
+ self.interval_s = interval_s
78
+ self._due = threading.Event()
79
+ self._stopped = threading.Event()
80
+
81
+ def start(self) -> Heartbeat:
82
+ threading.Thread(target=self._beat, name="world-host-heartbeat", daemon=True).start()
83
+ return self
84
+
85
+ def wake(self) -> None:
86
+ self._due.set()
87
+
88
+ def stop(self) -> None:
89
+ self._stopped.set()
90
+ self._due.set()
91
+
92
+ def _beat(self) -> None:
93
+ heartbeat_once(self.world_host, self.platform_url, self.secret)
94
+ while not self._stopped.is_set():
95
+ self._due.wait(self.interval_s)
96
+ self._due.clear()
97
+ heartbeat_once(self.world_host, self.platform_url, self.secret)