@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""The platform's half of a World Host: ``/__host/*``, behind ``X-Host-Secret``.
|
|
2
|
+
|
|
3
|
+
Only a request that reached the pod from inside the network gets that far (:meth:`ControlPlane.serves`),
|
|
4
|
+
and the secret is checked before any body is read. An admitted request is answered as ``(status, body)``
|
|
5
|
+
or, for a refusal, ``(status, body, headers)``: the host's own verbs (stats, turns, drain, versions,
|
|
6
|
+
opens) and every session verb, which the host binds to its own operations and hands in."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hmac
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Callable, Mapping
|
|
14
|
+
from email.message import Message
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
from .handler_errors import carried
|
|
18
|
+
from .host_surfaces import public_host
|
|
19
|
+
from .host_workers import WORKER_EXITED, refusal
|
|
20
|
+
from .validation import ContractError, fail
|
|
21
|
+
from .verbs import session_verb, unknown_verb
|
|
22
|
+
from .workers import WorkerGone
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from .world_host import WorldHost
|
|
26
|
+
|
|
27
|
+
Answer = tuple[int, Any] | tuple[int, Any, dict[str, str]]
|
|
28
|
+
HOST_HEADER = re.compile(r"(?:[A-Za-z0-9._-]+|\[[0-9A-Fa-f:.]+\])(?::[0-9]{1,5})?")
|
|
29
|
+
LABEL = r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?"
|
|
30
|
+
DOMAIN = re.compile(rf"(?P<name>(?:{LABEL}\.)+(?=[a-z0-9-]*[a-z]){LABEL})\.?(?::[0-9]{{1,5}})?")
|
|
31
|
+
FORWARDED_BY = ("X-Forwarded-For", "Forwarded")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ControlPlane:
|
|
35
|
+
"""One host's control plane. A host with no secret has no control plane at all rather than an
|
|
36
|
+
open one, and one with no worlds domain could not tell its own address from a session's."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, host: WorldHost, secret: str, operations: Mapping[str, Callable[..., dict]], worlds_domain: str):
|
|
39
|
+
if not secret:
|
|
40
|
+
raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
|
|
41
|
+
self.host = host
|
|
42
|
+
self.operations = operations
|
|
43
|
+
self.domain = self.domain_of(worlds_domain)
|
|
44
|
+
self._secret = secret.encode()
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def domain_of(value: str | None) -> str:
|
|
48
|
+
"""``WORLDS_INGRESS_DOMAIN`` as the gate compares it, without surrounding space, a port or a trailing dot.
|
|
49
|
+
Anything but a dotted name under a lettered top label is refused: against a malformed one every Host would
|
|
50
|
+
read as the pod's own, and against ``local`` or an address the platform's own calls would read as public."""
|
|
51
|
+
named = DOMAIN.fullmatch((value or "").strip().lower())
|
|
52
|
+
if named is None:
|
|
53
|
+
fail("host", f"WORLDS_INGRESS_DOMAIN must be the domain session hostnames live under, not {value!r}")
|
|
54
|
+
return named["name"]
|
|
55
|
+
|
|
56
|
+
def internal(self, target: str, headers: Message) -> bool:
|
|
57
|
+
"""Whether a request names the pod itself: an origin-form target and exactly one Host, a well-formed
|
|
58
|
+
name or address outside the worlds domain. ``/healthz`` answers any such request, one an edge
|
|
59
|
+
forwarded included, which learns only that the host is up."""
|
|
60
|
+
hosts = headers.get_all("Host") or []
|
|
61
|
+
if not target.startswith("/") or len(hosts) != 1 or not HOST_HEADER.fullmatch(hosts[0]):
|
|
62
|
+
return False
|
|
63
|
+
return not public_host(hosts[0], self.domain)
|
|
64
|
+
|
|
65
|
+
def serves(self, target: str, headers: Message) -> bool:
|
|
66
|
+
"""``/__host/*`` and ``/metrics`` also refuse what a proxy forwarded: the ALB and ingress-nginx add
|
|
67
|
+
``X-Forwarded-For`` whatever Host they pass on; the platform's calls and a scrape carry neither header."""
|
|
68
|
+
return self.internal(target, headers) and not any(headers.get(name) is not None for name in FORWARDED_BY)
|
|
69
|
+
|
|
70
|
+
def admits(self, presented: str | None) -> bool:
|
|
71
|
+
"""Compared as bytes in constant time, so a header no ASCII string can hold is refused, not raised."""
|
|
72
|
+
return hmac.compare_digest((presented or "").encode(), self._secret)
|
|
73
|
+
|
|
74
|
+
def answer(self, method: str, path: str, raw: bytes) -> Answer:
|
|
75
|
+
try:
|
|
76
|
+
body = json.loads(raw) if raw else {}
|
|
77
|
+
except ValueError as error:
|
|
78
|
+
return 400, {"error": "malformed json", "detail": str(error)}
|
|
79
|
+
if not isinstance(body, dict):
|
|
80
|
+
return 400, {"error": "malformed json", "detail": "the body must be a json object"}
|
|
81
|
+
parts = path.strip("/").split("/")
|
|
82
|
+
named = parts[2] if len(parts) >= 3 and parts[:2] == ["__host", "sessions"] else None
|
|
83
|
+
try:
|
|
84
|
+
return self._route(method, parts, body)
|
|
85
|
+
except ContractError as error:
|
|
86
|
+
return refusal(self.host.forwarding.instead_of(error, named))
|
|
87
|
+
except KeyError as error:
|
|
88
|
+
return 400, {"error": "missing field", "field": str(error)}
|
|
89
|
+
except WorkerGone as error:
|
|
90
|
+
return 502, {"error": WORKER_EXITED, "detail": str(error)}
|
|
91
|
+
|
|
92
|
+
def _route(self, method: str, parts: list[str], body: dict) -> Answer:
|
|
93
|
+
host = self.host
|
|
94
|
+
if method == "GET" and parts == ["__host", "stats"]:
|
|
95
|
+
return 200, host.stats()
|
|
96
|
+
if method == "GET" and parts == ["__host", "turns"]:
|
|
97
|
+
return 200, {"turns": host.admission.held_ages()}
|
|
98
|
+
if method == "POST" and parts == ["__host", "drain"]:
|
|
99
|
+
host.drain()
|
|
100
|
+
return 200, {"draining": True, "sessions": host.session_count()}
|
|
101
|
+
if method == "GET" and parts == ["__host", "versions"]:
|
|
102
|
+
return 200, {"versions": host.cache.version_ids()}
|
|
103
|
+
if method == "POST" and parts == ["__host", "versions"]:
|
|
104
|
+
return 200, host.warm(body["versionId"], body["bundleUrl"], body.get("snapshot"))
|
|
105
|
+
if method == "POST" and parts == ["__host", "sessions"]:
|
|
106
|
+
return 200, self._open(body)
|
|
107
|
+
if len(parts) >= 3 and parts[:2] == ["__host", "sessions"]:
|
|
108
|
+
return self._session(method, parts, body)
|
|
109
|
+
return 404, {"error": "unknown control route"}
|
|
110
|
+
|
|
111
|
+
def _open(self, body: dict) -> dict:
|
|
112
|
+
return self.host.open(
|
|
113
|
+
session_id=body["sessionId"],
|
|
114
|
+
version_id=body["versionId"],
|
|
115
|
+
bundle_url=body["bundleUrl"],
|
|
116
|
+
task=body.get("task"),
|
|
117
|
+
token=body.get("token"),
|
|
118
|
+
snapshot=body.get("snapshot"),
|
|
119
|
+
browser=body.get("browser"),
|
|
120
|
+
restore=body.get("restore"),
|
|
121
|
+
task_spec=body.get("taskSpec"),
|
|
122
|
+
dependencies=body.get("dependencies"),
|
|
123
|
+
pending=body.get("pending") is True,
|
|
124
|
+
auth=body.get("auth"),
|
|
125
|
+
ledger_source=body.get("ledgerSource"),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
def _session(self, method: str, parts: list[str], body: dict) -> Answer:
|
|
129
|
+
sid = parts[2]
|
|
130
|
+
self.host.forwarding.refuse(sid)
|
|
131
|
+
verb = parts[3] if len(parts) > 3 else None
|
|
132
|
+
if method == "PUT" and verb == "browser" and len(parts) == 4:
|
|
133
|
+
return 200, self.host.set_browser(sid, body.get("browser"))
|
|
134
|
+
if method == "POST" and verb == "export" and len(parts) == 5:
|
|
135
|
+
return self._upload(sid, parts[4], body.get("url"))
|
|
136
|
+
row = session_verb(method, verb)
|
|
137
|
+
if row is None or len(parts) > 4:
|
|
138
|
+
return 404, unknown_verb(method, verb, hosted=True, prefix=f"/__host/sessions/{sid}/")
|
|
139
|
+
return 200, carried(self.operations[row.route](self.host, sid, **row.read(body)))
|
|
140
|
+
|
|
141
|
+
def _upload(self, sid: str, export_id: str, url: object) -> Answer:
|
|
142
|
+
if not isinstance(url, str) or not url.startswith(("https://", "http://")):
|
|
143
|
+
return 400, {"error": "url must be the presigned upload url"}
|
|
144
|
+
return 200, self.host.upload_export(sid, export_id, url)
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""A World Host told to stop hands every session it holds to another host through the platform.
|
|
2
|
+
|
|
3
|
+
:class:`SessionHandoff` moves one session a step at a time: it fences it (:class:`SessionMove`),
|
|
4
|
+
waits for the requests already inside and for a call of it still running on its worker, settles
|
|
5
|
+
its request ledger, and asks the platform (:class:`PlatformHandoff`) to take its final export from
|
|
6
|
+
this host and reopen it elsewhere. The platform answers ``moved``, ``stayed``, ``closed`` or
|
|
7
|
+
``in_progress``. :class:`DrainHandoff` steps every session the host holds, :data:`MOVES_AT_ONCE`
|
|
8
|
+
at a time, until all have gone or the drain ends; a session that stayed serves here and is asked
|
|
9
|
+
again after a backoff without holding a step's place. What is left when the drain ends is shut
|
|
10
|
+
(its requests told to come back) and logged and counted; the fleet reopens it from its last
|
|
11
|
+
export once the host is gone, the path a host loss takes."""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import heapq
|
|
16
|
+
import http.client
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import queue
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.parse
|
|
24
|
+
import urllib.request
|
|
25
|
+
from collections import Counter
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from typing import TYPE_CHECKING, Protocol
|
|
29
|
+
|
|
30
|
+
from .handler_errors import log_line
|
|
31
|
+
from .host_owners import PROXY_TIMEOUT_S
|
|
32
|
+
from .request_ledger import LEDGER_STOP_S
|
|
33
|
+
from .session_move import same_host
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from .host_session import Session
|
|
37
|
+
from .world_host import WorldHost
|
|
38
|
+
|
|
39
|
+
HANDOFF_ROUTE = "/api/public/world-hosts/handoff"
|
|
40
|
+
QUIET_S = 50.0
|
|
41
|
+
MOVES_AT_ONCE = 8
|
|
42
|
+
ASK_TIMEOUT_S = 120.0
|
|
43
|
+
ASK_AGAIN_S = 2.0
|
|
44
|
+
BACKOFF_S = (2.0, 4.0, 8.0, 15.0)
|
|
45
|
+
TURN_POLL_S = 0.2
|
|
46
|
+
ENLIST_S = 1.0
|
|
47
|
+
LINGER_S = 5.0
|
|
48
|
+
IN_FLIGHT_S = PROXY_TIMEOUT_S
|
|
49
|
+
EXIT_S = 110.0
|
|
50
|
+
assert LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + 5.0 <= EXIT_S, "a stopping host's exit fits the grace past its drain"
|
|
51
|
+
|
|
52
|
+
MOVED = "moved"
|
|
53
|
+
STAYED = "stayed"
|
|
54
|
+
CLOSED = "closed"
|
|
55
|
+
IN_PROGRESS = "in_progress"
|
|
56
|
+
REFUSED = "refused"
|
|
57
|
+
UNANSWERED = "unanswered"
|
|
58
|
+
GONE = "gone"
|
|
59
|
+
FALLBACK = "fallback"
|
|
60
|
+
UNDECIDED = "undecided"
|
|
61
|
+
OUTCOMES = (MOVED, CLOSED, GONE, FALLBACK, UNDECIDED)
|
|
62
|
+
NOT_HANDED_OFF = {
|
|
63
|
+
FALLBACK: "it was not handed off (fallback); it goes with this host",
|
|
64
|
+
UNDECIDED: "it was not handed off (undecided); it goes with this host",
|
|
65
|
+
GONE: "it left the host another way during the drain",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def bare_origin(value: object) -> str | None:
|
|
70
|
+
"""``value`` as an ``http(s)://host[:port]`` origin, or None for anything else: a moved
|
|
71
|
+
session's requests go only to a host the platform named."""
|
|
72
|
+
if not isinstance(value, str):
|
|
73
|
+
return None
|
|
74
|
+
parts = urllib.parse.urlsplit(value)
|
|
75
|
+
bare = parts.scheme in ("http", "https") and parts.hostname and not (parts.username or parts.password)
|
|
76
|
+
if not bare or parts.path not in ("", "/") or parts.query or parts.fragment:
|
|
77
|
+
return None
|
|
78
|
+
return f"{parts.scheme}://{parts.netloc}"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class HandoffAnswer:
|
|
83
|
+
"""What the platform answered a handoff: ``moved`` (to ``owner``), ``stayed``, ``closed``,
|
|
84
|
+
``in_progress``; ``refused`` when it turned the ask away or was never reached, so this ask moved
|
|
85
|
+
nothing; or ``unanswered`` when no answer it could read came back."""
|
|
86
|
+
|
|
87
|
+
outcome: str
|
|
88
|
+
owner: str | None = None
|
|
89
|
+
reason: str = ""
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def of(cls, answer: object) -> HandoffAnswer:
|
|
93
|
+
body = answer if isinstance(answer, dict) else {}
|
|
94
|
+
outcome, reason = body.get("outcome"), str(body.get("reason") or "")[:500]
|
|
95
|
+
owner = bare_origin(body.get("hostUrl"))
|
|
96
|
+
if outcome == MOVED and owner is not None:
|
|
97
|
+
return cls(MOVED, owner, reason)
|
|
98
|
+
if outcome in (STAYED, CLOSED, IN_PROGRESS):
|
|
99
|
+
return cls(outcome, None, reason)
|
|
100
|
+
return cls.unanswered("an answer off the handoff contract")
|
|
101
|
+
|
|
102
|
+
@classmethod
|
|
103
|
+
def unanswered(cls, reason: str) -> HandoffAnswer:
|
|
104
|
+
return cls(UNANSWERED, None, reason)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class HandoffDoor(Protocol):
|
|
108
|
+
"""Where a host asks to have a session moved off it."""
|
|
109
|
+
|
|
110
|
+
def ask(self, request: dict, timeout_s: float) -> HandoffAnswer: ...
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class PlatformHandoff:
|
|
114
|
+
"""The platform's handoff door (:data:`HANDOFF_ROUTE`), behind the control-plane secret. A 4xx,
|
|
115
|
+
or a request that never reached it, is refused: this ask moved nothing, though an earlier one
|
|
116
|
+
may still. No answer to a request it took, or a 5xx, may have moved the session."""
|
|
117
|
+
|
|
118
|
+
def __init__(self, platform_url: str, secret: str) -> None:
|
|
119
|
+
self.url = f"{platform_url.rstrip('/')}{HANDOFF_ROUTE}"
|
|
120
|
+
self.secret = secret
|
|
121
|
+
|
|
122
|
+
def ask(self, request: dict, timeout_s: float) -> HandoffAnswer:
|
|
123
|
+
headers = {"content-type": "application/json", "x-host-secret": self.secret}
|
|
124
|
+
try:
|
|
125
|
+
body = json.dumps(request, allow_nan=False).encode("utf-8")
|
|
126
|
+
sent = urllib.request.Request(self.url, data=body, method="POST", headers=headers)
|
|
127
|
+
with urllib.request.urlopen(sent, timeout=timeout_s) as answer:
|
|
128
|
+
return HandoffAnswer.of(json.loads(answer.read() or b"null"))
|
|
129
|
+
except urllib.error.HTTPError as error:
|
|
130
|
+
return self._refused(error.code)
|
|
131
|
+
except urllib.error.URLError as error:
|
|
132
|
+
return HandoffAnswer(REFUSED, None, f"the platform was not reached: {type(error.reason).__name__}")
|
|
133
|
+
except (OSError, ValueError, http.client.HTTPException) as error:
|
|
134
|
+
return HandoffAnswer.unanswered(type(error).__name__)
|
|
135
|
+
|
|
136
|
+
@staticmethod
|
|
137
|
+
def _refused(status: int) -> HandoffAnswer:
|
|
138
|
+
if status < 500:
|
|
139
|
+
return HandoffAnswer(REFUSED, None, f"the platform refused the handoff ({status})")
|
|
140
|
+
return HandoffAnswer.unanswered(f"the platform answered {status}")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(frozen=True)
|
|
144
|
+
class Step:
|
|
145
|
+
"""Where one step left a session's handoff: its final ``outcome``, or None and when it is due again."""
|
|
146
|
+
|
|
147
|
+
outcome: str | None
|
|
148
|
+
again_at: float = 0.0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@dataclass
|
|
152
|
+
class SessionHandoff:
|
|
153
|
+
"""One session's handoff, a step at a time until ``until`` (monotonic). After a stay the session
|
|
154
|
+
serves here until its next step; from its ask until the platform answers, it is held, so no
|
|
155
|
+
write lands on this copy after the export the platform reopens from."""
|
|
156
|
+
|
|
157
|
+
host: WorldHost
|
|
158
|
+
door: HandoffDoor
|
|
159
|
+
session_id: str
|
|
160
|
+
until: float
|
|
161
|
+
sleep: Callable[[float], None] = time.sleep
|
|
162
|
+
request: dict | None = None
|
|
163
|
+
stays: int = 0
|
|
164
|
+
|
|
165
|
+
def step(self) -> Step:
|
|
166
|
+
session = self.host.held(self.session_id)
|
|
167
|
+
if session is None:
|
|
168
|
+
return Step(GONE)
|
|
169
|
+
try:
|
|
170
|
+
return self._step(session)
|
|
171
|
+
except Exception as error: # noqa: BLE001 - one session's failed step never stops the drain
|
|
172
|
+
log_line(f"handoff: session {self.session_id} step failed: {type(error).__name__}")
|
|
173
|
+
return self._after_failure(session)
|
|
174
|
+
|
|
175
|
+
def final(self) -> str:
|
|
176
|
+
return UNDECIDED if self.request is not None else FALLBACK
|
|
177
|
+
|
|
178
|
+
def _left(self) -> float:
|
|
179
|
+
return max(0.0, self.until - time.monotonic())
|
|
180
|
+
|
|
181
|
+
def _step(self, session: Session) -> Step:
|
|
182
|
+
if self.request is None:
|
|
183
|
+
self.request = self._prepare(session)
|
|
184
|
+
if self.request is None:
|
|
185
|
+
return self._unfence(session, "it did not go quiet in time")
|
|
186
|
+
answer = self.door.ask(self.request, min(ASK_TIMEOUT_S, self._left()))
|
|
187
|
+
return self._settle(session, answer)
|
|
188
|
+
|
|
189
|
+
def _prepare(self, session: Session) -> dict | None:
|
|
190
|
+
"""The platform's question, once the session is fenced and quiet and its ledger settled."""
|
|
191
|
+
if not self._quiet(session):
|
|
192
|
+
return None
|
|
193
|
+
account = self.host.ledger.settle(session, time.monotonic() + min(ASK_TIMEOUT_S, self._left()))
|
|
194
|
+
session.move.ask()
|
|
195
|
+
return {
|
|
196
|
+
"hostUrl": self.host.advertise_url,
|
|
197
|
+
"sessionId": session.session_id,
|
|
198
|
+
"ledger": account.as_json(),
|
|
199
|
+
"usage": {"openedAt": session.opened_at, "seconds": session.seconds},
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
def _quiet(self, session: Session) -> bool:
|
|
203
|
+
"""Fenced, with no request inside the session and no call of it still running on its worker,
|
|
204
|
+
within :data:`QUIET_S`; a worker that does not say counts as running one."""
|
|
205
|
+
quiet_by = time.monotonic() + min(QUIET_S, self._left())
|
|
206
|
+
if not session.move.fence(quiet_by - time.monotonic()):
|
|
207
|
+
return False
|
|
208
|
+
while self.host.turn_held(session):
|
|
209
|
+
if time.monotonic() >= quiet_by:
|
|
210
|
+
return False
|
|
211
|
+
self.sleep(TURN_POLL_S)
|
|
212
|
+
return True
|
|
213
|
+
|
|
214
|
+
def _settle(self, session: Session, answer: HandoffAnswer) -> Step:
|
|
215
|
+
settle: dict[str, Callable[[Session, HandoffAnswer], Step]] = {
|
|
216
|
+
MOVED: self._moved,
|
|
217
|
+
STAYED: lambda held, said: self._stay(held, said.reason),
|
|
218
|
+
REFUSED: lambda held, said: self._unfence(held, said.reason),
|
|
219
|
+
CLOSED: self._closed,
|
|
220
|
+
}
|
|
221
|
+
return settle.get(answer.outcome, self._ask_again)(session, answer)
|
|
222
|
+
|
|
223
|
+
def _moved(self, session: Session, answer: HandoffAnswer) -> Step:
|
|
224
|
+
assert answer.owner is not None
|
|
225
|
+
if same_host(answer.owner, self.host.advertise_url):
|
|
226
|
+
return self._stay(session, "the platform named this host")
|
|
227
|
+
self.host.forwarding.record(session.session_id, answer.owner)
|
|
228
|
+
session.move.moved(answer.owner)
|
|
229
|
+
self.host.close(session.session_id, reason="moved")
|
|
230
|
+
return Step(MOVED)
|
|
231
|
+
|
|
232
|
+
def _closed(self, session: Session, _answer: HandoffAnswer) -> Step:
|
|
233
|
+
session.move.closed()
|
|
234
|
+
self.host.close(session.session_id)
|
|
235
|
+
return Step(CLOSED)
|
|
236
|
+
|
|
237
|
+
def _ask_again(self, _session: Session, _answer: HandoffAnswer) -> Step:
|
|
238
|
+
return Step(None, time.monotonic() + ASK_AGAIN_S)
|
|
239
|
+
|
|
240
|
+
def _unfence(self, session: Session, reason: str) -> Step:
|
|
241
|
+
"""Back as it was on this host's own word, a copy the platform never named still pending; held and
|
|
242
|
+
asked again instead while the platform holds its final export."""
|
|
243
|
+
if not session.move.unfence():
|
|
244
|
+
return self._ask_again(session, HandoffAnswer.unanswered(reason))
|
|
245
|
+
return self._served_here(session, reason)
|
|
246
|
+
|
|
247
|
+
def _stay(self, session: Session, reason: str) -> Step:
|
|
248
|
+
session.move.stay()
|
|
249
|
+
return self._served_here(session, reason)
|
|
250
|
+
|
|
251
|
+
def _served_here(self, session: Session, reason: str) -> Step:
|
|
252
|
+
self.request = None
|
|
253
|
+
pause = BACKOFF_S[min(self.stays, len(BACKOFF_S) - 1)]
|
|
254
|
+
self.stays += 1
|
|
255
|
+
log_line(f"handoff: session {session.session_id} stayed: {reason}")
|
|
256
|
+
return Step(None, time.monotonic() + pause)
|
|
257
|
+
|
|
258
|
+
def _after_failure(self, session: Session) -> Step:
|
|
259
|
+
if self.request is not None:
|
|
260
|
+
return self._ask_again(session, HandoffAnswer.unanswered("a step failed"))
|
|
261
|
+
return self._unfence(session, "a step failed")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class _Schedule:
|
|
265
|
+
"""The drain's sessions, each due for its next step at a time, at most ``at_once`` stepping at
|
|
266
|
+
once on daemon threads, so a step stuck past the drain never holds the host up."""
|
|
267
|
+
|
|
268
|
+
def __init__(self, drain: DrainHandoff, door: HandoffDoor, until: float) -> None:
|
|
269
|
+
self.drain = drain
|
|
270
|
+
self.door = door
|
|
271
|
+
self.until = until
|
|
272
|
+
self.handoffs: dict[str, SessionHandoff] = {}
|
|
273
|
+
self.due: list[tuple[float, str]] = []
|
|
274
|
+
self.stepping: set[str] = set()
|
|
275
|
+
self.finished: dict[str, str] = {}
|
|
276
|
+
self.done: queue.Queue[tuple[str, Step]] = queue.Queue()
|
|
277
|
+
|
|
278
|
+
def run(self) -> dict[str, str]:
|
|
279
|
+
while time.monotonic() < self.until:
|
|
280
|
+
self._enlist()
|
|
281
|
+
if not (self.due or self.stepping):
|
|
282
|
+
break
|
|
283
|
+
self._start_due()
|
|
284
|
+
self._collect()
|
|
285
|
+
for sid, handoff in self.handoffs.items():
|
|
286
|
+
self.finished.setdefault(sid, handoff.final())
|
|
287
|
+
return self.finished
|
|
288
|
+
|
|
289
|
+
def _enlist(self) -> None:
|
|
290
|
+
for sid in self.drain.host.session_ids():
|
|
291
|
+
if sid in self.handoffs:
|
|
292
|
+
continue
|
|
293
|
+
self.handoffs[sid] = SessionHandoff(self.drain.host, self.door, sid, self.until, self.drain.sleep)
|
|
294
|
+
heapq.heappush(self.due, (time.monotonic(), sid))
|
|
295
|
+
|
|
296
|
+
def _start_due(self) -> None:
|
|
297
|
+
while self.due and self.due[0][0] <= time.monotonic() and len(self.stepping) < self.drain.at_once:
|
|
298
|
+
_, sid = heapq.heappop(self.due)
|
|
299
|
+
self.stepping.add(sid)
|
|
300
|
+
threading.Thread(target=self._step, args=(sid,), name="world-host-handoff", daemon=True).start()
|
|
301
|
+
|
|
302
|
+
def _step(self, sid: str) -> None:
|
|
303
|
+
self.done.put((sid, self.handoffs[sid].step()))
|
|
304
|
+
|
|
305
|
+
def _collect(self) -> None:
|
|
306
|
+
next_due = self.due[0][0] if self.due and len(self.stepping) < self.drain.at_once else math.inf
|
|
307
|
+
wake_at = min(next_due, self.until, time.monotonic() + ENLIST_S)
|
|
308
|
+
try:
|
|
309
|
+
sid, step = self.done.get(timeout=max(0.0, wake_at - time.monotonic()))
|
|
310
|
+
except queue.Empty:
|
|
311
|
+
return
|
|
312
|
+
self.stepping.discard(sid)
|
|
313
|
+
if step.outcome is None:
|
|
314
|
+
heapq.heappush(self.due, (step.again_at, sid))
|
|
315
|
+
return
|
|
316
|
+
self.finished[sid] = step.outcome
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class DrainHandoff:
|
|
320
|
+
"""A stopping host's drain: every session it holds handed off within ``drain_s``; then what is
|
|
321
|
+
left is shut, each logged and counted, and the host lingers so requests still on their way, and
|
|
322
|
+
those it sent on, are answered before it stops."""
|
|
323
|
+
|
|
324
|
+
def __init__(
|
|
325
|
+
self,
|
|
326
|
+
host: WorldHost,
|
|
327
|
+
door: HandoffDoor | None,
|
|
328
|
+
drain_s: float,
|
|
329
|
+
*,
|
|
330
|
+
at_once: int = MOVES_AT_ONCE,
|
|
331
|
+
linger_s: float = LINGER_S,
|
|
332
|
+
in_flight_s: float = IN_FLIGHT_S,
|
|
333
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
334
|
+
) -> None:
|
|
335
|
+
self.host = host
|
|
336
|
+
self.door = door
|
|
337
|
+
self.drain_s = drain_s
|
|
338
|
+
self.at_once = at_once
|
|
339
|
+
self.linger_s = linger_s
|
|
340
|
+
self.in_flight_s = in_flight_s
|
|
341
|
+
self.sleep = sleep
|
|
342
|
+
|
|
343
|
+
def run(self, idle: Callable[[], bool]) -> Counter[str]:
|
|
344
|
+
until = time.monotonic() + self.drain_s
|
|
345
|
+
finished = _Schedule(self, self.door, until).run() if self.door is not None else self._serve_out(until)
|
|
346
|
+
self._report(finished)
|
|
347
|
+
self._shut()
|
|
348
|
+
self._linger(idle)
|
|
349
|
+
return Counter(finished.values())
|
|
350
|
+
|
|
351
|
+
def _serve_out(self, until: float) -> dict[str, str]:
|
|
352
|
+
while self.host.session_count() > 0 and time.monotonic() < until:
|
|
353
|
+
self.sleep(0.5)
|
|
354
|
+
return {}
|
|
355
|
+
|
|
356
|
+
def _report(self, finished: dict[str, str]) -> None:
|
|
357
|
+
for session_id, outcome in finished.items():
|
|
358
|
+
self.host.tally(f"handoffs.{outcome}")
|
|
359
|
+
if outcome in NOT_HANDED_OFF:
|
|
360
|
+
log_line(f"handoff: session {session_id}: {NOT_HANDED_OFF[outcome]}")
|
|
361
|
+
|
|
362
|
+
def _shut(self) -> None:
|
|
363
|
+
for session in filter(None, map(self.host.held, self.host.session_ids())):
|
|
364
|
+
session.move.shut()
|
|
365
|
+
|
|
366
|
+
def _linger(self, idle: Callable[[], bool]) -> None:
|
|
367
|
+
self.sleep(self.linger_s)
|
|
368
|
+
quiet_by = time.monotonic() + self.in_flight_s
|
|
369
|
+
while not idle() and time.monotonic() < quiet_by:
|
|
370
|
+
self.sleep(0.1)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""What a World Host does on its own clock: the lease sweep closes the sessions whose lease ran
|
|
2
|
+
out and sweeps the version workers (:meth:`WorldHost.reap_workers`), and the heartbeat tells the
|
|
3
|
+
platform what the host is and holds. A step that fails is logged and the next one goes on."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import threading
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from typing import TYPE_CHECKING
|
|
13
|
+
|
|
14
|
+
from .handler_errors import log_line
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from .world_host import WorldHost
|
|
18
|
+
|
|
19
|
+
LEASE_SWEEP_S = 30.0
|
|
20
|
+
HEARTBEAT_S = 10.0
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def swept(step: Callable[[], list[str]], done: str) -> list[str]:
|
|
24
|
+
"""One step of the lease sweep and what it did, each logged; a step that raises did nothing."""
|
|
25
|
+
try:
|
|
26
|
+
names = step()
|
|
27
|
+
except Exception as error: # noqa: BLE001 - one failing step must not stop the steps or the sweeps after it
|
|
28
|
+
log_line(f"sweep step failed: {type(error).__name__}")
|
|
29
|
+
return []
|
|
30
|
+
for name in names:
|
|
31
|
+
log_line(f"{done} {name}")
|
|
32
|
+
return names
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def start_lease_sweeper(world_host: WorldHost, interval_s: float = LEASE_SWEEP_S) -> threading.Event:
|
|
36
|
+
"""Expire leases and sweep the version workers every ``interval_s`` until the returned event is set."""
|
|
37
|
+
stop = threading.Event()
|
|
38
|
+
|
|
39
|
+
def sweep() -> None:
|
|
40
|
+
while not stop.wait(interval_s):
|
|
41
|
+
swept(world_host.expire_leases, "lease expired, closed session")
|
|
42
|
+
swept(world_host.reap_workers, "stopped idle worker for version")
|
|
43
|
+
|
|
44
|
+
threading.Thread(target=sweep, name="world-host-leases", daemon=True).start()
|
|
45
|
+
return stop
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeout_s: float = 5.0) -> bool:
|
|
49
|
+
"""Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
|
|
50
|
+
The platform's registry is what places new sessions: a host it has not
|
|
51
|
+
heard from lately gets none. Returns whether the platform took it."""
|
|
52
|
+
body = json.dumps(world_host.stats()).encode("utf-8")
|
|
53
|
+
req = urllib.request.Request(
|
|
54
|
+
f"{platform_url.rstrip('/')}/api/public/world-hosts",
|
|
55
|
+
data=body,
|
|
56
|
+
method="POST",
|
|
57
|
+
headers={"content-type": "application/json", "x-host-secret": secret},
|
|
58
|
+
)
|
|
59
|
+
try:
|
|
60
|
+
with urllib.request.urlopen(req, timeout=timeout_s) as r:
|
|
61
|
+
return 200 <= r.status < 300
|
|
62
|
+
except (urllib.error.URLError, OSError, ValueError) as error:
|
|
63
|
+
log_line(f"heartbeat failed: {error}")
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Heartbeat:
|
|
68
|
+
"""Tells the platform what this host is and holds every ``interval_s``: at once on
|
|
69
|
+
:meth:`start`, so a fresh replica is placeable immediately; at once on :meth:`wake`,
|
|
70
|
+
so a drain is seen as soon as it begins while the host, still alive, keeps beating
|
|
71
|
+
through its handoff; and once more on :meth:`stop`."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, world_host: WorldHost, platform_url: str, secret: str, interval_s: float = HEARTBEAT_S):
|
|
74
|
+
self.world_host = world_host
|
|
75
|
+
self.platform_url = platform_url
|
|
76
|
+
self.secret = secret
|
|
77
|
+
self.interval_s = interval_s
|
|
78
|
+
self._due = threading.Event()
|
|
79
|
+
self._stopped = threading.Event()
|
|
80
|
+
|
|
81
|
+
def start(self) -> Heartbeat:
|
|
82
|
+
threading.Thread(target=self._beat, name="world-host-heartbeat", daemon=True).start()
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
def wake(self) -> None:
|
|
86
|
+
self._due.set()
|
|
87
|
+
|
|
88
|
+
def stop(self) -> None:
|
|
89
|
+
self._stopped.set()
|
|
90
|
+
self._due.set()
|
|
91
|
+
|
|
92
|
+
def _beat(self) -> None:
|
|
93
|
+
heartbeat_once(self.world_host, self.platform_url, self.secret)
|
|
94
|
+
while not self._stopped.is_set():
|
|
95
|
+
self._due.wait(self.interval_s)
|
|
96
|
+
self._due.clear()
|
|
97
|
+
heartbeat_once(self.world_host, self.platform_url, self.secret)
|