@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,953 @@
|
|
|
1
|
+
"""The session registry and the control operations of a World Host (:class:`WorldHost`): the
|
|
2
|
+
sessions it holds, their leases, exports and moves, and the platform's session verbs
|
|
3
|
+
(:data:`HOSTED_OPERATIONS`), which :mod:`.host` serves over HTTP."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import functools
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
import threading
|
|
11
|
+
import time
|
|
12
|
+
import uuid
|
|
13
|
+
from collections import Counter
|
|
14
|
+
from collections.abc import Callable, Iterator
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Optional
|
|
18
|
+
|
|
19
|
+
from ..world_session_contract import OPEN_DOOR_FEATURE, REQUEST_LOG_PAGE_MAX
|
|
20
|
+
from .admission import RETRY_AFTER_S, Admission
|
|
21
|
+
from .cache_layout import SESSIONS_DIR, VERSIONS_DIR
|
|
22
|
+
from .door import VENDOR_MODE, session_mode
|
|
23
|
+
from .host_handoff import OUTCOMES
|
|
24
|
+
from .host_loops import swept
|
|
25
|
+
from .host_owners import PlatformOwners
|
|
26
|
+
from .host_requests import BrowserRelay, DependencyRef, parse_dependency_refs, restore_sha256
|
|
27
|
+
from .host_session import Export, Session
|
|
28
|
+
from .host_surfaces import SessionHost
|
|
29
|
+
from .host_transfers import check_object_store_url, download_verified, file_digest, upload_file
|
|
30
|
+
from .host_versions import DEFAULT_MAX_BUNDLE_MB, CachedVersion, VersionCache
|
|
31
|
+
from .host_workers import CALL_TIMEOUT_S, EXPORT_TIMEOUT_S, LOST, UNKNOWN_SESSION, VersionWorkers
|
|
32
|
+
from .journal import WorldJournal
|
|
33
|
+
from .lanes import Lanes
|
|
34
|
+
from .request_ledger import HostLedger, LedgerLife, NoLedger
|
|
35
|
+
from .runtime import WorldRuntime
|
|
36
|
+
from .session_files import (
|
|
37
|
+
RECOVERED_DIR,
|
|
38
|
+
ExportFile,
|
|
39
|
+
RecoveredSession,
|
|
40
|
+
SessionExport,
|
|
41
|
+
real_dir,
|
|
42
|
+
)
|
|
43
|
+
from .session_move import PENDING_LEASE_S, Forwarding, platform_close
|
|
44
|
+
from .tasks import TaskSpec, parse_task_spec
|
|
45
|
+
from .tools_world import DependencyWorld
|
|
46
|
+
from .validation import fail
|
|
47
|
+
from .verbs import SESSION_VERBS
|
|
48
|
+
from .workers import Finishing, Worker, WorkerPool
|
|
49
|
+
|
|
50
|
+
DEFAULT_LEASE_S = 3600.0
|
|
51
|
+
DEFAULT_MAX_SESSIONS = 1500
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class WorldHost:
|
|
55
|
+
"""The session registry and the control operations, in one of two roles:
|
|
56
|
+
|
|
57
|
+
* the **supervisor** (the deployed host): serves code-free worlds in
|
|
58
|
+
process and places worlds with handlers on its :class:`WorkerPool`;
|
|
59
|
+
* a **worker** (``run_handlers=True``, ``pinned_version=...``): serves one
|
|
60
|
+
version, handlers included, for its supervisor.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
cache_root: Path,
|
|
66
|
+
*,
|
|
67
|
+
lease_s: float = DEFAULT_LEASE_S,
|
|
68
|
+
advertise_url: str | None = None,
|
|
69
|
+
workers: WorkerPool | None = None,
|
|
70
|
+
versions_root: Path | None = None,
|
|
71
|
+
run_handlers: bool = False,
|
|
72
|
+
pinned_version: str | None = None,
|
|
73
|
+
limits: dict[str, int] | None = None,
|
|
74
|
+
max_sessions: int = DEFAULT_MAX_SESSIONS,
|
|
75
|
+
max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB,
|
|
76
|
+
object_store: frozenset[str] | None = None,
|
|
77
|
+
admission: Admission | None = None,
|
|
78
|
+
build: str | None = None,
|
|
79
|
+
owners: PlatformOwners | None = None,
|
|
80
|
+
clock: Callable[[], float] = time.time,
|
|
81
|
+
):
|
|
82
|
+
#: Every presigned URL the control plane hands this host must point here; None admits any.
|
|
83
|
+
self.object_store = object_store
|
|
84
|
+
self.build = build
|
|
85
|
+
self.owners = owners
|
|
86
|
+
self.forwarding = Forwarding(advertise_url)
|
|
87
|
+
self.clock = clock
|
|
88
|
+
self.admission = admission or Admission()
|
|
89
|
+
self.cache = VersionCache(
|
|
90
|
+
versions_root or Path(cache_root) / VERSIONS_DIR, run_handlers=run_handlers, max_bundle_mb=max_bundle_mb
|
|
91
|
+
)
|
|
92
|
+
self.sessions_root = Path(cache_root) / SESSIONS_DIR
|
|
93
|
+
# Sessions live in this process, so whatever a previous one left on disk is unreachable.
|
|
94
|
+
shutil.rmtree(self.sessions_root, ignore_errors=True)
|
|
95
|
+
self.sessions_root.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
self.lease_s = lease_s
|
|
97
|
+
self.advertise_url = advertise_url
|
|
98
|
+
self.workers = workers
|
|
99
|
+
self.placement = VersionWorkers(workers, self) if workers is not None else None
|
|
100
|
+
self.run_handlers = run_handlers
|
|
101
|
+
self.pinned_version = pinned_version
|
|
102
|
+
self.limits = limits
|
|
103
|
+
self.max_sessions = max_sessions
|
|
104
|
+
self.draining = False
|
|
105
|
+
self.ledger: HostLedger = NoLedger()
|
|
106
|
+
self.finishing = Finishing()
|
|
107
|
+
self.started_at = time.time()
|
|
108
|
+
self._lock = threading.Lock()
|
|
109
|
+
self._sessions: dict[str, Session] = {}
|
|
110
|
+
#: Each holds a slot from admission, so concurrent opens cannot all pass the capacity check.
|
|
111
|
+
self._opening: set[str] = set()
|
|
112
|
+
#: Opens the platform closed before they finished: closed the moment they exist, not at lease end.
|
|
113
|
+
self._abandoned: set[str] = set()
|
|
114
|
+
#: Sessions out of the registry and not closed yet: their files are theirs until the close is done.
|
|
115
|
+
self._draining: dict[str, Session] = {}
|
|
116
|
+
self._opens = Lanes()
|
|
117
|
+
self._counters: Counter[str] = Counter()
|
|
118
|
+
self._session_seconds = 0.0
|
|
119
|
+
|
|
120
|
+
def open(
|
|
121
|
+
self,
|
|
122
|
+
session_id: str,
|
|
123
|
+
version_id: str,
|
|
124
|
+
bundle_url: str,
|
|
125
|
+
task: str | None,
|
|
126
|
+
token: str | None,
|
|
127
|
+
snapshot: dict[str, str] | None,
|
|
128
|
+
browser: dict[str, str] | None = None,
|
|
129
|
+
restore: dict[str, str] | None = None,
|
|
130
|
+
task_spec: Any = None,
|
|
131
|
+
dependencies: list[dict[str, Any]] | None = None,
|
|
132
|
+
pending: bool = False,
|
|
133
|
+
auth: str | None = None,
|
|
134
|
+
ledger_source: str | None = None,
|
|
135
|
+
) -> dict:
|
|
136
|
+
"""Open ``session_id`` on ``version_id``, either on the bundled ``task``
|
|
137
|
+
or on ``task_spec``, a session task of its own (:mod:`tasks`); exactly
|
|
138
|
+
one of the two. ``dependencies`` are the versions the world's
|
|
139
|
+
``[dependencies]`` resolved to, one ``{alias, versionId, bundleUrl,
|
|
140
|
+
snapshot?}`` per alias the manifest declares, opened beside it. A
|
|
141
|
+
``pending`` open is a session moving here from another host: it serves
|
|
142
|
+
nothing until the platform's row names this host (:class:`SessionMove`).
|
|
143
|
+
``auth`` (``vendor`` or ``open``) overrides the world's ``[auth] mode``."""
|
|
144
|
+
if self.pinned_version is not None and version_id != self.pinned_version:
|
|
145
|
+
fail("host", f"this worker serves version {self.pinned_version} only, not {version_id}")
|
|
146
|
+
if (task is None) == (task_spec is None):
|
|
147
|
+
fail("host", "an open names a bundled task or carries a task spec, one of the two")
|
|
148
|
+
if task is not None and not isinstance(task, str):
|
|
149
|
+
fail("host", "task is the bundled task's name")
|
|
150
|
+
spec = parse_task_spec(task_spec) if task_spec is not None else None
|
|
151
|
+
deps = parse_dependency_refs(dependencies)
|
|
152
|
+
life = LedgerLife.of(ledger_source)
|
|
153
|
+
self._replace_pending(session_id, restore, browser, pending)
|
|
154
|
+
with self._lock:
|
|
155
|
+
existing = self._sessions.get(session_id)
|
|
156
|
+
if existing is None:
|
|
157
|
+
# A retry of an open already served is answered even while draining: refusing it would
|
|
158
|
+
# send the platform to open the same id elsewhere while this copy holds the live writes.
|
|
159
|
+
if self.draining:
|
|
160
|
+
fail("draining", "this host is draining and takes no new sessions")
|
|
161
|
+
self._refuse_closing(session_id)
|
|
162
|
+
self._admit(session_id)
|
|
163
|
+
if existing is not None:
|
|
164
|
+
wanted = (version_id, (task, spec.raw if spec is not None else None), token)
|
|
165
|
+
return self._reopened(existing, wanted, restore, pending, auth)
|
|
166
|
+
try:
|
|
167
|
+
session = self._open(
|
|
168
|
+
session_id, version_id, bundle_url, task, spec, token, snapshot, browser, restore, deps, pending, auth
|
|
169
|
+
)
|
|
170
|
+
session.life = life
|
|
171
|
+
self._register(session)
|
|
172
|
+
described = self._describe(session)
|
|
173
|
+
except Exception:
|
|
174
|
+
self.tally("open_failures")
|
|
175
|
+
with self._lock:
|
|
176
|
+
self._opening.discard(session_id)
|
|
177
|
+
self._abandoned.discard(session_id)
|
|
178
|
+
raise
|
|
179
|
+
with self._lock:
|
|
180
|
+
self._opening.discard(session_id)
|
|
181
|
+
abandoned = session_id in self._abandoned
|
|
182
|
+
self._abandoned.discard(session_id)
|
|
183
|
+
if abandoned:
|
|
184
|
+
self.close(session_id)
|
|
185
|
+
fail(UNKNOWN_SESSION, f"session {session_id} was closed while it opened")
|
|
186
|
+
return described
|
|
187
|
+
|
|
188
|
+
def _replace_pending(self, session_id: str, restore: dict[str, str] | None, browser: Any, pending: bool) -> None:
|
|
189
|
+
"""A draining host takes no move, refused before anything here changes. A copy pending a move here that
|
|
190
|
+
a later open reuses gets a fresh lease; one from another export or browser is closed first."""
|
|
191
|
+
if pending and self.draining:
|
|
192
|
+
fail("draining", f"this host is draining and takes no session moving here; place {session_id} elsewhere")
|
|
193
|
+
existing = self.held(session_id)
|
|
194
|
+
if existing is None or not existing.move.pending:
|
|
195
|
+
return
|
|
196
|
+
existing.lease_until = max(existing.lease_until, time.time() + PENDING_LEASE_S)
|
|
197
|
+
if (existing.restore_sha256, existing.browser) != (restore_sha256(restore), BrowserRelay.parse(browser)):
|
|
198
|
+
self.close(session_id, reason="replaced")
|
|
199
|
+
|
|
200
|
+
def _reopened(
|
|
201
|
+
self, existing: Session, wanted: tuple, restore: dict[str, str] | None, pending: bool, auth: str | None
|
|
202
|
+
) -> dict:
|
|
203
|
+
"""A retried open gets the same session, never a rebuild that drops its writes. One moving off
|
|
204
|
+
this host is another host's to open, and one opened from another export or auth is refused."""
|
|
205
|
+
if existing.move.leaving:
|
|
206
|
+
fail("draining", f"session {existing.session_id} is moving to another host; place it elsewhere")
|
|
207
|
+
if (existing.version_id, existing.task_key, existing.token) != wanted or auth not in (None, existing.auth):
|
|
208
|
+
fail("host", f"session {existing.session_id} is already open as another version, task, token or auth")
|
|
209
|
+
if restore and restore_sha256(restore) != existing.restore_sha256:
|
|
210
|
+
fail("host", f"session {existing.session_id} is already open from another export")
|
|
211
|
+
if not pending:
|
|
212
|
+
existing.move.commit()
|
|
213
|
+
return self._describe(existing)
|
|
214
|
+
|
|
215
|
+
def _refuse_closing(self, session_id: str) -> None:
|
|
216
|
+
"""Under ``_lock``: an id whose earlier open still has a request inside keeps its files until it leaves."""
|
|
217
|
+
if session_id in self._draining:
|
|
218
|
+
fail(
|
|
219
|
+
"busy",
|
|
220
|
+
f"session {session_id} is still closing; retry after {RETRY_AFTER_S} s",
|
|
221
|
+
retryAfter=RETRY_AFTER_S,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def _admit(self, session_id: str) -> None:
|
|
225
|
+
"""Reserve one of this host's ``max_sessions`` for ``session_id`` while it
|
|
226
|
+
opens. Under ``_lock``."""
|
|
227
|
+
if session_id in self._opening:
|
|
228
|
+
fail("host", f"session {session_id} is being opened already")
|
|
229
|
+
if len(self._sessions) + len(self._opening) >= self.max_sessions:
|
|
230
|
+
fail("at_capacity", f"this host holds its {self.max_sessions} sessions; place elsewhere")
|
|
231
|
+
self._opening.add(session_id)
|
|
232
|
+
|
|
233
|
+
def _open(
|
|
234
|
+
self,
|
|
235
|
+
session_id: str,
|
|
236
|
+
version_id: str,
|
|
237
|
+
bundle_url: str,
|
|
238
|
+
task: str | None,
|
|
239
|
+
spec: TaskSpec | None,
|
|
240
|
+
token: str | None,
|
|
241
|
+
snapshot: dict[str, str] | None,
|
|
242
|
+
browser: dict[str, str] | None,
|
|
243
|
+
restore: dict[str, str] | None,
|
|
244
|
+
deps: list[DependencyRef],
|
|
245
|
+
pending: bool,
|
|
246
|
+
auth: str | None = None,
|
|
247
|
+
) -> Session:
|
|
248
|
+
cached = self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
|
|
249
|
+
mode = session_mode(auth, token)
|
|
250
|
+
dependencies = self._dependency_worlds(cached, deps)
|
|
251
|
+
if spec is not None:
|
|
252
|
+
cached.served_names(dependencies).check_allowed(spec.tools)
|
|
253
|
+
relay = BrowserRelay.parse(browser)
|
|
254
|
+
if cached.needs_worker and not self.run_handlers:
|
|
255
|
+
return self._open_on_worker(
|
|
256
|
+
cached, session_id, bundle_url, task, spec, token, relay, restore, deps, pending, mode
|
|
257
|
+
)
|
|
258
|
+
session_dir = self.sessions_root / session_id
|
|
259
|
+
with self._opens.turn(version_id):
|
|
260
|
+
runtime = WorldRuntime.open(
|
|
261
|
+
cached,
|
|
262
|
+
session_dir,
|
|
263
|
+
task=task,
|
|
264
|
+
spec=spec,
|
|
265
|
+
token=token,
|
|
266
|
+
dependencies=dependencies,
|
|
267
|
+
restore=lambda load: self._restore_into(restore, session_dir, load, cached),
|
|
268
|
+
journaled=self.run_handlers,
|
|
269
|
+
)
|
|
270
|
+
runtime.calls.outbox.on_pressure = lambda: self.ledger.wake(session_id)
|
|
271
|
+
session = Session(
|
|
272
|
+
session_id=session_id,
|
|
273
|
+
version_id=version_id,
|
|
274
|
+
task=task,
|
|
275
|
+
spec=cached.spec,
|
|
276
|
+
token=token,
|
|
277
|
+
bundle=cached.bundle,
|
|
278
|
+
tree=cached.tree,
|
|
279
|
+
lease_until=time.time() + (PENDING_LEASE_S if pending else self.lease_s),
|
|
280
|
+
engine=cached.engine,
|
|
281
|
+
runtime=runtime,
|
|
282
|
+
door=cached.door(token, self.clock),
|
|
283
|
+
ui=cached.ui,
|
|
284
|
+
browser=relay,
|
|
285
|
+
restored=runtime.restored,
|
|
286
|
+
task_spec=spec,
|
|
287
|
+
settings=cached.settings,
|
|
288
|
+
restore_sha256=restore_sha256(restore),
|
|
289
|
+
pending=pending,
|
|
290
|
+
auth=mode,
|
|
291
|
+
)
|
|
292
|
+
return session
|
|
293
|
+
|
|
294
|
+
def _open_on_worker(
|
|
295
|
+
self,
|
|
296
|
+
cached: CachedVersion,
|
|
297
|
+
session_id: str,
|
|
298
|
+
bundle_url: str,
|
|
299
|
+
task: str | None,
|
|
300
|
+
spec: TaskSpec | None,
|
|
301
|
+
token: str | None,
|
|
302
|
+
relay: BrowserRelay | None,
|
|
303
|
+
restore: dict[str, str] | None,
|
|
304
|
+
deps: list[DependencyRef],
|
|
305
|
+
pending: bool,
|
|
306
|
+
mode: str = VENDOR_MODE,
|
|
307
|
+
) -> Session:
|
|
308
|
+
"""Place the session on the version's worker (:class:`VersionWorkers`), which opens it from the bundle
|
|
309
|
+
this process cached; the supervisor keeps the ui's files and the browser tunnel, relays the ui's api
|
|
310
|
+
calls as they came (``via: ui``), and hands an export it fetched and verified to the worker as a local
|
|
311
|
+
file with its sha256, checked again there. No worker past the pool's memory budget: ``at_capacity``."""
|
|
312
|
+
if self.placement is None:
|
|
313
|
+
fail(
|
|
314
|
+
"host",
|
|
315
|
+
f"version {cached.version_id} runs world-authored code ({cached.authored_code}); "
|
|
316
|
+
"this host runs none in its process and has no version workers",
|
|
317
|
+
)
|
|
318
|
+
body = {
|
|
319
|
+
"sessionId": session_id,
|
|
320
|
+
"versionId": cached.version_id,
|
|
321
|
+
"bundleUrl": bundle_url,
|
|
322
|
+
"task": task,
|
|
323
|
+
"taskSpec": spec.raw if spec is not None else None,
|
|
324
|
+
"token": token,
|
|
325
|
+
"dependencies": [dep.raw for dep in deps],
|
|
326
|
+
"auth": mode,
|
|
327
|
+
}
|
|
328
|
+
fetch = functools.partial(self._restore_source, restore)
|
|
329
|
+
worker = self.placement.place(cached.version_id, session_id, body, fetch, self._opens)
|
|
330
|
+
session = Session(
|
|
331
|
+
session_id=session_id,
|
|
332
|
+
version_id=cached.version_id,
|
|
333
|
+
task=task,
|
|
334
|
+
spec=cached.spec,
|
|
335
|
+
token=token,
|
|
336
|
+
bundle=cached.bundle,
|
|
337
|
+
tree=cached.tree,
|
|
338
|
+
lease_until=time.time() + (PENDING_LEASE_S if pending else self.lease_s),
|
|
339
|
+
engine=cached.engine,
|
|
340
|
+
worker=worker,
|
|
341
|
+
door=cached.door(token, self.clock),
|
|
342
|
+
ui=cached.ui,
|
|
343
|
+
browser=relay,
|
|
344
|
+
restored=bool(restore),
|
|
345
|
+
task_spec=spec,
|
|
346
|
+
placement=body,
|
|
347
|
+
settings=cached.settings,
|
|
348
|
+
restore_sha256=restore_sha256(restore),
|
|
349
|
+
pending=pending,
|
|
350
|
+
auth=mode,
|
|
351
|
+
)
|
|
352
|
+
return session
|
|
353
|
+
|
|
354
|
+
def _restore_into(
|
|
355
|
+
self,
|
|
356
|
+
restore: dict[str, str] | None,
|
|
357
|
+
session_dir: Path,
|
|
358
|
+
load: Callable[[SessionExport], None],
|
|
359
|
+
cached: CachedVersion,
|
|
360
|
+
) -> bool:
|
|
361
|
+
"""Load the checkpoint ``restore`` names into a freshly opened world (an export's
|
|
362
|
+
bytes checked against the recorded sha256 here, where they are opened), then drop
|
|
363
|
+
what was fetched or set aside. False when there was nothing to restore."""
|
|
364
|
+
source = self._restore_source(restore, session_dir)
|
|
365
|
+
if source is None:
|
|
366
|
+
return False
|
|
367
|
+
try:
|
|
368
|
+
load(source.checkpoint(cached.engine, cached.runtime_class))
|
|
369
|
+
finally:
|
|
370
|
+
source.discard()
|
|
371
|
+
return True
|
|
372
|
+
|
|
373
|
+
def _restore_source(
|
|
374
|
+
self, restore: dict[str, str] | None, session_dir: Path
|
|
375
|
+
) -> ExportFile | RecoveredSession | None:
|
|
376
|
+
"""The checkpoint a session continues from, or None for a fresh session. The
|
|
377
|
+
platform names an export by ``{url, sha256}`` and the file is fetched into the
|
|
378
|
+
session's directory and verified; a supervisor names one to its worker by
|
|
379
|
+
``{path, sha256}``, or a dead worker's session directory it set aside by
|
|
380
|
+
``{recovered}``, and only the supervisor reaches a worker's control socket."""
|
|
381
|
+
if not restore:
|
|
382
|
+
return None
|
|
383
|
+
if restore.get("recovered") is not None:
|
|
384
|
+
return self._recovered_source(Path(str(restore["recovered"])))
|
|
385
|
+
path = restore.get("path")
|
|
386
|
+
if path is not None:
|
|
387
|
+
return self._worker_restore_source(Path(path), restore.get("sha256"))
|
|
388
|
+
url, sha256 = restore.get("url"), restore.get("sha256")
|
|
389
|
+
if not isinstance(url, str) or not isinstance(sha256, str) or len(sha256) != 64:
|
|
390
|
+
fail("host", "restore must be {url: <presigned export url>, sha256: <hex digest>}")
|
|
391
|
+
check_object_store_url(url, self.object_store, "restore.url")
|
|
392
|
+
return ExportFile(download_verified(url, sha256, session_dir / "restore.db"), sha256)
|
|
393
|
+
|
|
394
|
+
def _recovered_source(self, directory: Path) -> RecoveredSession:
|
|
395
|
+
root = self.sessions_root.parent / RECOVERED_DIR
|
|
396
|
+
inside = directory.parent == root and directory.name not in ("", ".", "..")
|
|
397
|
+
if not (self.run_handlers and inside and real_dir(root) and real_dir(directory)):
|
|
398
|
+
fail("host", "a recovered session is a worker's, from its own recovered directory")
|
|
399
|
+
return RecoveredSession(directory)
|
|
400
|
+
|
|
401
|
+
def _worker_restore_source(self, path: Path, sha256: object) -> ExportFile:
|
|
402
|
+
if not self.run_handlers:
|
|
403
|
+
fail("host", "restore by path is a worker's; name the export by {url, sha256}")
|
|
404
|
+
if not path.is_file():
|
|
405
|
+
fail("host", f"restore file {path} is missing")
|
|
406
|
+
if not isinstance(sha256, str) or len(sha256) != 64:
|
|
407
|
+
fail("host", "restore by path names the export's sha256 beside it")
|
|
408
|
+
return ExportFile(path, sha256)
|
|
409
|
+
|
|
410
|
+
def _trusted_version(self, bundle_url: str, snapshot: dict[str, str] | None) -> tuple[str, dict[str, str] | None]:
|
|
411
|
+
check_object_store_url(bundle_url, self.object_store, "bundleUrl")
|
|
412
|
+
if snapshot:
|
|
413
|
+
check_object_store_url(snapshot.get("url"), self.object_store, "snapshot.url")
|
|
414
|
+
return bundle_url, snapshot
|
|
415
|
+
|
|
416
|
+
def _register(self, session: Session) -> None:
|
|
417
|
+
with self._lock:
|
|
418
|
+
replaced = self._retire(self._sessions.pop(session.session_id, None), "replaced")
|
|
419
|
+
self._sessions[session.session_id] = session
|
|
420
|
+
self._counters["opened"] += 1
|
|
421
|
+
if replaced is not None:
|
|
422
|
+
self._finish(replaced)
|
|
423
|
+
|
|
424
|
+
def _describe(self, session: Session) -> dict:
|
|
425
|
+
return {
|
|
426
|
+
"sessionId": session.session_id,
|
|
427
|
+
"versionId": session.version_id,
|
|
428
|
+
"task": session.task,
|
|
429
|
+
"taskName": session.task if session.task_spec is None else session.task_spec.name,
|
|
430
|
+
"taskSource": "bundled" if session.task_spec is None else "inline",
|
|
431
|
+
"tools": (
|
|
432
|
+
list(session.task_spec.tools)
|
|
433
|
+
if session.task_spec is not None and session.task_spec.tools is not None
|
|
434
|
+
else None
|
|
435
|
+
),
|
|
436
|
+
"ready": True,
|
|
437
|
+
"hostUrl": self.advertise_url,
|
|
438
|
+
"leaseSeconds": self.lease_s,
|
|
439
|
+
"leaseUntil": session.lease_until,
|
|
440
|
+
"workerPid": session.worker.pid if session.worker is not None else None,
|
|
441
|
+
"ui": {"entry": session.ui.entry, "apiPath": session.ui.api_path} if session.ui is not None else None,
|
|
442
|
+
"browser": session.browser is not None,
|
|
443
|
+
"restored": session.restored,
|
|
444
|
+
"openedAt": session.opened_at,
|
|
445
|
+
"sessionSeconds": session.seconds,
|
|
446
|
+
"touchedAt": session.touched_at,
|
|
447
|
+
"ledgerSource": session.life.source,
|
|
448
|
+
"auth": session.auth,
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
def close(self, session_id: str, reason: str = "closed") -> dict:
|
|
452
|
+
with self._lock:
|
|
453
|
+
session = self._sessions.pop(session_id, None)
|
|
454
|
+
idle = self._retire(session, reason)
|
|
455
|
+
if session is None and session_id in self._opening:
|
|
456
|
+
self._abandoned.add(session_id)
|
|
457
|
+
if idle is not None:
|
|
458
|
+
self._finish(idle)
|
|
459
|
+
if session is None:
|
|
460
|
+
return {"closed": True, "openedAt": None, "sessionSeconds": 0.0}
|
|
461
|
+
return {"closed": True, "openedAt": session.opened_at, "sessionSeconds": session.seconds}
|
|
462
|
+
|
|
463
|
+
def set_browser(self, session_id: str, browser: Any) -> dict:
|
|
464
|
+
"""Point the session's browser surface at another pool relay: the one
|
|
465
|
+
it had died and the platform leased a replacement. Connections open
|
|
466
|
+
on the old relay end with it; every connection after this reaches the
|
|
467
|
+
new one. A session opened without a browser never gains one here."""
|
|
468
|
+
relay = BrowserRelay.parse(browser)
|
|
469
|
+
if relay is None:
|
|
470
|
+
fail("host", "browser must be {url: 'ws://host:port', token: <relay token>}")
|
|
471
|
+
with self._lock:
|
|
472
|
+
session = self._sessions.get(session_id)
|
|
473
|
+
if session is None:
|
|
474
|
+
fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
|
|
475
|
+
if session.browser is None:
|
|
476
|
+
fail("host", f"session {session_id} was opened without a browser surface")
|
|
477
|
+
session.browser = relay
|
|
478
|
+
return {"sessionId": session_id, "browser": True}
|
|
479
|
+
|
|
480
|
+
def export(self, session_id: str, known_sha256: str | None = None, handoff: bool = False) -> dict:
|
|
481
|
+
"""Copy the session's state to a file on this host and hash it, one export of
|
|
482
|
+
a session at a time; it waits for :meth:`upload_export`, and a new one discards
|
|
483
|
+
the last. A copy hashing to ``known_sha256`` (the platform's last export of the
|
|
484
|
+
session) is thrown away and reported ``unchanged``: nothing to upload. In a
|
|
485
|
+
version worker it names its ``path``, for the supervisor that uploads it. A
|
|
486
|
+
``handoff`` export is the final one of a session this host is moving, and is
|
|
487
|
+
refused for any other: only the host that holds a session hands it off."""
|
|
488
|
+
with self.using(session_id, touch=False, confirm=True) as session, session.exporting:
|
|
489
|
+
session.move.check_export(handoff)
|
|
490
|
+
engine = session.engine
|
|
491
|
+
if session.worker is not None:
|
|
492
|
+
answer = self.placement.control(session, "POST", "export", {}, timeout=EXPORT_TIMEOUT_S)
|
|
493
|
+
path = self.placement.export_path(session, answer)
|
|
494
|
+
else:
|
|
495
|
+
exports = self.sessions_root / session_id / "exports"
|
|
496
|
+
shutil.rmtree(exports, ignore_errors=True)
|
|
497
|
+
path = exports / f"{uuid.uuid4().hex}{Path(engine.snapshot_name).suffix}"
|
|
498
|
+
assert session.runtime is not None
|
|
499
|
+
session.runtime.export(path)
|
|
500
|
+
sha256, size = file_digest(path)
|
|
501
|
+
if known_sha256 is not None and sha256 == known_sha256:
|
|
502
|
+
path.unlink(missing_ok=True)
|
|
503
|
+
with self._lock:
|
|
504
|
+
session.export = None
|
|
505
|
+
self._counters["exports_unchanged"] += 1
|
|
506
|
+
return {"exportId": None, "sha256": sha256, "bytes": size, "unchanged": True}
|
|
507
|
+
export = Export(export_id=path.stem, path=path, sha256=sha256, bytes=size, content_type=engine.content_type)
|
|
508
|
+
with self._lock:
|
|
509
|
+
session.export = export
|
|
510
|
+
self._counters["exports"] += 1
|
|
511
|
+
described = export.describe()
|
|
512
|
+
if self.run_handlers:
|
|
513
|
+
described["path"] = str(path)
|
|
514
|
+
return described
|
|
515
|
+
|
|
516
|
+
def upload_export(self, session_id: str, export_id: str, url: str) -> dict:
|
|
517
|
+
"""PUT the export :meth:`export` took to the presigned URL the platform
|
|
518
|
+
signed for its hash and size; the local copy goes once it is up."""
|
|
519
|
+
check_object_store_url(url, self.object_store, "url")
|
|
520
|
+
with self.using(session_id, touch=False, confirm=True) as session:
|
|
521
|
+
export = session.export
|
|
522
|
+
if export is None or export.export_id != export_id:
|
|
523
|
+
fail("host", f"no export {export_id} for session {session_id}; take one first")
|
|
524
|
+
upload_file(url, export.path, export.sha256, export.bytes, export.content_type)
|
|
525
|
+
export.path.unlink(missing_ok=True)
|
|
526
|
+
with self._lock:
|
|
527
|
+
session.export = None
|
|
528
|
+
self._counters["exports_uploaded"] += 1
|
|
529
|
+
return {"uploaded": True, **export.describe()}
|
|
530
|
+
|
|
531
|
+
def renew(self, session_id: str) -> dict:
|
|
532
|
+
with self.using(session_id, touch=False, confirm=True) as session:
|
|
533
|
+
return self._describe(session)
|
|
534
|
+
|
|
535
|
+
def expire_leases(self, now: float | None = None) -> list[str]:
|
|
536
|
+
"""Close the sessions whose lease has run out and return their ids, under one lock so a renewal that
|
|
537
|
+
lands in between keeps its session; one on a worker is finished in the background (:class:`Finishing`)."""
|
|
538
|
+
now = time.time() if now is None else now
|
|
539
|
+
with self._lock:
|
|
540
|
+
expired = [s for s in self._sessions.values() if s.lease_until <= now and s.inflight == 0]
|
|
541
|
+
for session in expired:
|
|
542
|
+
self._retire(self._sessions.pop(session.session_id), "expired")
|
|
543
|
+
for session in expired:
|
|
544
|
+
self._finish_later(session)
|
|
545
|
+
return [s.session_id for s in expired]
|
|
546
|
+
|
|
547
|
+
def _retire(self, session: Session | None, reason: str = "closed") -> Session | None:
|
|
548
|
+
"""Mark a session that has left the registry, for ``reason``, its id held from a new open
|
|
549
|
+
until :meth:`_finish` has closed it; returns it when nobody is inside it, so the caller
|
|
550
|
+
closes the state now. Otherwise the last request out does (see :meth:`_release`).
|
|
551
|
+
Call under the lock."""
|
|
552
|
+
if session is None:
|
|
553
|
+
return None
|
|
554
|
+
session.closing = True
|
|
555
|
+
session.close_reason = reason
|
|
556
|
+
self._draining[session.session_id] = session
|
|
557
|
+
return None if session.inflight else session
|
|
558
|
+
|
|
559
|
+
def _finish(self, session: Session) -> None:
|
|
560
|
+
"""Free a session that has left the registry and has nobody inside it, once: close its
|
|
561
|
+
state here, or tell its worker to; its id takes a new open once that is done. One a
|
|
562
|
+
recovery holds is left to it (:meth:`given_back`), since it may still act on that id."""
|
|
563
|
+
if not self._claimed(session):
|
|
564
|
+
return
|
|
565
|
+
try:
|
|
566
|
+
self.ledger.leave(session)
|
|
567
|
+
self._close_state(session)
|
|
568
|
+
finally:
|
|
569
|
+
with self._lock:
|
|
570
|
+
if self._draining.get(session.session_id) is session:
|
|
571
|
+
del self._draining[session.session_id]
|
|
572
|
+
|
|
573
|
+
def _claimed(self, session: Session) -> bool:
|
|
574
|
+
with self._lock:
|
|
575
|
+
if session.finish_claimed or session.recoveries:
|
|
576
|
+
return False
|
|
577
|
+
session.finish_claimed = True
|
|
578
|
+
self._count_closed(session)
|
|
579
|
+
return True
|
|
580
|
+
|
|
581
|
+
def _count_closed(self, session: Session) -> None:
|
|
582
|
+
"""Under ``_lock``."""
|
|
583
|
+
self._counters[f"closed.{session.close_reason}"] += 1
|
|
584
|
+
self._session_seconds += session.seconds
|
|
585
|
+
|
|
586
|
+
def _finish_later(self, session: Session) -> None:
|
|
587
|
+
""":meth:`_finish` handed to :class:`Finishing`: off the caller's thread, behind its version's other
|
|
588
|
+
finishes, for a session on a live worker, whose tenant code answers the finish's drain and DELETE at
|
|
589
|
+
its own pace; at once for one served in this process or whose worker is gone, which no tenant code
|
|
590
|
+
paces (a dead worker's journal is read up to ``LEFT_JOURNAL_BYTES``)."""
|
|
591
|
+
self.finishing.leave(session.live_worker, functools.partial(self._finish, session))
|
|
592
|
+
|
|
593
|
+
def _close_state(self, session: Session) -> None:
|
|
594
|
+
if session.worker is None:
|
|
595
|
+
assert session.runtime is not None
|
|
596
|
+
session.runtime.close()
|
|
597
|
+
return
|
|
598
|
+
assert self.placement is not None
|
|
599
|
+
self.placement.finish(session)
|
|
600
|
+
|
|
601
|
+
def recovering(self, worker: Worker) -> tuple[list[Session], list[Session]]:
|
|
602
|
+
"""What a recovery of ``worker`` takes until it gives each back (:meth:`given_back`): the sessions
|
|
603
|
+
it serves, and those closing on it that no finish has claimed. A session may be held by a second
|
|
604
|
+
recovery as well, of the worker that replaced this one; no other finish runs it while any holds it."""
|
|
605
|
+
with self._lock:
|
|
606
|
+
serving = [s for s in self._sessions.values() if s.worker is worker and not s.closing]
|
|
607
|
+
closing = [s for s in self._draining.values() if s.worker is worker and not s.finish_claimed]
|
|
608
|
+
for session in (*serving, *closing):
|
|
609
|
+
session.recoveries += 1
|
|
610
|
+
return serving, closing
|
|
611
|
+
|
|
612
|
+
def given_back(self, session: Session) -> None:
|
|
613
|
+
"""A recovery is done with ``session``: one that closed meanwhile and that nobody is inside is
|
|
614
|
+
finished, unless another recovery still holds it (:meth:`_claimed`), off the recovery's thread when
|
|
615
|
+
its worker runs tenant code (:meth:`_finish_later`), so no sweep waits on it; its id takes a new open
|
|
616
|
+
only after the last action on it."""
|
|
617
|
+
with self._lock:
|
|
618
|
+
session.recoveries -= 1
|
|
619
|
+
left = self._draining.get(session.session_id) is session and session.inflight == 0
|
|
620
|
+
if left:
|
|
621
|
+
self._finish_later(session)
|
|
622
|
+
|
|
623
|
+
def worker_died(self, session: Session, entries: list[dict]) -> None:
|
|
624
|
+
self.ledger.adopt(session.session_id, entries)
|
|
625
|
+
|
|
626
|
+
def forget_lost(self, session: Session) -> None:
|
|
627
|
+
"""A session its dead worker took with it: gone from this host (the platform learns of it
|
|
628
|
+
on the next renew and reopens it from its last export). One a request is still inside keeps
|
|
629
|
+
its id from a new open until the last one out has finished it, since that finish acts on the
|
|
630
|
+
session's id: its spill account and its directory on the worker; that finish counts its close."""
|
|
631
|
+
with self._lock:
|
|
632
|
+
if self._sessions.get(session.session_id) is not session:
|
|
633
|
+
return
|
|
634
|
+
del self._sessions[session.session_id]
|
|
635
|
+
session.closing = True
|
|
636
|
+
session.close_reason = LOST
|
|
637
|
+
if session.inflight:
|
|
638
|
+
self._draining[session.session_id] = session
|
|
639
|
+
return
|
|
640
|
+
self._count_closed(session)
|
|
641
|
+
|
|
642
|
+
def reap_workers(self) -> list[str]:
|
|
643
|
+
"""Recover version workers that died holding sessions, recycle those holding a turn past
|
|
644
|
+
``STUCK_TURN_S``, and stop those idle for a while, a step that fails never stopping the
|
|
645
|
+
next; returns the versions whose idle workers were stopped."""
|
|
646
|
+
if self.workers is None or self.placement is None:
|
|
647
|
+
return []
|
|
648
|
+
swept(self.placement.recover_dead, "recovered the dead worker of version")
|
|
649
|
+
swept(self.placement.recover_stuck, "recycled the stuck worker of version")
|
|
650
|
+
return self.workers.reap_idle()
|
|
651
|
+
|
|
652
|
+
def session_count(self) -> int:
|
|
653
|
+
with self._lock:
|
|
654
|
+
return len(self._sessions)
|
|
655
|
+
|
|
656
|
+
def held(self, session_id: str) -> Session | None:
|
|
657
|
+
with self._lock:
|
|
658
|
+
return self._sessions.get(session_id)
|
|
659
|
+
|
|
660
|
+
def turn_held(self, session: Session) -> bool:
|
|
661
|
+
"""Whether a call of the session still holds its turn on its worker (one past its time)."""
|
|
662
|
+
return self.placement is not None and session.worker is not None and self.placement.holds_turn(session)
|
|
663
|
+
|
|
664
|
+
def worker_count(self) -> int:
|
|
665
|
+
return len(self.workers._workers) if self.workers is not None else 0
|
|
666
|
+
|
|
667
|
+
def warm(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> dict:
|
|
668
|
+
"""Cache a version before any session asks for it, so the first open
|
|
669
|
+
pays nothing. Idempotent: a version already here is reported, not
|
|
670
|
+
rebuilt."""
|
|
671
|
+
already = self.cache.has(version_id)
|
|
672
|
+
if not already:
|
|
673
|
+
self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
|
|
674
|
+
return {"versionId": version_id, "cached": True, "alreadyCached": already}
|
|
675
|
+
|
|
676
|
+
def drain(self) -> None:
|
|
677
|
+
"""Stop taking sessions; the ones here are served until they close."""
|
|
678
|
+
self.draining = True
|
|
679
|
+
|
|
680
|
+
def tally(self, key: str) -> None:
|
|
681
|
+
with self._lock:
|
|
682
|
+
self._counters[key] += 1
|
|
683
|
+
|
|
684
|
+
def stats(self) -> dict:
|
|
685
|
+
"""What this host is and holds, for the heartbeat and ``/__host/stats``:
|
|
686
|
+
the placement inputs (address, load, capacity, draining, which versions
|
|
687
|
+
are already cached) plus the lifetime counters."""
|
|
688
|
+
with self._lock:
|
|
689
|
+
sessions = len(self._sessions)
|
|
690
|
+
versions_in_use = sorted({s.version_id for s in self._sessions.values()})
|
|
691
|
+
counters = dict(self._counters)
|
|
692
|
+
session_seconds = self._session_seconds
|
|
693
|
+
return {
|
|
694
|
+
"hostUrl": self.advertise_url,
|
|
695
|
+
"sessions": sessions,
|
|
696
|
+
"capacity": self.max_sessions,
|
|
697
|
+
"draining": self.draining,
|
|
698
|
+
"workers": self.worker_count(),
|
|
699
|
+
"maxWorkers": self.workers.max_workers if self.workers is not None else 0,
|
|
700
|
+
"versions": self.cache.version_ids(),
|
|
701
|
+
"versionsInUse": versions_in_use,
|
|
702
|
+
"uptimeSeconds": time.time() - self.started_at,
|
|
703
|
+
"startedAt": self.started_at,
|
|
704
|
+
"build": self.build,
|
|
705
|
+
"counters": counters,
|
|
706
|
+
"sessionSeconds": session_seconds,
|
|
707
|
+
"pid": os.getpid(),
|
|
708
|
+
"limits": self.limits,
|
|
709
|
+
"features": [OPEN_DOOR_FEATURE],
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
def metrics_text(self) -> str:
|
|
713
|
+
"""The prometheus exposition of :meth:`stats`. ``worlds_host_sessions``
|
|
714
|
+
is the number the fleet scales on (sessions per host, not cpu)."""
|
|
715
|
+
stats = self.stats()
|
|
716
|
+
counters: dict[str, int] = stats["counters"]
|
|
717
|
+
lines = [
|
|
718
|
+
"# TYPE worlds_host_sessions gauge",
|
|
719
|
+
f"worlds_host_sessions {stats['sessions']}",
|
|
720
|
+
"# TYPE worlds_host_capacity gauge",
|
|
721
|
+
f"worlds_host_capacity {stats['capacity']}",
|
|
722
|
+
"# TYPE worlds_host_draining gauge",
|
|
723
|
+
f"worlds_host_draining {int(stats['draining'])}",
|
|
724
|
+
"# TYPE worlds_host_workers gauge",
|
|
725
|
+
f"worlds_host_workers {stats['workers']}",
|
|
726
|
+
"# TYPE worlds_host_versions_cached gauge",
|
|
727
|
+
f"worlds_host_versions_cached {len(stats['versions'])}",
|
|
728
|
+
"# TYPE worlds_host_uptime_seconds gauge",
|
|
729
|
+
f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
|
|
730
|
+
"# TYPE worlds_host_sessions_opened_total counter",
|
|
731
|
+
f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
|
|
732
|
+
"# TYPE worlds_host_open_failures_total counter",
|
|
733
|
+
f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
|
|
734
|
+
"# TYPE worlds_host_session_seconds_total counter",
|
|
735
|
+
f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
|
|
736
|
+
"# TYPE worlds_host_exports_total counter",
|
|
737
|
+
f"worlds_host_exports_total {counters.get('exports', 0)}",
|
|
738
|
+
"# TYPE worlds_host_exports_uploaded_total counter",
|
|
739
|
+
f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
|
|
740
|
+
"# TYPE worlds_host_sessions_closed_total counter",
|
|
741
|
+
]
|
|
742
|
+
for reason in ("closed", "expired", "lost", "replaced", "moved"):
|
|
743
|
+
lines.append(
|
|
744
|
+
f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}'
|
|
745
|
+
)
|
|
746
|
+
lines.append("# TYPE worlds_host_handoffs_total counter")
|
|
747
|
+
for outcome in OUTCOMES:
|
|
748
|
+
lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
|
|
749
|
+
for name in ("workers_replaced", "workers_recycled", "sessions_recovered", "sessions_unrecovered"):
|
|
750
|
+
lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
|
|
751
|
+
lines.append("# TYPE worlds_host_requests_total counter")
|
|
752
|
+
for surface in ("api", "ui", "browser", "proxied"):
|
|
753
|
+
lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
|
|
754
|
+
lines += [
|
|
755
|
+
"# TYPE worlds_host_platform_bearer_requests_total counter",
|
|
756
|
+
f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
|
|
757
|
+
]
|
|
758
|
+
return "\n".join(lines) + "\n"
|
|
759
|
+
|
|
760
|
+
def _dependency_worlds(self, cached: CachedVersion, deps: list[DependencyRef]) -> dict[str, DependencyWorld]:
|
|
761
|
+
"""The dependency versions a session opens beside ``cached``, cached the
|
|
762
|
+
way the root is (a worker imports their handlers too). Every alias the
|
|
763
|
+
root's manifest declares must be resolved, and nothing else: a version
|
|
764
|
+
without ``[dependencies]`` opened with any is refused, as is a
|
|
765
|
+
dependency that is not a schema world."""
|
|
766
|
+
declared = cached.declared_dependencies
|
|
767
|
+
if not deps:
|
|
768
|
+
if declared:
|
|
769
|
+
names = ", ".join(sorted(declared))
|
|
770
|
+
fail("host", f"version {cached.version_id} declares dependencies ({names}); the open resolved none")
|
|
771
|
+
return {}
|
|
772
|
+
worlds: dict[str, DependencyWorld] = {}
|
|
773
|
+
for ref in deps:
|
|
774
|
+
dependency = declared.get(ref.alias)
|
|
775
|
+
if dependency is None:
|
|
776
|
+
known = ", ".join(sorted(declared)) or "none"
|
|
777
|
+
fail("host", f"version {cached.version_id} declares no dependency '{ref.alias}' (it declares: {known})")
|
|
778
|
+
dep = self.cache.ensure(ref.version_id, *self._trusted_version(ref.bundle_url, ref.snapshot))
|
|
779
|
+
if dep.tools is None:
|
|
780
|
+
fail("host", f"dependency '{ref.alias}' ({ref.version_id}) is an api bundle, not a schema world")
|
|
781
|
+
worlds[ref.alias] = DependencyWorld.of(dependency, dep.tools)
|
|
782
|
+
cached.served_names(worlds)
|
|
783
|
+
return worlds
|
|
784
|
+
|
|
785
|
+
@contextmanager
|
|
786
|
+
def using(
|
|
787
|
+
self, session_id: str, *, touch: bool = True, ledger: bool = False, confirm: bool = False
|
|
788
|
+
) -> Iterator[Session]:
|
|
789
|
+
"""The session for the duration of one request: its lease renewed unless it is a ``ledger`` drain
|
|
790
|
+
(which must not keep a session alive) or a copy pending a move here, its ``touched_at`` moved
|
|
791
|
+
unless ``touch`` is off (a renewal or export observing the session, not using it), and its
|
|
792
|
+
teardown held off until the block ends, then run by the last one out; a ``ledger`` drain hands it
|
|
793
|
+
to :class:`Finishing`, which runs a worker session's finish off the drain's thread, so a close that
|
|
794
|
+
lands inside the spiller's drain never holds the spiller on tenant code, and an in-process
|
|
795
|
+
session's at once, inside its take's 5 s, as it does one whose worker is gone.
|
|
796
|
+
A platform request (``confirm``) goes only to the host the platform's row names, so it confirms
|
|
797
|
+
a copy pending a move here."""
|
|
798
|
+
with self._lock:
|
|
799
|
+
session = self._sessions.get(session_id)
|
|
800
|
+
if session is not None:
|
|
801
|
+
self._enter(session, touch=touch, renew=not ledger, confirm=confirm)
|
|
802
|
+
if session is None:
|
|
803
|
+
fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
|
|
804
|
+
try:
|
|
805
|
+
yield session
|
|
806
|
+
finally:
|
|
807
|
+
self._release(session, self._finish_later if ledger else self._finish)
|
|
808
|
+
|
|
809
|
+
def _enter(self, session: Session, *, touch: bool, renew: bool, confirm: bool) -> None:
|
|
810
|
+
if confirm:
|
|
811
|
+
session.move.commit()
|
|
812
|
+
renewed = renew and not session.move.pending
|
|
813
|
+
session.lease_until = time.time() + self.lease_s if renewed else session.lease_until
|
|
814
|
+
session.touched_at = time.time() if touch else session.touched_at
|
|
815
|
+
session.inflight += 1
|
|
816
|
+
|
|
817
|
+
def _release(self, session: Session, finish: Callable[[Session], None]) -> None:
|
|
818
|
+
with self._lock:
|
|
819
|
+
session.inflight -= 1
|
|
820
|
+
last_out = session.closing and session.inflight == 0
|
|
821
|
+
if last_out:
|
|
822
|
+
finish(session)
|
|
823
|
+
|
|
824
|
+
def owns(self, session_id: str) -> bool:
|
|
825
|
+
with self._lock:
|
|
826
|
+
return session_id in self._sessions
|
|
827
|
+
|
|
828
|
+
def reached_by(self, named: SessionHost) -> bool:
|
|
829
|
+
"""``named`` reaches the session it names: a ``w-`` host always (its door decides),
|
|
830
|
+
an ``o-`` host only on the open session whose label it carries. Asked before the
|
|
831
|
+
session is used, so a wrong label renews and touches nothing."""
|
|
832
|
+
if named.capability is None:
|
|
833
|
+
return True
|
|
834
|
+
session = self.held(named.session_id)
|
|
835
|
+
return session is not None and session.opened_by(named)
|
|
836
|
+
|
|
837
|
+
@contextmanager
|
|
838
|
+
def admitted(self, session: Session, hopped: bool = False) -> Iterator[None]:
|
|
839
|
+
"""One of the session's version's turns (:class:`Admission`) for a data-plane request, once its
|
|
840
|
+
move lets it in (:class:`SessionMove`): held while the session moves, counted against the
|
|
841
|
+
version's waiting bounds, served by a copy pending a move here once the platform names it."""
|
|
842
|
+
waiting = functools.partial(self.admission.waiting, session.version_id, session.session_id)
|
|
843
|
+
owner_now = functools.partial(self._owner_now, session.session_id)
|
|
844
|
+
with session.move.request(waiting, owner_now, self.advertise_url, hopped):
|
|
845
|
+
with self.admission.turn(session.version_id, session.session_id):
|
|
846
|
+
yield
|
|
847
|
+
|
|
848
|
+
def _owner_now(self, session_id: str) -> str | None:
|
|
849
|
+
return self.owners.current(session_id) if self.owners is not None else self.advertise_url
|
|
850
|
+
|
|
851
|
+
def _verb(self, session_id: str, method: str, verb: str, body: dict | None, local: Callable[[Any], dict]) -> dict:
|
|
852
|
+
"""One session verb, the platform's: never held while the session moves (:meth:`SessionMove.verb`),
|
|
853
|
+
admitted on its version, forwarded to the session's worker when it has one, else answered by its
|
|
854
|
+
in-process runtime through ``local``."""
|
|
855
|
+
with (
|
|
856
|
+
self.using(session_id, confirm=True) as session,
|
|
857
|
+
session.move.verb(),
|
|
858
|
+
self.admission.turn(session.version_id, session.session_id),
|
|
859
|
+
):
|
|
860
|
+
if session.worker is not None:
|
|
861
|
+
assert self.placement is not None
|
|
862
|
+
return self.placement.control(session, method, verb, body)
|
|
863
|
+
assert session.runtime is not None
|
|
864
|
+
return local(session.runtime)
|
|
865
|
+
|
|
866
|
+
def tools(self, session_id: str) -> dict:
|
|
867
|
+
return self._verb(session_id, "GET", "tools", None, lambda runtime: {"tools": runtime.served()})
|
|
868
|
+
|
|
869
|
+
def task(self, session_id: str) -> dict:
|
|
870
|
+
return self._verb(session_id, "GET", "task", None, lambda runtime: runtime.task_card())
|
|
871
|
+
|
|
872
|
+
def reset(self, session_id: str) -> dict:
|
|
873
|
+
return self._verb(session_id, "POST", "reset", {}, lambda runtime: runtime.reset())
|
|
874
|
+
|
|
875
|
+
def seed(self, session_id: str, rows: dict, mode: str = "append", redact: str = "off") -> dict:
|
|
876
|
+
args = {"rows": rows, "mode": mode, **({"redact": redact} if redact != "off" else {})}
|
|
877
|
+
return self._verb(session_id, "POST", "seed", {"args": args}, lambda runtime: runtime.seed(rows, mode, redact))
|
|
878
|
+
|
|
879
|
+
def state(self, session_id: str) -> dict:
|
|
880
|
+
return self._verb(session_id, "GET", "state", None, lambda runtime: runtime.state())
|
|
881
|
+
|
|
882
|
+
def evidence(self, session_id: str) -> dict:
|
|
883
|
+
return self._verb(session_id, "GET", "evidence", None, lambda runtime: runtime.grader_state())
|
|
884
|
+
|
|
885
|
+
def requests(
|
|
886
|
+
self,
|
|
887
|
+
session_id: str,
|
|
888
|
+
cursor: str | None = None,
|
|
889
|
+
limit: int = REQUEST_LOG_PAGE_MAX,
|
|
890
|
+
timeout: float | None = None,
|
|
891
|
+
) -> dict:
|
|
892
|
+
"""The next page of the session's request ledger outbox, everything up to ``cursor``
|
|
893
|
+
stored: the ledger's flush, which takes no turn, moves no clock and renews no lease."""
|
|
894
|
+
with self.using(session_id, touch=False, ledger=True) as session:
|
|
895
|
+
return self.ledger_page(session, cursor, limit, timeout)
|
|
896
|
+
|
|
897
|
+
def ledger_page(
|
|
898
|
+
self, session: Session, cursor: str | None, limit: int = REQUEST_LOG_PAGE_MAX, timeout: float | None = None
|
|
899
|
+
) -> dict:
|
|
900
|
+
if session.worker is not None:
|
|
901
|
+
assert self.placement is not None
|
|
902
|
+
asked = {"cursor": cursor, "limit": limit}
|
|
903
|
+
return session.life.stamp(self.placement.outbox(session, asked, timeout or CALL_TIMEOUT_S))
|
|
904
|
+
assert session.runtime is not None and session.runtime.calls.outbox is not None
|
|
905
|
+
return session.life.stamp(session.runtime.calls.outbox.drain(cursor, limit))
|
|
906
|
+
|
|
907
|
+
def session_ids(self, draining: bool = False) -> list[str]:
|
|
908
|
+
with self._lock:
|
|
909
|
+
return [*self._sessions, *(self._draining if draining else ())]
|
|
910
|
+
|
|
911
|
+
def grade(
|
|
912
|
+
self,
|
|
913
|
+
session_id: str,
|
|
914
|
+
report: Optional[str] = None,
|
|
915
|
+
context: Optional[dict] = None,
|
|
916
|
+
journal: Optional[dict] = None,
|
|
917
|
+
grader: Optional[dict] = None,
|
|
918
|
+
) -> dict:
|
|
919
|
+
sent = (("report", report), ("context", context), ("journal", journal), ("grader", grader))
|
|
920
|
+
body = {key: value for key, value in sent if value is not None}
|
|
921
|
+
parsed = WorldJournal.of(journal)
|
|
922
|
+
return self._verb(
|
|
923
|
+
session_id, "POST", "grade", body, lambda runtime: runtime.grade(report, context, parsed, grader)
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
def flush(self, session_id: str) -> dict:
|
|
927
|
+
with self.using(session_id, touch=False, ledger=True) as session:
|
|
928
|
+
return self.ledger.barrier(session)
|
|
929
|
+
|
|
930
|
+
def call(self, session_id: str, tool: str, args: dict) -> dict:
|
|
931
|
+
"""One of the session's declared tools by name, a dependency's as
|
|
932
|
+
``alias.tool``; a route's name is refused (routes answer over HTTP)."""
|
|
933
|
+
body = {"tool": tool, "args": args}
|
|
934
|
+
return self._verb(session_id, "POST", "call", body, lambda runtime: runtime.call(tool, args))
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
HOSTED_OPERATIONS: dict[str, Callable[..., dict]] = {
|
|
938
|
+
"GET tools": WorldHost.tools,
|
|
939
|
+
"GET task": WorldHost.task,
|
|
940
|
+
"GET state": WorldHost.state,
|
|
941
|
+
"GET evidence": WorldHost.evidence,
|
|
942
|
+
"POST requests": WorldHost.requests,
|
|
943
|
+
"POST flush": WorldHost.flush,
|
|
944
|
+
"POST grade": WorldHost.grade,
|
|
945
|
+
"POST reset": WorldHost.reset,
|
|
946
|
+
"POST seed": WorldHost.seed,
|
|
947
|
+
"POST call": WorldHost.call,
|
|
948
|
+
"POST close": platform_close,
|
|
949
|
+
"POST renew": WorldHost.renew,
|
|
950
|
+
"POST export": WorldHost.export,
|
|
951
|
+
"DELETE <session>": platform_close,
|
|
952
|
+
}
|
|
953
|
+
assert set(HOSTED_OPERATIONS) == {verb.route for verb in SESSION_VERBS}, "the host serves every session verb"
|