@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -7,13 +7,14 @@ use; unsupported assertions fail rather than becoming unenforced promises.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import datetime as dt
|
|
10
|
+
import functools
|
|
10
11
|
import hashlib
|
|
11
12
|
import json
|
|
12
13
|
import math
|
|
13
14
|
import re
|
|
14
15
|
import struct
|
|
15
16
|
from pathlib import Path
|
|
16
|
-
from typing import Any, NoReturn
|
|
17
|
+
from typing import Any, Mapping, NoReturn
|
|
17
18
|
|
|
18
19
|
from .._toml import tomllib
|
|
19
20
|
|
|
@@ -283,6 +284,63 @@ OPERATION_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
|
|
|
283
284
|
#: hyphens kept (``reference-count``), never a dot or slash, which address nested fields.
|
|
284
285
|
#: The TypeScript ``schema entity add`` pins the same rule (``PROPERTY_NAME`` in scaffoldJson.ts).
|
|
285
286
|
FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
|
|
287
|
+
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def table_of(value: Any, location: str) -> dict:
|
|
291
|
+
"""A connector.toml table, or the refusal naming where one was expected."""
|
|
292
|
+
if not isinstance(value, dict):
|
|
293
|
+
fail("spec", "Expected a table", location)
|
|
294
|
+
return value
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def string_of(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
|
|
298
|
+
"""``table[key]`` as a non-empty string, stripped; ``default`` when an optional key is absent."""
|
|
299
|
+
if key not in table and required:
|
|
300
|
+
fail("spec", f"Missing {key}", location)
|
|
301
|
+
if key not in table:
|
|
302
|
+
return default
|
|
303
|
+
value = table[key]
|
|
304
|
+
if not isinstance(value, str) or not value.strip():
|
|
305
|
+
fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
|
|
306
|
+
return value.strip()
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def route_path(path: str, location: str) -> str:
|
|
313
|
+
"""``path`` as a route declares it: from ``/``, with no query or fragment, each
|
|
314
|
+
``{argument}`` a segment of its own at most once."""
|
|
315
|
+
if not path.startswith("/") or "?" in path or "#" in path:
|
|
316
|
+
fail("spec", "path must start with / and carry no query", location)
|
|
317
|
+
placeholders = PLACEHOLDER.findall(path)
|
|
318
|
+
repeated = [p for p in placeholders if placeholders.count(p) > 1]
|
|
319
|
+
if repeated:
|
|
320
|
+
fail(
|
|
321
|
+
"spec",
|
|
322
|
+
f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
|
|
323
|
+
location,
|
|
324
|
+
argument=repeated[0],
|
|
325
|
+
)
|
|
326
|
+
return path
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@functools.lru_cache(maxsize=4096)
|
|
330
|
+
def path_pattern(path: str) -> re.Pattern:
|
|
331
|
+
"""A declared path as a pattern over a request's path, each ``{name}`` one segment."""
|
|
332
|
+
escaped = re.escape(path)
|
|
333
|
+
return re.compile("^" + re.sub(r"\\\{([A-Za-z_][A-Za-z0-9_]*)\\\}", r"(?P<\1>[^/]+)", escaped) + "$")
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def names_of(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
|
|
337
|
+
"""``table[key]`` as a list of distinct names ``pattern`` accepts, empty when absent."""
|
|
338
|
+
value = table.get(key, [])
|
|
339
|
+
if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
|
|
340
|
+
fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
|
|
341
|
+
if len(set(value)) != len(value):
|
|
342
|
+
fail("spec", f"{key} repeats a name", f"{location}/{key}")
|
|
343
|
+
return tuple(value)
|
|
286
344
|
|
|
287
345
|
|
|
288
346
|
def parse_toml(text: str) -> dict:
|
|
@@ -784,3 +842,13 @@ class Resolver:
|
|
|
784
842
|
name = sorted(set(self.documents) - self.checked)[0]
|
|
785
843
|
self.resolve(self.documents[name], name, f"{name}#", root=True)
|
|
786
844
|
self.checked.add(name)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def ttl_seconds_of(table: Mapping[str, Any], location: str) -> int | None:
|
|
848
|
+
"""``ttl_seconds`` when a world opts in to expiry, else None: it never expires."""
|
|
849
|
+
value = table.get("ttl_seconds")
|
|
850
|
+
if value is None:
|
|
851
|
+
return None
|
|
852
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
|
853
|
+
fail("spec", "ttl_seconds is a positive whole number of seconds", f"{location}/ttl_seconds")
|
|
854
|
+
return value
|
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
"""The
|
|
2
|
-
|
|
3
|
-
through this table — the local session (``local.py``, ``/__session/<verb>``)
|
|
4
|
-
and the World Host control plane (``host.py``, ``/__host/sessions/<sid>/
|
|
5
|
-
<verb>``) — so the two cannot drift: a verb added here is served by both, a
|
|
6
|
-
body shape accepted here is accepted by both. The transports stay distinct
|
|
7
|
-
(loopback with no secret against a fleet with leases, workers and exports);
|
|
8
|
-
only the verbs are shared.
|
|
9
|
-
|
|
10
|
-
A verb marked ``hosted`` is the platform's rendezvous with a session it keeps
|
|
11
|
-
alive across hosts (the lease, the state export, DELETE as the platform's
|
|
12
|
-
close); the local session, which lives as long as the process serving it,
|
|
13
|
-
refuses them by name."""
|
|
1
|
+
"""The one verb table local.py and host.py both dispatch through, so the transports cannot drift; a ``hosted`` verb
|
|
2
|
+
is control plane only (lease, export, DELETE, the ledger's flush and outbox), refused by name locally."""
|
|
14
3
|
|
|
15
4
|
from __future__ import annotations
|
|
16
5
|
|
|
@@ -18,17 +7,14 @@ from collections.abc import Callable
|
|
|
18
7
|
from dataclasses import dataclass
|
|
19
8
|
from typing import Any
|
|
20
9
|
|
|
10
|
+
from ..world_session_contract import REQUEST_LOG_PAGE_MAX
|
|
21
11
|
from .redaction import parse_mode
|
|
12
|
+
from .request_ledger import OUTBOX_CURSOR
|
|
22
13
|
from .validation import ContractError, fail
|
|
23
14
|
|
|
24
15
|
|
|
25
16
|
@dataclass(frozen=True)
|
|
26
17
|
class SessionVerb:
|
|
27
|
-
"""One row of the protocol: ``method`` and ``name`` are the route (``name``
|
|
28
|
-
None addresses the session itself, as ``DELETE`` does); ``read`` turns the
|
|
29
|
-
request body into the keyword arguments the session operation takes, or
|
|
30
|
-
refuses the shape (a :class:`ContractError` the transport answers 400)."""
|
|
31
|
-
|
|
32
18
|
method: str
|
|
33
19
|
name: str | None
|
|
34
20
|
read: Callable[[dict], dict[str, Any]]
|
|
@@ -47,9 +33,8 @@ def read_nothing(_body: dict) -> dict[str, Any]:
|
|
|
47
33
|
|
|
48
34
|
|
|
49
35
|
def read_seed(body: dict) -> dict[str, Any]:
|
|
50
|
-
"""``{rows, mode, redact}``, the same under ``args``
|
|
51
|
-
|
|
52
|
-
file has — taken as the rows of an append."""
|
|
36
|
+
"""``{rows, mode, redact}``, the same under ``args`` as the platform forwards it, or a bare rows file
|
|
37
|
+
``{entity: [row, ...]}`` taken as an append."""
|
|
53
38
|
args = body["args"] if isinstance(body.get("args"), dict) else body
|
|
54
39
|
if "rows" not in args and "mode" not in args and args and all(isinstance(v, list) for v in args.values()):
|
|
55
40
|
args = {"rows": args}
|
|
@@ -67,18 +52,20 @@ def read_seed(body: dict) -> dict[str, Any]:
|
|
|
67
52
|
|
|
68
53
|
|
|
69
54
|
def read_grade(body: dict) -> dict[str, Any]:
|
|
70
|
-
"""``report``
|
|
71
|
-
|
|
72
|
-
anything but a string is not. ``context`` is the evidence of the task's
|
|
73
|
-
other worlds (``{alias: state}``, each what that session's ``evidence``
|
|
74
|
-
answers), read by the cross-world verifiers anchored on this session."""
|
|
55
|
+
"""``report`` the agent's own account (a string), ``context`` the task's other worlds as ``{alias: state}``,
|
|
56
|
+
``journal`` the session's journal as the platform stored it, ``grader`` a candidate graded instead."""
|
|
75
57
|
report = body.get("report")
|
|
76
58
|
if report is not None and not isinstance(report, str):
|
|
77
59
|
fail("input", "report must be a string", "/report")
|
|
78
60
|
context = body.get("context")
|
|
79
|
-
if context is not None and (
|
|
61
|
+
if context is not None and (
|
|
62
|
+
not isinstance(context, dict) or any(not isinstance(v, dict) for v in context.values())
|
|
63
|
+
):
|
|
80
64
|
fail("input", "context must be {alias: state} of the task's other worlds", "/context")
|
|
81
|
-
|
|
65
|
+
for key in ("journal", "grader"):
|
|
66
|
+
if body.get(key) is not None and not isinstance(body[key], dict):
|
|
67
|
+
fail("input", f"{key} must be an object", f"/{key}")
|
|
68
|
+
return {"report": report, "context": context, "journal": body.get("journal"), "grader": body.get("grader")}
|
|
82
69
|
|
|
83
70
|
|
|
84
71
|
def read_call(body: dict) -> dict[str, Any]:
|
|
@@ -92,8 +79,22 @@ def read_call(body: dict) -> dict[str, Any]:
|
|
|
92
79
|
|
|
93
80
|
|
|
94
81
|
def read_export(body: dict) -> dict[str, Any]:
|
|
82
|
+
"""``knownSha256``: the platform's last export of the session; ``handoff``: this is the final
|
|
83
|
+
export of a session its host is handing off, which only that host takes."""
|
|
95
84
|
known = body.get("knownSha256")
|
|
96
|
-
return {"known_sha256": known if isinstance(known, str) else None}
|
|
85
|
+
return {"known_sha256": known if isinstance(known, str) else None, "handoff": body.get("handoff") is True}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def read_requests(body: dict) -> dict[str, Any]:
|
|
89
|
+
"""``cursor`` names the last outbox page that was stored (absent before the first); ``limit`` is 1 to
|
|
90
|
+
REQUEST_LOG_PAGE_MAX, the max when absent."""
|
|
91
|
+
cursor = body.get("cursor")
|
|
92
|
+
limit = body.get("limit", REQUEST_LOG_PAGE_MAX)
|
|
93
|
+
if cursor is not None and not (isinstance(cursor, str) and OUTBOX_CURSOR.fullmatch(cursor)):
|
|
94
|
+
fail("input", "cursor must be the cursor a page of the request ledger outbox answered", "/cursor")
|
|
95
|
+
if type(limit) is not int or not 1 <= limit <= REQUEST_LOG_PAGE_MAX:
|
|
96
|
+
fail("input", f"limit must be an integer from 1 to {REQUEST_LOG_PAGE_MAX}", "/limit")
|
|
97
|
+
return {"cursor": cursor, "limit": limit}
|
|
97
98
|
|
|
98
99
|
|
|
99
100
|
SESSION_VERBS: tuple[SessionVerb, ...] = (
|
|
@@ -101,6 +102,8 @@ SESSION_VERBS: tuple[SessionVerb, ...] = (
|
|
|
101
102
|
SessionVerb("GET", "task", read_nothing),
|
|
102
103
|
SessionVerb("GET", "state", read_nothing),
|
|
103
104
|
SessionVerb("GET", "evidence", read_nothing),
|
|
105
|
+
SessionVerb("POST", "requests", read_requests, hosted=True),
|
|
106
|
+
SessionVerb("POST", "flush", read_nothing, hosted=True),
|
|
104
107
|
SessionVerb("POST", "grade", read_grade),
|
|
105
108
|
SessionVerb("POST", "reset", read_nothing),
|
|
106
109
|
SessionVerb("POST", "seed", read_seed),
|
|
@@ -117,14 +120,10 @@ def session_verb(method: str, name: str | None) -> SessionVerb | None:
|
|
|
117
120
|
|
|
118
121
|
|
|
119
122
|
def served_verbs(*, hosted: bool) -> tuple[SessionVerb, ...]:
|
|
120
|
-
"""The rows a transport answers: every one on the host, the session's own
|
|
121
|
-
(not the lease's) locally."""
|
|
122
123
|
return tuple(verb for verb in SESSION_VERBS if hosted or not verb.hosted)
|
|
123
124
|
|
|
124
125
|
|
|
125
126
|
def unknown_verb(method: str, name: str | None, *, hosted: bool, prefix: str) -> dict:
|
|
126
|
-
"""The 404 body for a route off the table, naming what is on it. A hosted
|
|
127
|
-
verb sent to the local session is refused by name, not as unknown."""
|
|
128
127
|
verb = session_verb(method, name)
|
|
129
128
|
if verb is not None and verb.hosted and not hosted:
|
|
130
129
|
return {"error": f"{verb.route} is a hosted session's verb (the platform's lease); the local session has none"}
|
|
@@ -136,7 +135,4 @@ def unknown_verb(method: str, name: str | None, *, hosted: bool, prefix: str) ->
|
|
|
136
135
|
|
|
137
136
|
|
|
138
137
|
def refusal(error: ContractError) -> dict:
|
|
139
|
-
"""A refused call as the answer a session gives it: an unknown tool,
|
|
140
|
-
arguments off the schema, a tool outside the task's allow-list, a write the
|
|
141
|
-
store will not take, all ``{status: 400, body: {error, detail}}``."""
|
|
142
138
|
return {"status": 400, "body": {"error": error.detail.get("code", "host"), "detail": str(error)}}
|
|
@@ -15,7 +15,8 @@ import socket
|
|
|
15
15
|
import sys
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
18
|
-
from .
|
|
18
|
+
from .cache_layout import VERSIONS_DIR, WORKERS_DIR
|
|
19
|
+
from .host import make_host_handler
|
|
19
20
|
from .workers import (
|
|
20
21
|
DEFAULT_WORKER_MEMORY_MB,
|
|
21
22
|
WORKER_LEASE_S,
|
|
@@ -23,6 +24,7 @@ from .workers import (
|
|
|
23
24
|
apply_limits,
|
|
24
25
|
seal_process,
|
|
25
26
|
)
|
|
27
|
+
from .world_host import WorldHost
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -39,7 +41,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
39
41
|
raise SystemExit("world worker: WORLDS_WORKER_SECRET is unset; a worker only answers its supervisor")
|
|
40
42
|
supervisor_pid: int | None = None
|
|
41
43
|
if args.socket_fd is not None:
|
|
42
|
-
supervisor_pid =
|
|
44
|
+
supervisor_pid = os.getppid()
|
|
43
45
|
listening: socket.socket | str = socket.socket(fileno=args.socket_fd)
|
|
44
46
|
else:
|
|
45
47
|
listening = args.socket
|
|
@@ -48,8 +50,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
48
50
|
|
|
49
51
|
cache_root = Path(args.cache_root)
|
|
50
52
|
world_host = WorldHost(
|
|
51
|
-
cache_root /
|
|
52
|
-
versions_root=cache_root /
|
|
53
|
+
cache_root / WORKERS_DIR / args.version,
|
|
54
|
+
versions_root=cache_root / VERSIONS_DIR,
|
|
53
55
|
lease_s=WORKER_LEASE_S,
|
|
54
56
|
run_handlers=True,
|
|
55
57
|
pinned_version=args.version,
|
|
@@ -35,6 +35,7 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
|
|
|
35
35
|
|
|
36
36
|
from __future__ import annotations
|
|
37
37
|
|
|
38
|
+
import collections
|
|
38
39
|
import ctypes
|
|
39
40
|
import hashlib
|
|
40
41
|
import http.client
|
|
@@ -49,13 +50,16 @@ import sys
|
|
|
49
50
|
import tempfile
|
|
50
51
|
import threading
|
|
51
52
|
import time
|
|
52
|
-
from collections.abc import Mapping
|
|
53
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
54
|
+
from contextlib import contextmanager, suppress
|
|
53
55
|
from dataclasses import dataclass, field
|
|
54
56
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
55
57
|
from pathlib import Path
|
|
56
58
|
from typing import Protocol
|
|
57
59
|
|
|
60
|
+
from .cache_layout import WORKERS_DIR
|
|
58
61
|
from .grading import VerifierSandbox
|
|
62
|
+
from .handler_errors import log_line
|
|
59
63
|
|
|
60
64
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
61
65
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
@@ -81,10 +85,12 @@ FLOOR_PRECISION_BYTES = 1 << 20
|
|
|
81
85
|
#: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
|
|
82
86
|
#: spells it so, Linux spells it -1.
|
|
83
87
|
MAX_RLIMIT = 2**63 - 1
|
|
84
|
-
#: How long a worker with no sessions is kept before it is stopped.
|
|
85
88
|
DEFAULT_WORKER_IDLE_S = 300.0
|
|
86
|
-
#: How long the supervisor waits for a fresh worker to answer on its socket.
|
|
87
89
|
WORKER_START_TIMEOUT_S = 30.0
|
|
90
|
+
#: How long one health probe waits: a worker that cannot answer it in this time is not serving.
|
|
91
|
+
HEALTH_TIMEOUT_S = 2.0
|
|
92
|
+
#: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
|
|
93
|
+
PROBE_LIMIT_BYTES = 64 << 10
|
|
88
94
|
#: A worker's sessions live as long as the supervisor's do; the supervisor
|
|
89
95
|
#: holds the leases, so the worker's own are effectively unbounded.
|
|
90
96
|
WORKER_LEASE_S = 365.0 * 86400.0
|
|
@@ -94,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
|
94
100
|
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
95
101
|
#: the supervisor's connection, and its own control-plane secret.
|
|
96
102
|
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
103
|
+
FINISH_THREADS = 8
|
|
104
|
+
FINISH_SETTLE_S = 10.0
|
|
97
105
|
|
|
98
106
|
|
|
99
107
|
class WorkerGone(Exception):
|
|
100
|
-
"""The worker did not answer: it exited,
|
|
108
|
+
"""The worker did not answer: it exited, never came up, or dropped the request."""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class WorkerTimeout(WorkerGone):
|
|
112
|
+
"""The worker did not answer in time; it may be alive and still working on the request."""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class WorkerUnreached(WorkerGone):
|
|
116
|
+
"""The worker took no connection: the request never reached it."""
|
|
101
117
|
|
|
102
118
|
|
|
103
119
|
class WorkerCapacity(Exception):
|
|
@@ -117,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
|
|
|
117
133
|
self.sock = sock
|
|
118
134
|
|
|
119
135
|
|
|
136
|
+
def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
|
|
137
|
+
cut.set()
|
|
138
|
+
with suppress(OSError):
|
|
139
|
+
sock.shutdown(socket.SHUT_RDWR)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@contextmanager
|
|
143
|
+
def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
|
|
144
|
+
"""Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
|
|
145
|
+
cut = threading.Event()
|
|
146
|
+
timer = threading.Timer(seconds, _hang_up, (sock, cut))
|
|
147
|
+
timer.start()
|
|
148
|
+
try:
|
|
149
|
+
yield cut
|
|
150
|
+
finally:
|
|
151
|
+
timer.cancel()
|
|
152
|
+
|
|
153
|
+
|
|
120
154
|
def bind_worker_socket(path: str | Path) -> socket.socket:
|
|
121
155
|
"""The supervisor's end of a worker: a listening unix socket at ``path``,
|
|
122
156
|
``0600`` inside a ``0700`` directory, ready to hand to the worker."""
|
|
@@ -333,7 +367,6 @@ def worker_environment(
|
|
|
333
367
|
WORLDS_WORKER_SECRET=secret,
|
|
334
368
|
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
335
369
|
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
336
|
-
WORLDS_SUPERVISOR_PID=str(os.getpid()),
|
|
337
370
|
)
|
|
338
371
|
return env
|
|
339
372
|
|
|
@@ -389,7 +422,6 @@ class Worker:
|
|
|
389
422
|
socket_path: Path
|
|
390
423
|
secret: str
|
|
391
424
|
sessions: set[str] = field(default_factory=set)
|
|
392
|
-
#: When the worker last had a session, for idle reaping.
|
|
393
425
|
idle_since: float = field(default_factory=time.time)
|
|
394
426
|
|
|
395
427
|
@property
|
|
@@ -408,25 +440,78 @@ class Worker:
|
|
|
408
440
|
*,
|
|
409
441
|
control: bool = False,
|
|
410
442
|
timeout: float = 60.0,
|
|
443
|
+
limit: int | None = None,
|
|
411
444
|
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
412
445
|
"""One request over the socket. ``control`` adds the worker's own
|
|
413
|
-
secret, so the worker's control plane answers. A worker that
|
|
414
|
-
|
|
446
|
+
secret, so the worker's control plane answers. A worker that takes no connection raises
|
|
447
|
+
:class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
|
|
448
|
+
:class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
|
|
449
|
+
sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
|
|
450
|
+
:class:`WorkerTimeout`."""
|
|
415
451
|
forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
|
|
416
452
|
if control:
|
|
417
453
|
forward["X-Host-Secret"] = self.secret
|
|
418
|
-
|
|
454
|
+
started = time.monotonic()
|
|
455
|
+
conn = self._connected(timeout)
|
|
456
|
+
cut = threading.Event()
|
|
419
457
|
try:
|
|
420
|
-
conn.
|
|
421
|
-
|
|
422
|
-
|
|
458
|
+
with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
|
|
459
|
+
conn.request(method, path, body=body or None, headers=forward)
|
|
460
|
+
answer = conn.getresponse()
|
|
461
|
+
payload = answer.read() if limit is None else answer.read(limit + 1)
|
|
462
|
+
if cut.is_set():
|
|
463
|
+
raise WorkerTimeout(
|
|
464
|
+
f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
|
|
465
|
+
)
|
|
466
|
+
if limit is not None and len(payload) > limit:
|
|
467
|
+
raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
|
|
423
468
|
return answer.status, answer.getheaders(), payload
|
|
424
469
|
except (OSError, http.client.HTTPException) as error:
|
|
425
|
-
|
|
426
|
-
|
|
470
|
+
gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
|
|
471
|
+
raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
|
|
472
|
+
finally:
|
|
473
|
+
conn.close()
|
|
474
|
+
|
|
475
|
+
def _connected(self, timeout: float) -> UnixHTTPConnection:
|
|
476
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
|
|
477
|
+
try:
|
|
478
|
+
conn.connect()
|
|
479
|
+
except OSError as error:
|
|
480
|
+
conn.close()
|
|
481
|
+
raise WorkerUnreached(
|
|
482
|
+
f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
|
|
427
483
|
) from error
|
|
484
|
+
return conn
|
|
485
|
+
|
|
486
|
+
def probe(
|
|
487
|
+
self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
|
|
488
|
+
) -> tuple[int, bytes]:
|
|
489
|
+
"""One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
|
|
490
|
+
The worker runs tenant code: an answer sent slowly or without end is cut off when the
|
|
491
|
+
deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
|
|
492
|
+
unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
|
|
493
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
|
|
494
|
+
try:
|
|
495
|
+
conn.connect()
|
|
496
|
+
with _deadline(conn.sock, deadline_s) as cut:
|
|
497
|
+
conn.request("GET", path, headers={"X-Host-Secret": self.secret})
|
|
498
|
+
answer = conn.getresponse()
|
|
499
|
+
payload = answer.read(limit + 1)
|
|
500
|
+
except (OSError, http.client.HTTPException) as error:
|
|
501
|
+
raise WorkerGone(unanswered) from error
|
|
428
502
|
finally:
|
|
429
503
|
conn.close()
|
|
504
|
+
if cut.is_set() or len(payload) > limit:
|
|
505
|
+
raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
|
|
506
|
+
return answer.status, payload
|
|
507
|
+
|
|
508
|
+
def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
|
|
509
|
+
"""Whether the worker is alive and answers its health check, whole, in time."""
|
|
510
|
+
try:
|
|
511
|
+
status, _ = self.probe("/healthz", deadline_s=timeout_s)
|
|
512
|
+
except WorkerGone:
|
|
513
|
+
return False
|
|
514
|
+
return status == 200
|
|
430
515
|
|
|
431
516
|
def stop(self, grace_s: float = 5.0) -> None:
|
|
432
517
|
if self.alive():
|
|
@@ -439,6 +524,69 @@ class Worker:
|
|
|
439
524
|
self.socket_path.unlink(missing_ok=True)
|
|
440
525
|
|
|
441
526
|
|
|
527
|
+
class Finishing:
|
|
528
|
+
"""What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
|
|
529
|
+
threads, one job at a time for a version in the order they came (so a worker that never answers
|
|
530
|
+
holds one thread, not the pool). A session with no worker is finished at once, where it is."""
|
|
531
|
+
|
|
532
|
+
def __init__(self, threads: int = FINISH_THREADS) -> None:
|
|
533
|
+
self.threads = threads
|
|
534
|
+
self._changed = threading.Condition()
|
|
535
|
+
self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
|
|
536
|
+
self._serving: set[str] = set()
|
|
537
|
+
self._started = 0
|
|
538
|
+
|
|
539
|
+
def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
|
|
540
|
+
if worker is None:
|
|
541
|
+
job()
|
|
542
|
+
return
|
|
543
|
+
with self._changed:
|
|
544
|
+
self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
|
|
545
|
+
self._staff()
|
|
546
|
+
self._changed.notify()
|
|
547
|
+
|
|
548
|
+
def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
|
|
549
|
+
"""Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
|
|
550
|
+
until = time.monotonic() + deadline_s
|
|
551
|
+
with self._changed:
|
|
552
|
+
while self._waiting:
|
|
553
|
+
remaining = until - time.monotonic()
|
|
554
|
+
if remaining <= 0:
|
|
555
|
+
return False
|
|
556
|
+
self._changed.wait(remaining)
|
|
557
|
+
return True
|
|
558
|
+
|
|
559
|
+
def _staff(self) -> None:
|
|
560
|
+
if self._started < self.threads:
|
|
561
|
+
self._started += 1
|
|
562
|
+
threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
|
|
563
|
+
|
|
564
|
+
def _run(self) -> None:
|
|
565
|
+
while True:
|
|
566
|
+
with self._changed:
|
|
567
|
+
self._changed.wait_for(lambda: self._next() is not None)
|
|
568
|
+
version = self._next()
|
|
569
|
+
assert version is not None
|
|
570
|
+
job = self._waiting[version].popleft()
|
|
571
|
+
self._serving.add(version)
|
|
572
|
+
self._work(version, job)
|
|
573
|
+
|
|
574
|
+
def _next(self) -> str | None:
|
|
575
|
+
return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
|
|
576
|
+
|
|
577
|
+
def _work(self, version: str, job: Callable[[], None]) -> None:
|
|
578
|
+
try:
|
|
579
|
+
job()
|
|
580
|
+
except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
|
|
581
|
+
log_line(f"world host: a session did not finish: {type(error).__name__}")
|
|
582
|
+
finally:
|
|
583
|
+
with self._changed:
|
|
584
|
+
self._serving.discard(version)
|
|
585
|
+
if not self._waiting[version]:
|
|
586
|
+
del self._waiting[version]
|
|
587
|
+
self._changed.notify_all()
|
|
588
|
+
|
|
589
|
+
|
|
442
590
|
class WorkerPool:
|
|
443
591
|
"""Every version worker on this replica, started on the first session of a
|
|
444
592
|
version and stopped once it has had none for a while."""
|
|
@@ -468,7 +616,7 @@ class WorkerPool:
|
|
|
468
616
|
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
469
617
|
)
|
|
470
618
|
self.root = Path(cache_root)
|
|
471
|
-
self.workers_root = self.root /
|
|
619
|
+
self.workers_root = self.root / WORKERS_DIR
|
|
472
620
|
# Workers live and die with the supervisor; whatever a previous one
|
|
473
621
|
# left (sockets, session files) is unreachable.
|
|
474
622
|
shutil.rmtree(self.workers_root, ignore_errors=True)
|
|
@@ -491,24 +639,54 @@ class WorkerPool:
|
|
|
491
639
|
def get(self, version_id: str) -> Worker:
|
|
492
640
|
"""The live worker for a version, started if there is none. Starting one
|
|
493
641
|
past the budget first stops workers holding no sessions (whatever their
|
|
494
|
-
idle time); when every one is in use, raises :class:`WorkerCapacity`.
|
|
642
|
+
idle time); when every one is in use, raises :class:`WorkerCapacity`. A
|
|
643
|
+
dead worker that still holds sessions is never replaced here: its
|
|
644
|
+
sessions' files are on disk, and the supervisor recovers them through
|
|
645
|
+
:meth:`replace` (raises :class:`WorkerGone`)."""
|
|
495
646
|
with self._lock:
|
|
496
647
|
worker = self._workers.get(version_id)
|
|
497
648
|
if worker is not None and worker.alive():
|
|
498
649
|
return worker
|
|
650
|
+
if worker is not None and worker.sessions:
|
|
651
|
+
raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
|
|
499
652
|
if worker is not None:
|
|
500
653
|
worker.stop()
|
|
501
654
|
del self._workers[version_id]
|
|
502
|
-
self.
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
655
|
+
return self._start(version_id)
|
|
656
|
+
|
|
657
|
+
def replace(self, dead: Worker) -> Worker:
|
|
658
|
+
"""A fresh worker in ``dead``'s place, its sessions left for the caller to
|
|
659
|
+
reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
|
|
660
|
+
when the pool no longer holds ``dead``."""
|
|
661
|
+
with self._lock:
|
|
662
|
+
if self._workers.get(dead.version_id) is not dead:
|
|
663
|
+
raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
|
|
664
|
+
del self._workers[dead.version_id]
|
|
665
|
+
dead.stop()
|
|
666
|
+
return self._start(dead.version_id)
|
|
667
|
+
|
|
668
|
+
def _start(self, version_id: str) -> Worker:
|
|
669
|
+
self._make_room()
|
|
670
|
+
worker = self._spawn(version_id)
|
|
671
|
+
self._workers[version_id] = worker
|
|
672
|
+
return worker
|
|
673
|
+
|
|
674
|
+
def dead(self) -> list[Worker]:
|
|
675
|
+
"""The workers that exited still holding sessions: theirs to recover."""
|
|
676
|
+
with self._lock:
|
|
677
|
+
return [w for w in self._workers.values() if w.sessions and not w.alive()]
|
|
678
|
+
|
|
679
|
+
def live(self) -> list[Worker]:
|
|
680
|
+
"""The workers still running, holding sessions or not."""
|
|
681
|
+
with self._lock:
|
|
682
|
+
return [w for w in self._workers.values() if w.alive()]
|
|
506
683
|
|
|
507
684
|
def _make_room(self) -> None:
|
|
508
685
|
"""Under ``_lock``: leave one worker's worth of the budget free, by
|
|
509
|
-
forgetting exited workers first, then stopping
|
|
510
|
-
longest idle first, and refusing when every live
|
|
511
|
-
|
|
686
|
+
forgetting exited workers that hold no sessions first, then stopping
|
|
687
|
+
session-less ones longest idle first, and refusing when every live
|
|
688
|
+
worker is in use."""
|
|
689
|
+
exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
|
|
512
690
|
for version_id in exited:
|
|
513
691
|
self._workers.pop(version_id).stop()
|
|
514
692
|
spare = sorted(
|
|
@@ -527,15 +705,6 @@ class WorkerPool:
|
|
|
527
705
|
with self._lock:
|
|
528
706
|
return self._workers.get(version_id)
|
|
529
707
|
|
|
530
|
-
def drop(self, version_id: str) -> Worker | None:
|
|
531
|
-
"""Stop and forget a version's worker; returns it (with the sessions
|
|
532
|
-
it held) so the caller can forget those too."""
|
|
533
|
-
with self._lock:
|
|
534
|
-
worker = self._workers.pop(version_id, None)
|
|
535
|
-
if worker is not None:
|
|
536
|
-
worker.stop()
|
|
537
|
-
return worker
|
|
538
|
-
|
|
539
708
|
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
540
709
|
"""Stop workers that have had no session for ``idle_s``; returns their versions."""
|
|
541
710
|
now = time.time() if now is None else now
|
|
@@ -604,13 +773,8 @@ class WorkerPool:
|
|
|
604
773
|
raise WorkerGone(
|
|
605
774
|
f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
|
|
606
775
|
)
|
|
607
|
-
if worker.socket_path.exists():
|
|
608
|
-
|
|
609
|
-
status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
|
|
610
|
-
except WorkerGone:
|
|
611
|
-
status = 0
|
|
612
|
-
if status == 200:
|
|
613
|
-
return
|
|
776
|
+
if worker.socket_path.exists() and worker.answers_health():
|
|
777
|
+
return
|
|
614
778
|
time.sleep(0.05)
|
|
615
779
|
worker.stop()
|
|
616
780
|
raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|