@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -7,13 +7,14 @@ use; unsupported assertions fail rather than becoming unenforced promises.
7
7
  from __future__ import annotations
8
8
 
9
9
  import datetime as dt
10
+ import functools
10
11
  import hashlib
11
12
  import json
12
13
  import math
13
14
  import re
14
15
  import struct
15
16
  from pathlib import Path
16
- from typing import Any, NoReturn
17
+ from typing import Any, Mapping, NoReturn
17
18
 
18
19
  from .._toml import tomllib
19
20
 
@@ -283,6 +284,63 @@ OPERATION_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
283
284
  #: hyphens kept (``reference-count``), never a dot or slash, which address nested fields.
284
285
  #: The TypeScript ``schema entity add`` pins the same rule (``PROPERTY_NAME`` in scaffoldJson.ts).
285
286
  FIELD_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
287
+ METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
288
+
289
+
290
+ def table_of(value: Any, location: str) -> dict:
291
+ """A connector.toml table, or the refusal naming where one was expected."""
292
+ if not isinstance(value, dict):
293
+ fail("spec", "Expected a table", location)
294
+ return value
295
+
296
+
297
+ def string_of(table: dict, key: str, location: str, *, default: str | None = None, required: bool = True) -> str | None:
298
+ """``table[key]`` as a non-empty string, stripped; ``default`` when an optional key is absent."""
299
+ if key not in table and required:
300
+ fail("spec", f"Missing {key}", location)
301
+ if key not in table:
302
+ return default
303
+ value = table[key]
304
+ if not isinstance(value, str) or not value.strip():
305
+ fail("spec", f"{key} must be a non-empty string", f"{location}/{key}")
306
+ return value.strip()
307
+
308
+
309
+ PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
310
+
311
+
312
+ def route_path(path: str, location: str) -> str:
313
+ """``path`` as a route declares it: from ``/``, with no query or fragment, each
314
+ ``{argument}`` a segment of its own at most once."""
315
+ if not path.startswith("/") or "?" in path or "#" in path:
316
+ fail("spec", "path must start with / and carry no query", location)
317
+ placeholders = PLACEHOLDER.findall(path)
318
+ repeated = [p for p in placeholders if placeholders.count(p) > 1]
319
+ if repeated:
320
+ fail(
321
+ "spec",
322
+ f"{{{repeated[0]}}} appears twice in path {path}; each path argument may appear once",
323
+ location,
324
+ argument=repeated[0],
325
+ )
326
+ return path
327
+
328
+
329
+ @functools.lru_cache(maxsize=4096)
330
+ def path_pattern(path: str) -> re.Pattern:
331
+ """A declared path as a pattern over a request's path, each ``{name}`` one segment."""
332
+ escaped = re.escape(path)
333
+ return re.compile("^" + re.sub(r"\\\{([A-Za-z_][A-Za-z0-9_]*)\\\}", r"(?P<\1>[^/]+)", escaped) + "$")
334
+
335
+
336
+ def names_of(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
337
+ """``table[key]`` as a list of distinct names ``pattern`` accepts, empty when absent."""
338
+ value = table.get(key, [])
339
+ if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
340
+ fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
341
+ if len(set(value)) != len(value):
342
+ fail("spec", f"{key} repeats a name", f"{location}/{key}")
343
+ return tuple(value)
286
344
 
287
345
 
288
346
  def parse_toml(text: str) -> dict:
@@ -784,3 +842,13 @@ class Resolver:
784
842
  name = sorted(set(self.documents) - self.checked)[0]
785
843
  self.resolve(self.documents[name], name, f"{name}#", root=True)
786
844
  self.checked.add(name)
845
+
846
+
847
+ def ttl_seconds_of(table: Mapping[str, Any], location: str) -> int | None:
848
+ """``ttl_seconds`` when a world opts in to expiry, else None: it never expires."""
849
+ value = table.get("ttl_seconds")
850
+ if value is None:
851
+ return None
852
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
853
+ fail("spec", "ttl_seconds is a positive whole number of seconds", f"{location}/ttl_seconds")
854
+ return value
@@ -1,16 +1,5 @@
1
- """The session protocol, once: every verb a world session answers, with the
2
- method it takes and how its body is read. Both HTTP transports dispatch
3
- through this table — the local session (``local.py``, ``/__session/<verb>``)
4
- and the World Host control plane (``host.py``, ``/__host/sessions/<sid>/
5
- <verb>``) — so the two cannot drift: a verb added here is served by both, a
6
- body shape accepted here is accepted by both. The transports stay distinct
7
- (loopback with no secret against a fleet with leases, workers and exports);
8
- only the verbs are shared.
9
-
10
- A verb marked ``hosted`` is the platform's rendezvous with a session it keeps
11
- alive across hosts (the lease, the state export, DELETE as the platform's
12
- close); the local session, which lives as long as the process serving it,
13
- refuses them by name."""
1
+ """The one verb table local.py and host.py both dispatch through, so the transports cannot drift; a ``hosted`` verb
2
+ is control plane only (lease, export, DELETE, the ledger's flush and outbox), refused by name locally."""
14
3
 
15
4
  from __future__ import annotations
16
5
 
@@ -18,17 +7,14 @@ from collections.abc import Callable
18
7
  from dataclasses import dataclass
19
8
  from typing import Any
20
9
 
10
+ from ..world_session_contract import REQUEST_LOG_PAGE_MAX
21
11
  from .redaction import parse_mode
12
+ from .request_ledger import OUTBOX_CURSOR
22
13
  from .validation import ContractError, fail
23
14
 
24
15
 
25
16
  @dataclass(frozen=True)
26
17
  class SessionVerb:
27
- """One row of the protocol: ``method`` and ``name`` are the route (``name``
28
- None addresses the session itself, as ``DELETE`` does); ``read`` turns the
29
- request body into the keyword arguments the session operation takes, or
30
- refuses the shape (a :class:`ContractError` the transport answers 400)."""
31
-
32
18
  method: str
33
19
  name: str | None
34
20
  read: Callable[[dict], dict[str, Any]]
@@ -47,9 +33,8 @@ def read_nothing(_body: dict) -> dict[str, Any]:
47
33
 
48
34
 
49
35
  def read_seed(body: dict) -> dict[str, Any]:
50
- """``{rows, mode, redact}``, the same under ``args`` (the platform wraps
51
- the seed it forwards), or bare ``{entity: [row, ...]}`` — the shape a rows
52
- file has — taken as the rows of an append."""
36
+ """``{rows, mode, redact}``, the same under ``args`` as the platform forwards it, or a bare rows file
37
+ ``{entity: [row, ...]}`` taken as an append."""
53
38
  args = body["args"] if isinstance(body.get("args"), dict) else body
54
39
  if "rows" not in args and "mode" not in args and args and all(isinstance(v, list) for v in args.values()):
55
40
  args = {"rows": args}
@@ -67,18 +52,20 @@ def read_seed(body: dict) -> dict[str, Any]:
67
52
 
68
53
 
69
54
  def read_grade(body: dict) -> dict[str, Any]:
70
- """``report`` is the agent's own account of the run, read by a rubric
71
- judge or put beside the state for a deterministic grader; absent is fine,
72
- anything but a string is not. ``context`` is the evidence of the task's
73
- other worlds (``{alias: state}``, each what that session's ``evidence``
74
- answers), read by the cross-world verifiers anchored on this session."""
55
+ """``report`` the agent's own account (a string), ``context`` the task's other worlds as ``{alias: state}``,
56
+ ``journal`` the session's journal as the platform stored it, ``grader`` a candidate graded instead."""
75
57
  report = body.get("report")
76
58
  if report is not None and not isinstance(report, str):
77
59
  fail("input", "report must be a string", "/report")
78
60
  context = body.get("context")
79
- if context is not None and (not isinstance(context, dict) or any(not isinstance(v, dict) for v in context.values())):
61
+ if context is not None and (
62
+ not isinstance(context, dict) or any(not isinstance(v, dict) for v in context.values())
63
+ ):
80
64
  fail("input", "context must be {alias: state} of the task's other worlds", "/context")
81
- return {"report": report, "context": context}
65
+ for key in ("journal", "grader"):
66
+ if body.get(key) is not None and not isinstance(body[key], dict):
67
+ fail("input", f"{key} must be an object", f"/{key}")
68
+ return {"report": report, "context": context, "journal": body.get("journal"), "grader": body.get("grader")}
82
69
 
83
70
 
84
71
  def read_call(body: dict) -> dict[str, Any]:
@@ -92,8 +79,22 @@ def read_call(body: dict) -> dict[str, Any]:
92
79
 
93
80
 
94
81
  def read_export(body: dict) -> dict[str, Any]:
82
+ """``knownSha256``: the platform's last export of the session; ``handoff``: this is the final
83
+ export of a session its host is handing off, which only that host takes."""
95
84
  known = body.get("knownSha256")
96
- return {"known_sha256": known if isinstance(known, str) else None}
85
+ return {"known_sha256": known if isinstance(known, str) else None, "handoff": body.get("handoff") is True}
86
+
87
+
88
+ def read_requests(body: dict) -> dict[str, Any]:
89
+ """``cursor`` names the last outbox page that was stored (absent before the first); ``limit`` is 1 to
90
+ REQUEST_LOG_PAGE_MAX, the max when absent."""
91
+ cursor = body.get("cursor")
92
+ limit = body.get("limit", REQUEST_LOG_PAGE_MAX)
93
+ if cursor is not None and not (isinstance(cursor, str) and OUTBOX_CURSOR.fullmatch(cursor)):
94
+ fail("input", "cursor must be the cursor a page of the request ledger outbox answered", "/cursor")
95
+ if type(limit) is not int or not 1 <= limit <= REQUEST_LOG_PAGE_MAX:
96
+ fail("input", f"limit must be an integer from 1 to {REQUEST_LOG_PAGE_MAX}", "/limit")
97
+ return {"cursor": cursor, "limit": limit}
97
98
 
98
99
 
99
100
  SESSION_VERBS: tuple[SessionVerb, ...] = (
@@ -101,6 +102,8 @@ SESSION_VERBS: tuple[SessionVerb, ...] = (
101
102
  SessionVerb("GET", "task", read_nothing),
102
103
  SessionVerb("GET", "state", read_nothing),
103
104
  SessionVerb("GET", "evidence", read_nothing),
105
+ SessionVerb("POST", "requests", read_requests, hosted=True),
106
+ SessionVerb("POST", "flush", read_nothing, hosted=True),
104
107
  SessionVerb("POST", "grade", read_grade),
105
108
  SessionVerb("POST", "reset", read_nothing),
106
109
  SessionVerb("POST", "seed", read_seed),
@@ -117,14 +120,10 @@ def session_verb(method: str, name: str | None) -> SessionVerb | None:
117
120
 
118
121
 
119
122
  def served_verbs(*, hosted: bool) -> tuple[SessionVerb, ...]:
120
- """The rows a transport answers: every one on the host, the session's own
121
- (not the lease's) locally."""
122
123
  return tuple(verb for verb in SESSION_VERBS if hosted or not verb.hosted)
123
124
 
124
125
 
125
126
  def unknown_verb(method: str, name: str | None, *, hosted: bool, prefix: str) -> dict:
126
- """The 404 body for a route off the table, naming what is on it. A hosted
127
- verb sent to the local session is refused by name, not as unknown."""
128
127
  verb = session_verb(method, name)
129
128
  if verb is not None and verb.hosted and not hosted:
130
129
  return {"error": f"{verb.route} is a hosted session's verb (the platform's lease); the local session has none"}
@@ -136,7 +135,4 @@ def unknown_verb(method: str, name: str | None, *, hosted: bool, prefix: str) ->
136
135
 
137
136
 
138
137
  def refusal(error: ContractError) -> dict:
139
- """A refused call as the answer a session gives it: an unknown tool,
140
- arguments off the schema, a tool outside the task's allow-list, a write the
141
- store will not take, all ``{status: 400, body: {error, detail}}``."""
142
138
  return {"status": 400, "body": {"error": error.detail.get("code", "host"), "detail": str(error)}}
@@ -15,7 +15,8 @@ import socket
15
15
  import sys
16
16
  from pathlib import Path
17
17
 
18
- from .host import WorldHost, make_host_handler
18
+ from .cache_layout import VERSIONS_DIR, WORKERS_DIR
19
+ from .host import make_host_handler
19
20
  from .workers import (
20
21
  DEFAULT_WORKER_MEMORY_MB,
21
22
  WORKER_LEASE_S,
@@ -23,6 +24,7 @@ from .workers import (
23
24
  apply_limits,
24
25
  seal_process,
25
26
  )
27
+ from .world_host import WorldHost
26
28
 
27
29
 
28
30
  def main(argv: list[str] | None = None) -> int:
@@ -39,7 +41,7 @@ def main(argv: list[str] | None = None) -> int:
39
41
  raise SystemExit("world worker: WORLDS_WORKER_SECRET is unset; a worker only answers its supervisor")
40
42
  supervisor_pid: int | None = None
41
43
  if args.socket_fd is not None:
42
- supervisor_pid = int(os.environ.get("WORLDS_SUPERVISOR_PID") or os.getppid())
44
+ supervisor_pid = os.getppid()
43
45
  listening: socket.socket | str = socket.socket(fileno=args.socket_fd)
44
46
  else:
45
47
  listening = args.socket
@@ -48,8 +50,8 @@ def main(argv: list[str] | None = None) -> int:
48
50
 
49
51
  cache_root = Path(args.cache_root)
50
52
  world_host = WorldHost(
51
- cache_root / "workers" / args.version,
52
- versions_root=cache_root / "versions",
53
+ cache_root / WORKERS_DIR / args.version,
54
+ versions_root=cache_root / VERSIONS_DIR,
53
55
  lease_s=WORKER_LEASE_S,
54
56
  run_handlers=True,
55
57
  pinned_version=args.version,
@@ -35,6 +35,7 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
35
35
 
36
36
  from __future__ import annotations
37
37
 
38
+ import collections
38
39
  import ctypes
39
40
  import hashlib
40
41
  import http.client
@@ -49,13 +50,16 @@ import sys
49
50
  import tempfile
50
51
  import threading
51
52
  import time
52
- from collections.abc import Mapping
53
+ from collections.abc import Callable, Iterator, Mapping
54
+ from contextlib import contextmanager, suppress
53
55
  from dataclasses import dataclass, field
54
56
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
55
57
  from pathlib import Path
56
58
  from typing import Protocol
57
59
 
60
+ from .cache_layout import WORKERS_DIR
58
61
  from .grading import VerifierSandbox
62
+ from .handler_errors import log_line
59
63
 
60
64
  #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
61
65
  DEFAULT_WORKER_MEMORY_MB = 2048
@@ -81,10 +85,12 @@ FLOOR_PRECISION_BYTES = 1 << 20
81
85
  #: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
82
86
  #: spells it so, Linux spells it -1.
83
87
  MAX_RLIMIT = 2**63 - 1
84
- #: How long a worker with no sessions is kept before it is stopped.
85
88
  DEFAULT_WORKER_IDLE_S = 300.0
86
- #: How long the supervisor waits for a fresh worker to answer on its socket.
87
89
  WORKER_START_TIMEOUT_S = 30.0
90
+ #: How long one health probe waits: a worker that cannot answer it in this time is not serving.
91
+ HEALTH_TIMEOUT_S = 2.0
92
+ #: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
93
+ PROBE_LIMIT_BYTES = 64 << 10
88
94
  #: A worker's sessions live as long as the supervisor's do; the supervisor
89
95
  #: holds the leases, so the worker's own are effectively unbounded.
90
96
  WORKER_LEASE_S = 365.0 * 86400.0
@@ -94,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
94
100
  #: Headers the supervisor never forwards to a worker: the ones that describe
95
101
  #: the supervisor's connection, and its own control-plane secret.
96
102
  UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
103
+ FINISH_THREADS = 8
104
+ FINISH_SETTLE_S = 10.0
97
105
 
98
106
 
99
107
  class WorkerGone(Exception):
100
- """The worker did not answer: it exited, or never came up."""
108
+ """The worker did not answer: it exited, never came up, or dropped the request."""
109
+
110
+
111
+ class WorkerTimeout(WorkerGone):
112
+ """The worker did not answer in time; it may be alive and still working on the request."""
113
+
114
+
115
+ class WorkerUnreached(WorkerGone):
116
+ """The worker took no connection: the request never reached it."""
101
117
 
102
118
 
103
119
  class WorkerCapacity(Exception):
@@ -117,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
117
133
  self.sock = sock
118
134
 
119
135
 
136
+ def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
137
+ cut.set()
138
+ with suppress(OSError):
139
+ sock.shutdown(socket.SHUT_RDWR)
140
+
141
+
142
+ @contextmanager
143
+ def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
144
+ """Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
145
+ cut = threading.Event()
146
+ timer = threading.Timer(seconds, _hang_up, (sock, cut))
147
+ timer.start()
148
+ try:
149
+ yield cut
150
+ finally:
151
+ timer.cancel()
152
+
153
+
120
154
  def bind_worker_socket(path: str | Path) -> socket.socket:
121
155
  """The supervisor's end of a worker: a listening unix socket at ``path``,
122
156
  ``0600`` inside a ``0700`` directory, ready to hand to the worker."""
@@ -333,7 +367,6 @@ def worker_environment(
333
367
  WORLDS_WORKER_SECRET=secret,
334
368
  WORLDS_WORKER_MEMORY_MB=str(memory_mb),
335
369
  WORLDS_INGRESS_DOMAIN=worlds_domain,
336
- WORLDS_SUPERVISOR_PID=str(os.getpid()),
337
370
  )
338
371
  return env
339
372
 
@@ -389,7 +422,6 @@ class Worker:
389
422
  socket_path: Path
390
423
  secret: str
391
424
  sessions: set[str] = field(default_factory=set)
392
- #: When the worker last had a session, for idle reaping.
393
425
  idle_since: float = field(default_factory=time.time)
394
426
 
395
427
  @property
@@ -408,25 +440,78 @@ class Worker:
408
440
  *,
409
441
  control: bool = False,
410
442
  timeout: float = 60.0,
443
+ limit: int | None = None,
411
444
  ) -> tuple[int, list[tuple[str, str]], bytes]:
412
445
  """One request over the socket. ``control`` adds the worker's own
413
- secret, so the worker's control plane answers. A worker that cannot be
414
- reached raises :class:`WorkerGone`."""
446
+ secret, so the worker's control plane answers. A worker that takes no connection raises
447
+ :class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
448
+ :class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
449
+ sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
450
+ :class:`WorkerTimeout`."""
415
451
  forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
416
452
  if control:
417
453
  forward["X-Host-Secret"] = self.secret
418
- conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
454
+ started = time.monotonic()
455
+ conn = self._connected(timeout)
456
+ cut = threading.Event()
419
457
  try:
420
- conn.request(method, path, body=body or None, headers=forward)
421
- answer = conn.getresponse()
422
- payload = answer.read()
458
+ with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
459
+ conn.request(method, path, body=body or None, headers=forward)
460
+ answer = conn.getresponse()
461
+ payload = answer.read() if limit is None else answer.read(limit + 1)
462
+ if cut.is_set():
463
+ raise WorkerTimeout(
464
+ f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
465
+ )
466
+ if limit is not None and len(payload) > limit:
467
+ raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
423
468
  return answer.status, answer.getheaders(), payload
424
469
  except (OSError, http.client.HTTPException) as error:
425
- raise WorkerGone(
426
- f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}"
470
+ gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
471
+ raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
472
+ finally:
473
+ conn.close()
474
+
475
+ def _connected(self, timeout: float) -> UnixHTTPConnection:
476
+ conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
477
+ try:
478
+ conn.connect()
479
+ except OSError as error:
480
+ conn.close()
481
+ raise WorkerUnreached(
482
+ f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
427
483
  ) from error
484
+ return conn
485
+
486
+ def probe(
487
+ self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
488
+ ) -> tuple[int, bytes]:
489
+ """One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
490
+ The worker runs tenant code: an answer sent slowly or without end is cut off when the
491
+ deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
492
+ unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
493
+ conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
494
+ try:
495
+ conn.connect()
496
+ with _deadline(conn.sock, deadline_s) as cut:
497
+ conn.request("GET", path, headers={"X-Host-Secret": self.secret})
498
+ answer = conn.getresponse()
499
+ payload = answer.read(limit + 1)
500
+ except (OSError, http.client.HTTPException) as error:
501
+ raise WorkerGone(unanswered) from error
428
502
  finally:
429
503
  conn.close()
504
+ if cut.is_set() or len(payload) > limit:
505
+ raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
506
+ return answer.status, payload
507
+
508
+ def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
509
+ """Whether the worker is alive and answers its health check, whole, in time."""
510
+ try:
511
+ status, _ = self.probe("/healthz", deadline_s=timeout_s)
512
+ except WorkerGone:
513
+ return False
514
+ return status == 200
430
515
 
431
516
  def stop(self, grace_s: float = 5.0) -> None:
432
517
  if self.alive():
@@ -439,6 +524,69 @@ class Worker:
439
524
  self.socket_path.unlink(missing_ok=True)
440
525
 
441
526
 
527
+ class Finishing:
528
+ """What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
529
+ threads, one job at a time for a version in the order they came (so a worker that never answers
530
+ holds one thread, not the pool). A session with no worker is finished at once, where it is."""
531
+
532
+ def __init__(self, threads: int = FINISH_THREADS) -> None:
533
+ self.threads = threads
534
+ self._changed = threading.Condition()
535
+ self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
536
+ self._serving: set[str] = set()
537
+ self._started = 0
538
+
539
+ def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
540
+ if worker is None:
541
+ job()
542
+ return
543
+ with self._changed:
544
+ self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
545
+ self._staff()
546
+ self._changed.notify()
547
+
548
+ def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
549
+ """Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
550
+ until = time.monotonic() + deadline_s
551
+ with self._changed:
552
+ while self._waiting:
553
+ remaining = until - time.monotonic()
554
+ if remaining <= 0:
555
+ return False
556
+ self._changed.wait(remaining)
557
+ return True
558
+
559
+ def _staff(self) -> None:
560
+ if self._started < self.threads:
561
+ self._started += 1
562
+ threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
563
+
564
+ def _run(self) -> None:
565
+ while True:
566
+ with self._changed:
567
+ self._changed.wait_for(lambda: self._next() is not None)
568
+ version = self._next()
569
+ assert version is not None
570
+ job = self._waiting[version].popleft()
571
+ self._serving.add(version)
572
+ self._work(version, job)
573
+
574
+ def _next(self) -> str | None:
575
+ return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
576
+
577
+ def _work(self, version: str, job: Callable[[], None]) -> None:
578
+ try:
579
+ job()
580
+ except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
581
+ log_line(f"world host: a session did not finish: {type(error).__name__}")
582
+ finally:
583
+ with self._changed:
584
+ self._serving.discard(version)
585
+ if not self._waiting[version]:
586
+ del self._waiting[version]
587
+ self._changed.notify_all()
588
+
589
+
442
590
  class WorkerPool:
443
591
  """Every version worker on this replica, started on the first session of a
444
592
  version and stopped once it has had none for a while."""
@@ -468,7 +616,7 @@ class WorkerPool:
468
616
  "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
469
617
  )
470
618
  self.root = Path(cache_root)
471
- self.workers_root = self.root / "workers"
619
+ self.workers_root = self.root / WORKERS_DIR
472
620
  # Workers live and die with the supervisor; whatever a previous one
473
621
  # left (sockets, session files) is unreachable.
474
622
  shutil.rmtree(self.workers_root, ignore_errors=True)
@@ -491,24 +639,54 @@ class WorkerPool:
491
639
  def get(self, version_id: str) -> Worker:
492
640
  """The live worker for a version, started if there is none. Starting one
493
641
  past the budget first stops workers holding no sessions (whatever their
494
- idle time); when every one is in use, raises :class:`WorkerCapacity`."""
642
+ idle time); when every one is in use, raises :class:`WorkerCapacity`. A
643
+ dead worker that still holds sessions is never replaced here: its
644
+ sessions' files are on disk, and the supervisor recovers them through
645
+ :meth:`replace` (raises :class:`WorkerGone`)."""
495
646
  with self._lock:
496
647
  worker = self._workers.get(version_id)
497
648
  if worker is not None and worker.alive():
498
649
  return worker
650
+ if worker is not None and worker.sessions:
651
+ raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
499
652
  if worker is not None:
500
653
  worker.stop()
501
654
  del self._workers[version_id]
502
- self._make_room()
503
- worker = self._spawn(version_id)
504
- self._workers[version_id] = worker
505
- return worker
655
+ return self._start(version_id)
656
+
657
+ def replace(self, dead: Worker) -> Worker:
658
+ """A fresh worker in ``dead``'s place, its sessions left for the caller to
659
+ reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
660
+ when the pool no longer holds ``dead``."""
661
+ with self._lock:
662
+ if self._workers.get(dead.version_id) is not dead:
663
+ raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
664
+ del self._workers[dead.version_id]
665
+ dead.stop()
666
+ return self._start(dead.version_id)
667
+
668
+ def _start(self, version_id: str) -> Worker:
669
+ self._make_room()
670
+ worker = self._spawn(version_id)
671
+ self._workers[version_id] = worker
672
+ return worker
673
+
674
+ def dead(self) -> list[Worker]:
675
+ """The workers that exited still holding sessions: theirs to recover."""
676
+ with self._lock:
677
+ return [w for w in self._workers.values() if w.sessions and not w.alive()]
678
+
679
+ def live(self) -> list[Worker]:
680
+ """The workers still running, holding sessions or not."""
681
+ with self._lock:
682
+ return [w for w in self._workers.values() if w.alive()]
506
683
 
507
684
  def _make_room(self) -> None:
508
685
  """Under ``_lock``: leave one worker's worth of the budget free, by
509
- forgetting exited workers first, then stopping session-less ones
510
- longest idle first, and refusing when every live worker is in use."""
511
- exited = [v for v, w in self._workers.items() if not w.alive()]
686
+ forgetting exited workers that hold no sessions first, then stopping
687
+ session-less ones longest idle first, and refusing when every live
688
+ worker is in use."""
689
+ exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
512
690
  for version_id in exited:
513
691
  self._workers.pop(version_id).stop()
514
692
  spare = sorted(
@@ -527,15 +705,6 @@ class WorkerPool:
527
705
  with self._lock:
528
706
  return self._workers.get(version_id)
529
707
 
530
- def drop(self, version_id: str) -> Worker | None:
531
- """Stop and forget a version's worker; returns it (with the sessions
532
- it held) so the caller can forget those too."""
533
- with self._lock:
534
- worker = self._workers.pop(version_id, None)
535
- if worker is not None:
536
- worker.stop()
537
- return worker
538
-
539
708
  def reap_idle(self, now: float | None = None) -> list[str]:
540
709
  """Stop workers that have had no session for ``idle_s``; returns their versions."""
541
710
  now = time.time() if now is None else now
@@ -604,13 +773,8 @@ class WorkerPool:
604
773
  raise WorkerGone(
605
774
  f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
606
775
  )
607
- if worker.socket_path.exists():
608
- try:
609
- status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
610
- except WorkerGone:
611
- status = 0
612
- if status == 200:
613
- return
776
+ if worker.socket_path.exists() and worker.answers_health():
777
+ return
614
778
  time.sleep(0.05)
615
779
  worker.stop()
616
780
  raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")