@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""``WorkerKey``: which of a version's workers serves a session.
|
|
2
|
+
|
|
3
|
+
A version's sessions on its own data (main) and its sessions on any of its world
|
|
4
|
+
states run in two workers, never one: a state's data cannot exhaust the process
|
|
5
|
+
main's sessions are served by. The key is also what a version's turns, opens,
|
|
6
|
+
recoveries and finishes queue on, so neither side waits behind the other.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
MAIN_LANE = "main"
|
|
15
|
+
STATES_LANE = "states"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, order=True)
|
|
19
|
+
class WorkerKey:
|
|
20
|
+
version_id: str
|
|
21
|
+
states: bool = False
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def of(cls, version_id: str, world_state: bool) -> WorkerKey:
|
|
25
|
+
return cls(version_id, states=world_state)
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def name(self) -> str:
|
|
29
|
+
"""``<version>`` for main, ``<version>@states`` for its world states: the key's home, socket and queues."""
|
|
30
|
+
return f"{self.version_id}@{STATES_LANE}" if self.states else self.version_id
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def lane(self) -> str:
|
|
34
|
+
return STATES_LANE if self.states else MAIN_LANE
|
|
35
|
+
|
|
36
|
+
def home(self, workers_root: Path) -> Path:
|
|
37
|
+
"""Where this key's worker runs: its own ``WorldHost`` cache root under the pool's ``workers/``."""
|
|
38
|
+
return Path(workers_root) / self.name
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""``PeakReport``: each worker key's peak memory, told to the platform as it grows.
|
|
2
|
+
|
|
3
|
+
A worker's peak (``VmHWM``) is what serving its sessions took: for a version's main, its own
|
|
4
|
+
data; for its states lane, every state it served at once. The platform keeps the largest it is
|
|
5
|
+
told for the version, so the next open is provisioned by what the world was seen to need. A
|
|
6
|
+
key is reported once its peak has grown ``STEP_MB`` past what the platform last took, so a
|
|
7
|
+
heartbeat carries only what changed and a missed one is sent again.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import threading
|
|
13
|
+
|
|
14
|
+
from .worker_key import STATES_LANE, WorkerKey
|
|
15
|
+
|
|
16
|
+
STEP_MB = 16
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PeakReport:
|
|
20
|
+
def __init__(self) -> None:
|
|
21
|
+
self._lock = threading.Lock()
|
|
22
|
+
self._taken: dict[WorkerKey, int] = {}
|
|
23
|
+
|
|
24
|
+
def pending(self, peaks: list[tuple[WorkerKey, int]]) -> list[dict]:
|
|
25
|
+
"""The peaks worth telling: grown ``STEP_MB`` past the last one the platform took for that key. A key
|
|
26
|
+
no worker runs for is forgotten, so a worker started for it again is measured from nothing."""
|
|
27
|
+
with self._lock:
|
|
28
|
+
running = {key for key, _ in peaks}
|
|
29
|
+
self._taken = {key: mb for key, mb in self._taken.items() if key in running}
|
|
30
|
+
grown = [(key, mb) for key, mb in peaks if mb >= self._taken.get(key, 0) + STEP_MB]
|
|
31
|
+
return [{"versionId": key.version_id, "lane": key.lane, "peakMb": mb} for key, mb in grown]
|
|
32
|
+
|
|
33
|
+
def taken(self, sent: list[dict]) -> None:
|
|
34
|
+
"""The platform took a heartbeat carrying ``sent``."""
|
|
35
|
+
with self._lock:
|
|
36
|
+
for peak in sent:
|
|
37
|
+
key = WorkerKey.of(peak["versionId"], peak["lane"] == STATES_LANE)
|
|
38
|
+
self._taken[key] = max(self._taken.get(key, 0), peak["peakMb"])
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""``WorkerMemoryWatch``: each version worker's address space against its own cap, read on the lease sweep.
|
|
2
|
+
|
|
3
|
+
A worker runs out when its address space (``VmSize``) reaches its ``RLIMIT_AS``; at ``NEAR_CAP`` of that
|
|
4
|
+
it is logged once as it crosses and counted (``workers_near_cap``), so an alert fires before the
|
|
5
|
+
``MemoryError`` does. The readings, which name versions, go to the secret-gated stats and the heartbeat;
|
|
6
|
+
``/metrics``, which world code on the pod can reach, carries only their totals.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
|
|
13
|
+
from .workers import WorkerPool
|
|
14
|
+
|
|
15
|
+
NEAR_CAP = 0.8
|
|
16
|
+
WORKERS_NEAR_CAP = "workers_near_cap"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class WorkerMemoryWatch:
|
|
20
|
+
def __init__(self, pool: WorkerPool, tally: Callable[[str], None]) -> None:
|
|
21
|
+
self.pool = pool
|
|
22
|
+
self.tally = tally
|
|
23
|
+
|
|
24
|
+
def readings(self) -> list[dict]:
|
|
25
|
+
"""Each running worker's address space, cap, measured need and charge, in MiB; ``sizeMb`` None with no
|
|
26
|
+
``/proc``."""
|
|
27
|
+
return [
|
|
28
|
+
{
|
|
29
|
+
"versionId": worker.version_id,
|
|
30
|
+
"lane": worker.key.lane,
|
|
31
|
+
"sizeMb": size,
|
|
32
|
+
"capMb": worker.cap_mb,
|
|
33
|
+
"needMb": worker.need_mb,
|
|
34
|
+
"chargeMb": worker.charge_mb,
|
|
35
|
+
"nearCap": worker.near_cap,
|
|
36
|
+
}
|
|
37
|
+
for worker, size in self.pool.sizes()
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
def sweep(self) -> list[str]:
|
|
41
|
+
"""Mark each worker near its cap or not, and count the ones that just crossed. Returns a line for each,
|
|
42
|
+
which the lease sweep logs."""
|
|
43
|
+
crossed = []
|
|
44
|
+
for worker, size in self.pool.sizes():
|
|
45
|
+
near = size is not None and size >= worker.cap_mb * NEAR_CAP
|
|
46
|
+
if near and not worker.near_cap:
|
|
47
|
+
crossed.append(f"{worker.key.name} (world {worker.world!r}): {size} of its {worker.cap_mb} MiB")
|
|
48
|
+
self.tally(WORKERS_NEAR_CAP)
|
|
49
|
+
worker.near_cap = near
|
|
50
|
+
return crossed
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
"""Version workers:
|
|
1
|
+
"""Version workers: subprocesses that run world-authored code, one per :class:`WorkerKey`.
|
|
2
2
|
|
|
3
3
|
A schema world with an ``api/handlers.py`` brings its own Python. The host
|
|
4
4
|
process is shared by every tenant on the replica, so that code never runs in
|
|
5
|
-
it: the supervisor (:mod:`host`) starts
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
it: the supervisor (:mod:`host`) starts a worker per version and lane (its
|
|
6
|
+
sessions on the version's own data, and those on any of its world states), over
|
|
7
|
+
a unix socket only the supervisor can reach, and forwards those sessions to it.
|
|
8
|
+
The worker is the same :class:`host.WorldHost` in a smaller room:
|
|
8
9
|
|
|
9
10
|
* its own process, so a handler that loops, leaks or crashes takes down that
|
|
10
|
-
version's sessions on this replica and nothing else;
|
|
11
|
+
version's sessions of that lane on this replica and nothing else;
|
|
11
12
|
* rlimits on address space, open files and file size, set before any world
|
|
12
13
|
code is imported (:class:`AddressSpaceLimit` says how macOS differs);
|
|
13
14
|
* a scrubbed environment: the control-plane secret, cloud credentials and the
|
|
@@ -43,6 +44,7 @@ import os
|
|
|
43
44
|
import resource
|
|
44
45
|
import secrets
|
|
45
46
|
import shutil
|
|
47
|
+
import signal
|
|
46
48
|
import socket
|
|
47
49
|
import struct
|
|
48
50
|
import subprocess
|
|
@@ -55,28 +57,37 @@ from contextlib import contextmanager, suppress
|
|
|
55
57
|
from dataclasses import dataclass, field
|
|
56
58
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
57
59
|
from pathlib import Path
|
|
60
|
+
from types import MappingProxyType
|
|
58
61
|
from typing import Protocol
|
|
59
62
|
|
|
60
63
|
from .cache_layout import WORKERS_DIR
|
|
61
64
|
from .grading import VerifierSandbox
|
|
62
65
|
from .handler_errors import log_line
|
|
66
|
+
from .process_memory import PEAK_RESIDENT, PEAK_SIZE, RESIDENT, RESIDENT_ANON, SIZE, MemoryNeed, ProcessMemory
|
|
67
|
+
from .worker_key import WorkerKey
|
|
63
68
|
|
|
64
69
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
65
70
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
66
71
|
#: Memory every worker together may use, in MiB, unless
|
|
67
|
-
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise.
|
|
68
|
-
#:
|
|
69
|
-
#:
|
|
70
|
-
#:
|
|
72
|
+
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. A worker starts only while the
|
|
73
|
+
#: workers already running, each counted as :meth:`WorkerPool.counted_mb` says,
|
|
74
|
+
#: leave room for its need (at least a footprint). A worker past that is refused
|
|
75
|
+
#: ``at_capacity`` and placed on another replica instead of the kernel choosing
|
|
76
|
+
#: which process to kill.
|
|
71
77
|
DEFAULT_WORKER_BUDGET_MB = 4096
|
|
72
|
-
#:
|
|
73
|
-
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise
|
|
78
|
+
#: The least one worker is counted at against the budget, in MiB, unless
|
|
79
|
+
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise; a worker whose resident memory
|
|
80
|
+
#: the kernel reports higher is counted at that. Unset, a worker is counted at
|
|
74
81
|
#: its whole address-space cap: the safe reading, and a 50x over-reservation of
|
|
75
82
|
#: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
|
|
76
83
|
#: many worlds at once sets the footprint it measured, keeps the address-space
|
|
77
84
|
#: cap as the rail each worker cannot cross, and sizes the container for
|
|
78
|
-
#:
|
|
85
|
+
#: the budget plus the supervisor's share.
|
|
79
86
|
DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
|
|
87
|
+
#: The most address space one worker is ever given, in MiB, unless ``WORLDS_WORKER_MEMORY_CEILING_MB`` says
|
|
88
|
+
#: otherwise; unset, ``WORLDS_WORKER_MEMORY_MB``, so no worker's cap grows. A world measured to need more than
|
|
89
|
+
#: the cap (its peak and half again) gets that much up to this; a raised ceiling must fit the budget.
|
|
90
|
+
DEFAULT_WORKER_MEMORY_CEILING_MB: int | None = None
|
|
80
91
|
#: Largest file a worker may write (its session databases), in MiB, unless ``WORLDS_WORKER_FILE_MB`` says otherwise.
|
|
81
92
|
DEFAULT_WORKER_FILE_MB = 4096
|
|
82
93
|
#: Open files per worker: one per live session plus the sockets serving them.
|
|
@@ -97,10 +108,15 @@ WORKER_LEASE_S = 365.0 * 86400.0
|
|
|
97
108
|
#: Environment the worker inherits. Nothing else crosses: no host secret, no
|
|
98
109
|
#: cloud credentials, no platform URL.
|
|
99
110
|
INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
111
|
+
#: glibc keeps a malloc arena per thread (64 MiB of address space each) after the thread ends, and
|
|
112
|
+
#: ``RLIMIT_AS`` counts them: without this cap a worker that served many threads reads as near its cap
|
|
113
|
+
#: while holding little, and runs out of address space with memory to spare.
|
|
114
|
+
WORKER_MALLOC_ARENAS = "2"
|
|
100
115
|
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
101
116
|
#: the supervisor's connection, and its own control-plane secret.
|
|
102
117
|
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
103
118
|
FINISH_THREADS = 8
|
|
119
|
+
NO_WORKERS = MappingProxyType({"maxWorkers": 0, "workerMemoryMb": 0, "workerKeysInUse": (), "workerCapMb": 0})
|
|
104
120
|
FINISH_SETTLE_S = 10.0
|
|
105
121
|
|
|
106
122
|
|
|
@@ -236,6 +252,7 @@ class UnixHTTPServer(ThreadingHTTPServer):
|
|
|
236
252
|
|
|
237
253
|
|
|
238
254
|
_PR_SET_DUMPABLE = 4
|
|
255
|
+
_PR_SET_PDEATHSIG = 1
|
|
239
256
|
|
|
240
257
|
|
|
241
258
|
def seal_process() -> bool:
|
|
@@ -253,6 +270,16 @@ def seal_process() -> bool:
|
|
|
253
270
|
return False
|
|
254
271
|
|
|
255
272
|
|
|
273
|
+
def die_with_parent(parent: int) -> None:
|
|
274
|
+
"""Have the kernel kill this process the moment ``parent`` exits (Linux), so no code of this process
|
|
275
|
+
can postpone it; and leave at once if ``parent`` is gone already. Elsewhere the caller keeps its own watch."""
|
|
276
|
+
if sys.platform.startswith("linux"):
|
|
277
|
+
with suppress(OSError, AttributeError):
|
|
278
|
+
ctypes.CDLL(None, use_errno=True).prctl(_PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0)
|
|
279
|
+
if os.getppid() != parent:
|
|
280
|
+
os._exit(1)
|
|
281
|
+
|
|
282
|
+
|
|
256
283
|
class Kernel(Protocol):
|
|
257
284
|
"""Where a process's limits are read and set: the ``resource`` module."""
|
|
258
285
|
|
|
@@ -369,6 +396,7 @@ def worker_environment(
|
|
|
369
396
|
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
370
397
|
WORLDS_WORKER_FILE_MB=str(file_mb),
|
|
371
398
|
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
399
|
+
MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS,
|
|
372
400
|
)
|
|
373
401
|
return env
|
|
374
402
|
|
|
@@ -380,7 +408,7 @@ def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None)
|
|
|
380
408
|
secret, no cloud credentials, no platform URL."""
|
|
381
409
|
inherited: Mapping[str, str] = os.environ if source is None else source
|
|
382
410
|
env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
|
|
383
|
-
env
|
|
411
|
+
env.update(HOME=str(home), MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS)
|
|
384
412
|
return env
|
|
385
413
|
|
|
386
414
|
|
|
@@ -419,12 +447,26 @@ class ConfinedVerifierSandbox(VerifierSandbox):
|
|
|
419
447
|
|
|
420
448
|
@dataclass
|
|
421
449
|
class Worker:
|
|
422
|
-
|
|
450
|
+
key: WorkerKey
|
|
423
451
|
process: subprocess.Popen
|
|
424
452
|
socket_path: Path
|
|
425
453
|
secret: str
|
|
426
454
|
sessions: set[str] = field(default_factory=set)
|
|
427
455
|
idle_since: float = field(default_factory=time.time)
|
|
456
|
+
world: str = ""
|
|
457
|
+
#: World code here raised ``MemoryError``: the worker serves no new request and is recycled.
|
|
458
|
+
poisoned: bool = False
|
|
459
|
+
#: Its address-space cap (``RLIMIT_AS``), given at start; what it is counted at is ``charge_mb`` or its RSS.
|
|
460
|
+
cap_mb: int = DEFAULT_WORKER_MEMORY_MB
|
|
461
|
+
#: What its world was measured to need, which sizes a replacement; ``charge_mb`` what the budget
|
|
462
|
+
#: counts it at, at most its cap and only as far as the budget had room when it was charged.
|
|
463
|
+
need_mb: int = 0
|
|
464
|
+
charge_mb: int = 0
|
|
465
|
+
near_cap: bool = False
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def version_id(self) -> str:
|
|
469
|
+
return self.key.version_id
|
|
428
470
|
|
|
429
471
|
@property
|
|
430
472
|
def pid(self) -> int:
|
|
@@ -433,6 +475,10 @@ class Worker:
|
|
|
433
475
|
def alive(self) -> bool:
|
|
434
476
|
return self.process.poll() is None
|
|
435
477
|
|
|
478
|
+
def serves(self) -> bool:
|
|
479
|
+
"""Alive and not poisoned: a request may go to it."""
|
|
480
|
+
return self.alive() and not self.poisoned
|
|
481
|
+
|
|
436
482
|
def request(
|
|
437
483
|
self,
|
|
438
484
|
method: str,
|
|
@@ -528,7 +574,7 @@ class Worker:
|
|
|
528
574
|
|
|
529
575
|
class Finishing:
|
|
530
576
|
"""What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
|
|
531
|
-
threads, one job at a time for a
|
|
577
|
+
threads, one job at a time for a worker key in the order they came (so a worker that never answers
|
|
532
578
|
holds one thread, not the pool). A session with no worker is finished at once, where it is."""
|
|
533
579
|
|
|
534
580
|
def __init__(self, threads: int = FINISH_THREADS) -> None:
|
|
@@ -543,7 +589,7 @@ class Finishing:
|
|
|
543
589
|
job()
|
|
544
590
|
return
|
|
545
591
|
with self._changed:
|
|
546
|
-
self._waiting.setdefault(worker.
|
|
592
|
+
self._waiting.setdefault(worker.key.name, collections.deque()).append(job)
|
|
547
593
|
self._staff()
|
|
548
594
|
self._changed.notify()
|
|
549
595
|
|
|
@@ -590,8 +636,8 @@ class Finishing:
|
|
|
590
636
|
|
|
591
637
|
|
|
592
638
|
class WorkerPool:
|
|
593
|
-
"""Every version worker on this replica,
|
|
594
|
-
|
|
639
|
+
"""Every version worker on this replica, one per :class:`WorkerKey`, started on
|
|
640
|
+
the key's first session and stopped once it has had none for a while."""
|
|
595
641
|
|
|
596
642
|
def __init__(
|
|
597
643
|
self,
|
|
@@ -604,6 +650,8 @@ class WorkerPool:
|
|
|
604
650
|
idle_s: float = DEFAULT_WORKER_IDLE_S,
|
|
605
651
|
python: str = sys.executable,
|
|
606
652
|
file_mb: int = DEFAULT_WORKER_FILE_MB,
|
|
653
|
+
memory: ProcessMemory | None = None,
|
|
654
|
+
ceiling_mb: int | None = DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
607
655
|
):
|
|
608
656
|
if file_mb <= 0:
|
|
609
657
|
raise ValueError(f"a worker's file cap ({file_mb} MiB) must be at least 1 MiB (WORLDS_WORKER_FILE_MB)")
|
|
@@ -615,11 +663,22 @@ class WorkerPool:
|
|
|
615
663
|
f"address-space cap ({memory_mb} MiB) (WORLDS_WORKER_FOOTPRINT_MB, "
|
|
616
664
|
"WORLDS_WORKER_MEMORY_MB)"
|
|
617
665
|
)
|
|
666
|
+
ceiling_mb = memory_mb if ceiling_mb is None else ceiling_mb
|
|
667
|
+
if ceiling_mb < memory_mb:
|
|
668
|
+
raise ValueError(
|
|
669
|
+
f"the worker ceiling ({ceiling_mb} MiB) is under the worker cap ({memory_mb} MiB) "
|
|
670
|
+
"(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_MEMORY_MB)"
|
|
671
|
+
)
|
|
618
672
|
if budget_mb < footprint_mb:
|
|
619
673
|
raise ValueError(
|
|
620
674
|
f"the worker budget ({budget_mb} MiB) holds no worker of {footprint_mb} MiB "
|
|
621
675
|
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
622
676
|
)
|
|
677
|
+
if ceiling_mb > memory_mb and ceiling_mb > budget_mb:
|
|
678
|
+
raise ValueError(
|
|
679
|
+
f"the worker ceiling ({ceiling_mb} MiB) is over the worker budget ({budget_mb} MiB) "
|
|
680
|
+
"(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_BUDGET_MB)"
|
|
681
|
+
)
|
|
623
682
|
self.root = Path(cache_root)
|
|
624
683
|
self.workers_root = self.root / WORKERS_DIR
|
|
625
684
|
# Workers live and die with the supervisor; whatever a previous one
|
|
@@ -633,48 +692,98 @@ class WorkerPool:
|
|
|
633
692
|
self.sockets = Path(self._sockets.name)
|
|
634
693
|
self.worlds_domain = worlds_domain
|
|
635
694
|
self.memory_mb = memory_mb
|
|
695
|
+
self.ceiling_mb = ceiling_mb
|
|
636
696
|
self.budget_mb = budget_mb
|
|
637
697
|
self.footprint_mb = footprint_mb
|
|
638
698
|
self.file_mb = file_mb
|
|
639
699
|
self.max_workers = budget_mb // footprint_mb
|
|
640
700
|
self.idle_s = idle_s
|
|
641
701
|
self.python = python
|
|
702
|
+
self.memory = memory or ProcessMemory()
|
|
642
703
|
self._lock = threading.Lock()
|
|
643
|
-
self._workers: dict[
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
704
|
+
self._workers: dict[WorkerKey, Worker] = {}
|
|
705
|
+
self._roster: tuple[Worker, ...] = ()
|
|
706
|
+
|
|
707
|
+
def cap_for(self, need_mb: int) -> int:
|
|
708
|
+
"""The address space a worker needing ``need_mb`` is given: the cap, or more up to the ceiling."""
|
|
709
|
+
return min(self.ceiling_mb, max(self.memory_mb, need_mb))
|
|
710
|
+
|
|
711
|
+
def _bounded(self, need_mb: int) -> int:
|
|
712
|
+
"""A need as the budget charges it: never more than the ceiling, which is all a worker can hold, nor
|
|
713
|
+
than the whole budget."""
|
|
714
|
+
return min(need_mb, self.ceiling_mb, self.budget_mb)
|
|
715
|
+
|
|
716
|
+
def get(self, key: WorkerKey, *, world: str = "", need_mb: int = 0) -> Worker:
|
|
717
|
+
"""The live worker for a key, started if there is none (labelled with its ``world``, given
|
|
718
|
+
:meth:`cap_for` its ``need_mb`` and charged it); a running one keeps the larger need and is
|
|
719
|
+
charged it up to the cap it keeps (a process's cap only goes down) as far as the budget has
|
|
720
|
+
room, never refusing its own lane's open. Starting one
|
|
647
721
|
past the budget first stops workers holding no sessions (whatever their
|
|
648
722
|
idle time); when every one is in use, raises :class:`WorkerCapacity`. A
|
|
649
723
|
dead worker that still holds sessions is never replaced here: its
|
|
650
724
|
sessions' files are on disk, and the supervisor recovers them through
|
|
651
725
|
:meth:`replace` (raises :class:`WorkerGone`)."""
|
|
726
|
+
need_mb = self._bounded(need_mb)
|
|
652
727
|
with self._lock:
|
|
653
|
-
worker = self._workers.get(
|
|
728
|
+
worker = self._workers.get(key)
|
|
654
729
|
if worker is not None and worker.alive():
|
|
730
|
+
self._raise_need(worker, need_mb)
|
|
655
731
|
return worker
|
|
656
732
|
if worker is not None and worker.sessions:
|
|
657
|
-
raise WorkerGone(f"worker for
|
|
733
|
+
raise WorkerGone(f"worker for {key.name} (pid {worker.pid}) exited holding sessions")
|
|
658
734
|
if worker is not None:
|
|
659
735
|
worker.stop()
|
|
660
|
-
|
|
661
|
-
|
|
736
|
+
self._drop(key)
|
|
737
|
+
self._make_room(need_mb)
|
|
738
|
+
return self._start(key, world, need_mb)
|
|
662
739
|
|
|
663
740
|
def replace(self, dead: Worker) -> Worker:
|
|
664
741
|
"""A fresh worker in ``dead``'s place, its sessions left for the caller to
|
|
665
742
|
reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
|
|
666
|
-
when the pool no longer holds ``dead
|
|
743
|
+
when the pool no longer holds ``dead``, never by the budget: it takes the
|
|
744
|
+
place of a worker the budget held, so sessions are never lost to it. It needs
|
|
745
|
+
what ``dead`` was measured to need (:meth:`outgrown` raises that for a worker that ran out), but is
|
|
746
|
+
charged no more than the budget leaves it (at least a footprint), and its cap is never under
|
|
747
|
+
``dead``'s."""
|
|
667
748
|
with self._lock:
|
|
668
|
-
if self._workers.get(dead.
|
|
669
|
-
raise WorkerGone(f"worker for
|
|
670
|
-
|
|
749
|
+
if self._workers.get(dead.key) is not dead:
|
|
750
|
+
raise WorkerGone(f"worker for {dead.key.name} (pid {dead.pid}) was replaced already")
|
|
751
|
+
self._drop(dead.key)
|
|
671
752
|
dead.stop()
|
|
672
|
-
|
|
753
|
+
self._evict_spare(dead.need_mb)
|
|
754
|
+
charge = min(dead.need_mb, max(self.footprint_mb, self._free_mb(self._roster)))
|
|
755
|
+
return self._start(dead.key, dead.world, dead.need_mb, charge_mb=charge, cap_floor_mb=dead.cap_mb)
|
|
756
|
+
|
|
757
|
+
def outgrown(self, worker: Worker) -> None:
|
|
758
|
+
"""``worker`` ran out of address space: its peak address space (``VmPeak``) and half again is what
|
|
759
|
+
it needs from now on, so its replacement gets a larger cap up to the ceiling. Read before it stops."""
|
|
760
|
+
peak = self.memory.status(worker.pid).mb(PEAK_SIZE)
|
|
761
|
+
grown = MemoryNeed(peak).mb if peak is not None else 0
|
|
762
|
+
worker.need_mb = max(worker.need_mb, self._bounded(grown))
|
|
763
|
+
|
|
764
|
+
def _raise_need(self, worker: Worker, need_mb: int) -> None:
|
|
765
|
+
"""Under ``_lock``: ``worker`` asked for ``need_mb`` by another open of its lane keeps the larger need
|
|
766
|
+
and is charged it up to its cap as far as what the budget leaves allows. The open is never refused:
|
|
767
|
+
placement sends a lane's sessions to the replica that runs it."""
|
|
768
|
+
worker.need_mb = max(worker.need_mb, need_mb)
|
|
769
|
+
room = self._counted(worker) + max(0, self._free_mb(self._roster))
|
|
770
|
+
worker.charge_mb = max(worker.charge_mb, min(worker.need_mb, worker.cap_mb, room))
|
|
771
|
+
|
|
772
|
+
def _start(
|
|
773
|
+
self, key: WorkerKey, world: str, need_mb: int, charge_mb: int | None = None, cap_floor_mb: int = 0
|
|
774
|
+
) -> Worker:
|
|
775
|
+
charge = need_mb if charge_mb is None else charge_mb
|
|
776
|
+
worker = self._spawn(key, world, max(cap_floor_mb, self.cap_for(charge)))
|
|
777
|
+
worker.need_mb = need_mb
|
|
778
|
+
worker.charge_mb = min(charge, worker.cap_mb)
|
|
779
|
+
self._workers[key] = worker
|
|
780
|
+
self._roster = tuple(self._workers.values())
|
|
781
|
+
return worker
|
|
673
782
|
|
|
674
|
-
def
|
|
675
|
-
|
|
676
|
-
worker = self.
|
|
677
|
-
self.
|
|
783
|
+
def _drop(self, key: WorkerKey) -> Worker:
|
|
784
|
+
"""Under ``_lock``."""
|
|
785
|
+
worker = self._workers.pop(key)
|
|
786
|
+
self._roster = tuple(self._workers.values())
|
|
678
787
|
return worker
|
|
679
788
|
|
|
680
789
|
def dead(self) -> list[Worker]:
|
|
@@ -687,55 +796,114 @@ class WorkerPool:
|
|
|
687
796
|
with self._lock:
|
|
688
797
|
return [w for w in self._workers.values() if w.alive()]
|
|
689
798
|
|
|
690
|
-
def _make_room(self) -> None:
|
|
691
|
-
"""Under ``_lock``: leave
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
for
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
799
|
+
def _make_room(self, need_mb: int) -> None:
|
|
800
|
+
"""Under ``_lock``: leave room for a worker charged ``need_mb`` (at least a footprint) of the budget
|
|
801
|
+
(:meth:`_evict_spare`), and refuse when what the workers left are counted at leaves less; refused
|
|
802
|
+
before anything is stopped when stopping every worker holding no session would not free it, so
|
|
803
|
+
an open that cannot fit here stops no other world's warm worker."""
|
|
804
|
+
room = max(self.footprint_mb, need_mb)
|
|
805
|
+
spare = sum(self._counted(worker) for worker in self._roster if not worker.sessions)
|
|
806
|
+
self._refuse_short(room, self._free_mb(self._roster) + spare)
|
|
807
|
+
self._evict_spare(need_mb)
|
|
808
|
+
self._refuse_short(room, self._free_mb(self._roster))
|
|
809
|
+
|
|
810
|
+
def _refuse_short(self, room: int, free: int) -> None:
|
|
811
|
+
if free >= room:
|
|
812
|
+
return
|
|
813
|
+
raise WorkerCapacity(
|
|
814
|
+
f"this host's {len(self._workers)} version workers are counted at {self.counted_mb()} of "
|
|
815
|
+
f"{self.budget_mb} MiB (each what it needs or holds, at least {self.footprint_mb} MiB), which "
|
|
816
|
+
f"leaves no room for one needing {room} MiB; place elsewhere"
|
|
701
817
|
)
|
|
702
|
-
while len(self._workers) >= self.max_workers and spare:
|
|
703
|
-
self._workers.pop(spare.pop(0).version_id).stop()
|
|
704
|
-
if len(self._workers) >= self.max_workers:
|
|
705
|
-
raise WorkerCapacity(
|
|
706
|
-
f"this host's {self.max_workers} version workers ({self.budget_mb} MiB at "
|
|
707
|
-
f"{self.footprint_mb} MiB each) all hold sessions; place elsewhere"
|
|
708
|
-
)
|
|
709
818
|
|
|
710
|
-
def
|
|
819
|
+
def _evict_spare(self, need_mb: int = 0) -> None:
|
|
820
|
+
"""Under ``_lock``: forget exited workers that hold no sessions, then stop session-less ones,
|
|
821
|
+
longest idle first, until ``need_mb`` (at least a footprint) of the budget is free or none is
|
|
822
|
+
left to stop."""
|
|
823
|
+
room = max(self.footprint_mb, need_mb)
|
|
824
|
+
exited = [k for k, w in self._workers.items() if not w.alive() and not w.sessions]
|
|
825
|
+
for key in exited:
|
|
826
|
+
self._drop(key).stop()
|
|
827
|
+
spare = sorted((w for w in self._workers.values() if not w.sessions), key=lambda w: w.idle_since)
|
|
828
|
+
while self._free_mb(self._roster) < room and spare:
|
|
829
|
+
self._drop(spare.pop(0).key).stop()
|
|
830
|
+
|
|
831
|
+
def _counted(self, worker: Worker) -> int:
|
|
832
|
+
"""What ``worker`` takes of the budget: what it was charged, or the memory the kernel counts as its own
|
|
833
|
+
while it runs (``RssAnon``, else ``VmRSS``), whichever is more, never under the footprint."""
|
|
834
|
+
status = self.memory.status(worker.pid) if worker.alive() else None
|
|
835
|
+
measured = (status.mb(RESIDENT_ANON) or status.mb(RESIDENT)) if status is not None else None
|
|
836
|
+
return max(self.footprint_mb, worker.charge_mb, measured or 0)
|
|
837
|
+
|
|
838
|
+
def _free_mb(self, workers: tuple[Worker, ...]) -> int:
|
|
839
|
+
return self.budget_mb - sum(self._counted(worker) for worker in workers)
|
|
840
|
+
|
|
841
|
+
def counted_mb(self) -> int:
|
|
842
|
+
"""What every worker here is counted at against the budget, in MiB. Read off the last roster, never
|
|
843
|
+
under ``_lock``, which a starting or stopping worker holds for seconds: the heartbeat must not wait."""
|
|
844
|
+
return sum(self._counted(worker) for worker in self._roster)
|
|
845
|
+
|
|
846
|
+
def capacity(self) -> int:
|
|
847
|
+
"""The workers this replica could hold now: those it runs plus as many more as fit at the footprint,
|
|
848
|
+
so a replica whose workers are heavy advertises fewer slots than ``max_workers``. Lock-free, as
|
|
849
|
+
:meth:`counted_mb`."""
|
|
850
|
+
roster = self._roster
|
|
851
|
+
return len(roster) + max(0, self._free_mb(roster)) // self.footprint_mb
|
|
852
|
+
|
|
853
|
+
def peaks(self) -> list[tuple[WorkerKey, int]]:
|
|
854
|
+
"""Each running worker's peak resident memory (``VmHWM``) by key, where the kernel says. Lock-free."""
|
|
855
|
+
live = [worker for worker in self._roster if worker.alive()]
|
|
856
|
+
readings = [(worker.key, self.memory.status(worker.pid).mb(PEAK_RESIDENT)) for worker in live]
|
|
857
|
+
return sorted((key, mb) for key, mb in readings if mb is not None)
|
|
858
|
+
|
|
859
|
+
def stats(self) -> dict:
|
|
860
|
+
"""What the heartbeat says of this pool, lock-free: the slots that fit now, what the workers are
|
|
861
|
+
counted at, the keys they run for and the largest worker this host gives."""
|
|
862
|
+
return {
|
|
863
|
+
"maxWorkers": self.capacity(),
|
|
864
|
+
"workerMemoryMb": self.counted_mb(),
|
|
865
|
+
"workerKeysInUse": [{"versionId": key.version_id, "lane": key.lane} for key in self.keys_in_use()],
|
|
866
|
+
"workerCapMb": self.ceiling_mb,
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
def keys_in_use(self) -> list[WorkerKey]:
|
|
870
|
+
"""The keys a worker runs for here, so placement knows which opens need no new worker. Lock-free."""
|
|
871
|
+
return sorted(worker.key for worker in self._roster if worker.alive())
|
|
872
|
+
|
|
873
|
+
def peek(self, key: WorkerKey) -> Worker | None:
|
|
711
874
|
with self._lock:
|
|
712
|
-
return self._workers.get(
|
|
875
|
+
return self._workers.get(key)
|
|
713
876
|
|
|
714
877
|
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
715
|
-
"""Stop workers that have had no session for ``idle_s``; returns their
|
|
878
|
+
"""Stop workers that have had no session for ``idle_s``; returns their keys' names."""
|
|
716
879
|
now = time.time() if now is None else now
|
|
717
880
|
with self._lock:
|
|
718
881
|
idle = [
|
|
719
|
-
|
|
720
|
-
for w in self._workers.
|
|
882
|
+
key
|
|
883
|
+
for key, w in self._workers.items()
|
|
721
884
|
if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
|
|
722
885
|
]
|
|
723
|
-
stopped = [self.
|
|
886
|
+
stopped = [self._drop(key) for key in idle]
|
|
724
887
|
for worker in stopped:
|
|
725
888
|
worker.stop()
|
|
726
|
-
return idle
|
|
889
|
+
return [key.name for key in idle]
|
|
727
890
|
|
|
728
891
|
def stop_all(self) -> None:
|
|
729
892
|
with self._lock:
|
|
730
893
|
workers = list(self._workers.values())
|
|
731
894
|
self._workers.clear()
|
|
895
|
+
self._roster = ()
|
|
732
896
|
for worker in workers:
|
|
733
897
|
worker.stop()
|
|
734
898
|
|
|
735
|
-
def
|
|
736
|
-
|
|
899
|
+
def sizes(self) -> list[tuple[Worker, int | None]]:
|
|
900
|
+
"""Each running worker with the address space the kernel counts it at (``VmSize``). Lock-free."""
|
|
901
|
+
return [(worker, self.memory.status(worker.pid).mb(SIZE)) for worker in self._roster if worker.alive()]
|
|
902
|
+
|
|
903
|
+
def _spawn(self, key: WorkerKey, world: str, cap_mb: int) -> Worker:
|
|
904
|
+
home = key.home(self.workers_root)
|
|
737
905
|
home.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
738
|
-
socket_path = self.sockets / f"{hashlib.sha256(
|
|
906
|
+
socket_path = self.sockets / f"{hashlib.sha256(key.name.encode()).hexdigest()[:16]}.sock"
|
|
739
907
|
secret = secrets.token_urlsafe(24)
|
|
740
908
|
# The supervisor binds and listens; the worker inherits the descriptor
|
|
741
909
|
# and serves on it, so the socket is the supervisor's from the start.
|
|
@@ -749,7 +917,8 @@ class WorkerPool:
|
|
|
749
917
|
"--cache-root",
|
|
750
918
|
str(self.root),
|
|
751
919
|
"--version",
|
|
752
|
-
version_id,
|
|
920
|
+
key.version_id,
|
|
921
|
+
*(["--states"] if key.states else []),
|
|
753
922
|
"--socket-fd",
|
|
754
923
|
str(listening.fileno()),
|
|
755
924
|
],
|
|
@@ -757,7 +926,7 @@ class WorkerPool:
|
|
|
757
926
|
secret=secret,
|
|
758
927
|
home=home,
|
|
759
928
|
worlds_domain=self.worlds_domain,
|
|
760
|
-
memory_mb=
|
|
929
|
+
memory_mb=cap_mb,
|
|
761
930
|
file_mb=self.file_mb,
|
|
762
931
|
),
|
|
763
932
|
cwd=str(home),
|
|
@@ -769,7 +938,7 @@ class WorkerPool:
|
|
|
769
938
|
)
|
|
770
939
|
finally:
|
|
771
940
|
listening.close()
|
|
772
|
-
worker = Worker(
|
|
941
|
+
worker = Worker(key, process, socket_path, secret, world=world, cap_mb=cap_mb)
|
|
773
942
|
self._await_ready(worker)
|
|
774
943
|
return worker
|
|
775
944
|
|
|
@@ -778,10 +947,10 @@ class WorkerPool:
|
|
|
778
947
|
while time.time() < deadline:
|
|
779
948
|
if not worker.alive():
|
|
780
949
|
raise WorkerGone(
|
|
781
|
-
f"worker for
|
|
950
|
+
f"worker for {worker.key.name} exited with {worker.process.returncode} on start"
|
|
782
951
|
)
|
|
783
952
|
if worker.socket_path.exists() and worker.answers_health():
|
|
784
953
|
return
|
|
785
954
|
time.sleep(0.05)
|
|
786
955
|
worker.stop()
|
|
787
|
-
raise WorkerGone(f"worker for
|
|
956
|
+
raise WorkerGone(f"worker for {worker.key.name} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|