@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
"""Version workers:
|
|
1
|
+
"""Version workers: subprocesses that run world-authored code, one per :class:`WorkerKey`.
|
|
2
2
|
|
|
3
3
|
A schema world with an ``api/handlers.py`` brings its own Python. The host
|
|
4
4
|
process is shared by every tenant on the replica, so that code never runs in
|
|
5
|
-
it: the supervisor (:mod:`host`) starts
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
it: the supervisor (:mod:`host`) starts a worker per version and lane (its
|
|
6
|
+
sessions on the version's own data, and those on any of its world states), over
|
|
7
|
+
a unix socket only the supervisor can reach, and forwards those sessions to it.
|
|
8
|
+
The worker is the same :class:`host.WorldHost` in a smaller room:
|
|
8
9
|
|
|
9
10
|
* its own process, so a handler that loops, leaks or crashes takes down that
|
|
10
|
-
version's sessions on this replica and nothing else;
|
|
11
|
+
version's sessions of that lane on this replica and nothing else;
|
|
11
12
|
* rlimits on address space, open files and file size, set before any world
|
|
12
13
|
code is imported (:class:`AddressSpaceLimit` says how macOS differs);
|
|
13
14
|
* a scrubbed environment: the control-plane secret, cloud credentials and the
|
|
@@ -43,6 +44,7 @@ import os
|
|
|
43
44
|
import resource
|
|
44
45
|
import secrets
|
|
45
46
|
import shutil
|
|
47
|
+
import signal
|
|
46
48
|
import socket
|
|
47
49
|
import struct
|
|
48
50
|
import subprocess
|
|
@@ -55,29 +57,38 @@ from contextlib import contextmanager, suppress
|
|
|
55
57
|
from dataclasses import dataclass, field
|
|
56
58
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
57
59
|
from pathlib import Path
|
|
60
|
+
from types import MappingProxyType
|
|
58
61
|
from typing import Protocol
|
|
59
62
|
|
|
60
63
|
from .cache_layout import WORKERS_DIR
|
|
61
64
|
from .grading import VerifierSandbox
|
|
62
65
|
from .handler_errors import log_line
|
|
66
|
+
from .process_memory import PEAK_RESIDENT, PEAK_SIZE, RESIDENT, RESIDENT_ANON, SIZE, MemoryNeed, ProcessMemory
|
|
67
|
+
from .worker_key import WorkerKey
|
|
63
68
|
|
|
64
69
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
65
70
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
66
71
|
#: Memory every worker together may use, in MiB, unless
|
|
67
|
-
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise.
|
|
68
|
-
#:
|
|
69
|
-
#:
|
|
70
|
-
#:
|
|
72
|
+
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. A worker starts only while the
|
|
73
|
+
#: workers already running, each counted as :meth:`WorkerPool.counted_mb` says,
|
|
74
|
+
#: leave room for its need (at least a footprint). A worker past that is refused
|
|
75
|
+
#: ``at_capacity`` and placed on another replica instead of the kernel choosing
|
|
76
|
+
#: which process to kill.
|
|
71
77
|
DEFAULT_WORKER_BUDGET_MB = 4096
|
|
72
|
-
#:
|
|
73
|
-
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise
|
|
78
|
+
#: The least one worker is counted at against the budget, in MiB, unless
|
|
79
|
+
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise; a worker whose resident memory
|
|
80
|
+
#: the kernel reports higher is counted at that. Unset, a worker is counted at
|
|
74
81
|
#: its whole address-space cap: the safe reading, and a 50x over-reservation of
|
|
75
82
|
#: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
|
|
76
83
|
#: many worlds at once sets the footprint it measured, keeps the address-space
|
|
77
84
|
#: cap as the rail each worker cannot cross, and sizes the container for
|
|
78
|
-
#:
|
|
85
|
+
#: the budget plus the supervisor's share.
|
|
79
86
|
DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
|
|
80
|
-
#:
|
|
87
|
+
#: The most address space one worker is ever given, in MiB, unless ``WORLDS_WORKER_MEMORY_CEILING_MB`` says
|
|
88
|
+
#: otherwise; unset, ``WORLDS_WORKER_MEMORY_MB``, so no worker's cap grows. A world measured to need more than
|
|
89
|
+
#: the cap (its peak and half again) gets that much up to this; a raised ceiling must fit the budget.
|
|
90
|
+
DEFAULT_WORKER_MEMORY_CEILING_MB: int | None = None
|
|
91
|
+
#: Largest file a worker may write (its session databases), in MiB, unless ``WORLDS_WORKER_FILE_MB`` says otherwise.
|
|
81
92
|
DEFAULT_WORKER_FILE_MB = 4096
|
|
82
93
|
#: Open files per worker: one per live session plus the sockets serving them.
|
|
83
94
|
DEFAULT_WORKER_NOFILE = 4096
|
|
@@ -97,10 +108,15 @@ WORKER_LEASE_S = 365.0 * 86400.0
|
|
|
97
108
|
#: Environment the worker inherits. Nothing else crosses: no host secret, no
|
|
98
109
|
#: cloud credentials, no platform URL.
|
|
99
110
|
INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
111
|
+
#: glibc keeps a malloc arena per thread (64 MiB of address space each) after the thread ends, and
|
|
112
|
+
#: ``RLIMIT_AS`` counts them: without this cap a worker that served many threads reads as near its cap
|
|
113
|
+
#: while holding little, and runs out of address space with memory to spare.
|
|
114
|
+
WORKER_MALLOC_ARENAS = "2"
|
|
100
115
|
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
101
116
|
#: the supervisor's connection, and its own control-plane secret.
|
|
102
117
|
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
103
118
|
FINISH_THREADS = 8
|
|
119
|
+
NO_WORKERS = MappingProxyType({"maxWorkers": 0, "workerMemoryMb": 0, "workerKeysInUse": (), "workerCapMb": 0})
|
|
104
120
|
FINISH_SETTLE_S = 10.0
|
|
105
121
|
|
|
106
122
|
|
|
@@ -236,6 +252,7 @@ class UnixHTTPServer(ThreadingHTTPServer):
|
|
|
236
252
|
|
|
237
253
|
|
|
238
254
|
_PR_SET_DUMPABLE = 4
|
|
255
|
+
_PR_SET_PDEATHSIG = 1
|
|
239
256
|
|
|
240
257
|
|
|
241
258
|
def seal_process() -> bool:
|
|
@@ -253,6 +270,16 @@ def seal_process() -> bool:
|
|
|
253
270
|
return False
|
|
254
271
|
|
|
255
272
|
|
|
273
|
+
def die_with_parent(parent: int) -> None:
|
|
274
|
+
"""Have the kernel kill this process the moment ``parent`` exits (Linux), so no code of this process
|
|
275
|
+
can postpone it; and leave at once if ``parent`` is gone already. Elsewhere the caller keeps its own watch."""
|
|
276
|
+
if sys.platform.startswith("linux"):
|
|
277
|
+
with suppress(OSError, AttributeError):
|
|
278
|
+
ctypes.CDLL(None, use_errno=True).prctl(_PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0)
|
|
279
|
+
if os.getppid() != parent:
|
|
280
|
+
os._exit(1)
|
|
281
|
+
|
|
282
|
+
|
|
256
283
|
class Kernel(Protocol):
|
|
257
284
|
"""Where a process's limits are read and set: the ``resource`` module."""
|
|
258
285
|
|
|
@@ -355,6 +382,7 @@ def worker_environment(
|
|
|
355
382
|
home: Path,
|
|
356
383
|
worlds_domain: str,
|
|
357
384
|
memory_mb: int,
|
|
385
|
+
file_mb: int = DEFAULT_WORKER_FILE_MB,
|
|
358
386
|
source: Mapping[str, str] | None = None,
|
|
359
387
|
) -> dict[str, str]:
|
|
360
388
|
"""The environment a worker is started with: the interpreter's own needs,
|
|
@@ -366,7 +394,9 @@ def worker_environment(
|
|
|
366
394
|
HOME=str(home),
|
|
367
395
|
WORLDS_WORKER_SECRET=secret,
|
|
368
396
|
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
397
|
+
WORLDS_WORKER_FILE_MB=str(file_mb),
|
|
369
398
|
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
399
|
+
MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS,
|
|
370
400
|
)
|
|
371
401
|
return env
|
|
372
402
|
|
|
@@ -378,7 +408,7 @@ def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None)
|
|
|
378
408
|
secret, no cloud credentials, no platform URL."""
|
|
379
409
|
inherited: Mapping[str, str] = os.environ if source is None else source
|
|
380
410
|
env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
|
|
381
|
-
env
|
|
411
|
+
env.update(HOME=str(home), MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS)
|
|
382
412
|
return env
|
|
383
413
|
|
|
384
414
|
|
|
@@ -417,12 +447,26 @@ class ConfinedVerifierSandbox(VerifierSandbox):
|
|
|
417
447
|
|
|
418
448
|
@dataclass
|
|
419
449
|
class Worker:
|
|
420
|
-
|
|
450
|
+
key: WorkerKey
|
|
421
451
|
process: subprocess.Popen
|
|
422
452
|
socket_path: Path
|
|
423
453
|
secret: str
|
|
424
454
|
sessions: set[str] = field(default_factory=set)
|
|
425
455
|
idle_since: float = field(default_factory=time.time)
|
|
456
|
+
world: str = ""
|
|
457
|
+
#: World code here raised ``MemoryError``: the worker serves no new request and is recycled.
|
|
458
|
+
poisoned: bool = False
|
|
459
|
+
#: Its address-space cap (``RLIMIT_AS``), given at start; what it is counted at is ``charge_mb`` or its RSS.
|
|
460
|
+
cap_mb: int = DEFAULT_WORKER_MEMORY_MB
|
|
461
|
+
#: What its world was measured to need, which sizes a replacement; ``charge_mb`` what the budget
|
|
462
|
+
#: counts it at, at most its cap and only as far as the budget had room when it was charged.
|
|
463
|
+
need_mb: int = 0
|
|
464
|
+
charge_mb: int = 0
|
|
465
|
+
near_cap: bool = False
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def version_id(self) -> str:
|
|
469
|
+
return self.key.version_id
|
|
426
470
|
|
|
427
471
|
@property
|
|
428
472
|
def pid(self) -> int:
|
|
@@ -431,6 +475,10 @@ class Worker:
|
|
|
431
475
|
def alive(self) -> bool:
|
|
432
476
|
return self.process.poll() is None
|
|
433
477
|
|
|
478
|
+
def serves(self) -> bool:
|
|
479
|
+
"""Alive and not poisoned: a request may go to it."""
|
|
480
|
+
return self.alive() and not self.poisoned
|
|
481
|
+
|
|
434
482
|
def request(
|
|
435
483
|
self,
|
|
436
484
|
method: str,
|
|
@@ -526,7 +574,7 @@ class Worker:
|
|
|
526
574
|
|
|
527
575
|
class Finishing:
|
|
528
576
|
"""What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
|
|
529
|
-
threads, one job at a time for a
|
|
577
|
+
threads, one job at a time for a worker key in the order they came (so a worker that never answers
|
|
530
578
|
holds one thread, not the pool). A session with no worker is finished at once, where it is."""
|
|
531
579
|
|
|
532
580
|
def __init__(self, threads: int = FINISH_THREADS) -> None:
|
|
@@ -541,7 +589,7 @@ class Finishing:
|
|
|
541
589
|
job()
|
|
542
590
|
return
|
|
543
591
|
with self._changed:
|
|
544
|
-
self._waiting.setdefault(worker.
|
|
592
|
+
self._waiting.setdefault(worker.key.name, collections.deque()).append(job)
|
|
545
593
|
self._staff()
|
|
546
594
|
self._changed.notify()
|
|
547
595
|
|
|
@@ -588,8 +636,8 @@ class Finishing:
|
|
|
588
636
|
|
|
589
637
|
|
|
590
638
|
class WorkerPool:
|
|
591
|
-
"""Every version worker on this replica,
|
|
592
|
-
|
|
639
|
+
"""Every version worker on this replica, one per :class:`WorkerKey`, started on
|
|
640
|
+
the key's first session and stopped once it has had none for a while."""
|
|
593
641
|
|
|
594
642
|
def __init__(
|
|
595
643
|
self,
|
|
@@ -601,7 +649,12 @@ class WorkerPool:
|
|
|
601
649
|
footprint_mb: int | None = DEFAULT_WORKER_FOOTPRINT_MB,
|
|
602
650
|
idle_s: float = DEFAULT_WORKER_IDLE_S,
|
|
603
651
|
python: str = sys.executable,
|
|
652
|
+
file_mb: int = DEFAULT_WORKER_FILE_MB,
|
|
653
|
+
memory: ProcessMemory | None = None,
|
|
654
|
+
ceiling_mb: int | None = DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
604
655
|
):
|
|
656
|
+
if file_mb <= 0:
|
|
657
|
+
raise ValueError(f"a worker's file cap ({file_mb} MiB) must be at least 1 MiB (WORLDS_WORKER_FILE_MB)")
|
|
605
658
|
if footprint_mb is None:
|
|
606
659
|
footprint_mb = memory_mb
|
|
607
660
|
if footprint_mb <= 0 or footprint_mb > memory_mb:
|
|
@@ -610,11 +663,22 @@ class WorkerPool:
|
|
|
610
663
|
f"address-space cap ({memory_mb} MiB) (WORLDS_WORKER_FOOTPRINT_MB, "
|
|
611
664
|
"WORLDS_WORKER_MEMORY_MB)"
|
|
612
665
|
)
|
|
666
|
+
ceiling_mb = memory_mb if ceiling_mb is None else ceiling_mb
|
|
667
|
+
if ceiling_mb < memory_mb:
|
|
668
|
+
raise ValueError(
|
|
669
|
+
f"the worker ceiling ({ceiling_mb} MiB) is under the worker cap ({memory_mb} MiB) "
|
|
670
|
+
"(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_MEMORY_MB)"
|
|
671
|
+
)
|
|
613
672
|
if budget_mb < footprint_mb:
|
|
614
673
|
raise ValueError(
|
|
615
674
|
f"the worker budget ({budget_mb} MiB) holds no worker of {footprint_mb} MiB "
|
|
616
675
|
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
617
676
|
)
|
|
677
|
+
if ceiling_mb > memory_mb and ceiling_mb > budget_mb:
|
|
678
|
+
raise ValueError(
|
|
679
|
+
f"the worker ceiling ({ceiling_mb} MiB) is over the worker budget ({budget_mb} MiB) "
|
|
680
|
+
"(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_BUDGET_MB)"
|
|
681
|
+
)
|
|
618
682
|
self.root = Path(cache_root)
|
|
619
683
|
self.workers_root = self.root / WORKERS_DIR
|
|
620
684
|
# Workers live and die with the supervisor; whatever a previous one
|
|
@@ -628,47 +692,98 @@ class WorkerPool:
|
|
|
628
692
|
self.sockets = Path(self._sockets.name)
|
|
629
693
|
self.worlds_domain = worlds_domain
|
|
630
694
|
self.memory_mb = memory_mb
|
|
695
|
+
self.ceiling_mb = ceiling_mb
|
|
631
696
|
self.budget_mb = budget_mb
|
|
632
697
|
self.footprint_mb = footprint_mb
|
|
698
|
+
self.file_mb = file_mb
|
|
633
699
|
self.max_workers = budget_mb // footprint_mb
|
|
634
700
|
self.idle_s = idle_s
|
|
635
701
|
self.python = python
|
|
702
|
+
self.memory = memory or ProcessMemory()
|
|
636
703
|
self._lock = threading.Lock()
|
|
637
|
-
self._workers: dict[
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
704
|
+
self._workers: dict[WorkerKey, Worker] = {}
|
|
705
|
+
self._roster: tuple[Worker, ...] = ()
|
|
706
|
+
|
|
707
|
+
def cap_for(self, need_mb: int) -> int:
|
|
708
|
+
"""The address space a worker needing ``need_mb`` is given: the cap, or more up to the ceiling."""
|
|
709
|
+
return min(self.ceiling_mb, max(self.memory_mb, need_mb))
|
|
710
|
+
|
|
711
|
+
def _bounded(self, need_mb: int) -> int:
|
|
712
|
+
"""A need as the budget charges it: never more than the ceiling, which is all a worker can hold, nor
|
|
713
|
+
than the whole budget."""
|
|
714
|
+
return min(need_mb, self.ceiling_mb, self.budget_mb)
|
|
715
|
+
|
|
716
|
+
def get(self, key: WorkerKey, *, world: str = "", need_mb: int = 0) -> Worker:
|
|
717
|
+
"""The live worker for a key, started if there is none (labelled with its ``world``, given
|
|
718
|
+
:meth:`cap_for` its ``need_mb`` and charged it); a running one keeps the larger need and is
|
|
719
|
+
charged it up to the cap it keeps (a process's cap only goes down) as far as the budget has
|
|
720
|
+
room, never refusing its own lane's open. Starting one
|
|
641
721
|
past the budget first stops workers holding no sessions (whatever their
|
|
642
722
|
idle time); when every one is in use, raises :class:`WorkerCapacity`. A
|
|
643
723
|
dead worker that still holds sessions is never replaced here: its
|
|
644
724
|
sessions' files are on disk, and the supervisor recovers them through
|
|
645
725
|
:meth:`replace` (raises :class:`WorkerGone`)."""
|
|
726
|
+
need_mb = self._bounded(need_mb)
|
|
646
727
|
with self._lock:
|
|
647
|
-
worker = self._workers.get(
|
|
728
|
+
worker = self._workers.get(key)
|
|
648
729
|
if worker is not None and worker.alive():
|
|
730
|
+
self._raise_need(worker, need_mb)
|
|
649
731
|
return worker
|
|
650
732
|
if worker is not None and worker.sessions:
|
|
651
|
-
raise WorkerGone(f"worker for
|
|
733
|
+
raise WorkerGone(f"worker for {key.name} (pid {worker.pid}) exited holding sessions")
|
|
652
734
|
if worker is not None:
|
|
653
735
|
worker.stop()
|
|
654
|
-
|
|
655
|
-
|
|
736
|
+
self._drop(key)
|
|
737
|
+
self._make_room(need_mb)
|
|
738
|
+
return self._start(key, world, need_mb)
|
|
656
739
|
|
|
657
740
|
def replace(self, dead: Worker) -> Worker:
|
|
658
741
|
"""A fresh worker in ``dead``'s place, its sessions left for the caller to
|
|
659
742
|
reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
|
|
660
|
-
when the pool no longer holds ``dead
|
|
743
|
+
when the pool no longer holds ``dead``, never by the budget: it takes the
|
|
744
|
+
place of a worker the budget held, so sessions are never lost to it. It needs
|
|
745
|
+
what ``dead`` was measured to need (:meth:`outgrown` raises that for a worker that ran out), but is
|
|
746
|
+
charged no more than the budget leaves it (at least a footprint), and its cap is never under
|
|
747
|
+
``dead``'s."""
|
|
661
748
|
with self._lock:
|
|
662
|
-
if self._workers.get(dead.
|
|
663
|
-
raise WorkerGone(f"worker for
|
|
664
|
-
|
|
749
|
+
if self._workers.get(dead.key) is not dead:
|
|
750
|
+
raise WorkerGone(f"worker for {dead.key.name} (pid {dead.pid}) was replaced already")
|
|
751
|
+
self._drop(dead.key)
|
|
665
752
|
dead.stop()
|
|
666
|
-
|
|
753
|
+
self._evict_spare(dead.need_mb)
|
|
754
|
+
charge = min(dead.need_mb, max(self.footprint_mb, self._free_mb(self._roster)))
|
|
755
|
+
return self._start(dead.key, dead.world, dead.need_mb, charge_mb=charge, cap_floor_mb=dead.cap_mb)
|
|
756
|
+
|
|
757
|
+
def outgrown(self, worker: Worker) -> None:
|
|
758
|
+
"""``worker`` ran out of address space: its peak address space (``VmPeak``) and half again is what
|
|
759
|
+
it needs from now on, so its replacement gets a larger cap up to the ceiling. Read before it stops."""
|
|
760
|
+
peak = self.memory.status(worker.pid).mb(PEAK_SIZE)
|
|
761
|
+
grown = MemoryNeed(peak).mb if peak is not None else 0
|
|
762
|
+
worker.need_mb = max(worker.need_mb, self._bounded(grown))
|
|
763
|
+
|
|
764
|
+
def _raise_need(self, worker: Worker, need_mb: int) -> None:
|
|
765
|
+
"""Under ``_lock``: ``worker`` asked for ``need_mb`` by another open of its lane keeps the larger need
|
|
766
|
+
and is charged it up to its cap as far as what the budget leaves allows. The open is never refused:
|
|
767
|
+
placement sends a lane's sessions to the replica that runs it."""
|
|
768
|
+
worker.need_mb = max(worker.need_mb, need_mb)
|
|
769
|
+
room = self._counted(worker) + max(0, self._free_mb(self._roster))
|
|
770
|
+
worker.charge_mb = max(worker.charge_mb, min(worker.need_mb, worker.cap_mb, room))
|
|
771
|
+
|
|
772
|
+
def _start(
|
|
773
|
+
self, key: WorkerKey, world: str, need_mb: int, charge_mb: int | None = None, cap_floor_mb: int = 0
|
|
774
|
+
) -> Worker:
|
|
775
|
+
charge = need_mb if charge_mb is None else charge_mb
|
|
776
|
+
worker = self._spawn(key, world, max(cap_floor_mb, self.cap_for(charge)))
|
|
777
|
+
worker.need_mb = need_mb
|
|
778
|
+
worker.charge_mb = min(charge, worker.cap_mb)
|
|
779
|
+
self._workers[key] = worker
|
|
780
|
+
self._roster = tuple(self._workers.values())
|
|
781
|
+
return worker
|
|
667
782
|
|
|
668
|
-
def
|
|
669
|
-
|
|
670
|
-
worker = self.
|
|
671
|
-
self.
|
|
783
|
+
def _drop(self, key: WorkerKey) -> Worker:
|
|
784
|
+
"""Under ``_lock``."""
|
|
785
|
+
worker = self._workers.pop(key)
|
|
786
|
+
self._roster = tuple(self._workers.values())
|
|
672
787
|
return worker
|
|
673
788
|
|
|
674
789
|
def dead(self) -> list[Worker]:
|
|
@@ -681,55 +796,114 @@ class WorkerPool:
|
|
|
681
796
|
with self._lock:
|
|
682
797
|
return [w for w in self._workers.values() if w.alive()]
|
|
683
798
|
|
|
684
|
-
def _make_room(self) -> None:
|
|
685
|
-
"""Under ``_lock``: leave
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
for
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
799
|
+
def _make_room(self, need_mb: int) -> None:
|
|
800
|
+
"""Under ``_lock``: leave room for a worker charged ``need_mb`` (at least a footprint) of the budget
|
|
801
|
+
(:meth:`_evict_spare`), and refuse when what the workers left are counted at leaves less; refused
|
|
802
|
+
before anything is stopped when stopping every worker holding no session would not free it, so
|
|
803
|
+
an open that cannot fit here stops no other world's warm worker."""
|
|
804
|
+
room = max(self.footprint_mb, need_mb)
|
|
805
|
+
spare = sum(self._counted(worker) for worker in self._roster if not worker.sessions)
|
|
806
|
+
self._refuse_short(room, self._free_mb(self._roster) + spare)
|
|
807
|
+
self._evict_spare(need_mb)
|
|
808
|
+
self._refuse_short(room, self._free_mb(self._roster))
|
|
809
|
+
|
|
810
|
+
def _refuse_short(self, room: int, free: int) -> None:
|
|
811
|
+
if free >= room:
|
|
812
|
+
return
|
|
813
|
+
raise WorkerCapacity(
|
|
814
|
+
f"this host's {len(self._workers)} version workers are counted at {self.counted_mb()} of "
|
|
815
|
+
f"{self.budget_mb} MiB (each what it needs or holds, at least {self.footprint_mb} MiB), which "
|
|
816
|
+
f"leaves no room for one needing {room} MiB; place elsewhere"
|
|
695
817
|
)
|
|
696
|
-
while len(self._workers) >= self.max_workers and spare:
|
|
697
|
-
self._workers.pop(spare.pop(0).version_id).stop()
|
|
698
|
-
if len(self._workers) >= self.max_workers:
|
|
699
|
-
raise WorkerCapacity(
|
|
700
|
-
f"this host's {self.max_workers} version workers ({self.budget_mb} MiB at "
|
|
701
|
-
f"{self.footprint_mb} MiB each) all hold sessions; place elsewhere"
|
|
702
|
-
)
|
|
703
818
|
|
|
704
|
-
def
|
|
819
|
+
def _evict_spare(self, need_mb: int = 0) -> None:
|
|
820
|
+
"""Under ``_lock``: forget exited workers that hold no sessions, then stop session-less ones,
|
|
821
|
+
longest idle first, until ``need_mb`` (at least a footprint) of the budget is free or none is
|
|
822
|
+
left to stop."""
|
|
823
|
+
room = max(self.footprint_mb, need_mb)
|
|
824
|
+
exited = [k for k, w in self._workers.items() if not w.alive() and not w.sessions]
|
|
825
|
+
for key in exited:
|
|
826
|
+
self._drop(key).stop()
|
|
827
|
+
spare = sorted((w for w in self._workers.values() if not w.sessions), key=lambda w: w.idle_since)
|
|
828
|
+
while self._free_mb(self._roster) < room and spare:
|
|
829
|
+
self._drop(spare.pop(0).key).stop()
|
|
830
|
+
|
|
831
|
+
def _counted(self, worker: Worker) -> int:
|
|
832
|
+
"""What ``worker`` takes of the budget: what it was charged, or the memory the kernel counts as its own
|
|
833
|
+
while it runs (``RssAnon``, else ``VmRSS``), whichever is more, never under the footprint."""
|
|
834
|
+
status = self.memory.status(worker.pid) if worker.alive() else None
|
|
835
|
+
measured = (status.mb(RESIDENT_ANON) or status.mb(RESIDENT)) if status is not None else None
|
|
836
|
+
return max(self.footprint_mb, worker.charge_mb, measured or 0)
|
|
837
|
+
|
|
838
|
+
def _free_mb(self, workers: tuple[Worker, ...]) -> int:
|
|
839
|
+
return self.budget_mb - sum(self._counted(worker) for worker in workers)
|
|
840
|
+
|
|
841
|
+
def counted_mb(self) -> int:
|
|
842
|
+
"""What every worker here is counted at against the budget, in MiB. Read off the last roster, never
|
|
843
|
+
under ``_lock``, which a starting or stopping worker holds for seconds: the heartbeat must not wait."""
|
|
844
|
+
return sum(self._counted(worker) for worker in self._roster)
|
|
845
|
+
|
|
846
|
+
def capacity(self) -> int:
|
|
847
|
+
"""The workers this replica could hold now: those it runs plus as many more as fit at the footprint,
|
|
848
|
+
so a replica whose workers are heavy advertises fewer slots than ``max_workers``. Lock-free, as
|
|
849
|
+
:meth:`counted_mb`."""
|
|
850
|
+
roster = self._roster
|
|
851
|
+
return len(roster) + max(0, self._free_mb(roster)) // self.footprint_mb
|
|
852
|
+
|
|
853
|
+
def peaks(self) -> list[tuple[WorkerKey, int]]:
|
|
854
|
+
"""Each running worker's peak resident memory (``VmHWM``) by key, where the kernel says. Lock-free."""
|
|
855
|
+
live = [worker for worker in self._roster if worker.alive()]
|
|
856
|
+
readings = [(worker.key, self.memory.status(worker.pid).mb(PEAK_RESIDENT)) for worker in live]
|
|
857
|
+
return sorted((key, mb) for key, mb in readings if mb is not None)
|
|
858
|
+
|
|
859
|
+
def stats(self) -> dict:
|
|
860
|
+
"""What the heartbeat says of this pool, lock-free: the slots that fit now, what the workers are
|
|
861
|
+
counted at, the keys they run for and the largest worker this host gives."""
|
|
862
|
+
return {
|
|
863
|
+
"maxWorkers": self.capacity(),
|
|
864
|
+
"workerMemoryMb": self.counted_mb(),
|
|
865
|
+
"workerKeysInUse": [{"versionId": key.version_id, "lane": key.lane} for key in self.keys_in_use()],
|
|
866
|
+
"workerCapMb": self.ceiling_mb,
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
def keys_in_use(self) -> list[WorkerKey]:
|
|
870
|
+
"""The keys a worker runs for here, so placement knows which opens need no new worker. Lock-free."""
|
|
871
|
+
return sorted(worker.key for worker in self._roster if worker.alive())
|
|
872
|
+
|
|
873
|
+
def peek(self, key: WorkerKey) -> Worker | None:
|
|
705
874
|
with self._lock:
|
|
706
|
-
return self._workers.get(
|
|
875
|
+
return self._workers.get(key)
|
|
707
876
|
|
|
708
877
|
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
709
|
-
"""Stop workers that have had no session for ``idle_s``; returns their
|
|
878
|
+
"""Stop workers that have had no session for ``idle_s``; returns their keys' names."""
|
|
710
879
|
now = time.time() if now is None else now
|
|
711
880
|
with self._lock:
|
|
712
881
|
idle = [
|
|
713
|
-
|
|
714
|
-
for w in self._workers.
|
|
882
|
+
key
|
|
883
|
+
for key, w in self._workers.items()
|
|
715
884
|
if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
|
|
716
885
|
]
|
|
717
|
-
stopped = [self.
|
|
886
|
+
stopped = [self._drop(key) for key in idle]
|
|
718
887
|
for worker in stopped:
|
|
719
888
|
worker.stop()
|
|
720
|
-
return idle
|
|
889
|
+
return [key.name for key in idle]
|
|
721
890
|
|
|
722
891
|
def stop_all(self) -> None:
|
|
723
892
|
with self._lock:
|
|
724
893
|
workers = list(self._workers.values())
|
|
725
894
|
self._workers.clear()
|
|
895
|
+
self._roster = ()
|
|
726
896
|
for worker in workers:
|
|
727
897
|
worker.stop()
|
|
728
898
|
|
|
729
|
-
def
|
|
730
|
-
|
|
899
|
+
def sizes(self) -> list[tuple[Worker, int | None]]:
|
|
900
|
+
"""Each running worker with the address space the kernel counts it at (``VmSize``). Lock-free."""
|
|
901
|
+
return [(worker, self.memory.status(worker.pid).mb(SIZE)) for worker in self._roster if worker.alive()]
|
|
902
|
+
|
|
903
|
+
def _spawn(self, key: WorkerKey, world: str, cap_mb: int) -> Worker:
|
|
904
|
+
home = key.home(self.workers_root)
|
|
731
905
|
home.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
732
|
-
socket_path = self.sockets / f"{hashlib.sha256(
|
|
906
|
+
socket_path = self.sockets / f"{hashlib.sha256(key.name.encode()).hexdigest()[:16]}.sock"
|
|
733
907
|
secret = secrets.token_urlsafe(24)
|
|
734
908
|
# The supervisor binds and listens; the worker inherits the descriptor
|
|
735
909
|
# and serves on it, so the socket is the supervisor's from the start.
|
|
@@ -743,7 +917,8 @@ class WorkerPool:
|
|
|
743
917
|
"--cache-root",
|
|
744
918
|
str(self.root),
|
|
745
919
|
"--version",
|
|
746
|
-
version_id,
|
|
920
|
+
key.version_id,
|
|
921
|
+
*(["--states"] if key.states else []),
|
|
747
922
|
"--socket-fd",
|
|
748
923
|
str(listening.fileno()),
|
|
749
924
|
],
|
|
@@ -751,7 +926,8 @@ class WorkerPool:
|
|
|
751
926
|
secret=secret,
|
|
752
927
|
home=home,
|
|
753
928
|
worlds_domain=self.worlds_domain,
|
|
754
|
-
memory_mb=
|
|
929
|
+
memory_mb=cap_mb,
|
|
930
|
+
file_mb=self.file_mb,
|
|
755
931
|
),
|
|
756
932
|
cwd=str(home),
|
|
757
933
|
stdin=subprocess.DEVNULL,
|
|
@@ -762,7 +938,7 @@ class WorkerPool:
|
|
|
762
938
|
)
|
|
763
939
|
finally:
|
|
764
940
|
listening.close()
|
|
765
|
-
worker = Worker(
|
|
941
|
+
worker = Worker(key, process, socket_path, secret, world=world, cap_mb=cap_mb)
|
|
766
942
|
self._await_ready(worker)
|
|
767
943
|
return worker
|
|
768
944
|
|
|
@@ -771,10 +947,10 @@ class WorkerPool:
|
|
|
771
947
|
while time.time() < deadline:
|
|
772
948
|
if not worker.alive():
|
|
773
949
|
raise WorkerGone(
|
|
774
|
-
f"worker for
|
|
950
|
+
f"worker for {worker.key.name} exited with {worker.process.returncode} on start"
|
|
775
951
|
)
|
|
776
952
|
if worker.socket_path.exists() and worker.answers_health():
|
|
777
953
|
return
|
|
778
954
|
time.sleep(0.05)
|
|
779
955
|
worker.stop()
|
|
780
|
-
raise WorkerGone(f"worker for
|
|
956
|
+
raise WorkerGone(f"worker for {worker.key.name} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|