@withgateway/sdk 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/agents/index.d.mts +12 -4
  2. package/dist/agents/index.d.ts +12 -4
  3. package/dist/agents/index.js +30 -0
  4. package/dist/agents/index.js.map +1 -1
  5. package/dist/agents/index.mjs +30 -0
  6. package/dist/agents/index.mjs.map +1 -1
  7. package/dist/cli.js +2291 -945
  8. package/dist/cli.js.map +1 -1
  9. package/dist/cli.mjs +2290 -944
  10. package/dist/cli.mjs.map +1 -1
  11. package/dist/files/index.js.map +1 -1
  12. package/dist/files/index.mjs.map +1 -1
  13. package/dist/index.js +325 -184
  14. package/dist/index.js.map +1 -1
  15. package/dist/index.mjs +324 -183
  16. package/dist/index.mjs.map +1 -1
  17. package/dist/traces/index.js.map +1 -1
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +1 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +1 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +3 -1
  24. package/dist/worlds/index.d.ts +3 -1
  25. package/dist/worlds/index.js +178 -67
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +177 -66
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +1 -1
  30. package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
  31. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  32. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  33. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  34. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  35. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  36. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  37. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  38. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  39. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  40. package/runtime/gatewaysdk/environment/schema/host.py +21 -2
  41. package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
  42. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  43. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  44. package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
  45. package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
  46. package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
  47. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  48. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  49. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  51. package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
  52. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  53. package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
  54. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  55. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  56. package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
  57. package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
  58. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  59. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  60. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  61. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
  62. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  63. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  64. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
  65. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
  66. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  68. package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
  69. package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
  70. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  71. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  72. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  73. package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
  74. package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
  75. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  76. package/skills/connector-schema-authoring/SKILL.md +1 -0
  77. package/skills/connector-schema-authoring/connector-worlds.md +1 -0
  78. package/skills/connector-schema-authoring/world-schema.md +7 -0
  79. package/skills/factory-orchestrator/SKILL.md +147 -0
  80. package/skills/world-data-ingestion/ROWS.md +5 -1
  81. package/skills/world-data-ingestion/SKILL.md +4 -4
  82. package/skills/worlds-getting-started/SKILL.md +2 -2
  83. package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  84. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
@@ -0,0 +1,38 @@
1
+ """``WorkerKey``: which of a version's workers serves a session.
2
+
3
+ A version's sessions on its own data (main) and its sessions on any of its world
4
+ states run in two workers, never one: a state's data cannot exhaust the process
5
+ main's sessions are served by. The key is also what a version's turns, opens,
6
+ recoveries and finishes queue on, so neither side waits behind the other.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+
14
+ MAIN_LANE = "main"
15
+ STATES_LANE = "states"
16
+
17
+
18
+ @dataclass(frozen=True, order=True)
19
+ class WorkerKey:
20
+ version_id: str
21
+ states: bool = False
22
+
23
+ @classmethod
24
+ def of(cls, version_id: str, world_state: bool) -> WorkerKey:
25
+ return cls(version_id, states=world_state)
26
+
27
+ @property
28
+ def name(self) -> str:
29
+ """``<version>`` for main, ``<version>@states`` for its world states: the key's home, socket and queues."""
30
+ return f"{self.version_id}@{STATES_LANE}" if self.states else self.version_id
31
+
32
+ @property
33
+ def lane(self) -> str:
34
+ return STATES_LANE if self.states else MAIN_LANE
35
+
36
+ def home(self, workers_root: Path) -> Path:
37
+ """Where this key's worker runs: its own ``WorldHost`` cache root under the pool's ``workers/``."""
38
+ return Path(workers_root) / self.name
@@ -0,0 +1,38 @@
1
+ """``PeakReport``: each worker key's peak memory, told to the platform as it grows.
2
+
3
+ A worker's peak (``VmHWM``) is what serving its sessions took: for a version's main, its own
4
+ data; for its states lane, every state it served at once. The platform keeps the largest it is
5
+ told for the version, so the next open is provisioned by what the world was seen to need. A
6
+ key is reported once its peak has grown ``STEP_MB`` past what the platform last took, so a
7
+ heartbeat carries only what changed and a missed one is sent again.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import threading
13
+
14
+ from .worker_key import STATES_LANE, WorkerKey
15
+
16
+ STEP_MB = 16
17
+
18
+
19
+ class PeakReport:
20
+ def __init__(self) -> None:
21
+ self._lock = threading.Lock()
22
+ self._taken: dict[WorkerKey, int] = {}
23
+
24
+ def pending(self, peaks: list[tuple[WorkerKey, int]]) -> list[dict]:
25
+ """The peaks worth telling: grown ``STEP_MB`` past the last one the platform took for that key. A key
26
+ no worker runs for is forgotten, so a worker started for it again is measured from nothing."""
27
+ with self._lock:
28
+ running = {key for key, _ in peaks}
29
+ self._taken = {key: mb for key, mb in self._taken.items() if key in running}
30
+ grown = [(key, mb) for key, mb in peaks if mb >= self._taken.get(key, 0) + STEP_MB]
31
+ return [{"versionId": key.version_id, "lane": key.lane, "peakMb": mb} for key, mb in grown]
32
+
33
+ def taken(self, sent: list[dict]) -> None:
34
+ """The platform took a heartbeat carrying ``sent``."""
35
+ with self._lock:
36
+ for peak in sent:
37
+ key = WorkerKey.of(peak["versionId"], peak["lane"] == STATES_LANE)
38
+ self._taken[key] = max(self._taken.get(key, 0), peak["peakMb"])
@@ -0,0 +1,50 @@
1
+ """``WorkerMemoryWatch``: each version worker's address space against its own cap, read on the lease sweep.
2
+
3
+ A worker runs out when its address space (``VmSize``) reaches its ``RLIMIT_AS``; at ``NEAR_CAP`` of that
4
+ it is logged once as it crosses and counted (``workers_near_cap``), so an alert fires before the
5
+ ``MemoryError`` does. The readings, which name versions, go to the secret-gated stats and the heartbeat;
6
+ ``/metrics``, which world code on the pod can reach, carries only their totals.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Callable
12
+
13
+ from .workers import WorkerPool
14
+
15
+ NEAR_CAP = 0.8
16
+ WORKERS_NEAR_CAP = "workers_near_cap"
17
+
18
+
19
+ class WorkerMemoryWatch:
20
+ def __init__(self, pool: WorkerPool, tally: Callable[[str], None]) -> None:
21
+ self.pool = pool
22
+ self.tally = tally
23
+
24
+ def readings(self) -> list[dict]:
25
+ """Each running worker's address space, cap, measured need and charge, in MiB; ``sizeMb`` None with no
26
+ ``/proc``."""
27
+ return [
28
+ {
29
+ "versionId": worker.version_id,
30
+ "lane": worker.key.lane,
31
+ "sizeMb": size,
32
+ "capMb": worker.cap_mb,
33
+ "needMb": worker.need_mb,
34
+ "chargeMb": worker.charge_mb,
35
+ "nearCap": worker.near_cap,
36
+ }
37
+ for worker, size in self.pool.sizes()
38
+ ]
39
+
40
+ def sweep(self) -> list[str]:
41
+ """Mark each worker near its cap or not, and count the ones that just crossed. Returns a line for each,
42
+ which the lease sweep logs."""
43
+ crossed = []
44
+ for worker, size in self.pool.sizes():
45
+ near = size is not None and size >= worker.cap_mb * NEAR_CAP
46
+ if near and not worker.near_cap:
47
+ crossed.append(f"{worker.key.name} (world {worker.world!r}): {size} of its {worker.cap_mb} MiB")
48
+ self.tally(WORKERS_NEAR_CAP)
49
+ worker.near_cap = near
50
+ return crossed
@@ -1,13 +1,14 @@
1
- """Version workers: one subprocess per world version that runs world-authored code.
1
+ """Version workers: subprocesses that run world-authored code, one per :class:`WorkerKey`.
2
2
 
3
3
  A schema world with an ``api/handlers.py`` brings its own Python. The host
4
4
  process is shared by every tenant on the replica, so that code never runs in
5
- it: the supervisor (:mod:`host`) starts one worker per version, over a unix
6
- socket only the supervisor can reach, and forwards that version's sessions to
7
- it. The worker is the same :class:`host.WorldHost` in a smaller room:
5
+ it: the supervisor (:mod:`host`) starts a worker per version and lane (its
6
+ sessions on the version's own data, and those on any of its world states), over
7
+ a unix socket only the supervisor can reach, and forwards those sessions to it.
8
+ The worker is the same :class:`host.WorldHost` in a smaller room:
8
9
 
9
10
  * its own process, so a handler that loops, leaks or crashes takes down that
10
- version's sessions on this replica and nothing else;
11
+ version's sessions of that lane on this replica and nothing else;
11
12
  * rlimits on address space, open files and file size, set before any world
12
13
  code is imported (:class:`AddressSpaceLimit` says how macOS differs);
13
14
  * a scrubbed environment: the control-plane secret, cloud credentials and the
@@ -43,6 +44,7 @@ import os
43
44
  import resource
44
45
  import secrets
45
46
  import shutil
47
+ import signal
46
48
  import socket
47
49
  import struct
48
50
  import subprocess
@@ -55,28 +57,37 @@ from contextlib import contextmanager, suppress
55
57
  from dataclasses import dataclass, field
56
58
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
57
59
  from pathlib import Path
60
+ from types import MappingProxyType
58
61
  from typing import Protocol
59
62
 
60
63
  from .cache_layout import WORKERS_DIR
61
64
  from .grading import VerifierSandbox
62
65
  from .handler_errors import log_line
66
+ from .process_memory import PEAK_RESIDENT, PEAK_SIZE, RESIDENT, RESIDENT_ANON, SIZE, MemoryNeed, ProcessMemory
67
+ from .worker_key import WorkerKey
63
68
 
64
69
  #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
65
70
  DEFAULT_WORKER_MEMORY_MB = 2048
66
71
  #: Memory every worker together may use, in MiB, unless
67
- #: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. The number of workers a replica
68
- #: runs at once is this over the footprint each is counted at. A version past
69
- #: that is refused ``at_capacity`` and placed on another replica instead of
70
- #: the kernel choosing which process to kill.
72
+ #: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. A worker starts only while the
73
+ #: workers already running, each counted as :meth:`WorkerPool.counted_mb` says,
74
+ #: leave room for its need (at least a footprint). A worker past that is refused
75
+ #: ``at_capacity`` and placed on another replica instead of the kernel choosing
76
+ #: which process to kill.
71
77
  DEFAULT_WORKER_BUDGET_MB = 4096
72
- #: What one worker is counted at against the budget, in MiB, unless
73
- #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
78
+ #: The least one worker is counted at against the budget, in MiB, unless
79
+ #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise; a worker whose resident memory
80
+ #: the kernel reports higher is counted at that. Unset, a worker is counted at
74
81
  #: its whole address-space cap: the safe reading, and a 50x over-reservation of
75
82
  #: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
76
83
  #: many worlds at once sets the footprint it measured, keeps the address-space
77
84
  #: cap as the rail each worker cannot cross, and sizes the container for
78
- #: footprint x workers plus the supervisor's share.
85
+ #: the budget plus the supervisor's share.
79
86
  DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
87
+ #: The most address space one worker is ever given, in MiB, unless ``WORLDS_WORKER_MEMORY_CEILING_MB`` says
88
+ #: otherwise; unset, ``WORLDS_WORKER_MEMORY_MB``, so no worker's cap grows. A world measured to need more than
89
+ #: the cap (its peak and half again) gets that much up to this; a raised ceiling must fit the budget.
90
+ DEFAULT_WORKER_MEMORY_CEILING_MB: int | None = None
80
91
  #: Largest file a worker may write (its session databases), in MiB, unless ``WORLDS_WORKER_FILE_MB`` says otherwise.
81
92
  DEFAULT_WORKER_FILE_MB = 4096
82
93
  #: Open files per worker: one per live session plus the sockets serving them.
@@ -97,10 +108,15 @@ WORKER_LEASE_S = 365.0 * 86400.0
97
108
  #: Environment the worker inherits. Nothing else crosses: no host secret, no
98
109
  #: cloud credentials, no platform URL.
99
110
  INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
111
+ #: glibc keeps a malloc arena per thread (64 MiB of address space each) after the thread ends, and
112
+ #: ``RLIMIT_AS`` counts them: without this cap a worker that served many threads reads as near its cap
113
+ #: while holding little, and runs out of address space with memory to spare.
114
+ WORKER_MALLOC_ARENAS = "2"
100
115
  #: Headers the supervisor never forwards to a worker: the ones that describe
101
116
  #: the supervisor's connection, and its own control-plane secret.
102
117
  UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
103
118
  FINISH_THREADS = 8
119
+ NO_WORKERS = MappingProxyType({"maxWorkers": 0, "workerMemoryMb": 0, "workerKeysInUse": (), "workerCapMb": 0})
104
120
  FINISH_SETTLE_S = 10.0
105
121
 
106
122
 
@@ -236,6 +252,7 @@ class UnixHTTPServer(ThreadingHTTPServer):
236
252
 
237
253
 
238
254
  _PR_SET_DUMPABLE = 4
255
+ _PR_SET_PDEATHSIG = 1
239
256
 
240
257
 
241
258
  def seal_process() -> bool:
@@ -253,6 +270,16 @@ def seal_process() -> bool:
253
270
  return False
254
271
 
255
272
 
273
+ def die_with_parent(parent: int) -> None:
274
+ """Have the kernel kill this process the moment ``parent`` exits (Linux), so no code of this process
275
+ can postpone it; and leave at once if ``parent`` is gone already. Elsewhere the caller keeps its own watch."""
276
+ if sys.platform.startswith("linux"):
277
+ with suppress(OSError, AttributeError):
278
+ ctypes.CDLL(None, use_errno=True).prctl(_PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0)
279
+ if os.getppid() != parent:
280
+ os._exit(1)
281
+
282
+
256
283
  class Kernel(Protocol):
257
284
  """Where a process's limits are read and set: the ``resource`` module."""
258
285
 
@@ -369,6 +396,7 @@ def worker_environment(
369
396
  WORLDS_WORKER_MEMORY_MB=str(memory_mb),
370
397
  WORLDS_WORKER_FILE_MB=str(file_mb),
371
398
  WORLDS_INGRESS_DOMAIN=worlds_domain,
399
+ MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS,
372
400
  )
373
401
  return env
374
402
 
@@ -380,7 +408,7 @@ def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None)
380
408
  secret, no cloud credentials, no platform URL."""
381
409
  inherited: Mapping[str, str] = os.environ if source is None else source
382
410
  env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
383
- env["HOME"] = str(home)
411
+ env.update(HOME=str(home), MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS)
384
412
  return env
385
413
 
386
414
 
@@ -419,12 +447,26 @@ class ConfinedVerifierSandbox(VerifierSandbox):
419
447
 
420
448
  @dataclass
421
449
  class Worker:
422
- version_id: str
450
+ key: WorkerKey
423
451
  process: subprocess.Popen
424
452
  socket_path: Path
425
453
  secret: str
426
454
  sessions: set[str] = field(default_factory=set)
427
455
  idle_since: float = field(default_factory=time.time)
456
+ world: str = ""
457
+ #: World code here raised ``MemoryError``: the worker serves no new request and is recycled.
458
+ poisoned: bool = False
459
+ #: Its address-space cap (``RLIMIT_AS``), given at start; what it is counted at is ``charge_mb`` or its RSS.
460
+ cap_mb: int = DEFAULT_WORKER_MEMORY_MB
461
+ #: What its world was measured to need, which sizes a replacement; ``charge_mb`` what the budget
462
+ #: counts it at, at most its cap and only as far as the budget had room when it was charged.
463
+ need_mb: int = 0
464
+ charge_mb: int = 0
465
+ near_cap: bool = False
466
+
467
+ @property
468
+ def version_id(self) -> str:
469
+ return self.key.version_id
428
470
 
429
471
  @property
430
472
  def pid(self) -> int:
@@ -433,6 +475,10 @@ class Worker:
433
475
  def alive(self) -> bool:
434
476
  return self.process.poll() is None
435
477
 
478
+ def serves(self) -> bool:
479
+ """Alive and not poisoned: a request may go to it."""
480
+ return self.alive() and not self.poisoned
481
+
436
482
  def request(
437
483
  self,
438
484
  method: str,
@@ -528,7 +574,7 @@ class Worker:
528
574
 
529
575
  class Finishing:
530
576
  """What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
531
- threads, one job at a time for a version in the order they came (so a worker that never answers
577
+ threads, one job at a time for a worker key in the order they came (so a worker that never answers
532
578
  holds one thread, not the pool). A session with no worker is finished at once, where it is."""
533
579
 
534
580
  def __init__(self, threads: int = FINISH_THREADS) -> None:
@@ -543,7 +589,7 @@ class Finishing:
543
589
  job()
544
590
  return
545
591
  with self._changed:
546
- self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
592
+ self._waiting.setdefault(worker.key.name, collections.deque()).append(job)
547
593
  self._staff()
548
594
  self._changed.notify()
549
595
 
@@ -590,8 +636,8 @@ class Finishing:
590
636
 
591
637
 
592
638
  class WorkerPool:
593
- """Every version worker on this replica, started on the first session of a
594
- version and stopped once it has had none for a while."""
639
+ """Every version worker on this replica, one per :class:`WorkerKey`, started on
640
+ the key's first session and stopped once it has had none for a while."""
595
641
 
596
642
  def __init__(
597
643
  self,
@@ -604,6 +650,8 @@ class WorkerPool:
604
650
  idle_s: float = DEFAULT_WORKER_IDLE_S,
605
651
  python: str = sys.executable,
606
652
  file_mb: int = DEFAULT_WORKER_FILE_MB,
653
+ memory: ProcessMemory | None = None,
654
+ ceiling_mb: int | None = DEFAULT_WORKER_MEMORY_CEILING_MB,
607
655
  ):
608
656
  if file_mb <= 0:
609
657
  raise ValueError(f"a worker's file cap ({file_mb} MiB) must be at least 1 MiB (WORLDS_WORKER_FILE_MB)")
@@ -615,11 +663,22 @@ class WorkerPool:
615
663
  f"address-space cap ({memory_mb} MiB) (WORLDS_WORKER_FOOTPRINT_MB, "
616
664
  "WORLDS_WORKER_MEMORY_MB)"
617
665
  )
666
+ ceiling_mb = memory_mb if ceiling_mb is None else ceiling_mb
667
+ if ceiling_mb < memory_mb:
668
+ raise ValueError(
669
+ f"the worker ceiling ({ceiling_mb} MiB) is under the worker cap ({memory_mb} MiB) "
670
+ "(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_MEMORY_MB)"
671
+ )
618
672
  if budget_mb < footprint_mb:
619
673
  raise ValueError(
620
674
  f"the worker budget ({budget_mb} MiB) holds no worker of {footprint_mb} MiB "
621
675
  "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
622
676
  )
677
+ if ceiling_mb > memory_mb and ceiling_mb > budget_mb:
678
+ raise ValueError(
679
+ f"the worker ceiling ({ceiling_mb} MiB) is over the worker budget ({budget_mb} MiB) "
680
+ "(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_BUDGET_MB)"
681
+ )
623
682
  self.root = Path(cache_root)
624
683
  self.workers_root = self.root / WORKERS_DIR
625
684
  # Workers live and die with the supervisor; whatever a previous one
@@ -633,48 +692,98 @@ class WorkerPool:
633
692
  self.sockets = Path(self._sockets.name)
634
693
  self.worlds_domain = worlds_domain
635
694
  self.memory_mb = memory_mb
695
+ self.ceiling_mb = ceiling_mb
636
696
  self.budget_mb = budget_mb
637
697
  self.footprint_mb = footprint_mb
638
698
  self.file_mb = file_mb
639
699
  self.max_workers = budget_mb // footprint_mb
640
700
  self.idle_s = idle_s
641
701
  self.python = python
702
+ self.memory = memory or ProcessMemory()
642
703
  self._lock = threading.Lock()
643
- self._workers: dict[str, Worker] = {}
644
-
645
- def get(self, version_id: str) -> Worker:
646
- """The live worker for a version, started if there is none. Starting one
704
+ self._workers: dict[WorkerKey, Worker] = {}
705
+ self._roster: tuple[Worker, ...] = ()
706
+
707
+ def cap_for(self, need_mb: int) -> int:
708
+ """The address space a worker needing ``need_mb`` is given: the cap, or more up to the ceiling."""
709
+ return min(self.ceiling_mb, max(self.memory_mb, need_mb))
710
+
711
+ def _bounded(self, need_mb: int) -> int:
712
+ """A need as the budget charges it: never more than the ceiling, which is all a worker can hold, nor
713
+ than the whole budget."""
714
+ return min(need_mb, self.ceiling_mb, self.budget_mb)
715
+
716
+ def get(self, key: WorkerKey, *, world: str = "", need_mb: int = 0) -> Worker:
717
+ """The live worker for a key, started if there is none (labelled with its ``world``, given
718
+ :meth:`cap_for` its ``need_mb`` and charged it); a running one keeps the larger need and is
719
+ charged it up to the cap it keeps (a process's cap only goes down) as far as the budget has
720
+ room, never refusing its own lane's open. Starting one
647
721
  past the budget first stops workers holding no sessions (whatever their
648
722
  idle time); when every one is in use, raises :class:`WorkerCapacity`. A
649
723
  dead worker that still holds sessions is never replaced here: its
650
724
  sessions' files are on disk, and the supervisor recovers them through
651
725
  :meth:`replace` (raises :class:`WorkerGone`)."""
726
+ need_mb = self._bounded(need_mb)
652
727
  with self._lock:
653
- worker = self._workers.get(version_id)
728
+ worker = self._workers.get(key)
654
729
  if worker is not None and worker.alive():
730
+ self._raise_need(worker, need_mb)
655
731
  return worker
656
732
  if worker is not None and worker.sessions:
657
- raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
733
+ raise WorkerGone(f"worker for {key.name} (pid {worker.pid}) exited holding sessions")
658
734
  if worker is not None:
659
735
  worker.stop()
660
- del self._workers[version_id]
661
- return self._start(version_id)
736
+ self._drop(key)
737
+ self._make_room(need_mb)
738
+ return self._start(key, world, need_mb)
662
739
 
663
740
  def replace(self, dead: Worker) -> Worker:
664
741
  """A fresh worker in ``dead``'s place, its sessions left for the caller to
665
742
  reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
666
- when the pool no longer holds ``dead``."""
743
+ when the pool no longer holds ``dead``, never by the budget: it takes the
744
+ place of a worker the budget held, so sessions are never lost to it. It needs
745
+ what ``dead`` was measured to need (:meth:`outgrown` raises that for a worker that ran out), but is
746
+ charged no more than the budget leaves it (at least a footprint), and its cap is never under
747
+ ``dead``'s."""
667
748
  with self._lock:
668
- if self._workers.get(dead.version_id) is not dead:
669
- raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
670
- del self._workers[dead.version_id]
749
+ if self._workers.get(dead.key) is not dead:
750
+ raise WorkerGone(f"worker for {dead.key.name} (pid {dead.pid}) was replaced already")
751
+ self._drop(dead.key)
671
752
  dead.stop()
672
- return self._start(dead.version_id)
753
+ self._evict_spare(dead.need_mb)
754
+ charge = min(dead.need_mb, max(self.footprint_mb, self._free_mb(self._roster)))
755
+ return self._start(dead.key, dead.world, dead.need_mb, charge_mb=charge, cap_floor_mb=dead.cap_mb)
756
+
757
+ def outgrown(self, worker: Worker) -> None:
758
+ """``worker`` ran out of address space: its peak address space (``VmPeak``) and half again is what
759
+ it needs from now on, so its replacement gets a larger cap up to the ceiling. Read before it stops."""
760
+ peak = self.memory.status(worker.pid).mb(PEAK_SIZE)
761
+ grown = MemoryNeed(peak).mb if peak is not None else 0
762
+ worker.need_mb = max(worker.need_mb, self._bounded(grown))
763
+
764
+ def _raise_need(self, worker: Worker, need_mb: int) -> None:
765
+ """Under ``_lock``: ``worker`` asked for ``need_mb`` by another open of its lane keeps the larger need
766
+ and is charged it up to its cap as far as what the budget leaves allows. The open is never refused:
767
+ placement sends a lane's sessions to the replica that runs it."""
768
+ worker.need_mb = max(worker.need_mb, need_mb)
769
+ room = self._counted(worker) + max(0, self._free_mb(self._roster))
770
+ worker.charge_mb = max(worker.charge_mb, min(worker.need_mb, worker.cap_mb, room))
771
+
772
+ def _start(
773
+ self, key: WorkerKey, world: str, need_mb: int, charge_mb: int | None = None, cap_floor_mb: int = 0
774
+ ) -> Worker:
775
+ charge = need_mb if charge_mb is None else charge_mb
776
+ worker = self._spawn(key, world, max(cap_floor_mb, self.cap_for(charge)))
777
+ worker.need_mb = need_mb
778
+ worker.charge_mb = min(charge, worker.cap_mb)
779
+ self._workers[key] = worker
780
+ self._roster = tuple(self._workers.values())
781
+ return worker
673
782
 
674
- def _start(self, version_id: str) -> Worker:
675
- self._make_room()
676
- worker = self._spawn(version_id)
677
- self._workers[version_id] = worker
783
+ def _drop(self, key: WorkerKey) -> Worker:
784
+ """Under ``_lock``."""
785
+ worker = self._workers.pop(key)
786
+ self._roster = tuple(self._workers.values())
678
787
  return worker
679
788
 
680
789
  def dead(self) -> list[Worker]:
@@ -687,55 +796,114 @@ class WorkerPool:
687
796
  with self._lock:
688
797
  return [w for w in self._workers.values() if w.alive()]
689
798
 
690
- def _make_room(self) -> None:
691
- """Under ``_lock``: leave one worker's worth of the budget free, by
692
- forgetting exited workers that hold no sessions first, then stopping
693
- session-less ones longest idle first, and refusing when every live
694
- worker is in use."""
695
- exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
696
- for version_id in exited:
697
- self._workers.pop(version_id).stop()
698
- spare = sorted(
699
- (w for w in self._workers.values() if not w.sessions),
700
- key=lambda w: w.idle_since,
799
+ def _make_room(self, need_mb: int) -> None:
800
+ """Under ``_lock``: leave room for a worker charged ``need_mb`` (at least a footprint) of the budget
801
+ (:meth:`_evict_spare`), and refuse when what the workers left are counted at leaves less; refused
802
+ before anything is stopped when stopping every worker holding no session would not free it, so
803
+ an open that cannot fit here stops no other world's warm worker."""
804
+ room = max(self.footprint_mb, need_mb)
805
+ spare = sum(self._counted(worker) for worker in self._roster if not worker.sessions)
806
+ self._refuse_short(room, self._free_mb(self._roster) + spare)
807
+ self._evict_spare(need_mb)
808
+ self._refuse_short(room, self._free_mb(self._roster))
809
+
810
+ def _refuse_short(self, room: int, free: int) -> None:
811
+ if free >= room:
812
+ return
813
+ raise WorkerCapacity(
814
+ f"this host's {len(self._workers)} version workers are counted at {self.counted_mb()} of "
815
+ f"{self.budget_mb} MiB (each what it needs or holds, at least {self.footprint_mb} MiB), which "
816
+ f"leaves no room for one needing {room} MiB; place elsewhere"
701
817
  )
702
- while len(self._workers) >= self.max_workers and spare:
703
- self._workers.pop(spare.pop(0).version_id).stop()
704
- if len(self._workers) >= self.max_workers:
705
- raise WorkerCapacity(
706
- f"this host's {self.max_workers} version workers ({self.budget_mb} MiB at "
707
- f"{self.footprint_mb} MiB each) all hold sessions; place elsewhere"
708
- )
709
818
 
710
- def peek(self, version_id: str) -> Worker | None:
819
+ def _evict_spare(self, need_mb: int = 0) -> None:
820
+ """Under ``_lock``: forget exited workers that hold no sessions, then stop session-less ones,
821
+ longest idle first, until ``need_mb`` (at least a footprint) of the budget is free or none is
822
+ left to stop."""
823
+ room = max(self.footprint_mb, need_mb)
824
+ exited = [k for k, w in self._workers.items() if not w.alive() and not w.sessions]
825
+ for key in exited:
826
+ self._drop(key).stop()
827
+ spare = sorted((w for w in self._workers.values() if not w.sessions), key=lambda w: w.idle_since)
828
+ while self._free_mb(self._roster) < room and spare:
829
+ self._drop(spare.pop(0).key).stop()
830
+
831
+ def _counted(self, worker: Worker) -> int:
832
+ """What ``worker`` takes of the budget: what it was charged, or the memory the kernel counts as its own
833
+ while it runs (``RssAnon``, else ``VmRSS``), whichever is more, never under the footprint."""
834
+ status = self.memory.status(worker.pid) if worker.alive() else None
835
+ measured = (status.mb(RESIDENT_ANON) or status.mb(RESIDENT)) if status is not None else None
836
+ return max(self.footprint_mb, worker.charge_mb, measured or 0)
837
+
838
+ def _free_mb(self, workers: tuple[Worker, ...]) -> int:
839
+ return self.budget_mb - sum(self._counted(worker) for worker in workers)
840
+
841
+ def counted_mb(self) -> int:
842
+ """What every worker here is counted at against the budget, in MiB. Read off the last roster, never
843
+ under ``_lock``, which a starting or stopping worker holds for seconds: the heartbeat must not wait."""
844
+ return sum(self._counted(worker) for worker in self._roster)
845
+
846
+ def capacity(self) -> int:
847
+ """The workers this replica could hold now: those it runs plus as many more as fit at the footprint,
848
+ so a replica whose workers are heavy advertises fewer slots than ``max_workers``. Lock-free, as
849
+ :meth:`counted_mb`."""
850
+ roster = self._roster
851
+ return len(roster) + max(0, self._free_mb(roster)) // self.footprint_mb
852
+
853
+ def peaks(self) -> list[tuple[WorkerKey, int]]:
854
+ """Each running worker's peak resident memory (``VmHWM``) by key, where the kernel says. Lock-free."""
855
+ live = [worker for worker in self._roster if worker.alive()]
856
+ readings = [(worker.key, self.memory.status(worker.pid).mb(PEAK_RESIDENT)) for worker in live]
857
+ return sorted((key, mb) for key, mb in readings if mb is not None)
858
+
859
+ def stats(self) -> dict:
860
+ """What the heartbeat says of this pool, lock-free: the slots that fit now, what the workers are
861
+ counted at, the keys they run for and the largest worker this host gives."""
862
+ return {
863
+ "maxWorkers": self.capacity(),
864
+ "workerMemoryMb": self.counted_mb(),
865
+ "workerKeysInUse": [{"versionId": key.version_id, "lane": key.lane} for key in self.keys_in_use()],
866
+ "workerCapMb": self.ceiling_mb,
867
+ }
868
+
869
+ def keys_in_use(self) -> list[WorkerKey]:
870
+ """The keys a worker runs for here, so placement knows which opens need no new worker. Lock-free."""
871
+ return sorted(worker.key for worker in self._roster if worker.alive())
872
+
873
+ def peek(self, key: WorkerKey) -> Worker | None:
711
874
  with self._lock:
712
- return self._workers.get(version_id)
875
+ return self._workers.get(key)
713
876
 
714
877
  def reap_idle(self, now: float | None = None) -> list[str]:
715
- """Stop workers that have had no session for ``idle_s``; returns their versions."""
878
+ """Stop workers that have had no session for ``idle_s``; returns their keys' names."""
716
879
  now = time.time() if now is None else now
717
880
  with self._lock:
718
881
  idle = [
719
- w.version_id
720
- for w in self._workers.values()
882
+ key
883
+ for key, w in self._workers.items()
721
884
  if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
722
885
  ]
723
- stopped = [self._workers.pop(v) for v in idle]
886
+ stopped = [self._drop(key) for key in idle]
724
887
  for worker in stopped:
725
888
  worker.stop()
726
- return idle
889
+ return [key.name for key in idle]
727
890
 
728
891
  def stop_all(self) -> None:
729
892
  with self._lock:
730
893
  workers = list(self._workers.values())
731
894
  self._workers.clear()
895
+ self._roster = ()
732
896
  for worker in workers:
733
897
  worker.stop()
734
898
 
735
- def _spawn(self, version_id: str) -> Worker:
736
- home = self.workers_root / version_id
899
+ def sizes(self) -> list[tuple[Worker, int | None]]:
900
+ """Each running worker with the address space the kernel counts it at (``VmSize``). Lock-free."""
901
+ return [(worker, self.memory.status(worker.pid).mb(SIZE)) for worker in self._roster if worker.alive()]
902
+
903
+ def _spawn(self, key: WorkerKey, world: str, cap_mb: int) -> Worker:
904
+ home = key.home(self.workers_root)
737
905
  home.mkdir(parents=True, exist_ok=True, mode=0o700)
738
- socket_path = self.sockets / f"{hashlib.sha256(version_id.encode()).hexdigest()[:16]}.sock"
906
+ socket_path = self.sockets / f"{hashlib.sha256(key.name.encode()).hexdigest()[:16]}.sock"
739
907
  secret = secrets.token_urlsafe(24)
740
908
  # The supervisor binds and listens; the worker inherits the descriptor
741
909
  # and serves on it, so the socket is the supervisor's from the start.
@@ -749,7 +917,8 @@ class WorkerPool:
749
917
  "--cache-root",
750
918
  str(self.root),
751
919
  "--version",
752
- version_id,
920
+ key.version_id,
921
+ *(["--states"] if key.states else []),
753
922
  "--socket-fd",
754
923
  str(listening.fileno()),
755
924
  ],
@@ -757,7 +926,7 @@ class WorkerPool:
757
926
  secret=secret,
758
927
  home=home,
759
928
  worlds_domain=self.worlds_domain,
760
- memory_mb=self.memory_mb,
929
+ memory_mb=cap_mb,
761
930
  file_mb=self.file_mb,
762
931
  ),
763
932
  cwd=str(home),
@@ -769,7 +938,7 @@ class WorkerPool:
769
938
  )
770
939
  finally:
771
940
  listening.close()
772
- worker = Worker(version_id, process, socket_path, secret)
941
+ worker = Worker(key, process, socket_path, secret, world=world, cap_mb=cap_mb)
773
942
  self._await_ready(worker)
774
943
  return worker
775
944
 
@@ -778,10 +947,10 @@ class WorkerPool:
778
947
  while time.time() < deadline:
779
948
  if not worker.alive():
780
949
  raise WorkerGone(
781
- f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
950
+ f"worker for {worker.key.name} exited with {worker.process.returncode} on start"
782
951
  )
783
952
  if worker.socket_path.exists() and worker.answers_health():
784
953
  return
785
954
  time.sleep(0.05)
786
955
  worker.stop()
787
- raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
956
+ raise WorkerGone(f"worker for {worker.key.name} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")