@withgateway/sdk 0.6.0 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dist/agents/index.d.mts +581 -0
  2. package/dist/agents/index.d.ts +581 -0
  3. package/dist/agents/index.js +1473 -0
  4. package/dist/agents/index.js.map +1 -0
  5. package/dist/agents/index.mjs +1424 -0
  6. package/dist/agents/index.mjs.map +1 -0
  7. package/dist/apiError-DHpfbmdx.d.mts +17 -0
  8. package/dist/apiError-DHpfbmdx.d.ts +17 -0
  9. package/dist/cli.js +11611 -6845
  10. package/dist/cli.js.map +1 -1
  11. package/dist/cli.mjs +10090 -5325
  12. package/dist/cli.mjs.map +1 -1
  13. package/dist/files/index.d.mts +3 -1
  14. package/dist/files/index.d.ts +3 -1
  15. package/dist/files/index.js +371 -102
  16. package/dist/files/index.js.map +1 -1
  17. package/dist/files/index.mjs +371 -102
  18. package/dist/files/index.mjs.map +1 -1
  19. package/dist/hooks/index.js +346 -136
  20. package/dist/hooks/index.js.map +1 -1
  21. package/dist/hooks/index.mjs +346 -136
  22. package/dist/hooks/index.mjs.map +1 -1
  23. package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
  24. package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
  25. package/dist/index.d.mts +7 -4
  26. package/dist/index.d.ts +7 -4
  27. package/dist/index.js +3177 -748
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.mjs +2987 -590
  30. package/dist/index.mjs.map +1 -1
  31. package/dist/media-CQUC7TkS.d.mts +39 -0
  32. package/dist/media-DQnBloaX.d.ts +39 -0
  33. package/dist/traces/index.d.mts +3 -1
  34. package/dist/traces/index.d.ts +3 -1
  35. package/dist/traces/index.js +371 -102
  36. package/dist/traces/index.js.map +1 -1
  37. package/dist/traces/index.mjs +371 -102
  38. package/dist/traces/index.mjs.map +1 -1
  39. package/dist/tracing/index.d.mts +3 -2
  40. package/dist/tracing/index.d.ts +3 -2
  41. package/dist/tracing/index.js +257 -16
  42. package/dist/tracing/index.js.map +1 -1
  43. package/dist/tracing/index.mjs +257 -16
  44. package/dist/tracing/index.mjs.map +1 -1
  45. package/dist/transport-CpRe7zgd.d.mts +123 -0
  46. package/dist/transport-CpRe7zgd.d.ts +123 -0
  47. package/dist/worlds/index.d.mts +1986 -1746
  48. package/dist/worlds/index.d.ts +1986 -1746
  49. package/dist/worlds/index.js +1470 -290
  50. package/dist/worlds/index.js.map +1 -1
  51. package/dist/worlds/index.mjs +1465 -286
  52. package/dist/worlds/index.mjs.map +1 -1
  53. package/package.json +6 -1
  54. package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
  55. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  56. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  57. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  58. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  59. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  60. package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
  61. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
  62. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  63. package/runtime/gatewaysdk/environment/schema/api.py +3 -4
  64. package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
  65. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
  67. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  68. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  69. package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
  70. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  71. package/runtime/gatewaysdk/environment/schema/host.py +94 -12
  72. package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
  73. package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
  74. package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
  75. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  76. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  77. package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
  78. package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
  79. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  82. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  83. package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
  84. package/runtime/gatewaysdk/environment/schema/local.py +32 -7
  85. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  86. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  87. package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
  88. package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
  91. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  92. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  93. package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
  94. package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
  95. package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
  96. package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
  97. package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
  98. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  99. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  100. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  101. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
  102. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
  103. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  104. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  105. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
  106. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
  107. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  108. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
  109. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
  110. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  111. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  112. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
  113. package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
  114. package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
  115. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
  116. package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
  117. package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
  118. package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
  119. package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
  120. package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
  121. package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
  122. package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
  123. package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
  124. package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
  125. package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
  126. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  127. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  128. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  129. package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
  130. package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
  131. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  132. package/runtime/gatewaysdk/environment/secret_values.py +1 -1
  133. package/runtime/gatewaysdk/environment/world.py +34 -43
  134. package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
  135. package/skills/benchmarks-getting-started/SKILL.md +1 -1
  136. package/skills/connector-schema-authoring/SKILL.md +2 -2
  137. package/skills/connector-schema-authoring/connector-worlds.md +3 -2
  138. package/skills/connector-schema-authoring/world-schema.md +36 -23
  139. package/skills/factory-orchestrator/SKILL.md +147 -0
  140. package/skills/world-data-ingestion/ROWS.md +5 -1
  141. package/skills/world-data-ingestion/SKILL.md +8 -6
  142. package/skills/worlds-getting-started/SKILL.md +9 -9
  143. package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  144. package/skills/worlds-getting-started/references/gotchas.md +6 -6
  145. package/skills/worlds-getting-started/references/journal-checks.md +17 -2
  146. package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  147. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  148. package/skills/worlds-getting-started/references/ui-map.md +3 -2
  149. package/dist/media-BIApiPzS.d.mts +0 -102
  150. package/dist/media-BIApiPzS.d.ts +0 -102
@@ -1,13 +1,14 @@
1
- """Version workers: one subprocess per world version that runs world-authored code.
1
+ """Version workers: subprocesses that run world-authored code, one per :class:`WorkerKey`.
2
2
 
3
3
  A schema world with an ``api/handlers.py`` brings its own Python. The host
4
4
  process is shared by every tenant on the replica, so that code never runs in
5
- it: the supervisor (:mod:`host`) starts one worker per version, over a unix
6
- socket only the supervisor can reach, and forwards that version's sessions to
7
- it. The worker is the same :class:`host.WorldHost` in a smaller room:
5
+ it: the supervisor (:mod:`host`) starts a worker per version and lane (its
6
+ sessions on the version's own data, and those on any of its world states), over
7
+ a unix socket only the supervisor can reach, and forwards those sessions to it.
8
+ The worker is the same :class:`host.WorldHost` in a smaller room:
8
9
 
9
10
  * its own process, so a handler that loops, leaks or crashes takes down that
10
- version's sessions on this replica and nothing else;
11
+ version's sessions of that lane on this replica and nothing else;
11
12
  * rlimits on address space, open files and file size, set before any world
12
13
  code is imported (:class:`AddressSpaceLimit` says how macOS differs);
13
14
  * a scrubbed environment: the control-plane secret, cloud credentials and the
@@ -43,6 +44,7 @@ import os
43
44
  import resource
44
45
  import secrets
45
46
  import shutil
47
+ import signal
46
48
  import socket
47
49
  import struct
48
50
  import subprocess
@@ -55,29 +57,38 @@ from contextlib import contextmanager, suppress
55
57
  from dataclasses import dataclass, field
56
58
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
57
59
  from pathlib import Path
60
+ from types import MappingProxyType
58
61
  from typing import Protocol
59
62
 
60
63
  from .cache_layout import WORKERS_DIR
61
64
  from .grading import VerifierSandbox
62
65
  from .handler_errors import log_line
66
+ from .process_memory import PEAK_RESIDENT, PEAK_SIZE, RESIDENT, RESIDENT_ANON, SIZE, MemoryNeed, ProcessMemory
67
+ from .worker_key import WorkerKey
63
68
 
64
69
  #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
65
70
  DEFAULT_WORKER_MEMORY_MB = 2048
66
71
  #: Memory every worker together may use, in MiB, unless
67
- #: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. The number of workers a replica
68
- #: runs at once is this over the footprint each is counted at. A version past
69
- #: that is refused ``at_capacity`` and placed on another replica instead of
70
- #: the kernel choosing which process to kill.
72
+ #: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. A worker starts only while the
73
+ #: workers already running, each counted as :meth:`WorkerPool.counted_mb` says,
74
+ #: leave room for its need (at least a footprint). A worker past that is refused
75
+ #: ``at_capacity`` and placed on another replica instead of the kernel choosing
76
+ #: which process to kill.
71
77
  DEFAULT_WORKER_BUDGET_MB = 4096
72
- #: What one worker is counted at against the budget, in MiB, unless
73
- #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
78
+ #: The least one worker is counted at against the budget, in MiB, unless
79
+ #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise; a worker whose resident memory
80
+ #: the kernel reports higher is counted at that. Unset, a worker is counted at
74
81
  #: its whole address-space cap: the safe reading, and a 50x over-reservation of
75
82
  #: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
76
83
  #: many worlds at once sets the footprint it measured, keeps the address-space
77
84
  #: cap as the rail each worker cannot cross, and sizes the container for
78
- #: footprint x workers plus the supervisor's share.
85
+ #: the budget plus the supervisor's share.
79
86
  DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
80
- #: Largest file a worker may write (its session databases), in MiB.
87
+ #: The most address space one worker is ever given, in MiB, unless ``WORLDS_WORKER_MEMORY_CEILING_MB`` says
88
+ #: otherwise; unset, ``WORLDS_WORKER_MEMORY_MB``, so no worker's cap grows. A world measured to need more than
89
+ #: the cap (its peak and half again) gets that much up to this; a raised ceiling must fit the budget.
90
+ DEFAULT_WORKER_MEMORY_CEILING_MB: int | None = None
91
+ #: Largest file a worker may write (its session databases), in MiB, unless ``WORLDS_WORKER_FILE_MB`` says otherwise.
81
92
  DEFAULT_WORKER_FILE_MB = 4096
82
93
  #: Open files per worker: one per live session plus the sockets serving them.
83
94
  DEFAULT_WORKER_NOFILE = 4096
@@ -97,10 +108,15 @@ WORKER_LEASE_S = 365.0 * 86400.0
97
108
  #: Environment the worker inherits. Nothing else crosses: no host secret, no
98
109
  #: cloud credentials, no platform URL.
99
110
  INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
111
+ #: glibc keeps a malloc arena per thread (64 MiB of address space each) after the thread ends, and
112
+ #: ``RLIMIT_AS`` counts them: without this cap a worker that served many threads reads as near its cap
113
+ #: while holding little, and runs out of address space with memory to spare.
114
+ WORKER_MALLOC_ARENAS = "2"
100
115
  #: Headers the supervisor never forwards to a worker: the ones that describe
101
116
  #: the supervisor's connection, and its own control-plane secret.
102
117
  UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
103
118
  FINISH_THREADS = 8
119
+ NO_WORKERS = MappingProxyType({"maxWorkers": 0, "workerMemoryMb": 0, "workerKeysInUse": (), "workerCapMb": 0})
104
120
  FINISH_SETTLE_S = 10.0
105
121
 
106
122
 
@@ -236,6 +252,7 @@ class UnixHTTPServer(ThreadingHTTPServer):
236
252
 
237
253
 
238
254
  _PR_SET_DUMPABLE = 4
255
+ _PR_SET_PDEATHSIG = 1
239
256
 
240
257
 
241
258
  def seal_process() -> bool:
@@ -253,6 +270,16 @@ def seal_process() -> bool:
253
270
  return False
254
271
 
255
272
 
273
+ def die_with_parent(parent: int) -> None:
274
+ """Have the kernel kill this process the moment ``parent`` exits (Linux), so no code of this process
275
+ can postpone it; and leave at once if ``parent`` is gone already. Elsewhere the caller keeps its own watch."""
276
+ if sys.platform.startswith("linux"):
277
+ with suppress(OSError, AttributeError):
278
+ ctypes.CDLL(None, use_errno=True).prctl(_PR_SET_PDEATHSIG, signal.SIGKILL, 0, 0, 0)
279
+ if os.getppid() != parent:
280
+ os._exit(1)
281
+
282
+
256
283
  class Kernel(Protocol):
257
284
  """Where a process's limits are read and set: the ``resource`` module."""
258
285
 
@@ -355,6 +382,7 @@ def worker_environment(
355
382
  home: Path,
356
383
  worlds_domain: str,
357
384
  memory_mb: int,
385
+ file_mb: int = DEFAULT_WORKER_FILE_MB,
358
386
  source: Mapping[str, str] | None = None,
359
387
  ) -> dict[str, str]:
360
388
  """The environment a worker is started with: the interpreter's own needs,
@@ -366,7 +394,9 @@ def worker_environment(
366
394
  HOME=str(home),
367
395
  WORLDS_WORKER_SECRET=secret,
368
396
  WORLDS_WORKER_MEMORY_MB=str(memory_mb),
397
+ WORLDS_WORKER_FILE_MB=str(file_mb),
369
398
  WORLDS_INGRESS_DOMAIN=worlds_domain,
399
+ MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS,
370
400
  )
371
401
  return env
372
402
 
@@ -378,7 +408,7 @@ def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None)
378
408
  secret, no cloud credentials, no platform URL."""
379
409
  inherited: Mapping[str, str] = os.environ if source is None else source
380
410
  env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
381
- env["HOME"] = str(home)
411
+ env.update(HOME=str(home), MALLOC_ARENA_MAX=WORKER_MALLOC_ARENAS)
382
412
  return env
383
413
 
384
414
 
@@ -417,12 +447,26 @@ class ConfinedVerifierSandbox(VerifierSandbox):
417
447
 
418
448
  @dataclass
419
449
  class Worker:
420
- version_id: str
450
+ key: WorkerKey
421
451
  process: subprocess.Popen
422
452
  socket_path: Path
423
453
  secret: str
424
454
  sessions: set[str] = field(default_factory=set)
425
455
  idle_since: float = field(default_factory=time.time)
456
+ world: str = ""
457
+ #: World code here raised ``MemoryError``: the worker serves no new request and is recycled.
458
+ poisoned: bool = False
459
+ #: Its address-space cap (``RLIMIT_AS``), given at start; what it is counted at is ``charge_mb`` or its RSS.
460
+ cap_mb: int = DEFAULT_WORKER_MEMORY_MB
461
+ #: What its world was measured to need, which sizes a replacement; ``charge_mb`` what the budget
462
+ #: counts it at, at most its cap and only as far as the budget had room when it was charged.
463
+ need_mb: int = 0
464
+ charge_mb: int = 0
465
+ near_cap: bool = False
466
+
467
+ @property
468
+ def version_id(self) -> str:
469
+ return self.key.version_id
426
470
 
427
471
  @property
428
472
  def pid(self) -> int:
@@ -431,6 +475,10 @@ class Worker:
431
475
  def alive(self) -> bool:
432
476
  return self.process.poll() is None
433
477
 
478
+ def serves(self) -> bool:
479
+ """Alive and not poisoned: a request may go to it."""
480
+ return self.alive() and not self.poisoned
481
+
434
482
  def request(
435
483
  self,
436
484
  method: str,
@@ -526,7 +574,7 @@ class Worker:
526
574
 
527
575
  class Finishing:
528
576
  """What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
529
- threads, one job at a time for a version in the order they came (so a worker that never answers
577
+ threads, one job at a time for a worker key in the order they came (so a worker that never answers
530
578
  holds one thread, not the pool). A session with no worker is finished at once, where it is."""
531
579
 
532
580
  def __init__(self, threads: int = FINISH_THREADS) -> None:
@@ -541,7 +589,7 @@ class Finishing:
541
589
  job()
542
590
  return
543
591
  with self._changed:
544
- self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
592
+ self._waiting.setdefault(worker.key.name, collections.deque()).append(job)
545
593
  self._staff()
546
594
  self._changed.notify()
547
595
 
@@ -588,8 +636,8 @@ class Finishing:
588
636
 
589
637
 
590
638
  class WorkerPool:
591
- """Every version worker on this replica, started on the first session of a
592
- version and stopped once it has had none for a while."""
639
+ """Every version worker on this replica, one per :class:`WorkerKey`, started on
640
+ the key's first session and stopped once it has had none for a while."""
593
641
 
594
642
  def __init__(
595
643
  self,
@@ -601,7 +649,12 @@ class WorkerPool:
601
649
  footprint_mb: int | None = DEFAULT_WORKER_FOOTPRINT_MB,
602
650
  idle_s: float = DEFAULT_WORKER_IDLE_S,
603
651
  python: str = sys.executable,
652
+ file_mb: int = DEFAULT_WORKER_FILE_MB,
653
+ memory: ProcessMemory | None = None,
654
+ ceiling_mb: int | None = DEFAULT_WORKER_MEMORY_CEILING_MB,
604
655
  ):
656
+ if file_mb <= 0:
657
+ raise ValueError(f"a worker's file cap ({file_mb} MiB) must be at least 1 MiB (WORLDS_WORKER_FILE_MB)")
605
658
  if footprint_mb is None:
606
659
  footprint_mb = memory_mb
607
660
  if footprint_mb <= 0 or footprint_mb > memory_mb:
@@ -610,11 +663,22 @@ class WorkerPool:
610
663
  f"address-space cap ({memory_mb} MiB) (WORLDS_WORKER_FOOTPRINT_MB, "
611
664
  "WORLDS_WORKER_MEMORY_MB)"
612
665
  )
666
+ ceiling_mb = memory_mb if ceiling_mb is None else ceiling_mb
667
+ if ceiling_mb < memory_mb:
668
+ raise ValueError(
669
+ f"the worker ceiling ({ceiling_mb} MiB) is under the worker cap ({memory_mb} MiB) "
670
+ "(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_MEMORY_MB)"
671
+ )
613
672
  if budget_mb < footprint_mb:
614
673
  raise ValueError(
615
674
  f"the worker budget ({budget_mb} MiB) holds no worker of {footprint_mb} MiB "
616
675
  "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
617
676
  )
677
+ if ceiling_mb > memory_mb and ceiling_mb > budget_mb:
678
+ raise ValueError(
679
+ f"the worker ceiling ({ceiling_mb} MiB) is over the worker budget ({budget_mb} MiB) "
680
+ "(WORLDS_WORKER_MEMORY_CEILING_MB, WORLDS_WORKER_BUDGET_MB)"
681
+ )
618
682
  self.root = Path(cache_root)
619
683
  self.workers_root = self.root / WORKERS_DIR
620
684
  # Workers live and die with the supervisor; whatever a previous one
@@ -628,47 +692,98 @@ class WorkerPool:
628
692
  self.sockets = Path(self._sockets.name)
629
693
  self.worlds_domain = worlds_domain
630
694
  self.memory_mb = memory_mb
695
+ self.ceiling_mb = ceiling_mb
631
696
  self.budget_mb = budget_mb
632
697
  self.footprint_mb = footprint_mb
698
+ self.file_mb = file_mb
633
699
  self.max_workers = budget_mb // footprint_mb
634
700
  self.idle_s = idle_s
635
701
  self.python = python
702
+ self.memory = memory or ProcessMemory()
636
703
  self._lock = threading.Lock()
637
- self._workers: dict[str, Worker] = {}
638
-
639
- def get(self, version_id: str) -> Worker:
640
- """The live worker for a version, started if there is none. Starting one
704
+ self._workers: dict[WorkerKey, Worker] = {}
705
+ self._roster: tuple[Worker, ...] = ()
706
+
707
+ def cap_for(self, need_mb: int) -> int:
708
+ """The address space a worker needing ``need_mb`` is given: the cap, or more up to the ceiling."""
709
+ return min(self.ceiling_mb, max(self.memory_mb, need_mb))
710
+
711
+ def _bounded(self, need_mb: int) -> int:
712
+ """A need as the budget charges it: never more than the ceiling, which is all a worker can hold, nor
713
+ than the whole budget."""
714
+ return min(need_mb, self.ceiling_mb, self.budget_mb)
715
+
716
+ def get(self, key: WorkerKey, *, world: str = "", need_mb: int = 0) -> Worker:
717
+ """The live worker for a key, started if there is none (labelled with its ``world``, given
718
+ :meth:`cap_for` its ``need_mb`` and charged it); a running one keeps the larger need and is
719
+ charged it up to the cap it keeps (a process's cap only goes down) as far as the budget has
720
+ room, never refusing its own lane's open. Starting one
641
721
  past the budget first stops workers holding no sessions (whatever their
642
722
  idle time); when every one is in use, raises :class:`WorkerCapacity`. A
643
723
  dead worker that still holds sessions is never replaced here: its
644
724
  sessions' files are on disk, and the supervisor recovers them through
645
725
  :meth:`replace` (raises :class:`WorkerGone`)."""
726
+ need_mb = self._bounded(need_mb)
646
727
  with self._lock:
647
- worker = self._workers.get(version_id)
728
+ worker = self._workers.get(key)
648
729
  if worker is not None and worker.alive():
730
+ self._raise_need(worker, need_mb)
649
731
  return worker
650
732
  if worker is not None and worker.sessions:
651
- raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
733
+ raise WorkerGone(f"worker for {key.name} (pid {worker.pid}) exited holding sessions")
652
734
  if worker is not None:
653
735
  worker.stop()
654
- del self._workers[version_id]
655
- return self._start(version_id)
736
+ self._drop(key)
737
+ self._make_room(need_mb)
738
+ return self._start(key, world, need_mb)
656
739
 
657
740
  def replace(self, dead: Worker) -> Worker:
658
741
  """A fresh worker in ``dead``'s place, its sessions left for the caller to
659
742
  reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
660
- when the pool no longer holds ``dead``."""
743
+ when the pool no longer holds ``dead``, never by the budget: it takes the
744
+ place of a worker the budget held, so sessions are never lost to it. It needs
745
+ what ``dead`` was measured to need (:meth:`outgrown` raises that for a worker that ran out), but is
746
+ charged no more than the budget leaves it (at least a footprint), and its cap is never under
747
+ ``dead``'s."""
661
748
  with self._lock:
662
- if self._workers.get(dead.version_id) is not dead:
663
- raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
664
- del self._workers[dead.version_id]
749
+ if self._workers.get(dead.key) is not dead:
750
+ raise WorkerGone(f"worker for {dead.key.name} (pid {dead.pid}) was replaced already")
751
+ self._drop(dead.key)
665
752
  dead.stop()
666
- return self._start(dead.version_id)
753
+ self._evict_spare(dead.need_mb)
754
+ charge = min(dead.need_mb, max(self.footprint_mb, self._free_mb(self._roster)))
755
+ return self._start(dead.key, dead.world, dead.need_mb, charge_mb=charge, cap_floor_mb=dead.cap_mb)
756
+
757
+ def outgrown(self, worker: Worker) -> None:
758
+ """``worker`` ran out of address space: its peak address space (``VmPeak``) and half again is what
759
+ it needs from now on, so its replacement gets a larger cap up to the ceiling. Read before it stops."""
760
+ peak = self.memory.status(worker.pid).mb(PEAK_SIZE)
761
+ grown = MemoryNeed(peak).mb if peak is not None else 0
762
+ worker.need_mb = max(worker.need_mb, self._bounded(grown))
763
+
764
+ def _raise_need(self, worker: Worker, need_mb: int) -> None:
765
+ """Under ``_lock``: ``worker`` asked for ``need_mb`` by another open of its lane keeps the larger need
766
+ and is charged it up to its cap as far as what the budget leaves allows. The open is never refused:
767
+ placement sends a lane's sessions to the replica that runs it."""
768
+ worker.need_mb = max(worker.need_mb, need_mb)
769
+ room = self._counted(worker) + max(0, self._free_mb(self._roster))
770
+ worker.charge_mb = max(worker.charge_mb, min(worker.need_mb, worker.cap_mb, room))
771
+
772
+ def _start(
773
+ self, key: WorkerKey, world: str, need_mb: int, charge_mb: int | None = None, cap_floor_mb: int = 0
774
+ ) -> Worker:
775
+ charge = need_mb if charge_mb is None else charge_mb
776
+ worker = self._spawn(key, world, max(cap_floor_mb, self.cap_for(charge)))
777
+ worker.need_mb = need_mb
778
+ worker.charge_mb = min(charge, worker.cap_mb)
779
+ self._workers[key] = worker
780
+ self._roster = tuple(self._workers.values())
781
+ return worker
667
782
 
668
- def _start(self, version_id: str) -> Worker:
669
- self._make_room()
670
- worker = self._spawn(version_id)
671
- self._workers[version_id] = worker
783
+ def _drop(self, key: WorkerKey) -> Worker:
784
+ """Under ``_lock``."""
785
+ worker = self._workers.pop(key)
786
+ self._roster = tuple(self._workers.values())
672
787
  return worker
673
788
 
674
789
  def dead(self) -> list[Worker]:
@@ -681,55 +796,114 @@ class WorkerPool:
681
796
  with self._lock:
682
797
  return [w for w in self._workers.values() if w.alive()]
683
798
 
684
- def _make_room(self) -> None:
685
- """Under ``_lock``: leave one worker's worth of the budget free, by
686
- forgetting exited workers that hold no sessions first, then stopping
687
- session-less ones longest idle first, and refusing when every live
688
- worker is in use."""
689
- exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
690
- for version_id in exited:
691
- self._workers.pop(version_id).stop()
692
- spare = sorted(
693
- (w for w in self._workers.values() if not w.sessions),
694
- key=lambda w: w.idle_since,
799
+ def _make_room(self, need_mb: int) -> None:
800
+ """Under ``_lock``: leave room for a worker charged ``need_mb`` (at least a footprint) of the budget
801
+ (:meth:`_evict_spare`), and refuse when what the workers left are counted at leaves less; refused
802
+ before anything is stopped when stopping every worker holding no session would not free it, so
803
+ an open that cannot fit here stops no other world's warm worker."""
804
+ room = max(self.footprint_mb, need_mb)
805
+ spare = sum(self._counted(worker) for worker in self._roster if not worker.sessions)
806
+ self._refuse_short(room, self._free_mb(self._roster) + spare)
807
+ self._evict_spare(need_mb)
808
+ self._refuse_short(room, self._free_mb(self._roster))
809
+
810
+ def _refuse_short(self, room: int, free: int) -> None:
811
+ if free >= room:
812
+ return
813
+ raise WorkerCapacity(
814
+ f"this host's {len(self._workers)} version workers are counted at {self.counted_mb()} of "
815
+ f"{self.budget_mb} MiB (each what it needs or holds, at least {self.footprint_mb} MiB), which "
816
+ f"leaves no room for one needing {room} MiB; place elsewhere"
695
817
  )
696
- while len(self._workers) >= self.max_workers and spare:
697
- self._workers.pop(spare.pop(0).version_id).stop()
698
- if len(self._workers) >= self.max_workers:
699
- raise WorkerCapacity(
700
- f"this host's {self.max_workers} version workers ({self.budget_mb} MiB at "
701
- f"{self.footprint_mb} MiB each) all hold sessions; place elsewhere"
702
- )
703
818
 
704
- def peek(self, version_id: str) -> Worker | None:
819
+ def _evict_spare(self, need_mb: int = 0) -> None:
820
+ """Under ``_lock``: forget exited workers that hold no sessions, then stop session-less ones,
821
+ longest idle first, until ``need_mb`` (at least a footprint) of the budget is free or none is
822
+ left to stop."""
823
+ room = max(self.footprint_mb, need_mb)
824
+ exited = [k for k, w in self._workers.items() if not w.alive() and not w.sessions]
825
+ for key in exited:
826
+ self._drop(key).stop()
827
+ spare = sorted((w for w in self._workers.values() if not w.sessions), key=lambda w: w.idle_since)
828
+ while self._free_mb(self._roster) < room and spare:
829
+ self._drop(spare.pop(0).key).stop()
830
+
831
+ def _counted(self, worker: Worker) -> int:
832
+ """What ``worker`` takes of the budget: what it was charged, or the memory the kernel counts as its own
833
+ while it runs (``RssAnon``, else ``VmRSS``), whichever is more, never under the footprint."""
834
+ status = self.memory.status(worker.pid) if worker.alive() else None
835
+ measured = (status.mb(RESIDENT_ANON) or status.mb(RESIDENT)) if status is not None else None
836
+ return max(self.footprint_mb, worker.charge_mb, measured or 0)
837
+
838
+ def _free_mb(self, workers: tuple[Worker, ...]) -> int:
839
+ return self.budget_mb - sum(self._counted(worker) for worker in workers)
840
+
841
+ def counted_mb(self) -> int:
842
+ """What every worker here is counted at against the budget, in MiB. Read off the last roster, never
843
+ under ``_lock``, which a starting or stopping worker holds for seconds: the heartbeat must not wait."""
844
+ return sum(self._counted(worker) for worker in self._roster)
845
+
846
+ def capacity(self) -> int:
847
+ """The workers this replica could hold now: those it runs plus as many more as fit at the footprint,
848
+ so a replica whose workers are heavy advertises fewer slots than ``max_workers``. Lock-free, as
849
+ :meth:`counted_mb`."""
850
+ roster = self._roster
851
+ return len(roster) + max(0, self._free_mb(roster)) // self.footprint_mb
852
+
853
+ def peaks(self) -> list[tuple[WorkerKey, int]]:
854
+ """Each running worker's peak resident memory (``VmHWM``) by key, where the kernel says. Lock-free."""
855
+ live = [worker for worker in self._roster if worker.alive()]
856
+ readings = [(worker.key, self.memory.status(worker.pid).mb(PEAK_RESIDENT)) for worker in live]
857
+ return sorted((key, mb) for key, mb in readings if mb is not None)
858
+
859
+ def stats(self) -> dict:
860
+ """What the heartbeat says of this pool, lock-free: the slots that fit now, what the workers are
861
+ counted at, the keys they run for and the largest worker this host gives."""
862
+ return {
863
+ "maxWorkers": self.capacity(),
864
+ "workerMemoryMb": self.counted_mb(),
865
+ "workerKeysInUse": [{"versionId": key.version_id, "lane": key.lane} for key in self.keys_in_use()],
866
+ "workerCapMb": self.ceiling_mb,
867
+ }
868
+
869
+ def keys_in_use(self) -> list[WorkerKey]:
870
+ """The keys a worker runs for here, so placement knows which opens need no new worker. Lock-free."""
871
+ return sorted(worker.key for worker in self._roster if worker.alive())
872
+
873
+ def peek(self, key: WorkerKey) -> Worker | None:
705
874
  with self._lock:
706
- return self._workers.get(version_id)
875
+ return self._workers.get(key)
707
876
 
708
877
  def reap_idle(self, now: float | None = None) -> list[str]:
709
- """Stop workers that have had no session for ``idle_s``; returns their versions."""
878
+ """Stop workers that have had no session for ``idle_s``; returns their keys' names."""
710
879
  now = time.time() if now is None else now
711
880
  with self._lock:
712
881
  idle = [
713
- w.version_id
714
- for w in self._workers.values()
882
+ key
883
+ for key, w in self._workers.items()
715
884
  if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
716
885
  ]
717
- stopped = [self._workers.pop(v) for v in idle]
886
+ stopped = [self._drop(key) for key in idle]
718
887
  for worker in stopped:
719
888
  worker.stop()
720
- return idle
889
+ return [key.name for key in idle]
721
890
 
722
891
  def stop_all(self) -> None:
723
892
  with self._lock:
724
893
  workers = list(self._workers.values())
725
894
  self._workers.clear()
895
+ self._roster = ()
726
896
  for worker in workers:
727
897
  worker.stop()
728
898
 
729
- def _spawn(self, version_id: str) -> Worker:
730
- home = self.workers_root / version_id
899
+ def sizes(self) -> list[tuple[Worker, int | None]]:
900
+ """Each running worker with the address space the kernel counts it at (``VmSize``). Lock-free."""
901
+ return [(worker, self.memory.status(worker.pid).mb(SIZE)) for worker in self._roster if worker.alive()]
902
+
903
+ def _spawn(self, key: WorkerKey, world: str, cap_mb: int) -> Worker:
904
+ home = key.home(self.workers_root)
731
905
  home.mkdir(parents=True, exist_ok=True, mode=0o700)
732
- socket_path = self.sockets / f"{hashlib.sha256(version_id.encode()).hexdigest()[:16]}.sock"
906
+ socket_path = self.sockets / f"{hashlib.sha256(key.name.encode()).hexdigest()[:16]}.sock"
733
907
  secret = secrets.token_urlsafe(24)
734
908
  # The supervisor binds and listens; the worker inherits the descriptor
735
909
  # and serves on it, so the socket is the supervisor's from the start.
@@ -743,7 +917,8 @@ class WorkerPool:
743
917
  "--cache-root",
744
918
  str(self.root),
745
919
  "--version",
746
- version_id,
920
+ key.version_id,
921
+ *(["--states"] if key.states else []),
747
922
  "--socket-fd",
748
923
  str(listening.fileno()),
749
924
  ],
@@ -751,7 +926,8 @@ class WorkerPool:
751
926
  secret=secret,
752
927
  home=home,
753
928
  worlds_domain=self.worlds_domain,
754
- memory_mb=self.memory_mb,
929
+ memory_mb=cap_mb,
930
+ file_mb=self.file_mb,
755
931
  ),
756
932
  cwd=str(home),
757
933
  stdin=subprocess.DEVNULL,
@@ -762,7 +938,7 @@ class WorkerPool:
762
938
  )
763
939
  finally:
764
940
  listening.close()
765
- worker = Worker(version_id, process, socket_path, secret)
941
+ worker = Worker(key, process, socket_path, secret, world=world, cap_mb=cap_mb)
766
942
  self._await_ready(worker)
767
943
  return worker
768
944
 
@@ -771,10 +947,10 @@ class WorkerPool:
771
947
  while time.time() < deadline:
772
948
  if not worker.alive():
773
949
  raise WorkerGone(
774
- f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
950
+ f"worker for {worker.key.name} exited with {worker.process.returncode} on start"
775
951
  )
776
952
  if worker.socket_path.exists() and worker.answers_health():
777
953
  return
778
954
  time.sleep(0.05)
779
955
  worker.stop()
780
- raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
956
+ raise WorkerGone(f"worker for {worker.key.name} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")