@genex-ai/cli-demo 1.31.1 → 1.32.0-dev.650

package/templates/blender-service/pool.py

@@ -0,0 +1,910 @@
+"""Multi-seat pod: one Blender process per session, behind one router.
+
+    :8080  router (this process, threaded)   ->  127.0.0.1:8101+i  seat children
+                                                 each its own Blender, own scene
+
+WHY THREADS ARE SAFE HERE AND NOT IN server.py. bpy is main-thread-only, which is
+why `server.py` is a deliberately non-threading HTTPServer. This process contains
+NO bpy — it is pure stdlib that spawns Blender as children and relays HTTP to
+them. So a ThreadingHTTPServer is correct here, and it is what stops one slow
+seat from blocking every other tenant. Never import bpy into this file.
+
+ISOLATION is a process boundary, not a scene boundary. Two sessions never share
+`bpy.data`, a NAMESPACE dict, a checkpoint, or a deadline file — the collisions
+that made one-process-many-scenes unworkable. What they do share is the pod's
+filesystem, network namespace and GPU, so this is a density mechanism, not a
+security boundary against hostile code (see ops.py on why /exec cannot be
+sandboxed in-process).
+"""
+
+import hashlib
+import hmac
+import json
+import os
+import secrets
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+BLENDER = os.environ.get("GENEX_BLENDER_BIN", "blender")
+SERVER = os.environ.get("GENEX_BLENDER_SERVER", "/app/server.py")
+POD_SECRET = os.environ.get("GENEX_BLENDER_SECRET", "")
+STATE_ROOT = os.environ.get("GENEX_BLENDER_STATE", "/run/genex/seats")
+SEAT_BASE_PORT = int(os.environ.get("GENEX_BLENDER_SEAT_PORT", "8101"))
+SEAT_UID_BASE = int(os.environ.get("GENEX_BLENDER_SEAT_UID", "10000"))
+
+# MEASURED 2026-09-02 on an RTX 4090 pod, VmHWM (true peak RSS) per child:
+#   booted, empty scene                          1,103 MB
+#   castle (46 obj, 1,440 tris)                  1,313 MB
+#   heavy (12 obj, 612k tris, 8 x 4096^2 tex)    2,204 MB
+# Four seats each holding the heavy scene summed to 8,640 MB; the router is
+# 27 MB. The old guess of 2048 was ALREADY EXCEEDED by one moderately heavy
+# scene, and the shape is the lesson: ~1.1 GB is Blender's own floor before any
+# content, so 2048 left ~950 MB for the actual scene. 3072 keeps a heavy scene
+# plus a 40 MB sheet transient inside the budget with margin -- and this is an
+# OOM guard, where over-reserving is the safe direction.
+PER_SEAT_MB = int(os.environ.get("GENEX_BLENDER_PER_SEAT_MB", "3072"))
+# TWO, and the reason changed once it was measured. Memory was never the bind:
+# four heavy seats used 29% of a 29 GB pod and 22% of VRAM. What binds is GPU
+# COMPUTE. Four seats rendering at once ran 2.41x slower each (869 -> 2,094 ms
+# lit), and four concurrent renders finished in 3,100 ms wall against 3,476 ms
+# serial -- concurrency bought 11%, not 4x. So packing seats onto one card is
+# not a render-throughput win; it pays only by multiplexing the idle time
+# BETWEEN an agent's /exec calls. Seat count is therefore a DUTY-CYCLE choice,
+# not a memory one, and raising it needs an observed duty cycle, not a bigger
+# pod.
+HARD_CAP = int(os.environ.get("GENEX_BLENDER_SEATS", "2"))
+
+# A seat is BOOTING until its child answers /health. The GPU probe plus device
+# enumeration costs seconds, and on a cold pod the EEVEE shader compile can cost
+# 30-44s -- so the router NEVER waits for this. It answers 503 and the API turns
+# that into a 202 the caller retries. A blocking wait would share one thread
+# budget with live relays, which is the cross-tenant head-of-line blocking that
+# process-per-seat exists to prevent.
+SEAT_BOOT_BUDGET_S = float(os.environ.get("GENEX_BLENDER_SEAT_BOOT_S", "240"))
+RELAY_TIMEOUT_S = float(os.environ.get("GENEX_BLENDER_RELAY_S", "930"))
+MAX_BODY = 8 * 1024 * 1024
+
+# --- the R2 checkpoint push ---------------------------------------------------
+# THE ROUTER PUSHES, NOT THE CHILD, and that is a security choice before it is a
+# threading one: `/exec` runs arbitrary Python inside the child as a uid-switched
+# user, so an API-facing credential in that process is readable by the code it
+# runs. The router holds the pod secret already and is the only process here
+# that never imports bpy.
+#
+# The child's local checkpoint stays exactly as it was — synchronous, after every
+# mutating call. This lane only ships that finished file, off the request path.
+API_URL = os.environ.get("GENEX_API_URL", "").rstrip("/")
+POD_NAME = os.environ.get("GENEX_BLENDER_POD_NAME", "")
+# All three or nothing. On the local lane (`genex blender serve`) none of them is
+# set and this whole feature is inert without a flag to forget.
+PUSH_ENABLED = bool(API_URL and POD_NAME and POD_SECRET)
+# Quiet-time before a push: the agent's think time is where these seconds hide.
+CHECKPOINT_DEBOUNCE_S = float(os.environ.get("GENEX_BLENDER_PUSH_DEBOUNCE_S", "10"))
+# ...but never let a busy seat go unsaved forever chasing quiet.
+CHECKPOINT_MAX_DIRTY_S = float(os.environ.get("GENEX_BLENDER_PUSH_MAX_DIRTY_S", "60"))
+PUSH_RETRY_S = 30.0
+PUSH_TIMEOUT_S = 600.0
+RESTORE_DEADLINE_S = float(os.environ.get("GENEX_BLENDER_RESTORE_S", "120"))
+CLOSE_PUSH_TIMEOUT_S = 20.0
+# Cloudflare's error 1010 refuses the default Python UA (measured on the pod
+# proxy); the API host is behind Cloudflare too, so never rely on the default.
+USER_AGENT = "genex-blender-pool"
+# Bodies up to this size are read before a 503 so the poller keeps its socket.
+READ_AHEAD_MAX = 256 * 1024
+TERM_GRACE_S = 5.0
+FAST_FAIL_S = 20.0
+MAX_CONSECUTIVE_FAILURES = 5
+
+
+def log(msg):
+    sys.stderr.write("[pool] %s\n" % msg)
+    sys.stderr.flush()
+
+
+def cgroup_limit_mb():
+    """The container's real memory ceiling.
+
+    /proc/meminfo reports the HOST's memory, not the container's, so sizing from
+    it over-admits by however much bigger the host is -- and the failure is an
+    OOM kill of somebody else's Blender. cgroup v2 first, then v1, then a
+    configured fallback; 'max' means unlimited, which in a pod means "we cannot
+    tell", so treat it as the fallback rather than as infinity.
+    """
+    for path, div in (("/sys/fs/cgroup/memory.max", 1),
+                      ("/sys/fs/cgroup/memory/memory.limit_in_bytes", 1)):
+        try:
+            with open(path) as f:
+                raw = f.read().strip()
+            if raw == "max":
+                continue
+            n = int(raw) // (1024 * 1024)
+            # A v1 "unlimited" is a huge sentinel, not a real limit.
+            if 0 < n < 1024 * 1024:
+                return n
+        except (OSError, ValueError):
+            continue
+    return int(os.environ.get("GENEX_BLENDER_POD_MB", "0")) or None
+
+
+class Seat:
+    __slots__ = ("sid", "port", "uid", "proc", "token_sha", "child_secret", "run_dir",
+                 "workspace", "checkpoint", "opened_at", "last_request_at", "failures",
+                 "closing", "spawning", "_finish_close",
+                 # the R2 push/restore half
+                 "restore", "restoring", "seen_sig", "seen_at", "dirty_since",
+                 "push_sig", "pushing", "push_next_at", "push_state")
+
+    def __init__(self, sid, port, uid):
+        self.sid = sid
+        self.port = port
+        self.uid = uid
+        self.proc = None
+        self.token_sha = None
+        # The secret the CHILD checks. Distinct from the seat token the caller
+        # presents: the router translates one into the other, so a tenant never
+        # holds a credential its own Blender would accept directly.
+        self.child_secret = None
+        self.run_dir = os.path.join(STATE_ROOT, sid, "run")
+        self.workspace = os.path.join(STATE_ROOT, sid, "workspace")
+        self.checkpoint = os.path.join(STATE_ROOT, sid, "checkpoint.blend")
+        self.opened_at = time.time()
+        self.last_request_at = time.time()
+        self.failures = 0
+        self.closing = False
+        # True from the moment the seat is committed to the table until _spawn
+        # has set `proc`. The watcher reads `proc is None` as "dead child", and
+        # between `self.seats[sid] = seat` (under the lock) and Popen returning
+        # (outside it -- makedirs, chown, a fork) that is exactly what it sees.
+        # Its 1 s scan landing in that window spawned a SECOND Blender on the
+        # same port. Found by code-read on the first real pod run.
+        self.spawning = False
+        self._finish_close = None
+        # The chain's saved scene, handed over by the API at open (it probes R2;
+        # the pod is never allowed to ask for an arbitrary chain's bytes).
+        self.restore = None
+        self.restoring = False
+        # (st_mtime_ns, st_size) of the checkpoint as last SEEN and as last
+        # PUSHED. A signature rather than a timestamp: a scene can be rewritten
+        # to the same size within one clock tick.
+        self.seen_sig = None
+        self.seen_at = 0.0
+        self.dirty_since = None
+        self.push_sig = None
+        self.pushing = False
+        self.push_next_at = 0.0
+        self.push_state = None
+
+    def alive(self):
+        return self.proc is not None and self.proc.poll() is None
+
+    def ready(self):
+        """Answering, not merely running. The child binds its port only after
+        the GPU assertion and the checkpoint restore, so 'process exists' is not
+        the same question."""
+        if not self.alive():
+            return False
+        try:
+            with socket.create_connection(("127.0.0.1", self.port), timeout=0.4):
+                return True
+        except OSError:
+            return False
+
+
+def _sig(path):
+    """(st_mtime_ns, st_size), or None when the file is not there yet."""
+    try:
+        st = os.stat(path)
+    except OSError:
+        return None
+    return (st.st_mtime_ns, st.st_size)
+
+
+def restore_is_pointless(seat):
+    """True when there is nothing to wipe or restore — the seat has no saved
+    scene to fetch, so an existing local checkpoint (a restart in place) is the
+    better source and must not be deleted."""
+    return seat.restore is None
+
+
+class Pool:
+    def __init__(self):
+        self.seats = {}
+        self.lock = threading.Lock()
+        self.pod_mb = cgroup_limit_mb()
+        os.makedirs(STATE_ROOT, exist_ok=True)
+
+    def capacity(self):
+        """Seats this pod will admit. The lower of the hard cap and what memory
+        allows, never just the cap."""
+        if not self.pod_mb:
+            return HARD_CAP
+        # Leave one seat's worth of headroom for the router, the shared shader
+        # cache and page cache. Without it the last admitted seat is the one
+        # that OOMs, and it takes a stranger's session with it.
+        by_mem = max(0, (self.pod_mb - PER_SEAT_MB) // PER_SEAT_MB)
+        return max(1, min(HARD_CAP, by_mem))
+
+    def open(self, sid, token, restore=None):
+        with self.lock:
+            existing = self.seats.get(sid)
+            if existing and not existing.closing:
+                # Idempotent: a retried openSeat must not spawn a second child.
+                existing.token_sha = hashlib.sha256(token.encode()).hexdigest()
+                return existing, False
+            if len(self.seats) >= self.capacity():
+                return None, False
+            used = {s.port for s in self.seats.values()}
+            port = next(p for p in range(SEAT_BASE_PORT, SEAT_BASE_PORT + HARD_CAP + 8)
+                        if p not in used)
+            uid = SEAT_UID_BASE + (port - SEAT_BASE_PORT)
+            seat = Seat(sid, port, uid)
+            seat.restore = restore
+            seat.token_sha = hashlib.sha256(token.encode()).hexdigest()
+            # Flagged BEFORE it is visible in the table, so no scan can find it
+            # proc-less and unflagged.
+            seat.spawning = True
+            self.seats[sid] = seat
+        # A leftover directory from a wipe that never finished would make the
+        # restore below skip ("there is already a checkpoint") and the child boot
+        # from a stranger's scene — or from a stale one it then overwrites.
+        if not restore_is_pointless(seat):
+            shutil.rmtree(os.path.join(STATE_ROOT, sid), ignore_errors=True)
+            os.makedirs(seat.run_dir, exist_ok=True)
+        # THREADED, because _spawn may now download a checkpoint first and
+        # /pool/seats must still answer 201 `ready: false` immediately — the
+        # router never blocks, which is the property process-per-seat buys.
+        # `spawning` is already set under the lock, so the watcher skips the seat
+        # and a concurrent close hands the teardown to _spawn's finally.
+        threading.Thread(target=self._spawn, args=(seat,), daemon=True,
+                         name="spawn-%s" % sid).start()
+        return seat, True
+
+    def _spawn(self, seat):
+        with self.lock:
+            # A close that arrived first wins; a seat no longer in the table is
+            # not respawned. Both checks under the lock, so the watcher's restart
+            # and a concurrent DELETE cannot interleave into an orphan.
+            if seat.closing or self.seats.get(seat.sid) is not seat:
+                return
+            seat.spawning = True
+        try:
+            if seat.restore:
+                self._restore(seat)
+            tmpdir = os.path.join(seat.workspace, "tmp")
+            for d in (seat.run_dir, seat.workspace, tmpdir):
+                os.makedirs(d, exist_ok=True)
+            # Per-seat ownership so one tenant cannot read another's checkpoint or
+            # scripts through the shared filesystem. Best-effort: a pod that is not
+            # running as root still isolates by process, just not by uid. TMPDIR is
+            # created ABOVE, before this pass — created after it (the first cut) it
+            # belonged to root, the child could not write it, and tempfile fell
+            # back silently to the shared /tmp, which is the isolation this exists
+            # to provide (review finding).
+            try:
+                os.chown(os.path.dirname(seat.run_dir), seat.uid, seat.uid)
+                for d in (seat.run_dir, seat.workspace, tmpdir):
+                    os.chown(d, seat.uid, seat.uid)
+            except (OSError, AttributeError):
+                pass
+            env = dict(os.environ)
+            env.update({
+                "PORT": str(seat.port),
+                "GENEX_BLENDER_RUN_DIR": seat.run_dir,
+                "GENEX_BLENDER_CHECKPOINT": seat.checkpoint,
+                "HOME": seat.workspace,
+                "TMPDIR": os.path.join(seat.workspace, "tmp"),
+                # The seat's own secret. The pod secret authenticates the control
+                # plane; a seat token authenticates one session's routes, so seat A
+                # cannot drive seat B even though both are on this pod's loopback.
+                "GENEX_BLENDER_SECRET": secrets.token_hex(16),
+                # Children bind loopback only -- the router is the sole way in.
+                "GENEX_BLENDER_BIND": "127.0.0.1",
+            })
+            argv = [BLENDER, "--background", "--factory-startup", "-noaudio",
+                    "--python-exit-code", "1", "--python", SERVER,
+                    "--", "--port", str(seat.port), "--host", "127.0.0.1"]
+            extra = os.environ.get("GENEX_BLENDER_ARGS", "").split()
+            if extra:
+                # Same lever supervisor.spawn honours (e.g. --gpu-backend vulkan);
+                # the pooled path silently dropped it (review finding).
+                argv[1:1] = extra
+            kwargs = {}
+            if os.getuid() == 0:
+                kwargs["user"] = seat.uid
+                kwargs["group"] = seat.uid
+            seat.child_secret = env["GENEX_BLENDER_SECRET"]
+            seat.proc = subprocess.Popen(argv, env=env, **kwargs)
+            log("seat %s -> pid %s port %s uid %s" % (seat.sid, seat.proc.pid, seat.port, seat.uid))
+        finally:
+            # Cleared on EVERY exit. A raise here (Popen failing) must not
+            # leave a seat the watcher will skip forever.
+            seat.spawning = False
+            finish = seat._finish_close
+            if seat.closing and finish is not None:
+                # A DELETE landed while Popen was in flight: close() left the
+                # stop-and-wipe for us, because it could not see the child yet.
+                seat._finish_close = None
+                threading.Thread(target=finish, daemon=True, name="close-%s" % seat.sid).start()
+
+    def _restore(self, seat):
+        """Download the chain's saved scene, then arm the CHILD's own restore.
+
+        Zero changes to server.py: `_restore_if_recovering()` already loads
+        `GENEX_BLENDER_CHECKPOINT` at boot when `run_dir/restart-reason` exists,
+        and reports it as `sceneRestored` on the first response. Writing that
+        file is the whole integration.
+
+        Runs in the router BEFORE Popen, so the child's own BOOT_BUDGET never
+        pays for the download.
+        """
+        url = seat.restore.get("url")
+        want = int(seat.restore.get("bytes") or 0)
+        if not url or want <= 0:
+            return
+        if os.path.exists(seat.checkpoint):
+            # A restart in place: the local file is at least as new as R2's.
+            return
+        part = seat.checkpoint + ".part"
+        seat.restoring = True
+        started = time.time()
+        try:
+            os.makedirs(os.path.dirname(seat.checkpoint), exist_ok=True)
+            req = urllib.request.Request(url, method="GET")
+            req.add_header("User-Agent", USER_AGENT)
+            got = 0
+            with urllib.request.urlopen(req, timeout=RESTORE_DEADLINE_S) as r, open(part, "wb") as f:
+                while True:
+                    if seat.closing:
+                        raise OSError("seat closed during restore")
+                    if time.time() - started > RESTORE_DEADLINE_S:
+                        raise OSError("restore exceeded %.0fs" % RESTORE_DEADLINE_S)
+                    chunk = r.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+                    got += len(chunk)
+            if got != want:
+                raise OSError("restore truncated: %d of %d bytes" % (got, want))
+            os.replace(part, seat.checkpoint)
+            try:
+                os.chown(seat.checkpoint, seat.uid, seat.uid)
+            except (OSError, AttributeError):
+                pass
+            with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
+                json.dump({"at": time.time(), "reason": "restored_from_r2", "bytes": got}, f)
+            # The bytes we just wrote are R2's own — pushing them straight back
+            # would be a round trip for nothing.
+            seat.push_sig = _sig(seat.checkpoint)
+            log("seat %s restored %d bytes from R2" % (seat.sid, got))
+        except Exception as e:  # noqa: BLE001 - a failed restore must still boot
+            log("seat %s restore failed (%s) — booting empty" % (seat.sid, e))
+            for path in (part, seat.checkpoint):
+                try:
+                    os.unlink(path)
+                except OSError:
+                    pass
+            try:
+                with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
+                    json.dump({"at": time.time(), "reason": "restore_failed", "note": str(e)[:200]}, f)
+            except OSError:
+                pass
+        finally:
+            seat.restoring = False
+
+    def _maybe_push(self, seat, now):
+        """One seat's turn in the watcher: has its checkpoint changed, settled,
+        and not yet been pushed?"""
+        if not PUSH_ENABLED or seat.closing or seat.spawning or seat.pushing:
+            return
+        sig = _sig(seat.checkpoint)
+        if sig is None or sig == seat.push_sig:
+            return
+        if sig != seat.seen_sig:
+            seat.seen_sig = sig
+            seat.seen_at = now
+            if seat.dirty_since is None:
+                seat.dirty_since = now
+            return
+        if now < seat.push_next_at:
+            return
+        quiet = now - seat.seen_at >= CHECKPOINT_DEBOUNCE_S
+        stale = seat.dirty_since is not None and now - seat.dirty_since >= CHECKPOINT_MAX_DIRTY_S
+        if not (quiet or stale):
+            return
+        seat.pushing = True
+        threading.Thread(target=self._push, args=(seat, sig), daemon=True,
+                         name="push-%s" % seat.sid).start()
+
+    def _push(self, seat, sig, timeout=PUSH_TIMEOUT_S):
+        """Ask the API for a URL, then PUT the file.
+
+        The API mints per push because the presigned PUT signs the exact byte
+        length — a URL handed over at seat-open would be wrong the moment the
+        scene changed size.
+        """
+        try:
+            # Blender saves via write-then-rename, so a file whose signature is
+            # unchanged after the open cannot be a half-written one.
+            with open(seat.checkpoint, "rb") as f:
+                st = os.fstat(f.fileno())
+                if (st.st_mtime_ns, st.st_size) != sig:
+                    return
+                body = f.read()
+            payload = json.dumps({"sessionId": seat.sid, "bytes": len(body)}).encode()
+            req = urllib.request.Request(
+                API_URL + "/api/internal/blender/checkpoint-put", data=payload, method="POST")
+            req.add_header("Content-Type", "application/json")
+            req.add_header("User-Agent", USER_AGENT)
+            req.add_header("x-genex-pod", POD_NAME)
+            req.add_header("x-genex-pod-secret", POD_SECRET)
+            with urllib.request.urlopen(req, timeout=30) as r:
+                grant = json.load(r)
+            put = urllib.request.Request(grant["url"], data=body, method="PUT")
+            for k, v in (grant.get("headers") or {}).items():
+                put.add_header(k, v)
+            put.add_header("User-Agent", USER_AGENT)
+            with urllib.request.urlopen(put, timeout=timeout) as r:
+                if r.status not in (200, 201, 204):
+                    raise OSError("R2 answered %s" % r.status)
+            seat.push_sig = sig
+            seat.dirty_since = None
+            seat.push_state = {"pushedAt": time.time() * 1000.0, "bytes": len(body), "error": None}
+        except urllib.error.HTTPError as e:
+            # 409 superseded is not a failure: a newer episode of this chain owns
+            # the scene now, and this seat is a zombie that must stop writing.
+            detail = "%s %s" % (e.code, e.reason)
+            if e.code == 409:
+                seat.push_sig = sig
+                log("seat %s push superseded — a newer episode owns the chain" % seat.sid)
+            seat.push_state = {"pushedAt": (seat.push_state or {}).get("pushedAt"),
+                               "bytes": (seat.push_state or {}).get("bytes"), "error": detail}
+            seat.push_next_at = time.time() + PUSH_RETRY_S
+        except Exception as e:  # noqa: BLE001 - a failed push must never kill a seat
+            seat.push_state = {"pushedAt": (seat.push_state or {}).get("pushedAt"),
+                               "bytes": (seat.push_state or {}).get("bytes"), "error": str(e)[:200]}
+            seat.push_next_at = time.time() + PUSH_RETRY_S
+            log("seat %s push failed: %s" % (seat.sid, e))
+        finally:
+            seat.pushing = False
+
+    def close(self, sid, wipe=True, push=True):
+        with self.lock:
+            seat = self.seats.get(sid)
+            if not seat:
+                return False
+            # `closing` is set UNDER the lock and BEFORE the seat leaves the
+            # table. The first cut popped first and flagged second, and the
+            # watcher's scan — which reads the flag once off a snapshot — could
+            # land in between, see a dead child, and respawn a seat that was no
+            # longer in the table: an orphan Blender on a port the next tenant
+            # then could not bind (review finding).
+            seat.closing = True
+            self.seats.pop(sid, None)
+            spawning = seat.spawning
+
+        def finish():
+            _stop(seat.proc, "seat %s closed" % sid)
+            # THE LAST SAVE, before the local copy is deleted. Bounded, because
+            # the API is waiting on nothing here but the row is already closing;
+            # the route allows a short grace after close for exactly this.
+            if push and PUSH_ENABLED:
+                sig = _sig(seat.checkpoint)
+                if sig is not None and sig != seat.push_sig:
+                    seat.pushing = True
+                    self._push(seat, sig, timeout=CLOSE_PUSH_TIMEOUT_S)
+            if wipe:
+                # The push above is what keeps the scene; the pod's copy is
+                # scratch and must not outlive the seat, or the next tenant to
+                # land on this port inherits it.
+                shutil.rmtree(os.path.join(STATE_ROOT, sid), ignore_errors=True)
+
+        # A Popen in flight finishes the close itself (see _spawn's finally):
+        # stopping now would find proc None and leave the child it is about to
+        # create running with nobody to stop it.
+        if not spawning:
+            # Asynchronous, and the reason is the router's own thread budget: a
+            # close during a running /exec cannot interrupt bpy, so _stop burns
+            # the whole TERM_GRACE — five seconds a router thread must not spend
+            # holding a control-plane request open (review finding).
+            threading.Thread(target=finish, daemon=True, name="close-%s" % sid).start()
+        seat._finish_close = finish if spawning else None
+        return True
+
+    def get(self, sid):
+        with self.lock:
+            return self.seats.get(sid)
+
+    def status(self, with_seats=False):
+        with self.lock:
+            seats = list(self.seats.values())
+        # ready() opens a loopback socket per call: once per seat, not twice.
+        probed = [(s, s.alive(), s.ready()) for s in seats]
+        out = {
+            "capacity": self.capacity(),
+            "hardCap": HARD_CAP,
+            "podMemoryMb": self.pod_mb,
+            "perSeatMb": PER_SEAT_MB,
+            "open": len(seats),
+            # Counts only on the unauthenticated /health. Session ids appear ONLY
+            # on the authenticated /pool/status (with_seats), where the API reads
+            # per-seat readiness — what lets it bill a seat from the moment its
+            # Blender answers and never for a child that only crash-looped.
+            "ready": sum(1 for _, _, r in probed if r),
+            "booting": sum(1 for _, a, r in probed if a and not r),
+        }
+        if with_seats:
+            seats = {}
+            for st, a, r in probed:
+                entry = {"ready": r, "alive": a}
+                # Only when this pod actually has a durability half — an older
+                # API reads an absent key as "no report", which is the truth.
+                if PUSH_ENABLED or st.push_state or st.restoring:
+                    cp = dict(st.push_state or {"pushedAt": None, "bytes": None, "error": None})
+                    if st.restoring:
+                        cp["restoring"] = True
+                    entry["checkpoint"] = cp
+                seats[st.sid] = entry
+            out["seats"] = seats
+        return out
+
+
+def _stop(proc, why):
+    if proc is None:
+        return
+    log("stopping: %s" % why)
+    try:
+        proc.terminate()
+    except OSError:
+        pass
+    deadline = time.time() + TERM_GRACE_S
+    while time.time() < deadline:
+        if proc.poll() is not None:
+            return
+        time.sleep(0.1)
+    try:
+        proc.kill()
+    except OSError:
+        pass
+    proc.wait()
+
+
+POOL = Pool()
+
+
+def _watch_seats(stop_evt):
+    """One thread for the whole pod: enforce per-seat deadlines and restart dead
+    children. Per-seat threads would be tidier and would also mean N threads
+    blocked in waitpid; one scanner is enough at these seat counts."""
+    while not stop_evt.is_set():
+        now = time.time()
+        for sid, seat in list(POOL.seats.items()):
+            if seat.closing or seat.spawning:
+                continue
+            # Ship the finished checkpoint, off the request path. Cheap: one
+            # stat per seat per second, and a thread only when it settled.
+            try:
+                POOL._maybe_push(seat, now)
+            except Exception as e:  # noqa: BLE001 - never let the watcher die
+                log("seat %s push scheduling failed: %s" % (sid, e))
+            # A runaway request: the child writes a deadline file before every
+            # request and clears it after. Same protocol supervisor.py uses --
+            # the reason is unchanged, bpy cannot be interrupted in-process.
+            try:
+                with open(os.path.join(seat.run_dir, "deadline")) as f:
+                    d = json.load(f)
+                if time.time() > float(d["deadline"]):
+                    log("seat %s exceeded its %ss budget on %s"
+                        % (sid, d.get("budget"), d.get("route")))
+                    with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
+                        json.dump({"at": time.time(), "reason": "deadline_exceeded",
+                                   "route": d.get("route")}, f)
+                    _stop(seat.proc, "runaway in seat %s" % sid)
+                    try:
+                        os.unlink(os.path.join(seat.run_dir, "deadline"))
+                    except OSError:
+                        pass
+            except (OSError, ValueError, KeyError, TypeError):
+                pass
+            # A dead child is restarted in place: the seat, its port and its
+            # checkpoint survive, so the session keeps its scene. Only a crash
+            # LOOP gives up, and it closes one seat rather than the pod --
+            # the whole reason this is not supervisor.py.
+            if not seat.alive():
+                ran = time.time() - seat.opened_at
+                seat.failures = seat.failures + 1 if ran < FAST_FAIL_S else 1
+                if seat.failures >= MAX_CONSECUTIVE_FAILURES:
+                    log("seat %s failed %d times fast — closing it, pod stays up"
+                        % (sid, seat.failures))
+                    # push=False: this seat's checkpoint may be the very file
+                    # that kills the child at load, and R2 already holds the
+                    # last version that worked.
+                    POOL.close(sid, push=False)
+                    continue
+                rc = seat.proc.poll() if seat.proc else None
+                log("seat %s died (rc=%s), restarting (%d)" % (sid, rc, seat.failures))
+                # THE RESTORE KEYS ON THIS FILE. server.py restores the checkpoint
+                # only when `restart-reason` exists at boot; without it the child
+                # comes up EMPTY and the next /exec checkpoints that emptiness over
+                # the good scene. The first cut wrote it only for deadline kills, so
+                # every crash — OOM, a segfault in bpy, a script's sys.exit() —
+                # silently lost the session's work (review finding). supervisor.py
+                # has always written it on every exit path; this now matches.
+                try:
+                    with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
+                        json.dump({"at": time.time(), "code": rc,
+                                   "reason": "unexpected_clean_exit" if rc == 0 else "crashed"}, f)
+                except OSError:
+                    pass
+                seat.opened_at = time.time()
+                POOL._spawn(seat)
+        stop_evt.wait(1.0)
+
+
+class Router(BaseHTTPRequestHandler):
+    protocol_version = "HTTP/1.1"
+    server_version = "genex-blender-pool/0"
+    timeout = 30.0
+
+    def log_message(self, fmt, *args):
+        sys.stderr.write("[router] %s\n" % (fmt % args))
+
+    def handle_one_request(self):
+        # One handler instance serves EVERY request on a kept-alive socket, so
+        # per-request state is reset here, before parse_request runs.
+        self._body_consumed = False
+        try:
+            super().handle_one_request()
+        except (TimeoutError, OSError):
+            self.close_connection = True
+
+    def _send(self, code, payload):
+        self._respond(code, json.dumps(payload).encode("utf-8"))
+
+    def _respond(self, code, body, content_type="application/json"):
+        """THE response writer. Both _send and the relay's success path come
+        through here, so the keep-alive guard below lives in the writer rather
+        than in the order of calls that precede it (review finding)."""
+        self.send_response(code)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        if not self._body_consumed and self._announced_body():
+            # THE KEEP-ALIVE DESYNC, closed by construction rather than per branch.
+            #
+            # An early answer that never read the request body leaves those bytes
+            # in the socket, and on a kept-alive connection they are parsed as the
+            # NEXT request line -- measured on the first real pod: a 404 or 401
+            # followed by "400 Bad request syntax" on the same socket. The 503
+            # seat_not_ready branch had the same shape, and it is precisely the
+            # one the API retries on. Every pooled client (undici keep-alive,
+            # requests.Session, Go's default transport) hits it; urllib hid it
+            # only by never reusing a connection.
+            #
+            # So: any response written while an announced body is still unread
+            # closes the connection. send_header("Connection","close") also sets
+            # close_connection, which is what stops the handler loop. Reading the
+            # body instead would be worse here -- most of these branches answer an
+            # UNAUTHENTICATED caller, who does not get to make us read MAX_BODY.
+            self.send_header("Connection", "close")
+        self.end_headers()
+        try:
+            self.wfile.write(body)
+        except OSError:
+            pass
+
+    def _content_length(self):
+        """Parsed and sign-checked. int("-1") passes a `> MAX_BODY` test and turns
+        rfile.read(n) into a read-to-EOF that parks a router thread for the whole
+        socket timeout (review finding)."""
+        raw = self.headers.get("Content-Length")
+        if not raw:
+            return 0
+        try:
+            n = int(raw)
+        except ValueError:
+            raise ValueError("bad content-length")
+        if n < 0:
+            raise ValueError("negative content-length")
+        return n
+
+    def _announced_body(self):
+        """Did the request declare bytes we have not taken off the socket?"""
+        if self.headers.get("Transfer-Encoding"):
+            return True
+        try:
+            return self._content_length() != 0
+        except ValueError:
+            return True
+
+    def _pod_authed(self):
+        if not POD_SECRET:
+            return True
+        return hmac.compare_digest(self.headers.get("x-genex-internal", ""), POD_SECRET)
+
+    def _read_body(self):
+        n = self._content_length()
+        if n > MAX_BODY:
+            raise ValueError("body too large: %d" % n)
+        if self.headers.get("Transfer-Encoding"):
+            # Refused at the door rather than relayed: without a length we
+            # cannot enforce MAX_BODY, and the child would be the one to find out.
+            # NOT marked consumed: the 413 that follows must close the socket.
+            raise ValueError("chunked bodies are not accepted")
+        data = self.rfile.read(n) if n else b""
+        self._body_consumed = True
+        return data
+
+    def do_GET(self):
+        self._dispatch("GET")
+
+    def do_DELETE(self):
+        self._dispatch("DELETE")
+
+    def do_POST(self):
+        self._dispatch("POST")
+
+    def _dispatch(self, method):
+        route = (self.path or "").split("?")[0]
+        if route == "/health":
+            # Unauthenticated, and counts only — never session ids. This is what
+            # the container healthcheck and the API's liveness probe read.
+            return self._send(200, {"ok": True, "pool": POOL.status()})
+        if route.startswith("/pool/"):
+            if not self._pod_authed():
+                return self._send(401, {"error": "unauthorized"})
+            return self._control(method, route)
+        if route.startswith("/s/"):
+            return self._relay(method, route)
+        self._send(404, {"error": "not_found"})
+
+    def _control(self, method, route):
+        try:
+            raw = self._read_body()
+        except ValueError as e:
+            return self._send(413, {"error": "bad_body", "detail": str(e)})
+        body = json.loads(raw) if raw else {}
+        if route == "/pool/status":
+            return self._send(200, POOL.status(with_seats=True))
+        if route == "/pool/seats" and method == "POST":
+            sid = str(body.get("sessionId") or "").strip()
+            token = str(body.get("token") or "")
+            if not sid or not token:
+                return self._send(400, {"error": "sessionId and token required"})
+            # The chain's saved scene, resolved by the API (which knows the
+            # chain) and handed over here. Validated, not trusted: a url and a
+            # positive size, nothing else, and an unusable value is simply
+            # ignored — a seat that boots empty beats a seat that fails to open.
+            restore = body.get("restore")
+            if isinstance(restore, dict):
+                url = restore.get("url")
+                nbytes = restore.get("bytes")
+                ok = (isinstance(url, str) and url.startswith(("https://", "http://"))
+                      and isinstance(nbytes, int) and nbytes > 0)
+                restore = {"url": url, "bytes": nbytes} if ok else None
+            else:
+                restore = None
+            seat, created = POOL.open(sid, token, restore=restore)
+            if seat is None:
+                # The pod is full. The API allocates across pods and creates
+                # another; this is not the place to queue.
+                return self._send(409, {"error": "pool_full", **POOL.status()})
+            return self._send(201 if created else 200,
+                              {"sessionId": sid, "ready": seat.ready(), "created": created,
+                               # Which child, so the API's seat row can name the
+                               # process a VmHWM reading belongs to.
+                               "index": seat.port - SEAT_BASE_PORT})
+        if route.startswith("/pool/seats/") and method == "DELETE":
+            sid = route[len("/pool/seats/"):]
+            return self._send(200, {"closed": POOL.close(sid)})
+        self._send(404, {"error": "not_found"})
+
+    def _relay(self, method, route):
+        rest = route[len("/s/"):]
+        sid, _, child_route = rest.partition("/")
+        seat = POOL.get(sid)
+        if seat is None:
+            return self._send(404, {"error": "no_such_seat"})
+        token = self.headers.get("x-genex-seat", "")
+        if not seat.token_sha or not hmac.compare_digest(
+                hashlib.sha256(token.encode()).hexdigest(), seat.token_sha):
+            # Per-seat, so one tenant cannot drive another's Blender even though
+            # both live on this pod's loopback.
+            return self._send(401, {"error": "unauthorized"})
+        # The body is read HERE, before the readiness check, and the order is
+        # load-bearing: the caller has just proved it holds the seat token, so
+        # reading its body is its own budget -- and the 503 below is the answer
+        # the API polls every two seconds during a 21 s seat boot. Answering it
+        # with the body unread would close the socket on every poll (see _send),
+        # which is a reconnect through the vendor proxy per retry.
+        try:
+            announced = self._content_length()
+        except ValueError as e:
+            return self._send(413, {"error": "bad_body", "detail": str(e)})
+        if announced > READ_AHEAD_MAX and not seat.ready():
+            # Too big to swallow just to keep a socket: an inlined asset would be
+            # uploaded through the vendor proxy on every 2 s poll. Answer without
+            # reading; _respond closes the connection, one reconnect is cheaper.
+            return self._send(503, {"error": "seat_not_ready", "retryAfterMs": 2000})
+        try:
+            payload = self._read_body()
+        except ValueError as e:
+            return self._send(413, {"error": "bad_body", "detail": str(e)})
+        if not seat.ready():
+            # `spawning` covers the restore download too: before this, a seat
+            # fetching its scene read as `seat_down` and the API's retry gave up
+            # on a seat that was working perfectly.
+            booting = seat.spawning or (seat.alive() and (time.time() - seat.opened_at) < SEAT_BOOT_BUDGET_S)
+            # NEVER block waiting. The API turns this into a 202 the caller
+            # retries; holding the connection would tie up a router thread for
+            # the length of a cold shader compile.
+            return self._send(503, {"error": "seat_not_ready" if booting else "seat_down",
+                                    "retryAfterMs": 2000})
+        url = "http://127.0.0.1:%d/%s" % (seat.port, child_route)
+        req = urllib.request.Request(url, data=payload if method != "GET" else None,
+                                     method=method)
+        req.add_header("Content-Type", "application/json")
+        req.add_header("x-genex-internal", seat.child_secret)
+        # Connection: close on the loopback leg. The child is a single-threaded
+        # HTTPServer, so a kept-alive idle socket from the router would own it
+        # until the idle timeout and block this seat's own next request.
+        req.add_header("Connection", "close")
+        seat.last_request_at = time.time()
+        try:
+            with urllib.request.urlopen(req, timeout=RELAY_TIMEOUT_S) as r:
+                data = r.read()
+                code = r.status
+        except urllib.error.HTTPError as e:
+            data, code = e.read(), e.code
+        except (urllib.error.URLError, OSError, TimeoutError) as e:
+            # The child died mid-request; the watcher will restart it and the
+            # scene comes back from its checkpoint.
+            return self._send(502, {"error": "seat_unreachable", "detail": str(e)})
+        self._respond(code, data)
+
+
+def main():
+    if os.environ.get("GENEX_BLENDER_REQUIRE_SECRET") == "1" and not POD_SECRET:
+        sys.stderr.write("[pool] FATAL no_shared_secret: refusing to serve a "
+                         "multi-tenant pod unauthenticated.\n")
+        return 5
+    port = int(os.environ.get("PORT", "8080"))
+    stop_evt = threading.Event()
+    watcher = threading.Thread(target=_watch_seats, args=(stop_evt,), daemon=True)
+    watcher.start()
+    srv = ThreadingHTTPServer(("0.0.0.0", port), Router)
+    srv.daemon_threads = True
+    log("router on 0.0.0.0:%d | capacity %d (pod %s MB, %d MB/seat)"
+        % (port, POOL.capacity(), POOL.pod_mb, PER_SEAT_MB))
+
+    def bye(signum, _f):
+        log("signal %s — draining" % signum)
+        stop_evt.set()
+        threading.Thread(target=srv.shutdown, daemon=True).start()
+
+    signal.signal(signal.SIGTERM, bye)
+    signal.signal(signal.SIGINT, bye)
+    try:
+        srv.serve_forever(poll_interval=0.2)
+    finally:
+        stop_evt.set()
+        # wipe=False (the disk is going anyway) but push=True: a pod being torn
+        # down is exactly when the last minutes of work would otherwise be lost.
+        # Bounded by CLOSE_PUSH_TIMEOUT_S per seat inside close().
+        for sid in list(POOL.seats):
+            POOL.close(sid, wipe=False)
+        deadline = time.time() + CLOSE_PUSH_TIMEOUT_S
+        while time.time() < deadline and any(t.name.startswith(("close-", "push-"))
+                                             for t in threading.enumerate()):
+            time.sleep(0.2)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())