@genex-ai/cli-demo 1.30.0-dev.645 → 1.31.1

package/templates/blender-service/pool.py

@@ -1,910 +0,0 @@
-"""Multi-seat pod: one Blender process per session, behind one router.
-
-    :8080  router (this process, threaded)   ->  127.0.0.1:8101+i  seat children
-                                                 each its own Blender, own scene
-
-WHY THREADS ARE SAFE HERE AND NOT IN server.py. bpy is main-thread-only, which is
-why `server.py` is a deliberately non-threading HTTPServer. This process contains
-NO bpy — it is pure stdlib that spawns Blender as children and relays HTTP to
-them. So a ThreadingHTTPServer is correct here, and it is what stops one slow
-seat from blocking every other tenant. Never import bpy into this file.
-
-ISOLATION is a process boundary, not a scene boundary. Two sessions never share
-`bpy.data`, a NAMESPACE dict, a checkpoint, or a deadline file — the collisions
-that made one-process-many-scenes unworkable. What they do share is the pod's
-filesystem, network namespace and GPU, so this is a density mechanism, not a
-security boundary against hostile code (see ops.py on why /exec cannot be
-sandboxed in-process).
-"""
-
-import hashlib
-import hmac
-import json
-import os
-import secrets
-import shutil
-import signal
-import socket
-import subprocess
-import sys
-import threading
-import time
-import urllib.error
-import urllib.request
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-BLENDER = os.environ.get("GENEX_BLENDER_BIN", "blender")
-SERVER = os.environ.get("GENEX_BLENDER_SERVER", "/app/server.py")
-POD_SECRET = os.environ.get("GENEX_BLENDER_SECRET", "")
-STATE_ROOT = os.environ.get("GENEX_BLENDER_STATE", "/run/genex/seats")
-SEAT_BASE_PORT = int(os.environ.get("GENEX_BLENDER_SEAT_PORT", "8101"))
-SEAT_UID_BASE = int(os.environ.get("GENEX_BLENDER_SEAT_UID", "10000"))
-
-# MEASURED 2026-09-02 on an RTX 4090 pod, VmHWM (true peak RSS) per child:
-#   booted, empty scene                          1,103 MB
-#   castle (46 obj, 1,440 tris)                  1,313 MB
-#   heavy (12 obj, 612k tris, 8 x 4096^2 tex)    2,204 MB
-# Four seats each holding the heavy scene summed to 8,640 MB; the router is
-# 27 MB. The old guess of 2048 was ALREADY EXCEEDED by one moderately heavy
-# scene, and the shape is the lesson: ~1.1 GB is Blender's own floor before any
-# content, so 2048 left ~950 MB for the actual scene. 3072 keeps a heavy scene
-# plus a 40 MB sheet transient inside the budget with margin -- and this is an
-# OOM guard, where over-reserving is the safe direction.
-PER_SEAT_MB = int(os.environ.get("GENEX_BLENDER_PER_SEAT_MB", "3072"))
-# TWO, and the reason changed once it was measured. Memory was never the bind:
-# four heavy seats used 29% of a 29 GB pod and 22% of VRAM. What binds is GPU
-# COMPUTE. Four seats rendering at once ran 2.41x slower each (869 -> 2,094 ms
-# lit), and four concurrent renders finished in 3,100 ms wall against 3,476 ms
-# serial -- concurrency bought 11%, not 4x. So packing seats onto one card is
-# not a render-throughput win; it pays only by multiplexing the idle time
-# BETWEEN an agent's /exec calls. Seat count is therefore a DUTY-CYCLE choice,
-# not a memory one, and raising it needs an observed duty cycle, not a bigger
-# pod.
-HARD_CAP = int(os.environ.get("GENEX_BLENDER_SEATS", "2"))
-
-# A seat is BOOTING until its child answers /health. The GPU probe plus device
-# enumeration costs seconds, and on a cold pod the EEVEE shader compile can cost
-# 30-44s -- so the router NEVER waits for this. It answers 503 and the API turns
-# that into a 202 the caller retries. A blocking wait would share one thread
-# budget with live relays, which is the cross-tenant head-of-line blocking that
-# process-per-seat exists to prevent.
-SEAT_BOOT_BUDGET_S = float(os.environ.get("GENEX_BLENDER_SEAT_BOOT_S", "240"))
-RELAY_TIMEOUT_S = float(os.environ.get("GENEX_BLENDER_RELAY_S", "930"))
-MAX_BODY = 8 * 1024 * 1024
-
-# --- the R2 checkpoint push ---------------------------------------------------
-# THE ROUTER PUSHES, NOT THE CHILD, and that is a security choice before it is a
-# threading one: `/exec` runs arbitrary Python inside the child as a uid-switched
-# user, so an API-facing credential in that process is readable by the code it
-# runs. The router holds the pod secret already and is the only process here
-# that never imports bpy.
-#
-# The child's local checkpoint stays exactly as it was — synchronous, after every
-# mutating call. This lane only ships that finished file, off the request path.
-API_URL = os.environ.get("GENEX_API_URL", "").rstrip("/")
-POD_NAME = os.environ.get("GENEX_BLENDER_POD_NAME", "")
-# All three or nothing. On the local lane (`genex blender serve`) none of them is
-# set and this whole feature is inert without a flag to forget.
-PUSH_ENABLED = bool(API_URL and POD_NAME and POD_SECRET)
-# Quiet-time before a push: the agent's think time is where these seconds hide.
-CHECKPOINT_DEBOUNCE_S = float(os.environ.get("GENEX_BLENDER_PUSH_DEBOUNCE_S", "10"))
-# ...but never let a busy seat go unsaved forever chasing quiet.
-CHECKPOINT_MAX_DIRTY_S = float(os.environ.get("GENEX_BLENDER_PUSH_MAX_DIRTY_S", "60"))
-PUSH_RETRY_S = 30.0
-PUSH_TIMEOUT_S = 600.0
-RESTORE_DEADLINE_S = float(os.environ.get("GENEX_BLENDER_RESTORE_S", "120"))
-CLOSE_PUSH_TIMEOUT_S = 20.0
-# Cloudflare's error 1010 refuses the default Python UA (measured on the pod
-# proxy); the API host is behind Cloudflare too, so never rely on the default.
-USER_AGENT = "genex-blender-pool"
-# Bodies up to this size are read before a 503 so the poller keeps its socket.
-READ_AHEAD_MAX = 256 * 1024
-TERM_GRACE_S = 5.0
-FAST_FAIL_S = 20.0
-MAX_CONSECUTIVE_FAILURES = 5
-
-
-def log(msg):
-    sys.stderr.write("[pool] %s\n" % msg)
-    sys.stderr.flush()
-
-
-def cgroup_limit_mb():
-    """The container's real memory ceiling.
-
-    /proc/meminfo reports the HOST's memory, not the container's, so sizing from
-    it over-admits by however much bigger the host is -- and the failure is an
-    OOM kill of somebody else's Blender. cgroup v2 first, then v1, then a
-    configured fallback; 'max' means unlimited, which in a pod means "we cannot
-    tell", so treat it as the fallback rather than as infinity.
-    """
-    for path, div in (("/sys/fs/cgroup/memory.max", 1),
-                      ("/sys/fs/cgroup/memory/memory.limit_in_bytes", 1)):
-        try:
-            with open(path) as f:
-                raw = f.read().strip()
-            if raw == "max":
-                continue
-            n = int(raw) // (1024 * 1024)
-            # A v1 "unlimited" is a huge sentinel, not a real limit.
-            if 0 < n < 1024 * 1024:
-                return n
-        except (OSError, ValueError):
-            continue
-    return int(os.environ.get("GENEX_BLENDER_POD_MB", "0")) or None
-
-
-class Seat:
-    __slots__ = ("sid", "port", "uid", "proc", "token_sha", "child_secret", "run_dir",
-                 "workspace", "checkpoint", "opened_at", "last_request_at", "failures",
-                 "closing", "spawning", "_finish_close",
-                 # the R2 push/restore half
-                 "restore", "restoring", "seen_sig", "seen_at", "dirty_since",
-                 "push_sig", "pushing", "push_next_at", "push_state")
-
-    def __init__(self, sid, port, uid):
-        self.sid = sid
-        self.port = port
-        self.uid = uid
-        self.proc = None
-        self.token_sha = None
-        # The secret the CHILD checks. Distinct from the seat token the caller
-        # presents: the router translates one into the other, so a tenant never
-        # holds a credential its own Blender would accept directly.
-        self.child_secret = None
-        self.run_dir = os.path.join(STATE_ROOT, sid, "run")
-        self.workspace = os.path.join(STATE_ROOT, sid, "workspace")
-        self.checkpoint = os.path.join(STATE_ROOT, sid, "checkpoint.blend")
-        self.opened_at = time.time()
-        self.last_request_at = time.time()
-        self.failures = 0
-        self.closing = False
-        # True from the moment the seat is committed to the table until _spawn
-        # has set `proc`. The watcher reads `proc is None` as "dead child", and
-        # between `self.seats[sid] = seat` (under the lock) and Popen returning
-        # (outside it -- makedirs, chown, a fork) that is exactly what it sees.
-        # Its 1 s scan landing in that window spawned a SECOND Blender on the
-        # same port. Found by code-read on the first real pod run.
-        self.spawning = False
-        self._finish_close = None
-        # The chain's saved scene, handed over by the API at open (it probes R2;
-        # the pod is never allowed to ask for an arbitrary chain's bytes).
-        self.restore = None
-        self.restoring = False
-        # (st_mtime_ns, st_size) of the checkpoint as last SEEN and as last
-        # PUSHED. A signature rather than a timestamp: a scene can be rewritten
-        # to the same size within one clock tick.
-        self.seen_sig = None
-        self.seen_at = 0.0
-        self.dirty_since = None
-        self.push_sig = None
-        self.pushing = False
-        self.push_next_at = 0.0
-        self.push_state = None
-
-    def alive(self):
-        return self.proc is not None and self.proc.poll() is None
-
-    def ready(self):
-        """Answering, not merely running. The child binds its port only after
-        the GPU assertion and the checkpoint restore, so 'process exists' is not
-        the same question."""
-        if not self.alive():
-            return False
-        try:
-            with socket.create_connection(("127.0.0.1", self.port), timeout=0.4):
-                return True
-        except OSError:
-            return False
-
-
-def _sig(path):
-    """(st_mtime_ns, st_size), or None when the file is not there yet."""
-    try:
-        st = os.stat(path)
-    except OSError:
-        return None
-    return (st.st_mtime_ns, st.st_size)
-
-
-def restore_is_pointless(seat):
-    """True when there is nothing to wipe or restore — the seat has no saved
-    scene to fetch, so an existing local checkpoint (a restart in place) is the
-    better source and must not be deleted."""
-    return seat.restore is None
-
-
-class Pool:
-    def __init__(self):
-        self.seats = {}
-        self.lock = threading.Lock()
-        self.pod_mb = cgroup_limit_mb()
-        os.makedirs(STATE_ROOT, exist_ok=True)
-
-    def capacity(self):
-        """Seats this pod will admit. The lower of the hard cap and what memory
-        allows, never just the cap."""
-        if not self.pod_mb:
-            return HARD_CAP
-        # Leave one seat's worth of headroom for the router, the shared shader
-        # cache and page cache. Without it the last admitted seat is the one
-        # that OOMs, and it takes a stranger's session with it.
-        by_mem = max(0, (self.pod_mb - PER_SEAT_MB) // PER_SEAT_MB)
-        return max(1, min(HARD_CAP, by_mem))
-
-    def open(self, sid, token, restore=None):
-        with self.lock:
-            existing = self.seats.get(sid)
-            if existing and not existing.closing:
-                # Idempotent: a retried openSeat must not spawn a second child.
-                existing.token_sha = hashlib.sha256(token.encode()).hexdigest()
-                return existing, False
-            if len(self.seats) >= self.capacity():
-                return None, False
-            used = {s.port for s in self.seats.values()}
-            port = next(p for p in range(SEAT_BASE_PORT, SEAT_BASE_PORT + HARD_CAP + 8)
-                        if p not in used)
-            uid = SEAT_UID_BASE + (port - SEAT_BASE_PORT)
-            seat = Seat(sid, port, uid)
-            seat.restore = restore
-            seat.token_sha = hashlib.sha256(token.encode()).hexdigest()
-            # Flagged BEFORE it is visible in the table, so no scan can find it
-            # proc-less and unflagged.
-            seat.spawning = True
-            self.seats[sid] = seat
-        # A leftover directory from a wipe that never finished would make the
-        # restore below skip ("there is already a checkpoint") and the child boot
-        # from a stranger's scene — or from a stale one it then overwrites.
-        if not restore_is_pointless(seat):
-            shutil.rmtree(os.path.join(STATE_ROOT, sid), ignore_errors=True)
-            os.makedirs(seat.run_dir, exist_ok=True)
-        # THREADED, because _spawn may now download a checkpoint first and
-        # /pool/seats must still answer 201 `ready: false` immediately — the
-        # router never blocks, which is the property process-per-seat buys.
-        # `spawning` is already set under the lock, so the watcher skips the seat
-        # and a concurrent close hands the teardown to _spawn's finally.
-        threading.Thread(target=self._spawn, args=(seat,), daemon=True,
-                         name="spawn-%s" % sid).start()
-        return seat, True
-
-    def _spawn(self, seat):
-        with self.lock:
-            # A close that arrived first wins; a seat no longer in the table is
-            # not respawned. Both checks under the lock, so the watcher's restart
-            # and a concurrent DELETE cannot interleave into an orphan.
-            if seat.closing or self.seats.get(seat.sid) is not seat:
-                return
-            seat.spawning = True
-        try:
-            if seat.restore:
-                self._restore(seat)
-            tmpdir = os.path.join(seat.workspace, "tmp")
-            for d in (seat.run_dir, seat.workspace, tmpdir):
-                os.makedirs(d, exist_ok=True)
-            # Per-seat ownership so one tenant cannot read another's checkpoint or
-            # scripts through the shared filesystem. Best-effort: a pod that is not
-            # running as root still isolates by process, just not by uid. TMPDIR is
-            # created ABOVE, before this pass — created after it (the first cut) it
-            # belonged to root, the child could not write it, and tempfile fell
-            # back silently to the shared /tmp, which is the isolation this exists
-            # to provide (review finding).
-            try:
-                os.chown(os.path.dirname(seat.run_dir), seat.uid, seat.uid)
-                for d in (seat.run_dir, seat.workspace, tmpdir):
-                    os.chown(d, seat.uid, seat.uid)
-            except (OSError, AttributeError):
-                pass
-            env = dict(os.environ)
-            env.update({
-                "PORT": str(seat.port),
-                "GENEX_BLENDER_RUN_DIR": seat.run_dir,
-                "GENEX_BLENDER_CHECKPOINT": seat.checkpoint,
-                "HOME": seat.workspace,
-                "TMPDIR": os.path.join(seat.workspace, "tmp"),
-                # The seat's own secret. The pod secret authenticates the control
-                # plane; a seat token authenticates one session's routes, so seat A
-                # cannot drive seat B even though both are on this pod's loopback.
-                "GENEX_BLENDER_SECRET": secrets.token_hex(16),
-                # Children bind loopback only -- the router is the sole way in.
-                "GENEX_BLENDER_BIND": "127.0.0.1",
-            })
-            argv = [BLENDER, "--background", "--factory-startup", "-noaudio",
-                    "--python-exit-code", "1", "--python", SERVER,
-                    "--", "--port", str(seat.port), "--host", "127.0.0.1"]
-            extra = os.environ.get("GENEX_BLENDER_ARGS", "").split()
-            if extra:
-                # Same lever supervisor.spawn honours (e.g. --gpu-backend vulkan);
-                # the pooled path silently dropped it (review finding).
-                argv[1:1] = extra
-            kwargs = {}
-            if os.getuid() == 0:
-                kwargs["user"] = seat.uid
-                kwargs["group"] = seat.uid
-            seat.child_secret = env["GENEX_BLENDER_SECRET"]
-            seat.proc = subprocess.Popen(argv, env=env, **kwargs)
-            log("seat %s -> pid %s port %s uid %s" % (seat.sid, seat.proc.pid, seat.port, seat.uid))
-        finally:
-            # Cleared on EVERY exit. A raise here (Popen failing) must not
-            # leave a seat the watcher will skip forever.
-            seat.spawning = False
-            finish = seat._finish_close
-            if seat.closing and finish is not None:
-                # A DELETE landed while Popen was in flight: close() left the
-                # stop-and-wipe for us, because it could not see the child yet.
-                seat._finish_close = None
-                threading.Thread(target=finish, daemon=True, name="close-%s" % seat.sid).start()
-
-    def _restore(self, seat):
-        """Download the chain's saved scene, then arm the CHILD's own restore.
-
-        Zero changes to server.py: `_restore_if_recovering()` already loads
-        `GENEX_BLENDER_CHECKPOINT` at boot when `run_dir/restart-reason` exists,
-        and reports it as `sceneRestored` on the first response. Writing that
-        file is the whole integration.
-
-        Runs in the router BEFORE Popen, so the child's own BOOT_BUDGET never
-        pays for the download.
-        """
-        url = seat.restore.get("url")
-        want = int(seat.restore.get("bytes") or 0)
-        if not url or want <= 0:
-            return
-        if os.path.exists(seat.checkpoint):
-            # A restart in place: the local file is at least as new as R2's.
-            return
-        part = seat.checkpoint + ".part"
-        seat.restoring = True
-        started = time.time()
-        try:
-            os.makedirs(os.path.dirname(seat.checkpoint), exist_ok=True)
-            req = urllib.request.Request(url, method="GET")
-            req.add_header("User-Agent", USER_AGENT)
-            got = 0
-            with urllib.request.urlopen(req, timeout=RESTORE_DEADLINE_S) as r, open(part, "wb") as f:
-                while True:
-                    if seat.closing:
-                        raise OSError("seat closed during restore")
-                    if time.time() - started > RESTORE_DEADLINE_S:
-                        raise OSError("restore exceeded %.0fs" % RESTORE_DEADLINE_S)
-                    chunk = r.read(1024 * 1024)
-                    if not chunk:
-                        break
-                    f.write(chunk)
-                    got += len(chunk)
-            if got != want:
-                raise OSError("restore truncated: %d of %d bytes" % (got, want))
-            os.replace(part, seat.checkpoint)
-            try:
-                os.chown(seat.checkpoint, seat.uid, seat.uid)
-            except (OSError, AttributeError):
-                pass
-            with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
-                json.dump({"at": time.time(), "reason": "restored_from_r2", "bytes": got}, f)
-            # The bytes we just wrote are R2's own — pushing them straight back
-            # would be a round trip for nothing.
-            seat.push_sig = _sig(seat.checkpoint)
-            log("seat %s restored %d bytes from R2" % (seat.sid, got))
-        except Exception as e:  # noqa: BLE001 - a failed restore must still boot
-            log("seat %s restore failed (%s) — booting empty" % (seat.sid, e))
-            for path in (part, seat.checkpoint):
-                try:
-                    os.unlink(path)
-                except OSError:
-                    pass
-            try:
-                with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
-                    json.dump({"at": time.time(), "reason": "restore_failed", "note": str(e)[:200]}, f)
-            except OSError:
-                pass
-        finally:
-            seat.restoring = False
-
-    def _maybe_push(self, seat, now):
-        """One seat's turn in the watcher: has its checkpoint changed, settled,
-        and not yet been pushed?"""
-        if not PUSH_ENABLED or seat.closing or seat.spawning or seat.pushing:
-            return
-        sig = _sig(seat.checkpoint)
-        if sig is None or sig == seat.push_sig:
-            return
-        if sig != seat.seen_sig:
-            seat.seen_sig = sig
-            seat.seen_at = now
-            if seat.dirty_since is None:
-                seat.dirty_since = now
-            return
-        if now < seat.push_next_at:
-            return
-        quiet = now - seat.seen_at >= CHECKPOINT_DEBOUNCE_S
-        stale = seat.dirty_since is not None and now - seat.dirty_since >= CHECKPOINT_MAX_DIRTY_S
-        if not (quiet or stale):
-            return
-        seat.pushing = True
-        threading.Thread(target=self._push, args=(seat, sig), daemon=True,
-                         name="push-%s" % seat.sid).start()
-
-    def _push(self, seat, sig, timeout=PUSH_TIMEOUT_S):
-        """Ask the API for a URL, then PUT the file.
-
-        The API mints per push because the presigned PUT signs the exact byte
-        length — a URL handed over at seat-open would be wrong the moment the
-        scene changed size.
-        """
-        try:
-            # Blender saves via write-then-rename, so a file whose signature is
-            # unchanged after the open cannot be a half-written one.
-            with open(seat.checkpoint, "rb") as f:
-                st = os.fstat(f.fileno())
-                if (st.st_mtime_ns, st.st_size) != sig:
-                    return
-                body = f.read()
-            payload = json.dumps({"sessionId": seat.sid, "bytes": len(body)}).encode()
-            req = urllib.request.Request(
-                API_URL + "/api/internal/blender/checkpoint-put", data=payload, method="POST")
-            req.add_header("Content-Type", "application/json")
-            req.add_header("User-Agent", USER_AGENT)
-            req.add_header("x-genex-pod", POD_NAME)
-            req.add_header("x-genex-pod-secret", POD_SECRET)
-            with urllib.request.urlopen(req, timeout=30) as r:
-                grant = json.load(r)
-            put = urllib.request.Request(grant["url"], data=body, method="PUT")
-            for k, v in (grant.get("headers") or {}).items():
-                put.add_header(k, v)
-            put.add_header("User-Agent", USER_AGENT)
-            with urllib.request.urlopen(put, timeout=timeout) as r:
-                if r.status not in (200, 201, 204):
-                    raise OSError("R2 answered %s" % r.status)
-            seat.push_sig = sig
-            seat.dirty_since = None
-            seat.push_state = {"pushedAt": time.time() * 1000.0, "bytes": len(body), "error": None}
-        except urllib.error.HTTPError as e:
-            # 409 superseded is not a failure: a newer episode of this chain owns
-            # the scene now, and this seat is a zombie that must stop writing.
-            detail = "%s %s" % (e.code, e.reason)
-            if e.code == 409:
-                seat.push_sig = sig
-                log("seat %s push superseded — a newer episode owns the chain" % seat.sid)
-            seat.push_state = {"pushedAt": (seat.push_state or {}).get("pushedAt"),
-                               "bytes": (seat.push_state or {}).get("bytes"), "error": detail}
-            seat.push_next_at = time.time() + PUSH_RETRY_S
-        except Exception as e:  # noqa: BLE001 - a failed push must never kill a seat
-            seat.push_state = {"pushedAt": (seat.push_state or {}).get("pushedAt"),
-                               "bytes": (seat.push_state or {}).get("bytes"), "error": str(e)[:200]}
-            seat.push_next_at = time.time() + PUSH_RETRY_S
-            log("seat %s push failed: %s" % (seat.sid, e))
-        finally:
-            seat.pushing = False
-
-    def close(self, sid, wipe=True, push=True):
-        with self.lock:
-            seat = self.seats.get(sid)
-            if not seat:
-                return False
-            # `closing` is set UNDER the lock and BEFORE the seat leaves the
-            # table. The first cut popped first and flagged second, and the
-            # watcher's scan — which reads the flag once off a snapshot — could
-            # land in between, see a dead child, and respawn a seat that was no
-            # longer in the table: an orphan Blender on a port the next tenant
-            # then could not bind (review finding).
-            seat.closing = True
-            self.seats.pop(sid, None)
-            spawning = seat.spawning
-
-        def finish():
-            _stop(seat.proc, "seat %s closed" % sid)
-            # THE LAST SAVE, before the local copy is deleted. Bounded, because
-            # the API is waiting on nothing here but the row is already closing;
-            # the route allows a short grace after close for exactly this.
-            if push and PUSH_ENABLED:
-                sig = _sig(seat.checkpoint)
-                if sig is not None and sig != seat.push_sig:
-                    seat.pushing = True
-                    self._push(seat, sig, timeout=CLOSE_PUSH_TIMEOUT_S)
-            if wipe:
-                # The push above is what keeps the scene; the pod's copy is
-                # scratch and must not outlive the seat, or the next tenant to
-                # land on this port inherits it.
-                shutil.rmtree(os.path.join(STATE_ROOT, sid), ignore_errors=True)
-
-        # A Popen in flight finishes the close itself (see _spawn's finally):
-        # stopping now would find proc None and leave the child it is about to
-        # create running with nobody to stop it.
-        if not spawning:
-            # Asynchronous, and the reason is the router's own thread budget: a
-            # close during a running /exec cannot interrupt bpy, so _stop burns
-            # the whole TERM_GRACE — five seconds a router thread must not spend
-            # holding a control-plane request open (review finding).
-            threading.Thread(target=finish, daemon=True, name="close-%s" % sid).start()
-        seat._finish_close = finish if spawning else None
-        return True
-
-    def get(self, sid):
-        with self.lock:
-            return self.seats.get(sid)
-
-    def status(self, with_seats=False):
-        with self.lock:
-            seats = list(self.seats.values())
-        # ready() opens a loopback socket per call: once per seat, not twice.
-        probed = [(s, s.alive(), s.ready()) for s in seats]
-        out = {
-            "capacity": self.capacity(),
-            "hardCap": HARD_CAP,
-            "podMemoryMb": self.pod_mb,
-            "perSeatMb": PER_SEAT_MB,
-            "open": len(seats),
-            # Counts only on the unauthenticated /health. Session ids appear ONLY
-            # on the authenticated /pool/status (with_seats), where the API reads
-            # per-seat readiness — what lets it bill a seat from the moment its
-            # Blender answers and never for a child that only crash-looped.
-            "ready": sum(1 for _, _, r in probed if r),
-            "booting": sum(1 for _, a, r in probed if a and not r),
-        }
-        if with_seats:
-            seats = {}
-            for st, a, r in probed:
-                entry = {"ready": r, "alive": a}
-                # Only when this pod actually has a durability half — an older
-                # API reads an absent key as "no report", which is the truth.
-                if PUSH_ENABLED or st.push_state or st.restoring:
-                    cp = dict(st.push_state or {"pushedAt": None, "bytes": None, "error": None})
-                    if st.restoring:
-                        cp["restoring"] = True
-                    entry["checkpoint"] = cp
-                seats[st.sid] = entry
-            out["seats"] = seats
-        return out
-
-
-def _stop(proc, why):
-    if proc is None:
-        return
-    log("stopping: %s" % why)
-    try:
-        proc.terminate()
-    except OSError:
-        pass
-    deadline = time.time() + TERM_GRACE_S
-    while time.time() < deadline:
-        if proc.poll() is not None:
-            return
-        time.sleep(0.1)
-    try:
-        proc.kill()
-    except OSError:
-        pass
-    proc.wait()
-
-
-POOL = Pool()
-
-
-def _watch_seats(stop_evt):
-    """One thread for the whole pod: enforce per-seat deadlines and restart dead
-    children. Per-seat threads would be tidier and would also mean N threads
-    blocked in waitpid; one scanner is enough at these seat counts."""
-    while not stop_evt.is_set():
-        now = time.time()
-        for sid, seat in list(POOL.seats.items()):
-            if seat.closing or seat.spawning:
-                continue
-            # Ship the finished checkpoint, off the request path. Cheap: one
-            # stat per seat per second, and a thread only when it settled.
-            try:
-                POOL._maybe_push(seat, now)
-            except Exception as e:  # noqa: BLE001 - never let the watcher die
-                log("seat %s push scheduling failed: %s" % (sid, e))
-            # A runaway request: the child writes a deadline file before every
-            # request and clears it after. Same protocol supervisor.py uses --
-            # the reason is unchanged, bpy cannot be interrupted in-process.
-            try:
-                with open(os.path.join(seat.run_dir, "deadline")) as f:
-                    d = json.load(f)
-                if time.time() > float(d["deadline"]):
-                    log("seat %s exceeded its %ss budget on %s"
-                        % (sid, d.get("budget"), d.get("route")))
-                    with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
-                        json.dump({"at": time.time(), "reason": "deadline_exceeded",
-                                   "route": d.get("route")}, f)
-                    _stop(seat.proc, "runaway in seat %s" % sid)
-                    try:
-                        os.unlink(os.path.join(seat.run_dir, "deadline"))
-                    except OSError:
-                        pass
-            except (OSError, ValueError, KeyError, TypeError):
-                pass
-            # A dead child is restarted in place: the seat, its port and its
-            # checkpoint survive, so the session keeps its scene. Only a crash
-            # LOOP gives up, and it closes one seat rather than the pod --
-            # the whole reason this is not supervisor.py.
-            if not seat.alive():
-                ran = time.time() - seat.opened_at
-                seat.failures = seat.failures + 1 if ran < FAST_FAIL_S else 1
-                if seat.failures >= MAX_CONSECUTIVE_FAILURES:
-                    log("seat %s failed %d times fast — closing it, pod stays up"
-                        % (sid, seat.failures))
-                    # push=False: this seat's checkpoint may be the very file
-                    # that kills the child at load, and R2 already holds the
-                    # last version that worked.
-                    POOL.close(sid, push=False)
-                    continue
-                rc = seat.proc.poll() if seat.proc else None
-                log("seat %s died (rc=%s), restarting (%d)" % (sid, rc, seat.failures))
-                # THE RESTORE KEYS ON THIS FILE. server.py restores the checkpoint
-                # only when `restart-reason` exists at boot; without it the child
-                # comes up EMPTY and the next /exec checkpoints that emptiness over
-                # the good scene. The first cut wrote it only for deadline kills, so
-                # every crash — OOM, a segfault in bpy, a script's sys.exit() —
-                # silently lost the session's work (review finding). supervisor.py
-                # has always written it on every exit path; this now matches.
-                try:
-                    with open(os.path.join(seat.run_dir, "restart-reason"), "w") as f:
-                        json.dump({"at": time.time(), "code": rc,
-                                   "reason": "unexpected_clean_exit" if rc == 0 else "crashed"}, f)
-                except OSError:
-                    pass
-                seat.opened_at = time.time()
-                POOL._spawn(seat)
-        stop_evt.wait(1.0)
-
-
-class Router(BaseHTTPRequestHandler):
-    protocol_version = "HTTP/1.1"
-    server_version = "genex-blender-pool/0"
-    timeout = 30.0
-
-    def log_message(self, fmt, *args):
-        sys.stderr.write("[router] %s\n" % (fmt % args))
-
-    def handle_one_request(self):
-        # One handler instance serves EVERY request on a kept-alive socket, so
-        # per-request state is reset here, before parse_request runs.
-        self._body_consumed = False
-        try:
-            super().handle_one_request()
-        except (TimeoutError, OSError):
-            self.close_connection = True
-
-    def _send(self, code, payload):
-        self._respond(code, json.dumps(payload).encode("utf-8"))
-
-    def _respond(self, code, body, content_type="application/json"):
-        """THE response writer. Both _send and the relay's success path come
-        through here, so the keep-alive guard below lives in the writer rather
-        than in the order of calls that precede it (review finding)."""
-        self.send_response(code)
-        self.send_header("Content-Type", content_type)
-        self.send_header("Content-Length", str(len(body)))
-        if not self._body_consumed and self._announced_body():
-            # THE KEEP-ALIVE DESYNC, closed by construction rather than per branch.
-            #
-            # An early answer that never read the request body leaves those bytes
-            # in the socket, and on a kept-alive connection they are parsed as the
-            # NEXT request line -- measured on the first real pod: a 404 or 401
-            # followed by "400 Bad request syntax" on the same socket. The 503
-            # seat_not_ready branch had the same shape, and it is precisely the
-            # one the API retries on. Every pooled client (undici keep-alive,
-            # requests.Session, Go's default transport) hits it; urllib hid it
-            # only by never reusing a connection.
-            #
-            # So: any response written while an announced body is still unread
-            # closes the connection. send_header("Connection","close") also sets
-            # close_connection, which is what stops the handler loop. Reading the
-            # body instead would be worse here -- most of these branches answer an
-            # UNAUTHENTICATED caller, who does not get to make us read MAX_BODY.
-            self.send_header("Connection", "close")
-        self.end_headers()
-        try:
-            self.wfile.write(body)
-        except OSError:
-            pass
-
-    def _content_length(self):
-        """Parsed and sign-checked. int("-1") passes a `> MAX_BODY` test and turns
-        rfile.read(n) into a read-to-EOF that parks a router thread for the whole
-        socket timeout (review finding)."""
-        raw = self.headers.get("Content-Length")
-        if not raw:
-            return 0
-        try:
-            n = int(raw)
-        except ValueError:
-            raise ValueError("bad content-length")
-        if n < 0:
-            raise ValueError("negative content-length")
-        return n
-
-    def _announced_body(self):
-        """Did the request declare bytes we have not taken off the socket?"""
-        if self.headers.get("Transfer-Encoding"):
-            return True
-        try:
-            return self._content_length() != 0
-        except ValueError:
-            return True
-
-    def _pod_authed(self):
-        if not POD_SECRET:
-            return True
-        return hmac.compare_digest(self.headers.get("x-genex-internal", ""), POD_SECRET)
-
-    def _read_body(self):
-        n = self._content_length()
-        if n > MAX_BODY:
-            raise ValueError("body too large: %d" % n)
-        if self.headers.get("Transfer-Encoding"):
-            # Refused at the door rather than relayed: without a length we
-            # cannot enforce MAX_BODY, and the child would be the one to find out.
-            # NOT marked consumed: the 413 that follows must close the socket.
-            raise ValueError("chunked bodies are not accepted")
-        data = self.rfile.read(n) if n else b""
-        self._body_consumed = True
-        return data
-
-    def do_GET(self):
-        self._dispatch("GET")
-
-    def do_DELETE(self):
-        self._dispatch("DELETE")
-
-    def do_POST(self):
-        self._dispatch("POST")
-
-    def _dispatch(self, method):
-        route = (self.path or "").split("?")[0]
-        if route == "/health":
-            # Unauthenticated, and counts only — never session ids. This is what
-            # the container healthcheck and the API's liveness probe read.
-            return self._send(200, {"ok": True, "pool": POOL.status()})
-        if route.startswith("/pool/"):
-            if not self._pod_authed():
-                return self._send(401, {"error": "unauthorized"})
-            return self._control(method, route)
-        if route.startswith("/s/"):
-            return self._relay(method, route)
-        self._send(404, {"error": "not_found"})
-
-    def _control(self, method, route):
-        try:
-            raw = self._read_body()
-        except ValueError as e:
-            return self._send(413, {"error": "bad_body", "detail": str(e)})
-        body = json.loads(raw) if raw else {}
-        if route == "/pool/status":
-            return self._send(200, POOL.status(with_seats=True))
-        if route == "/pool/seats" and method == "POST":
-            sid = str(body.get("sessionId") or "").strip()
-            token = str(body.get("token") or "")
-            if not sid or not token:
-                return self._send(400, {"error": "sessionId and token required"})
-            # The chain's saved scene, resolved by the API (which knows the
-            # chain) and handed over here. Validated, not trusted: a url and a
-            # positive size, nothing else, and an unusable value is simply
-            # ignored — a seat that boots empty beats a seat that fails to open.
-            restore = body.get("restore")
-            if isinstance(restore, dict):
-                url = restore.get("url")
-                nbytes = restore.get("bytes")
-                ok = (isinstance(url, str) and url.startswith(("https://", "http://"))
-                      and isinstance(nbytes, int) and nbytes > 0)
-                restore = {"url": url, "bytes": nbytes} if ok else None
-            else:
-                restore = None
-            seat, created = POOL.open(sid, token, restore=restore)
-            if seat is None:
-                # The pod is full. The API allocates across pods and creates
-                # another; this is not the place to queue.
-                return self._send(409, {"error": "pool_full", **POOL.status()})
-            return self._send(201 if created else 200,
-                              {"sessionId": sid, "ready": seat.ready(), "created": created,
-                               # Which child, so the API's seat row can name the
-                               # process a VmHWM reading belongs to.
-                               "index": seat.port - SEAT_BASE_PORT})
-        if route.startswith("/pool/seats/") and method == "DELETE":
-            sid = route[len("/pool/seats/"):]
-            return self._send(200, {"closed": POOL.close(sid)})
-        self._send(404, {"error": "not_found"})
-
-    def _relay(self, method, route):
-        rest = route[len("/s/"):]
-        sid, _, child_route = rest.partition("/")
-        seat = POOL.get(sid)
-        if seat is None:
-            return self._send(404, {"error": "no_such_seat"})
-        token = self.headers.get("x-genex-seat", "")
-        if not seat.token_sha or not hmac.compare_digest(
-                hashlib.sha256(token.encode()).hexdigest(), seat.token_sha):
-            # Per-seat, so one tenant cannot drive another's Blender even though
-            # both live on this pod's loopback.
-            return self._send(401, {"error": "unauthorized"})
-        # The body is read HERE, before the readiness check, and the order is
-        # load-bearing: the caller has just proved it holds the seat token, so
-        # reading its body is its own budget -- and the 503 below is the answer
-        # the API polls every two seconds during a 21 s seat boot. Answering it
-        # with the body unread would close the socket on every poll (see _send),
-        # which is a reconnect through the vendor proxy per retry.
-        try:
-            announced = self._content_length()
-        except ValueError as e:
-            return self._send(413, {"error": "bad_body", "detail": str(e)})
-        if announced > READ_AHEAD_MAX and not seat.ready():
-            # Too big to swallow just to keep a socket: an inlined asset would be
-            # uploaded through the vendor proxy on every 2 s poll. Answer without
-            # reading; _respond closes the connection, one reconnect is cheaper.
-            return self._send(503, {"error": "seat_not_ready", "retryAfterMs": 2000})
-        try:
-            payload = self._read_body()
-        except ValueError as e:
-            return self._send(413, {"error": "bad_body", "detail": str(e)})
-        if not seat.ready():
-            # `spawning` covers the restore download too: before this, a seat
-            # fetching its scene read as `seat_down` and the API's retry gave up
-            # on a seat that was working perfectly.
-            booting = seat.spawning or (seat.alive() and (time.time() - seat.opened_at) < SEAT_BOOT_BUDGET_S)
-            # NEVER block waiting. The API turns this into a 202 the caller
-            # retries; holding the connection would tie up a router thread for
-            # the length of a cold shader compile.
-            return self._send(503, {"error": "seat_not_ready" if booting else "seat_down",
-                                    "retryAfterMs": 2000})
-        url = "http://127.0.0.1:%d/%s" % (seat.port, child_route)
-        req = urllib.request.Request(url, data=payload if method != "GET" else None,
-                                     method=method)
-        req.add_header("Content-Type", "application/json")
-        req.add_header("x-genex-internal", seat.child_secret)
-        # Connection: close on the loopback leg. The child is a single-threaded
-        # HTTPServer, so a kept-alive idle socket from the router would own it
-        # until the idle timeout and block this seat's own next request.
-        req.add_header("Connection", "close")
-        seat.last_request_at = time.time()
-        try:
-            with urllib.request.urlopen(req, timeout=RELAY_TIMEOUT_S) as r:
-                data = r.read()
-                code = r.status
-        except urllib.error.HTTPError as e:
-            data, code = e.read(), e.code
-        except (urllib.error.URLError, OSError, TimeoutError) as e:
-            # The child died mid-request; the watcher will restart it and the
-            # scene comes back from its checkpoint.
-            return self._send(502, {"error": "seat_unreachable", "detail": str(e)})
-        self._respond(code, data)
-
-
-def main():
-    if os.environ.get("GENEX_BLENDER_REQUIRE_SECRET") == "1" and not POD_SECRET:
-        sys.stderr.write("[pool] FATAL no_shared_secret: refusing to serve a "
-                         "multi-tenant pod unauthenticated.\n")
-        return 5
-    port = int(os.environ.get("PORT", "8080"))
-    stop_evt = threading.Event()
-    watcher = threading.Thread(target=_watch_seats, args=(stop_evt,), daemon=True)
-    watcher.start()
-    srv = ThreadingHTTPServer(("0.0.0.0", port), Router)
-    srv.daemon_threads = True
-    log("router on 0.0.0.0:%d | capacity %d (pod %s MB, %d MB/seat)"
-        % (port, POOL.capacity(), POOL.pod_mb, PER_SEAT_MB))
-
-    def bye(signum, _f):
-        log("signal %s — draining" % signum)
-        stop_evt.set()
-        threading.Thread(target=srv.shutdown, daemon=True).start()
-
-    signal.signal(signal.SIGTERM, bye)
-    signal.signal(signal.SIGINT, bye)
-    try:
-        srv.serve_forever(poll_interval=0.2)
-    finally:
-        stop_evt.set()
-        # wipe=False (the disk is going anyway) but push=True: a pod being torn
-        # down is exactly when the last minutes of work would otherwise be lost.
-        # Bounded by CLOSE_PUSH_TIMEOUT_S per seat inside close().
-        for sid in list(POOL.seats):
-            POOL.close(sid, wipe=False)
-        deadline = time.time() + CLOSE_PUSH_TIMEOUT_S
-        while time.time() < deadline and any(t.name.startswith(("close-", "push-"))
-                                             for t in threading.enumerate()):
-            time.sleep(0.2)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())