@mcoda/mswarm 0.1.134 → 0.1.135-codali.2

package/scripts/media_gpu_lock.py

@@ -0,0 +1,238 @@
+"""Serialize GPU1 media inference across the independent HTTP services."""
+
+from __future__ import annotations
+
+import fcntl
+import json
+import logging
+import math
+import os
+import stat
+import threading
+import time
+import uuid
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Callable, Iterator
+
+
+class MediaGpuError(RuntimeError):
+    status = 503
+    code = "media_gpu_unavailable"
+
+
+class MediaGpuBusy(MediaGpuError):
+    status = 429
+    code = "media_gpu_busy"
+
+
+class MediaGpuCancelled(MediaGpuError):
+    code = "service_shutting_down"
+
+
+class MediaGpuRecoveryRequired(MediaGpuError):
+    code = "media_gpu_recovery_required"
+
+
+def _open_scheduler() -> int:
+    runtime_dir = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}"))
+    path = os.environ.get("MCODA_MEDIA_GPU_LOCK_PATH", str(runtime_dir / "mcoda-media-gpu1.lock"))
+    fd = -1
+    try:
+        fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600)
+        metadata = os.fstat(fd)
+        if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.geteuid()
+                or metadata.st_nlink != 1 or stat.S_IMODE(metadata.st_mode) & 0o077):
+            os.close(fd)
+            raise MediaGpuError("shared media GPU lock must be a private, owner-owned regular file")
+        return fd
+    except OSError as error:
+        if fd >= 0:
+            os.close(fd)
+        raise MediaGpuError("cannot access the shared media GPU scheduler") from error
+
+
+def _read_lease(fd: int) -> dict | None:
+    raw = os.pread(fd, 16_385, 0)
+    if not raw:
+        return None
+    try:
+        if len(raw) > 16_384:
+            raise ValueError("oversized lease")
+        value = json.loads(raw)
+        if (not isinstance(value, dict) or value.get("version") != 1
+                or not isinstance(value.get("id"), str)
+                or value.get("status") not in ("active", "uncertain")):
+            raise ValueError("invalid lease")
+        return value
+    except (ValueError, UnicodeDecodeError) as error:
+        raise MediaGpuRecoveryRequired("media GPU lease is unreadable; reconcile native inference before recovery") from error
+
+
+def _write_lease(fd: int, value: dict | None) -> None:
+    data = b"" if value is None else json.dumps(value, separators=(",", ":")).encode("utf-8")
+    os.lseek(fd, 0, os.SEEK_SET)
+    offset = 0
+    while offset < len(data):
+        written = os.write(fd, data[offset:])
+        if written <= 0:
+            raise MediaGpuError("cannot persist the shared media GPU lease")
+        offset += written
+    os.ftruncate(fd, len(data))
+    os.fsync(fd)
+
+
+class MediaGpuLease:
+    """Host-owned receipt; never store prompts, credentials or client URLs."""
+
+    def __init__(self, fd: int, label: str):
+        self._fd = fd
+        self._closed = False
+        self._uncertain = False
+        self.record = {"version": 1, "id": str(uuid.uuid4()), "status": "active",
+                       "label": label[:128], "pid": os.getpid(), "started_at": time.time()}
+        _write_lease(fd, self.record)
+
+    def mark_uncertain(self, reason: str) -> None:
+        if self._closed:
+            raise MediaGpuError("media GPU lease is already closed")
+        self._uncertain = True
+        self.record.update(status="uncertain", reason=reason[:128])
+        _write_lease(self._fd, self.record)
+
+    def record_native_job(self, job_id: str) -> None:
+        if self._closed or not isinstance(job_id, str) or not job_id or len(job_id) > 256:
+            raise MediaGpuError("invalid native media job receipt")
+        self.record["native_job_id"] = job_id
+        _write_lease(self._fd, self.record)
+
+    def record_native_process(self, pid: int) -> None:
+        """Pin a Linux native child's identity without recording its arguments."""
+        if self._closed or not isinstance(pid, int) or pid <= 0:
+            raise MediaGpuError("invalid native media process receipt")
+        try:
+            fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()
+            boot_id = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
+            if int(fields[2]) != pid or not boot_id:
+                raise ValueError("native process must own its process group")
+            identity = {"pid": pid, "process_group": pid,
+                        "start_ticks": int(fields[19]), "boot_id": boot_id}
+        except (OSError, ValueError, IndexError) as error:
+            raise MediaGpuError("cannot establish native media process identity") from error
+        self.record["native_process"] = identity
+        _write_lease(self._fd, self.record)
+
+    def confirm_finished(self) -> None:
+        if self._closed:
+            raise MediaGpuError("media GPU lease is already closed")
+        self._uncertain = False
+        self.record.update(status="active")
+        self.record.pop("reason", None)
+        _write_lease(self._fd, self.record)
+
+
+def reconcile_media_gpu_slot(expected_id: str, terminal_probe: Callable[[dict], bool]) -> bool:
+    """Clear one abandoned receipt only after a trusted backend terminal probe.
+
+    Process death, elapsed time, HTTP disconnection and a cancel acknowledgement
+    are not terminal proof. The caller supplies a backend-specific probe or
+    evidence that the native engine was stopped and cannot retain that job.
+    """
+    fd = _open_scheduler()
+    try:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError as error:
+            raise MediaGpuBusy("media GPU lease is still owned by a running worker") from error
+        current = _read_lease(fd)
+        if current is None or current["id"] != expected_id:
+            return False
+        if terminal_probe(dict(current)) is not True:
+            return False
+        _write_lease(fd, None)
+        return True
+    finally:
+        os.close(fd)
+
+
+def media_gpu_health() -> str:
+    """Report admission state without exposing a receipt or clearing uncertainty."""
+    fd = -1
+    try:
+        fd = _open_scheduler()
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            return "busy"
+        return "ready" if _read_lease(fd) is None else "recovery_required"
+    except MediaGpuRecoveryRequired:
+        return "recovery_required"
+    except MediaGpuError:
+        return "unavailable"
+    finally:
+        if fd >= 0:
+            os.close(fd)
+
+
+@contextmanager
+def media_gpu_slot(
+    label: str, *, cancelled: threading.Event | None = None,
+    shutdown: threading.Event | None = None,
+) -> Iterator[MediaGpuLease]:
+    """Hold a kernel lock until native inference finishes, even if a client leaves.
+
+    Never unlink the lock file: waiters must keep referring to the same inode.
+    Each call opens its own file description so threads also exclude each other.
+    """
+    try:
+        timeout = float(os.environ.get("MCODA_MEDIA_GPU_WAIT_SECONDS", "600"))
+    except ValueError as error:
+        raise MediaGpuError("media GPU wait limit must be a number") from error
+    if not math.isfinite(timeout) or timeout < 0:
+        raise MediaGpuError("media GPU wait limit must be finite and nonnegative")
+    fd = _open_scheduler()
+    acquired = False
+    lease: MediaGpuLease | None = None
+    started = time.monotonic()
+    try:
+        while True:
+            if any(event is not None and event.is_set() for event in (cancelled, shutdown)):
+                raise MediaGpuCancelled("media generation stopped while waiting for the GPU")
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                acquired = True
+                break
+            except BlockingIOError:
+                remaining = timeout - (time.monotonic() - started)
+                if remaining <= 0:
+                    raise MediaGpuBusy("the media GPU is busy; retry the request")
+                if cancelled is not None:
+                    cancelled.wait(min(0.1, remaining))
+                else:
+                    time.sleep(min(0.1, remaining))
+        if any(event is not None and event.is_set() for event in (cancelled, shutdown)):
+            raise MediaGpuCancelled("media generation stopped before GPU inference")
+        if _read_lease(fd) is not None:
+            raise MediaGpuRecoveryRequired("an earlier media GPU lease has no confirmed outcome; reconcile native inference before retrying")
+        lease = MediaGpuLease(fd, label)
+        logging.info(
+            "media_gpu_acquired label=%s pid=%d wait_ms=%d monotonic=%.6f",
+            label, os.getpid(), int((time.monotonic() - started) * 1000), time.monotonic(),
+        )
+        yield lease
+    finally:
+        try:
+            if acquired:
+                if lease is not None:
+                    try:
+                        if not lease._uncertain:
+                            _write_lease(fd, None)
+                    finally:
+                        lease._closed = True
+                logging.info(
+                    "media_gpu_released label=%s pid=%d monotonic=%.6f",
+                    label, os.getpid(), time.monotonic(),
+                )
+                fcntl.flock(fd, fcntl.LOCK_UN)
+        finally:
+            os.close(fd)

package/scripts/stable-audio-3-openai-server.py

@@ -25,6 +25,8 @@ from pathlib import Path
 from socketserver import TCPServer
 from typing import Any, Protocol
 
+from media_gpu_lock import MediaGpuError, MediaGpuRecoveryRequired, media_gpu_health, media_gpu_slot
+
 
 DEFAULT_HOST = "127.0.0.1"
 DEFAULT_PORT = 11446
@@ -165,7 +167,7 @@ class SubprocessBackend:
         seed: int | None,
     ) -> tuple[bytes, dict[str, Any]]:
         effective_seed = seed if seed is not None else secrets.randbelow(2**31)
-        with tempfile.TemporaryDirectory(prefix="stable-audio-3-") as temp_dir:
+        with media_gpu_slot("audio") as lease, tempfile.TemporaryDirectory(prefix="stable-audio-3-") as temp_dir:
             output_path = Path(temp_dir) / "generation.wav"
             command = [
                 str(self.python),
@@ -189,26 +191,48 @@ class SubprocessBackend:
                 command.extend(("--precision", self.precision))
             if self.models_dir:
                 command.extend(("--models-dir", str(self.models_dir)))
-            process = subprocess.Popen(
+            lease.mark_uncertain("audio_launch_pending")
+            try:
+                process = subprocess.Popen(
                     command,
                     cwd=self.runtime_root,
                     stdin=subprocess.DEVNULL,
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.DEVNULL,
                     start_new_session=True,
-            )
+                )
+            except OSError:
+                # Popen reports failed exec only after reaping its failed child.
+                lease.confirm_finished()
+                raise
             try:
-                return_code = process.wait(timeout=self.timeout_seconds)
-            except subprocess.TimeoutExpired as error:
+                lease.record_native_process(process.pid)
                 try:
-                    os.killpg(process.pid, signal.SIGTERM)
-                    process.wait(timeout=5)
-                except subprocess.TimeoutExpired:
-                    os.killpg(process.pid, signal.SIGKILL)
-                    process.wait()
+                    return_code = process.wait(timeout=self.timeout_seconds)
+                except subprocess.TimeoutExpired as error:
+                    try:
+                        os.killpg(process.pid, signal.SIGTERM)
+                        process.wait(timeout=5)
+                    except subprocess.TimeoutExpired:
+                        os.killpg(process.pid, signal.SIGKILL)
+                        process.wait()
+                    except ProcessLookupError:
+                        process.wait()
+                    raise RuntimeError("Stable Audio CLI timed out") from error
+            finally:
+                # A reaped wrapper does not prove that its process group stopped.
+                # Any unexpected wait/identity failure keeps the durable fence.
+                group_alive = True
+                try:
+                    os.killpg(process.pid, 0)
                 except ProcessLookupError:
+                    group_alive = False
+                except PermissionError:
                     pass
-                raise RuntimeError("Stable Audio CLI timed out") from error
+                if process.poll() is not None and not group_alive:
+                    lease.confirm_finished()
+            if lease.record["status"] == "uncertain":
+                raise MediaGpuRecoveryRequired("audio process group has no confirmed terminal outcome")
             if return_code != 0:
                 raise RuntimeError(
                     f"Stable Audio CLI exited with status {return_code}"
@@ -244,7 +268,8 @@ class ServerState:
 
 
 class AudioHttpServer(ThreadingHTTPServer):
-    daemon_threads = True
+    # Let active inference finish before process exit releases the shared GPU.
+    daemon_threads = False
     request_queue_size = 8
 
     def server_bind(self) -> None:
@@ -438,8 +463,9 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
 
         def do_GET(self) -> None:
             if self.path == "/healthz":
-                if state.fatal_error.is_set():
-                    self._send_json(503, {"status": "unhealthy", "model": state.model_id})
+                gpu = media_gpu_health()
+                if state.fatal_error.is_set() or gpu not in ("ready", "busy"):
+                    self._send_json(503, {"status": "unhealthy", "model": state.model_id, "media_gpu": gpu})
                     return
                 self._send_json(
                     200,
@@ -447,6 +473,7 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
                         "status": "ok",
                         "model": state.model_id,
                         "backend": state.backend_name,
+                        "media_gpu": gpu,
                         "dit": state.dit,
                         "decoder": state.decoder,
                     },
@@ -568,6 +595,10 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
                 )
             except ApiError as error:
                 self._send_error(error)
+            except MediaGpuError as error:
+                self._send_error(
+                    ApiError(error.status, str(error), error_type="server_error", code=error.code)
+                )
             except Exception as error:
                 logging.error("generation failed error_type=%s", type(error).__name__)
                 self._send_error(

package/scripts/stable-diffusion-cpp-openai-server.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import argparse
 import base64
 import binascii
+import http.client
 import ipaddress
 import json
 import logging
@@ -28,6 +29,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from socketserver import TCPServer
 from typing import Any
 
+from media_gpu_lock import MediaGpuError, media_gpu_health, media_gpu_slot
+
 
 DEFAULT_HOST = "127.0.0.1"
 DEFAULT_PORT = 11449
@@ -116,7 +119,11 @@ class UpstreamError(Exception):
     pass
 
 
-class UpstreamTimeout(UpstreamError):
+class UpstreamTransportError(UpstreamError):
+    pass
+
+
+class UpstreamTimeout(UpstreamTransportError):
     pass
 
 
@@ -459,10 +466,10 @@ def _bounded_json_request(
         raw = error.read(max_bytes + 1)
         status = error.code
         content_type = error.headers.get_content_type()
-    except (OSError, urllib.error.URLError) as error:
+    except (OSError, urllib.error.URLError, http.client.HTTPException) as error:
         if _is_timeout_error(error):
             raise UpstreamTimeout("stable-diffusion.cpp request timed out") from error
-        raise UpstreamError("stable-diffusion.cpp is unreachable") from error
+        raise UpstreamTransportError("stable-diffusion.cpp is unreachable") from error
     if len(raw) > max_bytes:
         raise UpstreamError("stable-diffusion.cpp response exceeds the configured limit")
     if content_type != "application/json":
@@ -507,6 +514,19 @@ def _native_request(request: dict[str, Any]) -> dict[str, Any]:
 
 
 def _generate_image(state: ServerState, request: dict[str, Any]) -> dict[str, Any]:
+    with media_gpu_slot("image", cancelled=state.shutdown_requested) as lease:
+        try:
+            return _generate_image_locked(state, request)
+        except (UpstreamTransportError, OSError, urllib.error.URLError, http.client.HTTPException) as error:
+            # A lost HTTP connection does not prove the synchronous native
+            # renderer stopped. Keep a durable fence after this worker exits.
+            lease.mark_uncertain("image_transport_lost")
+            if isinstance(error, UpstreamError):
+                raise
+            raise UpstreamTransportError("native image outcome is uncertain") from error
+
+
+def _generate_image_locked(state: ServerState, request: dict[str, Any]) -> dict[str, Any]:
     max_json_bytes = math.ceil(state.max_output_bytes / 3) * 4 + 131_072
     status, payload = _bounded_json_request(
         f"{state.upstream_base_url}/v1/images/generations",
@@ -776,6 +796,10 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
                     ApiError(404, "not found", error_type="not_found_error")
                 )
                 return
+            gpu = media_gpu_health() if self.path == "/healthz" else None
+            if self.path == "/healthz" and gpu not in ("ready", "busy"):
+                self._send_json(503, {"status": "unhealthy", "model": state.model_id, "media_gpu": gpu})
+                return
             try:
                 _check_upstream_model(state)
             except UpstreamError:
@@ -801,6 +825,7 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
                         "status": "ok",
                         "model": state.model_id,
                         "upstream_model": state.upstream_model_id,
+                        "media_gpu": gpu,
                         "upstream": "stable-diffusion.cpp",
                     },
                 )
@@ -921,7 +946,8 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
             threading.Thread(
                 target=run_generation,
                 name="sd-cpp-image-generation",
-                daemon=True,
+                # Preserve the GPU lease until non-cancellable inference ends.
+                daemon=False,
             ).start()
             if not completed.wait(state.inference_timeout_seconds):
                 logging.error("generation exceeded client deadline; native request still active")
@@ -947,6 +973,10 @@ def _handler_for(state: ServerState) -> type[BaseHTTPRequestHandler]:
                 )
             elif isinstance(error, ApiError):
                 self._send_error(error)
+            elif isinstance(error, MediaGpuError):
+                self._send_error(
+                    ApiError(error.status, str(error), error_type="server_error", code=error.code)
+                )
             elif isinstance(error, UpstreamTimeout):
                 logging.error("generation failed reason=upstream timeout")
                 self._send_error(

package/scripts/stable-diffusion-cpp-webm-duration.patch

@@ -0,0 +1,22 @@
+--- a/examples/common/media_io.cpp
++++ b/examples/common/media_io.cpp
+@@ -1212,6 +1212,9 @@
+ 
+         const uint64_t frame_duration_ns = std::max<uint64_t>(
+             1, static_cast<uint64_t>(std::llround(1000000000.0 / static_cast<double>(fps))));
++        if (video_track != nullptr) {
++            video_track->set_default_duration(frame_duration_ns);
++        }
+         uint64_t timestamp_ns = 0;
+ 
+         for (int i = 0; i < num_images; ++i) {
+@@ -1256,6 +1259,9 @@
+             timestamp_ns += frame_duration_ns;
+         }
+ 
++        // Include the final frame's display interval, not only its timestamp.
++        segment.set_duration(static_cast<double>(timestamp_ns) /
++                             segment.GetSegmentInfo()->timecode_scale());
+         if (!segment.Finalize()) {
+             fprintf(stderr, "Error: Failed to finalize WebM output.\n");
+             return -1;