@camstack/addon-pipeline 1.2.359 → 1.2.361

package/python/test_webgpu_isolated.py

@@ -168,5 +168,137 @@ class IsolatedSessionTests(unittest.TestCase):
         proc.kill.assert_called()
 
 
+class _FakeEpDevice:
+    ep_name = "WebGpuExecutionProvider"
+
+
+class _FakeSessionOptions:
+    def __init__(self) -> None:
+        self.intra_op_num_threads = 0
+        self.devices: list = []
+
+    def add_provider_for_devices(self, devices, options) -> None:
+        self.devices = list(devices)
+
+
+class _FakeSession:
+    def __init__(self, path: str, sess_options=None, providers=None) -> None:
+        self.path = path
+        self.sess_options = sess_options
+        self._providers = (
+            list(providers)
+            if providers is not None
+            else ["WebGpuExecutionProvider", "CPUExecutionProvider"]
+        )
+
+    def get_providers(self) -> list[str]:
+        return self._providers
+
+
+class _FakeOrt:
+    """Mirrors the one ORT behaviour that matters: a second registration RAISES."""
+
+    SessionOptions = _FakeSessionOptions
+    InferenceSession = _FakeSession
+
+    def __init__(self) -> None:
+        self.register_calls = 0
+
+    def register_execution_provider_library(self, name: str, path: str) -> None:
+        self.register_calls += 1
+        if self.register_calls > 1:
+            raise RuntimeError(f"EP library is already registered under {name}")
+
+    def get_ep_devices(self) -> list:
+        return [_FakeEpDevice()]
+
+
+class LoaderTests(unittest.TestCase):
+    """D845: register once per process; refuse a render node that cannot be opened."""
+
+    def setUp(self) -> None:
+        wi._webgpu_registered = False
+        fake_plugin = MagicMock()
+        fake_plugin.get_library_path.return_value = "/fake/libwebgpu_ep.so"
+        fake_plugin.get_ep_name.return_value = "WebGpuExecutionProvider"
+        self._plugin_patch = patch.dict(sys.modules, {"onnxruntime_ep_webgpu": fake_plugin})
+        self._plugin_patch.start()
+
+    def tearDown(self) -> None:
+        self._plugin_patch.stop()
+        wi._webgpu_registered = False
+
+    def test_registers_the_plugin_once_across_sessions(self) -> None:
+        ort = _FakeOrt()
+        s1, p1 = wi.load_onnx_webgpu_session(ort, "/m/a.onnx", render_node_ok=True)
+        s2, p2 = wi.load_onnx_webgpu_session(ort, "/m/b.onnx", render_node_ok=True)
+        self.assertEqual(ort.register_calls, 1)
+        self.assertEqual(p1, ["WebGpuExecutionProvider", "CPUExecutionProvider"])
+        self.assertEqual(p2, p1)
+        # Both sessions were placed on the WebGPU device, not the CPU fallback.
+        self.assertEqual(len(s1.sess_options.devices), 1)
+        self.assertEqual(len(s2.sess_options.devices), 1)
+
+    def test_a_registration_that_raises_is_a_cpu_fallback_not_a_crash(self) -> None:
+        """Disarm check: without the guard the second load re-registers and the
+        fake raises — the loader must then hand back CPU providers, never a
+        WebGPU session built on a poisoned registration."""
+        ort = _FakeOrt()
+        wi.load_onnx_webgpu_session(ort, "/m/a.onnx", render_node_ok=True)
+        wi._webgpu_registered = False  # simulate the pre-D845 loader forgetting
+        _, providers = wi.load_onnx_webgpu_session(ort, "/m/b.onnx", render_node_ok=True)
+        self.assertEqual(ort.register_calls, 2)
+        self.assertEqual(providers, [wi.ONNX_CPU_EP])
+
+    def test_bounds_the_cpu_thread_pool(self) -> None:
+        ort = _FakeOrt()
+        session, _ = wi.load_onnx_webgpu_session(ort, "/m/a.onnx", render_node_ok=True)
+        self.assertEqual(session.sess_options.intra_op_num_threads, wi.WEBGPU_INTRA_OP_THREADS)
+
+    def test_refuses_without_an_openable_render_node(self) -> None:
+        ort = _FakeOrt()
+        with self.assertRaises(wi.WebGpuDeviceUnavailable) as ctx:
+            wi.load_onnx_webgpu_session(ort, "/m/a.onnx", render_node_ok=False)
+        self.assertIn("/dev/dri/renderD", str(ctx.exception))
+        self.assertIn("uid=", str(ctx.exception))
+        self.assertEqual(ort.register_calls, 0)
+
+    def test_no_render_node_check_outside_linux(self) -> None:
+        ort = _FakeOrt()
+        with patch.object(wi, "render_node_accessible", return_value=None):
+            _, providers = wi.load_onnx_webgpu_session(ort, "/m/a.onnx")
+        self.assertEqual(providers[0], "WebGpuExecutionProvider")
+
+
+class RenderNodeProbeTests(unittest.TestCase):
+    def test_non_linux_is_not_checked(self) -> None:
+        self.assertIsNone(wi.render_node_accessible(platform="darwin"))
+
+    def test_missing_dir_or_no_nodes(self) -> None:
+        def missing(_: str) -> list[str]:
+            raise FileNotFoundError
+
+        self.assertFalse(wi.render_node_accessible(platform="linux", listdir=missing))
+        self.assertFalse(
+            wi.render_node_accessible(platform="linux", listdir=lambda _: ["card1", "by-path"])
+        )
+
+    def test_denied_node_is_false_and_granted_is_true(self) -> None:
+        def nodes(_: str) -> list[str]:
+            return ["card1", "renderD128"]
+
+        self.assertFalse(
+            wi.render_node_accessible(platform="linux", listdir=nodes, access=lambda p, m: False)
+        )
+        seen: list[str] = []
+
+        def grant(p: str, m: int) -> bool:
+            seen.append(p)
+            return True
+
+        self.assertTrue(wi.render_node_accessible(platform="linux", listdir=nodes, access=grant))
+        self.assertEqual(seen, ["/dev/dri/renderD128"])
+
+
 if __name__ == "__main__":
     unittest.main()

package/python/webgpu_isolated.py

@@ -1,14 +1,30 @@
 """
 WebGPU ORT sessions in a dedicated OS process.
 
-ORT WebGPU EP (Dawn/Vulkan) SIGSEGVs when a second InferenceSession is created
-in the same Python process (measured amd-proxmox 2026-10-10). Two processes,
-one session each, work concurrently. This module:
+One child process per WebGPU model (D823). This module:
 
-  - loads a WebGPU session (or CPU fallback) — used by the child worker;
+  - loads a WebGPU session — used by the child worker;
   - hosts the child worker entrypoint (`python webgpu_isolated.py --ipc-in N --ipc-out M`);
   - exposes `WebGpuIsolatedSession` for the pool parent: one child per model.
 
+Two facts measured on amd-proxmox 2026-10-13 (D845) shape the loader:
+
+  1. The SIGSEGV that motivated D823 was NOT a second session. It is a second
+     `register_execution_provider_library` call for a name that is already
+     registered: ORT raises "EP library is already registered", and the next
+     `InferenceSession` on that EP then dies in `_create_inference_session`.
+     Register ONCE per process (`_ensure_webgpu_ep_registered`) and three
+     sessions in one process load and run — serialised; concurrent `run()` on
+     distinct sessions aborts the process, so a shared process would need one
+     lock. The child-per-model layout stays for crash isolation.
+  2. Dawn never reports a software adapter. With `/dev/dri/renderD*` not
+     openable (the entrypoint dropped the render group), the session still
+     answered `providers=[WebGpuExecutionProvider, …]` and ran at 224 ms per
+     yolov9t-320 frame with gpu_busy 0 % — 15× the GPU's 13 ms, for a week,
+     looking like a weak iGPU. A render node that cannot be opened is
+     therefore REFUSED before any session is built; the error names the
+     uid/gid so the fix (device ownership in the entrypoint) is one read away.
+
 IPC uses dedicated pipe fds (`pass_fds`), never stdin/stdout — ORT/Dawn log on
 fd 1 and a yolov9 raw head on a shared stdout pickle channel truncated the
 frame (2026-10-10). Pipe `read(n)` can also return short; framing uses
@@ -24,13 +40,77 @@ import struct
 import subprocess
 import sys
 import threading
-from typing import Any, BinaryIO, Optional
+from typing import Any, BinaryIO, Callable, Optional
 
 ONNX_CPU_EP = "CPUExecutionProvider"
 ONNX_WEBGPU_EP_REGISTRATION = "webgpu_ep_registration"
 
+# ORT's CPU thread pool in a WebGPU child only feeds buffer uploads and the
+# rare CPU-placed node; the per-frame cost is Dawn's dispatch on ONE thread.
+# Measured 2026-10-13 (yolov9t-320 warm p50): 14.2 ms at 1, 2 and 6 threads —
+# so the default (one spinning thread per core, per child, ten children on a
+# six-core node) buys nothing and costs CPU. Two keeps a CPU-placed node from
+# serialising fully.
+WEBGPU_INTRA_OP_THREADS = 2
+
+RENDER_NODE_DIR = "/dev/dri"
+RENDER_NODE_PREFIX = "renderD"
+
 _HDR = struct.Struct(">I")
 
+_webgpu_registered = False
+_webgpu_register_lock = threading.Lock()
+
+
+class WebGpuDeviceUnavailable(RuntimeError):
+    """The webgpu device was asked for and cannot reach a GPU. Never served from CPU."""
+
+
+def render_node_accessible(
+    *,
+    platform: str = sys.platform,
+    listdir: Callable[[str], list[str]] = os.listdir,
+    access: Callable[[str, int], bool] = os.access,
+) -> Optional[bool]:
+    """True when some /dev/dri/renderD* is readable+writable by this process.
+
+    None on a platform without DRM render nodes (nothing to check); False when
+    the directory is absent, empty of render nodes, or every node is denied.
+    """
+    if not platform.startswith("linux"):
+        return None
+    try:
+        names = [n for n in listdir(RENDER_NODE_DIR) if n.startswith(RENDER_NODE_PREFIX)]
+    except OSError:
+        return False
+    if not names:
+        return False
+    return any(access(os.path.join(RENDER_NODE_DIR, n), os.R_OK | os.W_OK) for n in names)
+
+
+def _identity() -> str:
+    try:
+        return f"uid={os.getuid()} gid={os.getgid()} groups={sorted(os.getgroups())}"
+    except (AttributeError, OSError):
+        return "uid=? gid=? groups=?"
+
+
+def _ensure_webgpu_ep_registered(ort: Any, webgpu_ep: Any) -> None:
+    """Register the plugin EP library at most once per process.
+
+    A second registration under the same name raises and leaves the next
+    session creation to SIGSEGV (the real cause behind D823's "second session
+    crashes"). Idempotent across callers and threads.
+    """
+    global _webgpu_registered
+    with _webgpu_register_lock:
+        if _webgpu_registered:
+            return
+        ort.register_execution_provider_library(
+            ONNX_WEBGPU_EP_REGISTRATION, webgpu_ep.get_library_path()
+        )
+        _webgpu_registered = True
+
 
 def _read_exact(stream: BinaryIO, n: int) -> bytes:
     """Read exactly n bytes; pipe read() may return short without EOF."""
@@ -59,11 +139,29 @@ def _read_msg(stream: BinaryIO) -> Any:
     return pickle.loads(data)
 
 
-def load_onnx_webgpu_session(ort: Any, path: str) -> tuple[Any, list[str]]:
-    """Build an InferenceSession on the WebGPU plugin EP, CPU fallback on miss.
+def load_onnx_webgpu_session(
+    ort: Any,
+    path: str,
+    *,
+    render_node_ok: Optional[bool] = None,
+) -> tuple[Any, list[str]]:
+    """Build an InferenceSession on the WebGPU plugin EP.
 
-    Same contract as the former inference_pool._load_onnx_webgpu_session.
+    Refuses (`WebGpuDeviceUnavailable`) when no DRM render node is openable —
+    Dawn would otherwise serve the session from software and still call it
+    WebGPU. A missing plugin or a missing Vulkan adapter still falls back to
+    the CPU EP, and says so in `providers`.
     """
+    accessible = render_node_accessible() if render_node_ok is None else render_node_ok
+    if accessible is False:
+        raise WebGpuDeviceUnavailable(
+            f"ONNX WebGPU: no openable {RENDER_NODE_DIR}/{RENDER_NODE_PREFIX}* "
+            f"({_identity()}) — Dawn would run this model on a software adapter at "
+            f"~15× the GPU latency while reporting WebGpuExecutionProvider; refusing "
+            f"device webgpu. The container entrypoint must own or group-grant the "
+            f"render node to the camstack user (compose group_add is dropped by gosu)."
+        )
+
     try:
         import onnxruntime_ep_webgpu as webgpu_ep  # type: ignore
     except Exception as err:
@@ -72,15 +170,11 @@ def load_onnx_webgpu_session(ort: Any, path: str) -> tuple[Any, list[str]]:
         return ort.InferenceSession(path, providers=[ONNX_CPU_EP]), [ONNX_CPU_EP]
 
     try:
-        ort.register_execution_provider_library(
-            ONNX_WEBGPU_EP_REGISTRATION, webgpu_ep.get_library_path()
-        )
+        _ensure_webgpu_ep_registered(ort, webgpu_ep)
     except Exception as err:
-        msg = str(err).lower()
-        if "already" not in msg and "exist" not in msg:
-            sys.stderr.write(f"ONNX WebGPU: register failed ({err}); CPU fallback\n")
-            sys.stderr.flush()
-            return ort.InferenceSession(path, providers=[ONNX_CPU_EP]), [ONNX_CPU_EP]
+        sys.stderr.write(f"ONNX WebGPU: register failed ({err}); CPU fallback\n")
+        sys.stderr.flush()
+        return ort.InferenceSession(path, providers=[ONNX_CPU_EP]), [ONNX_CPU_EP]
 
     ep_name = webgpu_ep.get_ep_name()
     webgpu_device = next(
@@ -96,10 +190,14 @@ def load_onnx_webgpu_session(ort: Any, path: str) -> tuple[Any, list[str]]:
         return ort.InferenceSession(path, providers=[ONNX_CPU_EP]), [ONNX_CPU_EP]
 
     sess_options = ort.SessionOptions()
+    sess_options.intra_op_num_threads = WEBGPU_INTRA_OP_THREADS
     sess_options.add_provider_for_devices([webgpu_device], {})
     session = ort.InferenceSession(path, sess_options=sess_options)
     active = list(session.get_providers())
-    sys.stderr.write(f"ONNX WebGPU: providers={active} ep={ep_name} device=webgpu\n")
+    sys.stderr.write(
+        f"ONNX WebGPU: providers={active} ep={ep_name} device=webgpu "
+        f"render-node=ok {_identity()}\n"
+    )
     sys.stderr.flush()
     return session, active