@camstack/addon-pipeline 1.2.615 → 1.2.617

package/python/openvino_compile_probe.py

@@ -0,0 +1,245 @@
+"""Compile an OpenVINO GPU/NPU model in a CHILD process before the pool loads it.
+
+Why (2026-10-22, hub with a Core Ultra 7 265K iGPU, OpenVINO 2025.4.1): the
+iGPU compiler SIGSEGVed the pool worker three times in 15 s while
+`compile_model()` ran for a model with no compiled blob (`malloc(): unsorted
+double linked list corrupted`, then `longjmp causes uninitialized stack
+frame`). The compile runs in the same process as the predict threads and every
+other model, so one native crash took the whole `openvino:gpu` worker (and the
+camera's detection with it) down, and the respawn went straight back into the
+same load. Python cannot catch a SIGSEGV. A warm load (blob present) never
+crashed.
+
+So the first load of each (model x device request x compile config) happens in
+a throwaway interpreter that runs the pool's OWN `_load_model` — same
+PrePostProcessor, same candidate list, same properties, same CACHE_DIR — and
+therefore writes the blob the pool then loads warm. Mirrors CoreML's
+`coreml_load_probe.py` (D742):
+
+  ok           -> load in-process (a cache HIT), never probe this key again
+                  while the blobs the probe wrote are still on disk
+  crashed      -> refuse with `OpenvinoLoadCrashed` (reason
+                  `native-load-crashed`), now and on every later load of the
+                  same key, without re-probing
+  inconclusive -> (a Python exception in the child, a probe past its bound, a
+                  child that could not start) remembered NOWHERE; the
+                  in-process load runs exactly as it did before this gate
+
+Only a death by SIGNAL is a crash. CPU-only loads are never probed.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from typing import Callable, Optional
+
+# One-way: inference_pool imports this module lazily, inside its OpenVINO branch.
+from coreml_load_probe import (
+    VERDICT_CRASHED,
+    VERDICT_INCONCLUSIVE,
+    VERDICT_OK,
+    _signal_name,
+    classify_probe_exit,
+)
+
+PROBE_VERDICT_VERSION = 1
+PROBE_MARKER = "@@CAMSTACK_OV_PROBE_OK@@"
+# A cold iGPU compile of a large detector takes tens of seconds; past this the
+# probe says nothing and the pool's own load bound (POOL_MODEL_LOAD_BOUNDS)
+# is the only fence, exactly as before the probe existed.
+PROBE_TIMEOUT_SEC = 300.0
+# The config keys that change what `compile_model` is handed (PrePostProcessor
+# plan, input shape, precision hint). A threshold or a class list does not.
+COMPILE_CONFIG_KEYS = (
+    "device",
+    "inputSize",
+    "inputWidth",
+    "inputHeight",
+    "inputChannels",
+    "preprocessMode",
+    "inputNormalization",
+    "gpuInferencePrecision",
+)
+PROBED_DEVICE_PREFIXES = ("GPU", "NPU")
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+
+# The child runs the pool's own loader, with the probe switched off in it.
+_CHILD_CODE = f"""
+import json, sys
+args = json.loads(sys.argv[1])
+sys.path.insert(0, args["pool_dir"])
+import inference_pool as ip
+ip._ov_probe_enabled = False
+ip._init_runtime("openvino", str(args["config"].get("device", "")))
+ip._load_model(ip.ModelSlot(), args["config"])
+print({PROBE_MARKER!r}, flush=True)
+"""
+
+ProbeRunner = Callable[[str, dict], "tuple[str, str]"]
+
+
+class OpenvinoLoadCrashed(Exception):
+    """The model kills the native compiler on this device; refused instead of loaded."""
+
+    reason = "native-load-crashed"
+
+    def __init__(self, model_path: str, device: str, detail: str) -> None:
+        self.model_path = model_path
+        self.device = device
+        self.detail = detail
+        super().__init__(
+            f"OpenVINO model {os.path.basename(model_path)} crashes the native compiler "
+            f"on device={device} ({detail}); refused so it cannot take the pool down. "
+            f"Run this model on another device, or replace it"
+        )
+
+
+def needs_probe(candidates: "list[str]") -> bool:
+    """True when any compile target is a GPU or an NPU (CPU-only loads are never probed)."""
+    return any(str(c).upper().startswith(PROBED_DEVICE_PREFIXES) for c in candidates)
+
+
+def run_probe(
+    model_path: str,
+    config: dict,
+    child_code: str = _CHILD_CODE,
+    timeout_sec: float = PROBE_TIMEOUT_SEC,
+) -> "tuple[str, str]":
+    """Compile `model_path` in a fresh interpreter; return (verdict, detail)."""
+    payload = json.dumps({"config": {**config, "path": model_path}, "pool_dir": _HERE})
+    try:
+        proc = subprocess.run(
+            [sys.executable, "-c", child_code, payload],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+            timeout=timeout_sec,
+        )
+    except subprocess.TimeoutExpired:
+        return VERDICT_INCONCLUSIVE, f"probe timed out after {timeout_sec:g}s"
+    except (OSError, TypeError, ValueError) as exc:
+        return VERDICT_INCONCLUSIVE, f"probe could not start ({exc})"
+    marker_seen = any(line.strip() == PROBE_MARKER for line in proc.stdout.splitlines())
+    verdict = classify_probe_exit(proc.returncode, marker_seen)
+    tail = " | ".join(proc.stderr.strip().splitlines()[-3:])
+    if proc.returncode < 0:
+        detail = f"probe child died by signal {-proc.returncode} ({_signal_name(-proc.returncode)})"
+    else:
+        detail = f"probe child exited {proc.returncode}"
+    return verdict, f"{detail}{': ' + tail if tail else ''}"
+
+
+# Looked up at call time, so a test can replace it.
+DEFAULT_RUNNER: ProbeRunner = run_probe
+
+
+def _file_fingerprint(path: str) -> dict:
+    """size + mtime of the model file and, for an IR, its `.bin` weights."""
+    out: dict = {}
+    for candidate in (path, os.path.splitext(path)[0] + ".bin"):
+        try:
+            st = os.stat(candidate)
+        except OSError:
+            continue
+        out[os.path.basename(candidate)] = [st.st_size, st.st_mtime_ns]
+    return out
+
+
+def probe_key(path: str, config: dict, candidates: "list[str]", env_tag: str) -> dict:
+    return {
+        "version": PROBE_VERDICT_VERSION,
+        "model": _file_fingerprint(path),
+        "candidates": [str(c).upper() for c in candidates],
+        "compile": {k: config.get(k) for k in COMPILE_CONFIG_KEYS},
+        # An OpenVINO / GPU-driver update may fix or break a model.
+        "env": env_tag,
+    }
+
+
+def probe_verdict_path(path: str, config: dict, cache_dir: str) -> str:
+    stem = os.path.splitext(os.path.basename(path))[0]
+    device = str(config.get("device", "AUTO")).upper() or "AUTO"
+    return os.path.join(cache_dir, f"{stem}.{device}.ovprobe.json")
+
+
+def _blob_names(cache_dir: str) -> "set[str]":
+    try:
+        return {n for n in os.listdir(cache_dir) if n.endswith(".blob")}
+    except OSError:
+        return set()
+
+
+def _read_verdict(path: str, key: dict) -> Optional[dict]:
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            stored = json.load(fh)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(stored, dict) or stored.get("key") != key:
+        return None
+    if stored.get("verdict") not in (VERDICT_OK, VERDICT_CRASHED):
+        return None
+    return stored
+
+
+def _write_verdict(path: str, key: dict, verdict: str, detail: str, blobs: "list[str]") -> None:
+    """Atomic replace; a verdict that cannot be written only costs a re-probe."""
+    tmp = f"{path}.tmp-{os.getpid()}"
+    try:
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump({"key": key, "verdict": verdict, "detail": detail, "blobs": blobs}, fh)
+        os.replace(tmp, path)
+    except OSError as exc:
+        sys.stderr.write(f"OpenVINO probe: could not persist the verdict for {path} ({exc})\n")
+        sys.stderr.flush()
+        try:
+            os.remove(tmp)
+        except OSError:
+            pass
+
+
+def _warm(known: dict, cache_dir: str) -> bool:
+    """An `ok` verdict holds only while the blobs its probe wrote are still on disk."""
+    blobs = known.get("blobs")
+    if not isinstance(blobs, list):
+        return False
+    return all(os.path.isfile(os.path.join(cache_dir, str(b))) for b in blobs)
+
+
+def ensure_openvino_loadable(
+    model_path: str,
+    config: dict,
+    candidates: "list[str]",
+    cache_dir: Optional[str],
+    env_tag: str = "",
+    run: Optional[ProbeRunner] = None,
+) -> None:
+    """Return when `model_path` may be compiled in-process; raise when it would crash."""
+    if not needs_probe(candidates):
+        return
+    name = os.path.basename(model_path)
+    device = str(config.get("device", "AUTO"))
+    key = probe_key(model_path, config, candidates, env_tag)
+    verdict_path = probe_verdict_path(model_path, config, cache_dir) if cache_dir else None
+    if verdict_path is not None:
+        known = _read_verdict(verdict_path, key)
+        if known is not None:
+            if known["verdict"] == VERDICT_CRASHED:
+                sys.stderr.write(
+                    f"OpenVINO probe: {name} device={device} is a KNOWN compiler crash — refused\n"
+                )
+                sys.stderr.flush()
+                raise OpenvinoLoadCrashed(model_path, device, str(known.get("detail") or "known crash"))
+            if cache_dir is not None and _warm(known, cache_dir):
+                return  # blob present: the in-process load is a cache hit
+    before = _blob_names(cache_dir) if cache_dir else set()
+    runner = run if run is not None else DEFAULT_RUNNER
+    verdict, detail = runner(model_path, config)
+    sys.stderr.write(f"OpenVINO probe: {name} device={device} -> {verdict} ({detail})\n")
+    sys.stderr.flush()
+    if verdict in (VERDICT_OK, VERDICT_CRASHED) and verdict_path is not None and cache_dir is not None:
+        written = sorted(_blob_names(cache_dir) - before) if verdict == VERDICT_OK else []
+        _write_verdict(verdict_path, key, verdict, detail, written)
+    if verdict == VERDICT_CRASHED:
+        raise OpenvinoLoadCrashed(model_path, device, detail)

package/python/test_openvino_compile_probe.py

@@ -0,0 +1,298 @@
+"""Unit tests for the OpenVINO child-process compile probe (openvino_compile_probe).
+
+An iGPU compiler SIGSEGV (2026-10-22, hub Core Ultra iGPU) takes the whole pool
+process with it. The probe compiles a (model x device request x compile config)
+key whose blob is absent in a CHILD first, remembers the verdict, and refuses a
+model whose child died by a signal — with an exception, not a crash.
+
+Covers:
+  - needs_probe — only GPU/NPU targets are probed; CPU-only never.
+  - run_probe — a REAL child that SIGSEGVs is `crashed`, one that prints the
+    marker is `ok`, a Python failure / a timeout is `inconclusive`.
+  - ensure_openvino_loadable — a crash refuses now and on the next call without
+    re-probing; a warm key (blob the probe wrote still present) skips the probe;
+    a wiped blob re-probes; a changed model / compile config / env re-probes.
+  - inference_pool._load_model (openvino branch) — a model the probe calls
+    crashed never reaches `core.compile_model`; the reply carries
+    reason `native-load-crashed`.
+
+Pure-python: stubs numpy / PIL / postprocessors when absent.
+
+Run: python3 -m unittest test_openvino_compile_probe -v
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import types
+import unittest
+
+try:
+    import numpy  # noqa: F401
+except ImportError:
+    _np = types.ModuleType("numpy")
+    _np.array = lambda seq, dtype=None: list(seq)  # noqa: E731
+    _np.float32 = "float32"
+    sys.modules["numpy"] = _np
+
+try:
+    import PIL.Image  # noqa: F401
+except ImportError:
+    _pil = types.ModuleType("PIL")
+    _pil_image = types.ModuleType("PIL.Image")
+    _pil.Image = _pil_image
+    sys.modules["PIL"] = _pil
+    sys.modules["PIL.Image"] = _pil_image
+
+if "postprocessors" not in sys.modules:
+    _pp = types.ModuleType("postprocessors")
+    _pp.POSTPROCESSORS = {}
+    sys.modules["postprocessors"] = _pp
+
+import openvino_compile_probe as probe
+import inference_pool as ip
+
+GPU_CONFIG = {"device": "gpu", "inputSize": 320}
+
+
+def _write(path: str, content: bytes = b"x") -> None:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "wb") as fh:
+        fh.write(content)
+
+
+class _Runner:
+    """A fake probe runner: answers a fixed verdict, optionally writes a blob."""
+
+    def __init__(self, verdict: str, detail: str = "", blob: "str | None" = None, cache_dir: str = "") -> None:
+        self.verdict = verdict
+        self.detail = detail
+        self.blob = blob
+        self.cache_dir = cache_dir
+        self.calls: "list[tuple[str, dict]]" = []
+
+    def __call__(self, model_path: str, config: dict) -> "tuple[str, str]":
+        self.calls.append((model_path, config))
+        if self.blob:
+            _write(os.path.join(self.cache_dir, self.blob))
+        return self.verdict, self.detail
+
+
+class NeedsProbeTest(unittest.TestCase):
+    def test_gpu_and_npu_are_probed(self) -> None:
+        self.assertTrue(probe.needs_probe(["GPU"]))
+        self.assertTrue(probe.needs_probe(["NPU", "GPU", "CPU"]))
+        self.assertTrue(probe.needs_probe(["GPU.0"]))
+
+    def test_cpu_only_is_never_probed(self) -> None:
+        self.assertFalse(probe.needs_probe(["CPU"]))
+        self.assertFalse(probe.needs_probe([]))
+
+
+class RunProbeRealChildTest(unittest.TestCase):
+    """The child is a real interpreter: a native SIGSEGV is seen as a crash."""
+
+    def test_child_that_segfaults_is_crashed(self) -> None:
+        code = "import os, signal; os.kill(os.getpid(), signal.SIGSEGV)"
+        verdict, detail = probe.run_probe("/nonexistent.xml", GPU_CONFIG, child_code=code)
+        self.assertEqual(verdict, probe.VERDICT_CRASHED)
+        self.assertIn("SIGSEGV", detail)
+
+    def test_child_that_prints_marker_is_ok(self) -> None:
+        code = f"print({probe.PROBE_MARKER!r}, flush=True)"
+        verdict, _ = probe.run_probe("/nonexistent.xml", GPU_CONFIG, child_code=code)
+        self.assertEqual(verdict, probe.VERDICT_OK)
+
+    def test_child_that_raises_is_inconclusive(self) -> None:
+        verdict, _ = probe.run_probe(
+            "/nonexistent.xml", GPU_CONFIG, child_code="raise RuntimeError('bad ir')",
+        )
+        self.assertEqual(verdict, probe.VERDICT_INCONCLUSIVE)
+
+    def test_child_past_its_bound_is_inconclusive(self) -> None:
+        verdict, detail = probe.run_probe(
+            "/nonexistent.xml", GPU_CONFIG, child_code="import time; time.sleep(30)",
+            timeout_sec=0.5,
+        )
+        self.assertEqual(verdict, probe.VERDICT_INCONCLUSIVE)
+        self.assertIn("timed out", detail)
+
+
+class EnsureOpenvinoLoadableTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.model = os.path.join(self._tmp.name, "yolov9m-320-int8.xml")
+        _write(self.model, b"<net/>")
+        _write(os.path.join(self._tmp.name, "yolov9m-320-int8.bin"), b"w")
+        self.cache = os.path.join(self._tmp.name, ".ov-cache")
+        os.makedirs(self.cache)
+
+    def tearDown(self) -> None:
+        self._tmp.cleanup()
+
+    def ensure(self, runner, config=None, candidates=("GPU",), env_tag="2025.4.1") -> None:
+        probe.ensure_openvino_loadable(
+            self.model, config or GPU_CONFIG, list(candidates), self.cache,
+            env_tag=env_tag, run=runner,
+        )
+
+    def test_crash_refuses_with_a_typed_reason(self) -> None:
+        runner = _Runner(probe.VERDICT_CRASHED, "probe child died by signal 11 (SIGSEGV)")
+        with self.assertRaises(probe.OpenvinoLoadCrashed) as ctx:
+            self.ensure(runner)
+        self.assertEqual(ctx.exception.reason, "native-load-crashed")
+        self.assertIn("yolov9m-320-int8", str(ctx.exception))
+
+    def test_crash_is_remembered_and_not_reprobed(self) -> None:
+        runner = _Runner(probe.VERDICT_CRASHED)
+        for _ in range(2):
+            with self.assertRaises(probe.OpenvinoLoadCrashed):
+                self.ensure(runner)
+        self.assertEqual(len(runner.calls), 1)
+
+    def test_cpu_only_load_does_not_probe(self) -> None:
+        runner = _Runner(probe.VERDICT_CRASHED)
+        self.ensure(runner, config={"device": "cpu"}, candidates=("CPU",))
+        self.assertEqual(runner.calls, [])
+
+    def test_warm_key_skips_the_probe(self) -> None:
+        runner = _Runner(probe.VERDICT_OK, blob="11aa.blob", cache_dir=self.cache)
+        self.ensure(runner)
+        self.ensure(runner)
+        self.assertEqual(len(runner.calls), 1)
+
+    def test_a_wiped_blob_is_probed_again(self) -> None:
+        runner = _Runner(probe.VERDICT_OK, blob="11aa.blob", cache_dir=self.cache)
+        self.ensure(runner)
+        os.remove(os.path.join(self.cache, "11aa.blob"))
+        self.ensure(runner)
+        self.assertEqual(len(runner.calls), 2)
+
+    def test_a_changed_model_is_reprobed(self) -> None:
+        crashed = _Runner(probe.VERDICT_CRASHED)
+        with self.assertRaises(probe.OpenvinoLoadCrashed):
+            self.ensure(crashed)
+        _write(self.model, b"<net>fixed</net>")
+        ok = _Runner(probe.VERDICT_OK)
+        self.ensure(ok)
+        self.assertEqual(len(ok.calls), 1)
+
+    def test_a_different_compile_config_is_reprobed(self) -> None:
+        crashed = _Runner(probe.VERDICT_CRASHED)
+        with self.assertRaises(probe.OpenvinoLoadCrashed):
+            self.ensure(crashed)
+        ok = _Runner(probe.VERDICT_OK)
+        self.ensure(ok, config={**GPU_CONFIG, "gpuInferencePrecision": "f32"})
+        self.assertEqual(len(ok.calls), 1)
+
+    def test_a_threshold_is_not_part_of_the_key(self) -> None:
+        crashed = _Runner(probe.VERDICT_CRASHED)
+        with self.assertRaises(probe.OpenvinoLoadCrashed):
+            self.ensure(crashed)
+        with self.assertRaises(probe.OpenvinoLoadCrashed):
+            self.ensure(crashed, config={**GPU_CONFIG, "confidence": 0.9})
+        self.assertEqual(len(crashed.calls), 1)
+
+    def test_a_different_runtime_build_is_reprobed(self) -> None:
+        crashed = _Runner(probe.VERDICT_CRASHED)
+        with self.assertRaises(probe.OpenvinoLoadCrashed):
+            self.ensure(crashed, env_tag="2025.4.1")
+        ok = _Runner(probe.VERDICT_OK)
+        self.ensure(ok, env_tag="2026.0.0")
+        self.assertEqual(len(ok.calls), 1)
+
+    def test_inconclusive_neither_refuses_nor_is_remembered(self) -> None:
+        runner = _Runner(probe.VERDICT_INCONCLUSIVE, "child exited 1")
+        self.ensure(runner)
+        self.ensure(runner)
+        self.assertEqual(len(runner.calls), 2)
+
+    def test_verdict_lives_in_the_ov_cache_dir(self) -> None:
+        self.ensure(_Runner(probe.VERDICT_OK))
+        path = probe.probe_verdict_path(self.model, GPU_CONFIG, self.cache)
+        self.assertEqual(os.path.dirname(path), self.cache)
+        with open(path, "r", encoding="utf-8") as fh:
+            self.assertEqual(json.load(fh)["verdict"], probe.VERDICT_OK)
+
+
+class _FakeCore:
+    """The slice of `openvino.Core` `_load_model` touches before it compiles."""
+
+    available_devices = ["CPU", "GPU"]
+
+    def __init__(self) -> None:
+        self.compile_calls = 0
+
+    def get_property(self, *_a, **_k):
+        raise RuntimeError("no property")
+
+    def read_model(self, *_a, **_k):
+        raise AssertionError("the model must not be read: the probe refuses first")
+
+    def compile_model(self, *_a, **_k):
+        self.compile_calls += 1
+        raise AssertionError("the native compiler must not be reached")
+
+
+class LoadModelGateTest(unittest.TestCase):
+    """`_load_model` asks the probe BEFORE the in-process compile."""
+
+    def test_crashed_model_never_reaches_the_native_compiler(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            model = os.path.join(tmp, "yolov9m-320-int8.xml")
+            _write(model, b"<net/>")
+            core = _FakeCore()
+            saved = (ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER)
+            try:
+                ip._runtime = "openvino"
+                ip._runtime_lib = core
+                probe.DEFAULT_RUNNER = _Runner(probe.VERDICT_CRASHED, "child died by signal 11")
+                with self.assertRaises(probe.OpenvinoLoadCrashed):
+                    ip._load_model(ip.ModelSlot(), {"path": model, "device": "gpu"})
+            finally:
+                (ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER) = saved
+            self.assertEqual(core.compile_calls, 0)
+
+    def test_the_child_itself_is_not_probed(self) -> None:
+        # The child runs `_load_model` with the probe switched off in it; without
+        # that switch it would fork a child of its own, forever.
+        with tempfile.TemporaryDirectory() as tmp:
+            model = os.path.join(tmp, "m.xml")
+            _write(model, b"<net/>")
+            runner = _Runner(probe.VERDICT_CRASHED)
+            saved = (ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER, ip._ov_probe_enabled)
+            try:
+                ip._runtime = "openvino"
+                ip._runtime_lib = _FakeCore()
+                ip._ov_probe_enabled = False
+                probe.DEFAULT_RUNNER = runner
+                with self.assertRaises(Exception) as ctx:
+                    ip._load_model(ip.ModelSlot(), {"path": model, "device": "gpu"})
+                self.assertNotIsInstance(ctx.exception, probe.OpenvinoLoadCrashed)
+            finally:
+                (ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER, ip._ov_probe_enabled) = saved
+            self.assertEqual(runner.calls, [])
+
+
+class ModelCommandReasonTest(unittest.TestCase):
+    def test_load_reply_carries_native_load_crashed(self) -> None:
+        import asyncio
+
+        def failing_load(_slot, config):
+            raise probe.OpenvinoLoadCrashed(config["path"], "gpu", "child died by signal 11")
+
+        async def run() -> dict:
+            loader = ip.ModelLoader(timeout_sec=5.0, load_fn=failing_load)
+            cmd = {"cmd": "load", "index": 0, "config": {"path": "/m/yolov9m-320-int8.xml"}}
+            return await ip._handle_model_command([], cmd, loader)
+
+        reply = asyncio.run(run())
+        self.assertEqual(reply["status"], "error")
+        self.assertEqual(reply["reason"], "native-load-crashed")
+        self.assertEqual(reply["modelId"], "yolov9m-320-int8")
+
+
+if __name__ == "__main__":
+    unittest.main()