@camstack/addon-pipeline 1.2.177 → 1.2.179

package/python/inference_pool.py

@@ -36,6 +36,21 @@ Runtime protocol:
     0x02 — infer_raw    payload = [1B model_idx][4B width][4B height]
                                   [1B fmt 0=RGB,1=BGR,2=GRAY][pixels]
 
+Wire-protocol versioning (D350): the startup config may declare
+`protocolVersion`; the effective wire version is `min(PROTOCOL_VERSION,
+declared)` and the ready reply reports this worker's own PROTOCOL_VERSION so
+the Node side can negotiate the same minimum. At v2, every SHEDDABLE request
+(infer_jpeg / infer_raw / infer_batch / infer_cached) prefixes its payload
+with the request's ABSOLUTE deadline — [8B epoch-ms uint64 LE] — and this
+worker checks it immediately before predict, answering
+`{"dropped": true, "shedReason": "deadline-expired"}` instead of executing
+work whose caller has already abandoned it (the 2026-09-04 storm: 4h15m of
+the accelerator at 100% on replies discarded as `unknown request id`).
+Commands and model loads NEVER carry a deadline at any version — a shed
+command desynchronizes model state. A host that declares nothing is an OLD
+host and gets byte-identical v1 behaviour; skew in either direction is the
+NORMAL state during a rolling deploy.
+
 Commands: load, unload, replace, reconfigure, status, uncache_frame, mem_stats.
 """
 from __future__ import annotations
@@ -87,6 +102,93 @@ RAW_FMT_RGB = 0x00
 RAW_FMT_BGR = 0x01
 RAW_FMT_GRAY = 0x02
 
+# ---------------------------------------------------------------------------
+# Wire-protocol versioning + the deadline on the wire (D350)
+# ---------------------------------------------------------------------------
+
+# Highest wire version THIS worker implements. Reported in the ready reply;
+# the effective version per connection is min() of both sides' declarations,
+# so a worker and a host at different versions (normal during a rolling
+# deploy) always agree on the layout.
+PROTOCOL_VERSION = 2
+
+# Opcodes that may carry (and be shed by) a deadline. Mirrors
+# SHEDDABLE_MSG_TYPES in shared-inference-pool.ts — commands and cacheFrame
+# are absent on BOTH sides: a shed command desynchronizes model state, a shed
+# cacheFrame strands the inferCached that follows it.
+SHEDDABLE_MSG_TYPES = frozenset({MSG_INFER_JPEG, MSG_INFER_RAW, MSG_INFER_BATCH, MSG_INFER_CACHED})
+
+# Bytes of the absolute-deadline prefix a v2 sheddable payload carries.
+DEADLINE_PREFIX_LEN = 8
+
+# Sampling stride for the deadline-expired shed stderr line. Under a storm
+# this fires at frame rate; one line per shed buried every other signal the
+# last time a shed path logged unsampled (45 741 rows/hour, 2026-08-01). The
+# counter (PoolCounters.shed_deadline_expired) keeps the exact total.
+DEADLINE_SHED_LOG_EVERY = 500
+
+
+def _negotiate_wire_version(config: dict) -> int:
+    """Effective wire version for this connection: min(ours, host declared).
+
+    No declaration (an OLD host) or a garbage value negotiates v1 — the
+    byte-identical historical layout. Pure; unit-tested.
+    """
+    try:
+        declared = int(config.get("protocolVersion", 1) or 1)
+    except (TypeError, ValueError):
+        declared = 1
+    return max(1, min(PROTOCOL_VERSION, declared))
+
+
+def _split_deadline(msg_type: int, payload: bytes, wire_version: int) -> "tuple[int, bytes]":
+    """Split a request payload into (deadline_ms, rest) by the NEGOTIATED
+    version. v1, and every non-sheddable opcode at any version, passes the
+    payload through untouched with deadline 0 (= no deadline, never sheds).
+    Raises ValueError on a v2 sheddable payload too short to carry its
+    prefix. Pure; unit-tested.
+    """
+    if wire_version < 2 or msg_type not in SHEDDABLE_MSG_TYPES:
+        return 0, payload
+    if len(payload) < DEADLINE_PREFIX_LEN:
+        raise ValueError(
+            f"truncated deadline prefix on msg_type={msg_type} "
+            f"({len(payload)} bytes, need {DEADLINE_PREFIX_LEN})"
+        )
+    (deadline_ms,) = struct.unpack("<Q", payload[:DEADLINE_PREFIX_LEN])
+    return deadline_ms, payload[DEADLINE_PREFIX_LEN:]
+
+
+def _deadline_expired(deadline_ms: int, now_ms: float) -> bool:
+    """0 means "no deadline" and never expires (v1 host / unstamped)."""
+    return deadline_ms > 0 and now_ms > deadline_ms
+
+
+def _dropped_deadline_payload() -> dict:
+    """The shed reply for an expired request — same shape the TS side already
+    recognises as flow control (`poolResultToEngineOutput`)."""
+    return {
+        "kind": "detections",
+        "detections": [],
+        "inferenceMs": 0,
+        "dropped": True,
+        "shedReason": "deadline-expired",
+    }
+
+
+def _log_deadline_shed(total: int, model_idx: int, late_ms: float) -> None:
+    """Sampled stderr line for a deadline shed — first occurrence and every
+    DEADLINE_SHED_LOG_EVERY-th, carrying the running total so nothing is
+    lost, only repeated. A branch that drops work silently is how the
+    2026-09-04 collapse produced not one line in four hours."""
+    if total != 1 and total % DEADLINE_SHED_LOG_EVERY != 0:
+        return
+    sys.stderr.write(
+        f"deadline-expired shed: model {model_idx}, {late_ms:.0f}ms past deadline "
+        f"(total {total}, sampled every {DEADLINE_SHED_LOG_EVERY})\n"
+    )
+    sys.stderr.flush()
+
 
 # ---------------------------------------------------------------------------
 # Preprocessing (unchanged from v1 — same shapes/math, moved into helpers)
@@ -1891,6 +1993,10 @@ class PoolCounters:
     infer_cached: int = 0
     commands: int = 0
     frames_cached: int = 0
+    # Requests answered `dropped` because their wire deadline had already
+    # passed when they would have run — dead work NOT executed (D350). For a
+    # batch, every item counts.
+    shed_deadline_expired: int = 0
 
     def as_dict(self) -> dict:
         return {
@@ -1900,6 +2006,7 @@ class PoolCounters:
             "inferCached": self.infer_cached,
             "commands": self.commands,
             "framesCached": self.frames_cached,
+            "shedDeadlineExpired": self.shed_deadline_expired,
             "totalInference": (
                 self.infer_jpeg + self.infer_raw + self.infer_batch + self.infer_cached
             ),
@@ -2008,6 +2115,58 @@ def _mem_stats_payload(
     }
 
 
+# ---------------------------------------------------------------------------
+# Single-frame inference execution — module-level so the deadline gate is
+# unit-testable (test_inference_pool_deadline.py) without driving the whole
+# event loop. `handle_inference` in _run() delegates here; the caller's
+# `finally` still owns the backpressure slot release.
+# ---------------------------------------------------------------------------
+
+
+async def _execute_inference(
+    req_id: int,
+    img: Any,
+    model_idx: int,
+    deadline_ms: int,
+    *,
+    models: "list[ModelSlot]",
+    dispatcher: Any,
+    writer: Any,
+    batch_mode: str,
+    get_accumulator: "Optional[Callable[[int], Any]]",
+    counters: "PoolCounters",
+) -> None:
+    """Deadline gate + predict for ONE admitted single-frame request.
+
+    The deadline is checked HERE — immediately before predict — rather than
+    only at arrival, because this coroutine is entered both on admission and
+    when the backpressure queue promotes a waiting frame: the wait is exactly
+    where a request goes stale. The gate comes first (before the model-loaded
+    check) on purpose — the cheapest question first, and an abandoned request
+    deserves a shed, not a diagnosis. `dispatcher` and `writer` are
+    duck-typed (RuntimeDispatcher / ResponseWriter in production).
+    """
+    now_ms = time.time() * 1000.0
+    if _deadline_expired(deadline_ms, now_ms):
+        counters.shed_deadline_expired += 1
+        _log_deadline_shed(counters.shed_deadline_expired, model_idx, now_ms - deadline_ms)
+        await writer.send(req_id, _dropped_deadline_payload())
+        return
+    if model_idx >= len(models) or not models[model_idx].loaded:
+        await writer.send(req_id, {
+            "error": f"Model {model_idx} not loaded",
+            "kind": "detections",
+            "detections": [],
+            "inferenceMs": 0,
+        })
+        return
+    if batch_mode == "window" and get_accumulator is not None:
+        await get_accumulator(model_idx).submit(req_id, img)
+        return
+    result = await dispatcher.run(models[model_idx], img)
+    await writer.send(req_id, result)
+
+
 # ---------------------------------------------------------------------------
 # IPC — binary framing with request_id multiplexing
 # ---------------------------------------------------------------------------
@@ -2192,6 +2351,10 @@ async def _run() -> None:
     config = json.loads(payload)
     runtime = config.get("runtime", "coreml")
     pool_device = str(config.get("device", "") or "")
+    # Wire version for THIS connection (D350): an old host declares nothing
+    # and negotiates v1 — byte-identical historical behaviour, no deadline
+    # prefix expected on any payload.
+    wire_version = _negotiate_wire_version(config)
     concurrency = int(config.get("concurrency", 1) or 1)
     batch_mode = str(config.get("batch_mode", "none"))
     if batch_mode not in ("none", "list", "window"):
@@ -2201,7 +2364,8 @@ async def _run() -> None:
 
     sys.stderr.write(
         f"Initializing runtime: {runtime} (concurrency={concurrency}, "
-        f"batch_mode={batch_mode}, window_ms={window_ms}, max_batch={max_batch_size})\n"
+        f"batch_mode={batch_mode}, window_ms={window_ms}, max_batch={max_batch_size}, "
+        f"wire_version={wire_version})\n"
     )
     sys.stderr.flush()
     _init_runtime(runtime, pool_device)
@@ -2266,6 +2430,10 @@ async def _run() -> None:
         "startupMs": startup_ms,
         "runtime": runtime,
         "workers": dispatcher.workers,
+        # Our own capability, NOT the negotiated minimum: the Node side runs
+        # the same min() against what it declared, so both ends derive the
+        # identical effective version (D350).
+        "protocolVersion": PROTOCOL_VERSION,
     })
 
     # ── Window accumulator (per-model) ──────────────────────────────
@@ -2358,21 +2526,23 @@ async def _run() -> None:
         return acc
 
     # ── Main loop ───────────────────────────────────────────────────
-    async def handle_inference(req_id: int, img: Image.Image, model_idx: int) -> None:
+    async def handle_inference(
+        req_id: int, img: Image.Image, model_idx: int, deadline_ms: int = 0,
+    ) -> None:
         try:
-            if model_idx >= len(models) or not models[model_idx].loaded:
-                await writer.send(req_id, {
-                    "error": f"Model {model_idx} not loaded",
-                    "kind": "detections",
-                    "detections": [],
-                    "inferenceMs": 0,
-                })
-                return
-            if batch_mode == "window":
-                await get_accumulator(model_idx).submit(req_id, img)
-                return
-            result = await dispatcher.run(models[model_idx], img)
-            await writer.send(req_id, result)
+            # The deadline gate lives in _execute_inference, immediately
+            # before predict — it runs here on admission AND again when a
+            # queued frame is promoted below, which is where a request goes
+            # stale (D350).
+            await _execute_inference(
+                req_id, img, model_idx, deadline_ms,
+                models=models,
+                dispatcher=dispatcher,
+                writer=writer,
+                batch_mode=batch_mode,
+                get_accumulator=get_accumulator,
+                counters=counters,
+            )
         except Exception as exc:
             sys.stderr.write(f"Inference error (model {model_idx}, req {req_id}): {exc}\n")
             sys.stderr.flush()
@@ -2385,8 +2555,10 @@ async def _run() -> None:
         finally:
             # Release this model's running slot; promote the next queued
             # frame (if any) into the freed slot immediately.
-            for next_req_id, next_img in backpressure.complete(model_idx):
-                asyncio.create_task(handle_inference(next_req_id, next_img, model_idx))
+            for next_req_id, next_img, next_deadline_ms in backpressure.complete(model_idx):
+                asyncio.create_task(
+                    handle_inference(next_req_id, next_img, model_idx, next_deadline_ms)
+                )
 
     def _send_dropped(req_id: int) -> None:
         # Fast shed response — returned in microseconds instead of queuing the
@@ -2399,19 +2571,36 @@ async def _run() -> None:
             "dropped": True,
         }))
 
-    def _dispatch_inference(req_id: int, img: Image.Image, model_idx: int) -> None:
+    def _dispatch_inference(
+        req_id: int, img: Image.Image, model_idx: int, deadline_ms: int = 0,
+    ) -> None:
         # Gate every single-frame inference through the per-model in-flight
         # bound. Slot accounting is synchronous (admit here, complete in the
         # handle_inference `finally`), so a slot can never leak past a task's
         # lifetime. Under overload the OLDEST queued frame is shed and answered
         # immediately so the caller never hangs on a frame we chose not to run.
-        to_run, to_drop = backpressure.admit(model_idx, (req_id, img))
-        for dropped_req_id, _dropped_img in to_drop:
+        # The wire deadline rides the queued tuple so the gate in
+        # _execute_inference sees it AFTER the wait, not just at arrival.
+        to_run, to_drop = backpressure.admit(model_idx, (req_id, img, deadline_ms))
+        for dropped_req_id, _dropped_img, _dropped_deadline in to_drop:
             _send_dropped(dropped_req_id)
-        for run_req_id, run_img in to_run:
-            asyncio.create_task(handle_inference(run_req_id, run_img, model_idx))
-
-    async def handle_batch(req_id: int, model_idx: int, items: list[Image.Image]) -> None:
+        for run_req_id, run_img, run_deadline_ms in to_run:
+            asyncio.create_task(handle_inference(run_req_id, run_img, model_idx, run_deadline_ms))
+
+    async def handle_batch(
+        req_id: int, model_idx: int, items: list[Image.Image], deadline_ms: int = 0,
+    ) -> None:
+        # Deadline gate first — the whole batch is one abandoned caller, so
+        # every item is answered as a well-formed dropped record and none of
+        # the work runs (D350).
+        now_ms = time.time() * 1000.0
+        if _deadline_expired(deadline_ms, now_ms):
+            counters.shed_deadline_expired += len(items)
+            _log_deadline_shed(counters.shed_deadline_expired, model_idx, now_ms - deadline_ms)
+            await writer.send(req_id, {
+                "results": [_dropped_deadline_payload() for _ in items],
+            })
+            return
         if model_idx >= len(models) or not models[model_idx].loaded:
             await writer.send(req_id, {
                 "error": f"Model {model_idx} not loaded",
@@ -2459,6 +2648,14 @@ async def _run() -> None:
         except asyncio.IncompleteReadError:
             return
 
+        # v2 sheddable payloads carry [8B absolute deadline] first; v1 (and
+        # every command) passes through untouched with deadline 0 (D350).
+        try:
+            deadline_ms, payload = _split_deadline(msg_type, payload, wire_version)
+        except ValueError as exc:
+            await writer.send(req_id, {"error": str(exc)})
+            continue
+
         if msg_type == MSG_COMMAND:
             counters.commands += 1
             try:
@@ -2505,7 +2702,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"jpeg decode failed: {exc}"})
                 continue
-            _dispatch_inference(req_id, img, model_idx)
+            _dispatch_inference(req_id, img, model_idx, deadline_ms)
 
         elif msg_type == MSG_INFER_RAW:
             counters.infer_raw += 1
@@ -2521,7 +2718,7 @@ async def _run() -> None:
             except Exception as exc:
                 await writer.send(req_id, {"error": f"raw wrap failed: {exc}"})
                 continue
-            _dispatch_inference(req_id, img, model_idx)
+            _dispatch_inference(req_id, img, model_idx, deadline_ms)
 
         elif msg_type == MSG_INFER_BATCH:
             # Header: [1B model_idx][1B count][4B frame_id]
@@ -2567,7 +2764,7 @@ async def _run() -> None:
             if parse_err is not None:
                 await writer.send(req_id, {"error": parse_err, "results": []})
                 continue
-            asyncio.create_task(handle_batch(req_id, model_idx, items))
+            asyncio.create_task(handle_batch(req_id, model_idx, items, deadline_ms))
 
         elif msg_type == MSG_CACHE_FRAME:
             counters.frames_cached += 1
@@ -2605,7 +2802,7 @@ async def _run() -> None:
             if model_idx >= len(models) or not models[model_idx].loaded:
                 await writer.send(req_id, {"error": f"model {model_idx} not loaded"})
                 continue
-            _dispatch_inference(req_id, img, model_idx)
+            _dispatch_inference(req_id, img, model_idx, deadline_ms)
 
         else:
             await writer.send(req_id, {"error": f"unknown msg_type: {msg_type}"})

package/python/test_inference_pool_deadline.py

@@ -0,0 +1,253 @@
+"""Unit tests for the deadline-on-the-wire shed (D350).
+
+On a pool timeout the TS runner drops the frame and returns null — but the
+Python worker still executed the abandoned request and its late reply was
+discarded as `Response for unknown request id`. Measured 2026-09-04: a 4h15m
+storm, 16 146 timeouts, the accelerator at 100% on results nobody read.
+
+Each SHEDDABLE request (infer_jpeg / infer_raw / infer_batch / infer_cached)
+now carries its ABSOLUTE deadline (epoch ms, uint64 LE) on the wire when both
+sides negotiated protocol v2; the worker checks it immediately before predict
+and answers `{dropped: true, shedReason: "deadline-expired"}` instead of
+running dead work. Commands and model loads NEVER carry a deadline — a shed
+command desynchronizes model state.
+
+Pure-python: stubs numpy / PIL / postprocessors in sys.modules so the module
+imports without the ML runtime installed (inference_pool uses
+`from __future__ import annotations`, so annotations never touch the stubs).
+
+Run: python3 -m unittest test_inference_pool_deadline -v
+"""
+from __future__ import annotations
+
+import struct
+import sys
+import time
+import types
+import unittest
+
+# ---------------------------------------------------------------------------
+# Stub the heavy third-party imports BEFORE importing inference_pool.
+# ---------------------------------------------------------------------------
+
+try:  # real numpy when the sandbox has it — a blind stub POISONS the whole
+    import numpy  # noqa: F401  # pytest session for every sibling that needs it
+except ImportError:
+    _np = types.ModuleType("numpy")
+    _np.float32 = float
+    _np.array = lambda values, dtype=None: values
+    sys.modules["numpy"] = _np
+
+try:  # real PIL when present, for the same reason
+    import PIL.Image  # noqa: F401
+except ImportError:
+    _pil = types.ModuleType("PIL")
+    _pil_image = types.ModuleType("PIL.Image")
+    _pil.Image = _pil_image
+    sys.modules["PIL"] = _pil
+    sys.modules["PIL.Image"] = _pil_image
+
+if "postprocessors" not in sys.modules:
+    _pp = types.ModuleType("postprocessors")
+    _pp.POSTPROCESSORS = {}
+    sys.modules["postprocessors"] = _pp
+
+from inference_pool import (  # noqa: E402
+    MSG_COMMAND,
+    MSG_INFER_BATCH,
+    MSG_INFER_CACHED,
+    MSG_INFER_JPEG,
+    MSG_INFER_RAW,
+    PROTOCOL_VERSION,
+    SHEDDABLE_MSG_TYPES,
+    ModelSlot,
+    PoolCounters,
+    _deadline_expired,
+    _execute_inference,
+    _negotiate_wire_version,
+    _split_deadline,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fakes — duck-typed stand-ins for the loop's writer / dispatcher.
+# ---------------------------------------------------------------------------
+
+
+class _FakeWriter:
+    """Collects (req_id, payload) pairs the loop would write to stdout."""
+
+    def __init__(self) -> None:
+        self.sent: list[tuple[int, dict]] = []
+
+    async def send(self, req_id: int, payload: dict) -> None:
+        self.sent.append((req_id, payload))
+
+
+class _FakeDispatcher:
+    """Counts predict dispatches — the thing an expired request must NOT do."""
+
+    def __init__(self) -> None:
+        self.run_calls = 0
+
+    async def run(self, slot: object, img: object) -> dict:
+        self.run_calls += 1
+        return {"kind": "detections", "detections": [], "inferenceMs": 1.0}
+
+
+def _loaded_models() -> list:
+    slot = ModelSlot()
+    slot.loaded = True
+    return [slot]
+
+
+def _now_ms() -> int:
+    return int(time.time() * 1000)
+
+
+# ---------------------------------------------------------------------------
+# The gate: expired ⇒ dropped, NO predict. Fresh ⇒ predict runs.
+# ---------------------------------------------------------------------------
+
+
+class DeadlineGateTest(unittest.IsolatedAsyncioTestCase):
+    async def test_expired_request_is_answered_dropped_and_no_predict_ran(self) -> None:
+        writer = _FakeWriter()
+        dispatcher = _FakeDispatcher()
+        counters = PoolCounters()
+        await _execute_inference(
+            7,
+            object(),
+            0,
+            _now_ms() - 5_000,  # expired 5s ago — the storm's steady state
+            models=_loaded_models(),
+            dispatcher=dispatcher,
+            writer=writer,
+            batch_mode="none",
+            get_accumulator=None,
+            counters=counters,
+        )
+        # THE fix: dead work is not executed.
+        self.assertEqual(dispatcher.run_calls, 0)
+        self.assertEqual(len(writer.sent), 1)
+        req_id, payload = writer.sent[0]
+        self.assertEqual(req_id, 7)
+        self.assertIs(payload.get("dropped"), True)
+        self.assertEqual(payload.get("shedReason"), "deadline-expired")
+        self.assertEqual(payload.get("kind"), "detections")
+        self.assertEqual(payload.get("detections"), [])
+        # The shed is COUNTABLE — a silent drop is what let the original
+        # collapse run for four hours.
+        self.assertEqual(counters.shed_deadline_expired, 1)
+        self.assertEqual(counters.as_dict().get("shedDeadlineExpired"), 1)
+
+    async def test_future_deadline_runs_normally(self) -> None:
+        writer = _FakeWriter()
+        dispatcher = _FakeDispatcher()
+        counters = PoolCounters()
+        await _execute_inference(
+            8,
+            object(),
+            0,
+            _now_ms() + 60_000,
+            models=_loaded_models(),
+            dispatcher=dispatcher,
+            writer=writer,
+            batch_mode="none",
+            get_accumulator=None,
+            counters=counters,
+        )
+        self.assertEqual(dispatcher.run_calls, 1)
+        req_id, payload = writer.sent[0]
+        self.assertEqual(req_id, 8)
+        self.assertNotIn("dropped", payload)
+        self.assertEqual(counters.shed_deadline_expired, 0)
+
+    async def test_no_deadline_runs_normally(self) -> None:
+        """deadline 0 = old host / no deadline — never sheds (skew safety)."""
+        writer = _FakeWriter()
+        dispatcher = _FakeDispatcher()
+        counters = PoolCounters()
+        await _execute_inference(
+            9,
+            object(),
+            0,
+            0,
+            models=_loaded_models(),
+            dispatcher=dispatcher,
+            writer=writer,
+            batch_mode="none",
+            get_accumulator=None,
+            counters=counters,
+        )
+        self.assertEqual(dispatcher.run_calls, 1)
+        self.assertEqual(counters.shed_deadline_expired, 0)
+
+
+# ---------------------------------------------------------------------------
+# Wire parsing — both layouts, chosen by the negotiated version.
+# ---------------------------------------------------------------------------
+
+
+class SplitDeadlineTest(unittest.TestCase):
+    def test_v1_payload_passes_through_untouched(self) -> None:
+        """An OLD host never stamps a deadline — the payload is the old layout
+        and must not lose its first 8 bytes to a prefix that is not there."""
+        payload = bytes([3]) + b"jpegdata"
+        self.assertEqual(_split_deadline(MSG_INFER_JPEG, payload, 1), (0, payload))
+
+    def test_v2_sheddable_payload_strips_the_deadline_prefix(self) -> None:
+        deadline = 1_757_000_000_123
+        rest = bytes([3]) + b"jpegdata"
+        payload = struct.pack("<Q", deadline) + rest
+        self.assertEqual(_split_deadline(MSG_INFER_JPEG, payload, 2), (deadline, rest))
+
+    def test_v2_applies_to_every_sheddable_opcode(self) -> None:
+        deadline = 42
+        rest = b"\x01payload"
+        for msg_type in (MSG_INFER_JPEG, MSG_INFER_RAW, MSG_INFER_BATCH, MSG_INFER_CACHED):
+            payload = struct.pack("<Q", deadline) + rest
+            self.assertEqual(_split_deadline(msg_type, payload, 2), (deadline, rest))
+            self.assertIn(msg_type, SHEDDABLE_MSG_TYPES)
+
+    def test_commands_never_carry_a_deadline_even_at_v2(self) -> None:
+        """A shed command desynchronizes model state — commands keep the old
+        layout at every version."""
+        payload = b'{"cmd": "status"}'
+        self.assertEqual(_split_deadline(MSG_COMMAND, payload, 2), (0, payload))
+        self.assertNotIn(MSG_COMMAND, SHEDDABLE_MSG_TYPES)
+
+    def test_truncated_deadline_prefix_raises(self) -> None:
+        with self.assertRaises(ValueError):
+            _split_deadline(MSG_INFER_JPEG, b"\x00\x01\x02", 2)
+
+
+class DeadlineExpiredTest(unittest.TestCase):
+    def test_zero_deadline_never_expires(self) -> None:
+        self.assertFalse(_deadline_expired(0, 1e15))
+
+    def test_future_deadline_is_not_expired(self) -> None:
+        self.assertFalse(_deadline_expired(2_000, 1_999.0))
+
+    def test_past_deadline_is_expired(self) -> None:
+        self.assertTrue(_deadline_expired(2_000, 2_001.0))
+
+
+class NegotiateWireVersionTest(unittest.TestCase):
+    def test_old_host_without_declaration_negotiates_v1(self) -> None:
+        """A host at a different version is the NORMAL state during a deploy —
+        no declaration means the old layout, always."""
+        self.assertEqual(_negotiate_wire_version({}), 1)
+
+    def test_matching_host_negotiates_v2(self) -> None:
+        self.assertEqual(_negotiate_wire_version({"protocolVersion": 2}), 2)
+
+    def test_newer_host_is_capped_at_our_version(self) -> None:
+        self.assertEqual(_negotiate_wire_version({"protocolVersion": 99}), PROTOCOL_VERSION)
+
+    def test_garbage_declaration_falls_back_to_v1(self) -> None:
+        self.assertEqual(_negotiate_wire_version({"protocolVersion": "banana"}), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()