@trillboards/edge-sdk 0.2.245 → 0.2.253

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/deploy/detector/bench/detect_frames.py +88 -0
  2. package/deploy/detector/bench/openvino_parity.py +152 -0
  3. package/deploy/detector/pyproject.toml +10 -1
  4. package/deploy/detector/tests/test_actor_tracking.py +168 -0
  5. package/deploy/detector/tests/test_actor_tracking_wire.py +22 -0
  6. package/deploy/detector/tests/test_loader.py +25 -0
  7. package/deploy/detector/tests/test_openvino_engines.py +251 -0
  8. package/deploy/detector/tests/test_registry.py +53 -0
  9. package/deploy/detector/tests/test_server.py +43 -0
  10. package/deploy/detector/tests/test_version_stamp.py +17 -4
  11. package/deploy/detector/trillboards_detector/_version.py +2 -2
  12. package/deploy/detector/trillboards_detector/actor_tracking.py +139 -2
  13. package/deploy/detector/trillboards_detector/cli.py +5 -0
  14. package/deploy/detector/trillboards_detector/detections.py +5 -0
  15. package/deploy/detector/trillboards_detector/execution.py +386 -0
  16. package/deploy/detector/trillboards_detector/loader.py +40 -1
  17. package/deploy/detector/trillboards_detector/registry.py +33 -1
  18. package/deploy/detector/trillboards_detector/server.py +43 -1
  19. package/deploy/detector/trillboards_detector/session.py +81 -3
  20. package/deploy/detector/trillboards_detector/track_records.py +113 -0
  21. package/deploy/windows/build-windows.ps1 +1 -0
  22. package/deploy/windows/ci/test-detector-looks-field.ps1 +15 -3
  23. package/deploy/windows/ci/test-detector-reinstall.ps1 +645 -0
  24. package/deploy/windows/ci/test-rollout-proofs.ps1 +3 -2
  25. package/deploy/windows/ensure-detector.ps1 +255 -0
  26. package/deploy/windows/install-update.ps1 +4 -3
  27. package/deploy/windows/setup-detector.ps1 +122 -29
  28. package/deploy/windows/stop-existing.ps1 +1 -0
  29. package/deploy/windows/trillboards-edge.cmd +10 -22
  30. package/dist/ActorTracking.d.ts +38 -0
  31. package/dist/ActorTracking.d.ts.map +1 -1
  32. package/dist/ActorTracking.js +40 -0
  33. package/dist/ActorTracking.js.map +1 -1
  34. package/dist/AlertClips.d.ts +4 -2
  35. package/dist/AlertClips.d.ts.map +1 -1
  36. package/dist/AlertClips.js +24 -9
  37. package/dist/AlertClips.js.map +1 -1
  38. package/dist/CloudSensingManager.d.ts +9 -0
  39. package/dist/CloudSensingManager.d.ts.map +1 -1
  40. package/dist/CloudSensingManager.js +52 -4
  41. package/dist/CloudSensingManager.js.map +1 -1
  42. package/dist/ConnectorStatusClient.d.ts +8 -0
  43. package/dist/ConnectorStatusClient.d.ts.map +1 -1
  44. package/dist/ConnectorStatusClient.js +3 -0
  45. package/dist/ConnectorStatusClient.js.map +1 -1
  46. package/dist/DurableVlmCandidateSpool.d.ts +11 -0
  47. package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
  48. package/dist/DurableVlmCandidateSpool.js +31 -2
  49. package/dist/DurableVlmCandidateSpool.js.map +1 -1
  50. package/dist/EdgeAgent.js +1 -1
  51. package/dist/FrameGroupingShadow.d.ts +57 -0
  52. package/dist/FrameGroupingShadow.d.ts.map +1 -1
  53. package/dist/FrameGroupingShadow.js +123 -5
  54. package/dist/FrameGroupingShadow.js.map +1 -1
  55. package/dist/LocalDetectorClient.d.ts +44 -1
  56. package/dist/LocalDetectorClient.d.ts.map +1 -1
  57. package/dist/LocalDetectorClient.js +88 -4
  58. package/dist/LocalDetectorClient.js.map +1 -1
  59. package/dist/LocalVideoBuffer.d.ts +14 -2
  60. package/dist/LocalVideoBuffer.d.ts.map +1 -1
  61. package/dist/LocalVideoBuffer.js +23 -3
  62. package/dist/LocalVideoBuffer.js.map +1 -1
  63. package/dist/NativeReadiness.d.ts +21 -2
  64. package/dist/NativeReadiness.d.ts.map +1 -1
  65. package/dist/NativeReadiness.js +25 -6
  66. package/dist/NativeReadiness.js.map +1 -1
  67. package/dist/OrderedReviewClient.d.ts +3 -0
  68. package/dist/OrderedReviewClient.d.ts.map +1 -1
  69. package/dist/OrderedReviewClient.js +3 -0
  70. package/dist/OrderedReviewClient.js.map +1 -1
  71. package/dist/SensingProfileClient.d.ts.map +1 -1
  72. package/dist/SensingProfileClient.js +3 -0
  73. package/dist/SensingProfileClient.js.map +1 -1
  74. package/dist/TemporalReviewClient.d.ts +3 -0
  75. package/dist/TemporalReviewClient.d.ts.map +1 -1
  76. package/dist/TemporalReviewClient.js +3 -0
  77. package/dist/TemporalReviewClient.js.map +1 -1
  78. package/dist/VideoSegmentUploader.d.ts +14 -7
  79. package/dist/VideoSegmentUploader.d.ts.map +1 -1
  80. package/dist/VideoSegmentUploader.js +34 -38
  81. package/dist/VideoSegmentUploader.js.map +1 -1
  82. package/dist/VlmCandidateBuffer.d.ts +13 -0
  83. package/dist/VlmCandidateBuffer.d.ts.map +1 -1
  84. package/dist/VlmCandidateBuffer.js.map +1 -1
  85. package/dist/autoSense.d.ts +4 -0
  86. package/dist/autoSense.d.ts.map +1 -1
  87. package/dist/autoSense.js +7 -0
  88. package/dist/autoSense.js.map +1 -1
  89. package/dist/cli.d.ts.map +1 -1
  90. package/dist/cli.js +3 -0
  91. package/dist/cli.js.map +1 -1
  92. package/dist/connectorUpdate.d.ts +18 -9
  93. package/dist/connectorUpdate.d.ts.map +1 -1
  94. package/dist/connectorUpdate.js +66 -3
  95. package/dist/connectorUpdate.js.map +1 -1
  96. package/dist/connectorUpdateApply.d.ts +20 -1
  97. package/dist/connectorUpdateApply.d.ts.map +1 -1
  98. package/dist/connectorUpdateApply.js +55 -1
  99. package/dist/connectorUpdateApply.js.map +1 -1
  100. package/dist/connectorUpdateApplyMac.d.ts +3 -1
  101. package/dist/connectorUpdateApplyMac.d.ts.map +1 -1
  102. package/dist/connectorUpdateApplyMac.js +6 -0
  103. package/dist/connectorUpdateApplyMac.js.map +1 -1
  104. package/dist/continuousReview.d.ts +6 -2
  105. package/dist/continuousReview.d.ts.map +1 -1
  106. package/dist/continuousReview.js.map +1 -1
  107. package/dist/detectorBackendRelay.d.ts +24 -0
  108. package/dist/detectorBackendRelay.d.ts.map +1 -0
  109. package/dist/detectorBackendRelay.js +70 -0
  110. package/dist/detectorBackendRelay.js.map +1 -0
  111. package/dist/detectorEndpointState.d.ts +30 -0
  112. package/dist/detectorEndpointState.d.ts.map +1 -0
  113. package/dist/detectorEndpointState.js +125 -0
  114. package/dist/detectorEndpointState.js.map +1 -0
  115. package/dist/detectorLauncher.d.ts +23 -0
  116. package/dist/detectorLauncher.d.ts.map +1 -0
  117. package/dist/detectorLauncher.js +94 -0
  118. package/dist/detectorLauncher.js.map +1 -0
  119. package/dist/diagnose.d.ts +1 -1
  120. package/dist/diagnose.js +1 -1
  121. package/dist/localSensing.d.ts +72 -0
  122. package/dist/localSensing.d.ts.map +1 -1
  123. package/dist/localSensing.js +280 -31
  124. package/dist/localSensing.js.map +1 -1
  125. package/dist/pipelineSwitches.d.ts +59 -0
  126. package/dist/pipelineSwitches.d.ts.map +1 -0
  127. package/dist/pipelineSwitches.js +240 -0
  128. package/dist/pipelineSwitches.js.map +1 -0
  129. package/package.json +4 -4
  130. package/retained/ProfileAuthorityClient.cjs +4 -0
  131. package/retained/RetainedCaptureClient.cjs +41 -2
  132. package/retained/WebsiteAdmissionCaptureClient.cjs +57 -2
  133. package/retained/__tests__/look-reuse.test.cjs +167 -0
  134. package/retained/manager.cjs +28 -2
@@ -60,6 +60,9 @@ class DetectionBatch:
60
60
  # Derived from the exact bytes used to create each ONNX session, not a
61
61
  # request pin, a catalogue entry, or a later read of the mutable model path.
62
62
  model_hashes: dict[str, str] = field(default_factory=dict)
63
+ #: Which engine answered (plan v7 W8): `ort`, `openvino_cpu`, `openvino_gpu`;
64
+ #: None for a model that runs on ORT only. Diagnostic, never a contract input.
65
+ engine: "str | None" = None
63
66
 
64
67
 
65
68
  def merge_batches(batches: "list[DetectionBatch]") -> DetectionBatch:
@@ -95,6 +98,8 @@ def merge_batches(batches: "list[DetectionBatch]") -> DetectionBatch:
95
98
  merged.degenerate_dropped += b.degenerate_dropped
96
99
  merged.unnamed_class_dropped += b.unnamed_class_dropped
97
100
  merged.models.extend(b.models)
101
+ if b.engine is not None:
102
+ merged.engine = b.engine if merged.engine in (None, b.engine) else f"{merged.engine}+{b.engine}"
98
103
  for model, digest in b.model_hashes.items():
99
104
  if model in merged.model_hashes and merged.model_hashes[model] != digest:
100
105
  raise ValueError("one frame cannot merge different artifacts under one model id")
@@ -6,6 +6,8 @@ so provider policy can be tested without downloading models or CUDA libraries.
6
6
  from __future__ import annotations
7
7
 
8
8
  import logging
9
+ import threading
10
+ import time
9
11
 
10
12
  LOG = logging.getLogger(__name__)
11
13
  CPU = "CPUExecutionProvider"
@@ -108,3 +110,387 @@ class LocalOrtSession:
108
110
 
109
111
  def __getattr__(self, name):
110
112
  return getattr(self._session, name)
113
+
114
+
115
+ # ── OpenVINO engines (plan v7 W8) ──────────────────────────────────────────
116
+ #
117
+ # The Texas store PC is an i7-7700 (AVX2, no AVX-512) with an HD 630 iGPU and no
118
+ # NVIDIA card, so every RF-DETR call ran on ORT's CPU kernels with the iGPU idle.
119
+ # Measured 2026-11-16 on 4 pinned devbox cores with the shipped artifact: OpenVINO
120
+ # 2026.4.1 on the CPU 2.32 frames/s against ORT 1.70 (+36%), person scores within
121
+ # 0.0004 of ORT on all 252 persons of 360 Texas stills. The ONNX Runtime OpenVINO
122
+ # plug-in on the pinned ORT 1.29.0 does not compile this artifact ("Input names
123
+ # mismatch"), so OpenVINO is used directly.
124
+ #
125
+ # ORT stays loaded as the reference and the fallback. An OpenVINO engine serves
126
+ # only after its parity probe passed on THIS host, on live frames, against ORT:
127
+ # a different kernel library, and on the iGPU a different precision, is a
128
+ # different numeric path, and the bar is identical detections at the operating
129
+ # thresholds with person scores within PROBE_PERSON_TOLERANCE.
130
+
131
+ BACKEND_MODES = ("ort", "openvino_probe", "openvino", "openvino_gpu")
132
+ #: Frames each OpenVINO engine is compared on before it may serve.
133
+ PROBE_FRAMES = 24
134
+ #: The parity bar on person scores (sigmoid probabilities), as in plan v7 W8.
135
+ PROBE_PERSON_TOLERANCE = 1e-3
136
+ #: A box both engines keep may move this much (fraction of the frame, ~6 px at 640) and still be the
137
+ #: same detection. Measured 2026-11-16: the largest shift on 494 Texas and corpus frames was 0.0063.
138
+ PROBE_BOX_TOLERANCE = 0.01
139
+ #: OpenVINO answers only a request whose detection floor is at least this. Below it the two engines
140
+ #: disagree on rare queries (9 of 5,535 kept pairs at a 0.05 floor on 60 Texas frames), so a box
141
+ #: method that asks at 0.05 (the uniform rule's screens and fixtures) keeps ONNX Runtime's answers.
142
+ OPENVINO_MIN_CONFIDENCE = 0.2
143
+ #: The iGPU is tried in FP16 first (its fast path), then FP32 if FP16 misses parity.
144
+ GPU_PRECISIONS = ("f16", "f32")
145
+
146
+
147
+ class EngineUnavailable(RuntimeError):
148
+ """This host cannot build the engine (no OpenVINO, no such device)."""
149
+
150
+
151
+ class BackendPolicy:
152
+ """The engine this service was asked for, shared by every detector session.
153
+
154
+ Set by `POST /backend` (the connector relays the server's runtime_config
155
+ CCTV_DETECTOR_BACKEND) or `serve --backend`. `ort` is today's behaviour.
156
+ """
157
+
158
+ def __init__(self, mode="ort"):
159
+ if mode not in BACKEND_MODES:
160
+ raise ValueError(f"backend must be one of {BACKEND_MODES}")
161
+ self._mode = mode
162
+ self._lock = threading.Lock()
163
+
164
+ @property
165
+ def mode(self):
166
+ with self._lock:
167
+ return self._mode
168
+
169
+ def set(self, mode):
170
+ if mode not in BACKEND_MODES:
171
+ raise ValueError(f"backend must be one of {BACKEND_MODES}")
172
+ with self._lock:
173
+ changed = mode != self._mode
174
+ self._mode = mode
175
+ if changed:
176
+ LOG.info("[detector] backend requested: %s", mode)
177
+ return changed
178
+
179
+
180
+ def openvino_version():
181
+ try:
182
+ import openvino # noqa: F401
183
+ return getattr(openvino, "__version__", None) or "unknown"
184
+ except Exception: # not installed on this host: ORT only
185
+ return None
186
+
187
+
188
+ class _NodeArg:
189
+ """The two NodeArg fields `session._read_graph_spec` reads."""
190
+
191
+ def __init__(self, name, shape):
192
+ self.name, self.shape = name, shape
193
+
194
+
195
+ class OpenVinoSession:
196
+ """One compiled OpenVINO model behind the ORT session surface the detector reads.
197
+
198
+ `get_inputs`/`get_outputs` (name, static shape), `run(output_names, feeds)`
199
+ and `get_providers`. The model is read from the SAME bytes ORT loaded,
200
+ re-verified by hash, so an OTA that replaced the file cannot slip a different
201
+ artifact under the engine. CPU runs FP32 explicitly: a host with BF16/AMX
202
+ would otherwise default to bf16, which is a different numeric path.
203
+ """
204
+
205
+ def __init__(self, model_bytes, device, precision, *, core=None, threads=None):
206
+ try:
207
+ import openvino as ov
208
+ except Exception as error:
209
+ raise EngineUnavailable(f"openvino is not installed: {error}") from None
210
+ core = core or ov.Core()
211
+ if device not in core.available_devices:
212
+ raise EngineUnavailable(f"{device} is not an OpenVINO device on this host ({core.available_devices})")
213
+ config = {"PERFORMANCE_HINT": "LATENCY", "INFERENCE_PRECISION_HINT": precision}
214
+ if device == "CPU" and threads:
215
+ config["INFERENCE_NUM_THREADS"] = int(threads)
216
+ model = core.read_model(model=model_bytes)
217
+ self._compiled = core.compile_model(model, device, config)
218
+ self._request = self._compiled.create_infer_request()
219
+ self.device, self.precision = device, precision
220
+ try:
221
+ self.device_name = str(core.get_property(device, "FULL_DEVICE_NAME")).strip()
222
+ except Exception:
223
+ self.device_name = device
224
+ shape = lambda port: [int(d.get_length()) if d.is_static else None for d in port.get_partial_shape()] # noqa: E731
225
+ self._inputs = [_NodeArg(p.get_any_name(), shape(p)) for p in self._compiled.inputs]
226
+ self._outputs = [_NodeArg(p.get_any_name(), shape(p)) for p in self._compiled.outputs]
227
+ self._ports = {p.get_any_name(): p for p in self._compiled.outputs}
228
+
229
+ def get_inputs(self):
230
+ return list(self._inputs)
231
+
232
+ def get_outputs(self):
233
+ return list(self._outputs)
234
+
235
+ def get_providers(self):
236
+ return [f"OpenVINO:{self.device}:{self.precision}"]
237
+
238
+ def run(self, output_names, feeds):
239
+ if len(feeds) != 1:
240
+ raise ValueError("the detector graph has exactly one input")
241
+ (blob,) = feeds.values()
242
+ self._request.infer({0: blob})
243
+ return [self._request.get_tensor(self._ports[name]).data.copy() for name in output_names]
244
+
245
+
246
+ class _Engine:
247
+ """One engine, its own lock, and what it has done (`busy_ms` is wall time inside run)."""
248
+
249
+ def __init__(self, name, session, state):
250
+ self.name, self.session, self.state = name, session, state
251
+ self.lock = threading.Lock()
252
+ self.calls = 0
253
+ self.busy_ms = 0.0
254
+ self.recent_ms = []
255
+ self.reason = None
256
+ self.probe = None
257
+ self.precision = getattr(session, "precision", None) if session is not None else None
258
+
259
+ def run(self, output_names, feeds):
260
+ started = time.perf_counter()
261
+ try:
262
+ return self.session.run(output_names, feeds)
263
+ finally:
264
+ elapsed = (time.perf_counter() - started) * 1000.0
265
+ self.calls += 1
266
+ self.busy_ms += elapsed
267
+ self.recent_ms = (self.recent_ms + [elapsed])[-64:]
268
+
269
+ def snapshot(self):
270
+ recent = sorted(self.recent_ms)
271
+ row = {"state": self.state, "calls": self.calls, "busy_ms": round(self.busy_ms, 1),
272
+ "ms_p50": round(recent[len(recent) // 2], 1) if recent else None}
273
+ if self.precision:
274
+ row["precision"] = self.precision
275
+ if self.session is not None and getattr(self.session, "device_name", None):
276
+ row["device"] = self.session.device_name
277
+ if self.reason:
278
+ row["reason"] = self.reason[:500]
279
+ if self.probe is not None:
280
+ row["probe"] = dict(self.probe)
281
+ return row
282
+
283
+
284
+ class DetectorEngines:
285
+ """Which engine answers each frame of one detector model (plan v7 W8).
286
+
287
+ ORT is always loaded: the reference every probe compares against, and the
288
+ fallback. Under a policy other than `ort`, OpenVINO engines (CPU, and the
289
+ iGPU when OpenVINO lists one) are built in the background and probed on the
290
+ next PROBE_FRAMES live frames: each frame is answered by ORT and ALSO run
291
+ through the probing engine, and `compare(reference, candidate)` grades the
292
+ pair. An engine that passed serves under `openvino` (CPU) or `openvino_gpu`
293
+ (CPU first, the iGPU as a second engine for a frame that arrives while the
294
+ first is busy). `openvino_probe` probes and reports, and never serves.
295
+
296
+ Never a negative observation: an OpenVINO failure at run time marks that
297
+ engine failed and the same input is answered by ORT.
298
+ """
299
+
300
+ def __init__(self, ort_session, model_reader, policy, compare, *, log=LOG.info,
301
+ session_factory=OpenVinoSession, probe_frames=PROBE_FRAMES, threads=None):
302
+ self._ort = _Engine("ort", ort_session, "reference")
303
+ self._reader = model_reader
304
+ self._policy = policy or BackendPolicy("ort")
305
+ self._compare = compare
306
+ self._log = log
307
+ self._factory = session_factory
308
+ self._probe_frames = probe_frames
309
+ self._threads = threads
310
+ self._lock = threading.Lock()
311
+ self._ov = {}
312
+ self._building = False
313
+ self._openvino = openvino_version()
314
+
315
+ # ── building ──────────────────────────────────────────────────────────
316
+ def _wanted(self, mode):
317
+ if mode == "ort":
318
+ return ()
319
+ return ("openvino_cpu", "openvino_gpu")
320
+
321
+ def _ensure_built(self, mode):
322
+ wanted = [name for name in self._wanted(mode) if name not in self._ov]
323
+ if not wanted or self._building:
324
+ return
325
+ if self._openvino is None:
326
+ with self._lock:
327
+ for name in wanted:
328
+ engine = _Engine(name, None, "unavailable")
329
+ engine.reason = "openvino is not installed on this host"
330
+ self._ov[name] = engine
331
+ return
332
+ self._building = True
333
+ threading.Thread(target=self._build, args=(wanted,), name="detector-openvino-build", daemon=True).start()
334
+
335
+ def _build(self, names):
336
+ try:
337
+ model_bytes = self._reader()
338
+ for name in names:
339
+ device = "CPU" if name == "openvino_cpu" else "GPU"
340
+ precisions = ("f32",) if device == "CPU" else GPU_PRECISIONS
341
+ engine = None
342
+ for precision in precisions:
343
+ try:
344
+ started = time.perf_counter()
345
+ session = self._factory(model_bytes, device, precision, threads=self._threads)
346
+ engine = _Engine(name, session, "probing")
347
+ engine.probe = {"frames": 0, "max_person_diff": 0.0, "max_score_diff": 0.0,
348
+ "mismatches": 0, "max_box_shift": 0.0,
349
+ "compile_ms": round((time.perf_counter() - started) * 1000.0)}
350
+ engine.next_precisions = [p for p in precisions if p != precision][:1]
351
+ self._log(f"[detector] {name} built ({session.get_providers()[0]}, "
352
+ f"{engine.probe['compile_ms']} ms); probing on {self._probe_frames} live frames")
353
+ break
354
+ except EngineUnavailable as error:
355
+ engine = _Engine(name, None, "unavailable")
356
+ engine.reason = str(error)
357
+ break
358
+ except Exception as error:
359
+ engine = _Engine(name, None, "failed")
360
+ engine.reason = f"compile {precision}: {type(error).__name__}: {error}"
361
+ with self._lock:
362
+ self._ov[name] = engine
363
+ except Exception as error:
364
+ with self._lock:
365
+ for name in names:
366
+ if name not in self._ov:
367
+ engine = _Engine(name, None, "failed")
368
+ engine.reason = f"model unavailable for OpenVINO: {error}"
369
+ self._ov[name] = engine
370
+ finally:
371
+ self._building = False
372
+
373
+ def _rebuild_gpu_precision(self, engine):
374
+ """FP16 missed parity on the iGPU: try the next precision once, in the background."""
375
+ rest = getattr(engine, "next_precisions", [])
376
+ if not rest:
377
+ return
378
+ precision = rest[0]
379
+
380
+ def build():
381
+ try:
382
+ session = self._factory(self._reader(), "GPU", precision, threads=None)
383
+ fresh = _Engine(engine.name, session, "probing")
384
+ fresh.probe = {"frames": 0, "max_person_diff": 0.0, "max_score_diff": 0.0, "mismatches": 0,
385
+ "max_box_shift": 0.0, "previous": {"precision": engine.precision, **(engine.probe or {})}}
386
+ fresh.next_precisions = []
387
+ with self._lock:
388
+ self._ov[engine.name] = fresh
389
+ self._log(f"[detector] {engine.name} rebuilt in {precision} after {engine.precision} missed parity")
390
+ except Exception as error:
391
+ engine.reason = f"{engine.reason}; {precision} rebuild failed: {error}"
392
+
393
+ threading.Thread(target=build, name="detector-openvino-rebuild", daemon=True).start()
394
+
395
+ # ── running ───────────────────────────────────────────────────────────
396
+ def _serving(self, mode):
397
+ """(primary, secondary): the engines that answer frames under `mode` now."""
398
+ with self._lock:
399
+ cpu, gpu = self._ov.get("openvino_cpu"), self._ov.get("openvino_gpu")
400
+ primary = cpu if mode in ("openvino", "openvino_gpu") and cpu is not None and cpu.state == "passed" else self._ort
401
+ secondary = gpu if mode == "openvino_gpu" and gpu is not None and gpu.state == "passed" else None
402
+ return primary, secondary
403
+
404
+ def run(self, output_names, feeds, min_confidence=None):
405
+ """Answer one frame; returns (outputs, engine name). `min_confidence` is the request's floor."""
406
+ mode = self._policy.mode
407
+ self._ensure_built(mode)
408
+ with self._lock:
409
+ probing = [e for e in self._ov.values() if e.state == "probing"]
410
+ if probing:
411
+ # While any engine is on probation ORT answers, and is the reference.
412
+ outputs = self._run_on(self._ort, output_names, feeds)
413
+ for engine in probing:
414
+ self._probe_one(engine, output_names, feeds, outputs)
415
+ return outputs, self._ort.name
416
+ primary, secondary = self._serving(mode)
417
+ if min_confidence is not None and min_confidence < OPENVINO_MIN_CONFIDENCE:
418
+ primary, secondary = self._ort, None
419
+ if secondary is None:
420
+ return self._run_on(primary, output_names, feeds), primary.name
421
+ # Two engines: whichever is free; both busy waits on the primary.
422
+ if primary.lock.acquire(blocking=False):
423
+ engine = primary
424
+ elif secondary.lock.acquire(blocking=False):
425
+ engine = secondary
426
+ else:
427
+ primary.lock.acquire()
428
+ engine = primary
429
+ return self._run_locked(engine, output_names, feeds), engine.name
430
+
431
+ def _run_locked(self, engine, output_names, feeds):
432
+ try:
433
+ return self._guarded(engine, output_names, feeds)
434
+ finally:
435
+ engine.lock.release()
436
+
437
+ def _run_on(self, engine, output_names, feeds):
438
+ with engine.lock:
439
+ return self._guarded(engine, output_names, feeds)
440
+
441
+ def _guarded(self, engine, output_names, feeds):
442
+ if engine is self._ort:
443
+ return engine.run(output_names, feeds)
444
+ try:
445
+ return engine.run(output_names, feeds)
446
+ except Exception as error:
447
+ engine.state = "failed"
448
+ engine.reason = f"run: {type(error).__name__}: {error}"
449
+ self._log(f"[detector] {engine.name} failed at run time; ONNX Runtime answers: {engine.reason}")
450
+ with self._ort.lock:
451
+ return self._ort.run(output_names, feeds)
452
+
453
+ def _probe_one(self, engine, output_names, feeds, reference):
454
+ try:
455
+ with engine.lock:
456
+ candidate = engine.run(output_names, feeds)
457
+ except Exception as error:
458
+ engine.state = "failed"
459
+ engine.reason = f"probe run: {type(error).__name__}: {error}"
460
+ return
461
+ graded = self._compare(reference, candidate)
462
+ probe = engine.probe
463
+ probe["frames"] += 1
464
+ probe["max_person_diff"] = max(probe["max_person_diff"], float(graded["person_diff"]))
465
+ probe["max_score_diff"] = max(probe["max_score_diff"], float(graded["score_diff"]))
466
+ probe["mismatches"] += int(graded["mismatches"])
467
+ probe["max_box_shift"] = max(probe.get("max_box_shift", 0.0), float(graded.get("box_shift", 0.0)))
468
+ probe["ort_ms_p50"] = self._ort.snapshot()["ms_p50"]
469
+ probe["ms_p50"] = engine.snapshot()["ms_p50"]
470
+ if probe["frames"] < self._probe_frames:
471
+ return
472
+ passed = (probe["max_person_diff"] <= PROBE_PERSON_TOLERANCE and probe["mismatches"] == 0
473
+ and probe.get("max_box_shift", 0.0) <= PROBE_BOX_TOLERANCE)
474
+ engine.state = "passed" if passed else "failed"
475
+ if not passed:
476
+ engine.reason = (f"parity: person diff {probe['max_person_diff']:.2e} (bar {PROBE_PERSON_TOLERANCE:.0e}), "
477
+ f"{probe['mismatches']} detection mismatch(es), box shift {probe.get('max_box_shift', 0.0):.4f} "
478
+ f"(bar {PROBE_BOX_TOLERANCE}) over {probe['frames']} frames")
479
+ self._log(f"[detector] {engine.name} parity probe {'PASSED' if passed else 'FAILED'}: {probe}")
480
+ if not passed and engine.name == "openvino_gpu":
481
+ self._rebuild_gpu_precision(engine)
482
+
483
+ # ── reporting ─────────────────────────────────────────────────────────
484
+ def snapshot(self):
485
+ """The `/health` view: what was asked, which engines answer now, and each engine's record."""
486
+ mode = self._policy.mode
487
+ with self._lock:
488
+ probing = any(e.state == "probing" for e in self._ov.values())
489
+ engines = {"ort": self._ort.snapshot(), **{name: e.snapshot() for name, e in self._ov.items()}}
490
+ primary, secondary = (self._ort, None) if probing else self._serving(mode)
491
+ return {"requested": mode, "openvino": self._openvino,
492
+ "serving": [primary.name] + ([secondary.name] if secondary is not None else []),
493
+ "engines": engines}
494
+
495
+ def get_providers(self):
496
+ return self._ort.session.get_providers()
@@ -79,6 +79,24 @@ def _warmup_jpeg() -> bytes:
79
79
  return buf.getvalue()
80
80
 
81
81
 
82
+ #: The frame size a live look sends (the connector's 640x480 analysis frame).
83
+ FULL_SIZE_WARMUP = (640, 480)
84
+
85
+
86
+ def _full_size_warmup_jpeg() -> bytes:
87
+ """A 640x480 JPEG with real texture, for the pinned model's full-size warm-up.
88
+
89
+ Noise, not a flat fill: a flat frame compresses to almost nothing and decodes
90
+ in no time, so it would warm the forward pass but not the decode a live frame
91
+ costs. See ModelRegistry._ensure_loaded for why this pass exists.
92
+ """
93
+ from PIL import Image
94
+
95
+ buf = io.BytesIO()
96
+ Image.effect_noise(FULL_SIZE_WARMUP, 48).convert("RGB").save(buf, format="JPEG", quality=80)
97
+ return buf.getvalue()
98
+
99
+
82
100
  def build_catalogue(
83
101
  default_model_name: str,
84
102
  default_model_path: "Path | None" = None,
@@ -137,6 +155,7 @@ class DetectorLoader:
137
155
  log=print,
138
156
  accelerator: str = "auto",
139
157
  task_models: tuple = BUILTIN_TASK_MODELS,
158
+ backend_policy=None,
140
159
  ):
141
160
  # `ram_budget_bytes=None` is NOT a default policy — it is the absence of
142
161
  # one, and the registry refuses every portfolio decision in that state
@@ -147,6 +166,10 @@ class DetectorLoader:
147
166
  self.model_name = model_name
148
167
  self._intra_op_threads = intra_op_threads
149
168
  self._accelerator = accelerator
169
+ # Plan v7 W8: the engine this service was asked for (POST /backend, serve
170
+ # --backend). Shared by every RF-DETR session; "ort" builds nothing new.
171
+ from .execution import BackendPolicy
172
+ self.backend_policy = backend_policy if backend_policy is not None else BackendPolicy("ort")
150
173
  self._poll_interval_s = poll_interval_s
151
174
  self._log = log
152
175
 
@@ -156,6 +179,7 @@ class DetectorLoader:
156
179
  probe=self._probe,
157
180
  ram_budget_bytes=ram_budget_bytes,
158
181
  log=log,
182
+ warm=self._warm_full_size,
159
183
  )
160
184
 
161
185
  self._stop = threading.Event()
@@ -177,7 +201,8 @@ class DetectorLoader:
177
201
  from .pose import PoseEstimator
178
202
  return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
179
203
  from .session import OnnxDetector
180
- return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator)
204
+ return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator,
205
+ backend_policy=self.backend_policy)
181
206
 
182
207
  def _probe(self, session: OnnxDetector) -> None:
183
208
  """PROVE IT. A session that constructs is not a session that infers.
@@ -192,6 +217,15 @@ class DetectorLoader:
192
217
  else:
193
218
  session.detect(_warmup_jpeg(), 0.99, None)
194
219
 
220
+ @staticmethod
221
+ def _warm_full_size(session) -> None:
222
+ """One more forward pass, on a live-sized frame, for the pinned cascade gate.
223
+
224
+ The registry runs this after `_probe` and never as a gate (see
225
+ ModelRegistry._ensure_loaded). `0.99` for the same reason as the probe.
226
+ """
227
+ session.detect(_full_size_warmup_jpeg(), 0.99, None)
228
+
195
229
  def start(self) -> None:
196
230
  self._thread.start()
197
231
 
@@ -307,9 +341,14 @@ class DetectorLoader:
307
341
  "acceleration",
308
342
  "output_layout",
309
343
  "preprocessing",
344
+ "backend",
310
345
  ):
311
346
  if key in pinned:
312
347
  payload[key] = pinned[key]
348
+ policy = getattr(self, "backend_policy", None)
349
+ if "backend" not in payload and policy is not None:
350
+ # Not loaded yet, or a pinned model on ORT only: still say what was asked.
351
+ payload["backend"] = {"requested": policy.mode, "serving": None, "engines": {}}
313
352
  return payload
314
353
 
315
354
  def prepare_models(self, models: list) -> list:
@@ -214,6 +214,7 @@ class ModelRegistry:
214
214
  ram_budget_bytes: "int | None",
215
215
  log=print,
216
216
  clock=time.monotonic,
217
+ warm=None,
217
218
  ):
218
219
  if not entries:
219
220
  raise ValueError("a registry with no models can answer nothing")
@@ -242,6 +243,8 @@ class ModelRegistry:
242
243
  self._entries = list(entries)
243
244
  self._session_factory = session_factory
244
245
  self._probe = probe
246
+ #: The pinned model's full-size warm-up (see `_ensure_loaded`), or None.
247
+ self._warm = warm
245
248
  #: `None` means NO CEILING WAS INJECTED — not "unlimited".
246
249
  self._budget = None if ram_budget_bytes is None else int(ram_budget_bytes)
247
250
  self._log = log
@@ -818,6 +821,26 @@ class ModelRegistry:
818
821
  rss_delta = max(0, after[0] - before[0])
819
822
  rss_source = after[1]
820
823
 
824
+ # THE PINNED MODEL WARMS AT FULL SIZE BEFORE IT IS READY (plan v7 W8 follow-up).
825
+ # The probe above proves a forward pass completes, on a 64x48 frame, and that
826
+ # frame is too small to warm the path a live look takes: after the 0.2.246
827
+ # detector swap at Texas the first live call took 3.2 s and the next 18 averaged
828
+ # 1.4 s. So the cascade gate runs once more on a full-size frame before READY.
829
+ # Only the pinned model: it loads at start or on a new artifact, ahead of the
830
+ # looks it gates, while a specialist loads inside the first request that needs
831
+ # it, where another pass would only lengthen that request. Never a gate: a
832
+ # failure here is logged and the model is ready on its proof. Not timed into
833
+ # warmup_ms (`warmup_inference_ms`), which stays the proof pass's own time, and
834
+ # after the RSS reading, so the admission number is unchanged.
835
+ warm_ms = None
836
+ if entry.pinned and self._warm is not None:
837
+ try:
838
+ warm_started = time.perf_counter()
839
+ self._warm(session)
840
+ warm_ms = (time.perf_counter() - warm_started) * 1000.0
841
+ except Exception as err:
842
+ self._log(f"[detector] {entry.name}: full-size warm-up failed; ready on its probe: {err}")
843
+
821
844
  with self._lock:
822
845
  state.session = session
823
846
  state.stamp = stamp
@@ -830,9 +853,10 @@ class ModelRegistry:
830
853
  state.rss_source = rss_source
831
854
  self._index = None
832
855
  measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
856
+ warmed = f", full-size warm-up {warm_ms:.0f} ms" if warm_ms is not None else ""
833
857
  self._log(
834
858
  f"[detector] ready — {entry.name} at {entry.path} "
835
- f"(probe {warmup_ms:.0f} ms, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
859
+ f"(probe {warmup_ms:.0f} ms{warmed}, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
836
860
  )
837
861
 
838
862
  def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
@@ -892,6 +916,8 @@ class ModelRegistry:
892
916
  if s.reason:
893
917
  row["reason"] = s.reason
894
918
  if s.warmup_ms is not None:
919
+ # The proof pass (64x48) only; the pinned model's full-size warm-up
920
+ # is in the ready log line, not here (see `_ensure_loaded`).
895
921
  row["warmup_inference_ms"] = round(s.warmup_ms, 1)
896
922
  if s.loaded_at is not None:
897
923
  row["loaded_at"] = s.loaded_at
@@ -912,6 +938,12 @@ class ModelRegistry:
912
938
  if isinstance(acceleration, dict):
913
939
  row["acceleration"] = acceleration
914
940
  row["providers"] = acceleration["providers"]
941
+ # Which engines answer this model, and each one's parity probe and
942
+ # busy time (plan v7 W8). Absent for a model that runs on ORT only.
943
+ backend = getattr(s.session, "backend_snapshot", None)
944
+ backend = backend() if callable(backend) else None
945
+ if isinstance(backend, dict):
946
+ row["backend"] = backend
915
947
  # WHICH DECODE AND WHICH PREPROCESSING PRODUCED THIS MODEL'S
916
948
  # NUMBERS. Three layouts now reach this service and their
917
949
  # preprocessors are not interchangeable, so a confidence with
@@ -247,6 +247,10 @@ def build_detect_response(batch) -> dict:
247
247
  if not isinstance(batch.inference_ms, (int, float)) or not math.isfinite(batch.inference_ms):
248
248
  raise ValueError(f"inference_ms is not a finite number: {batch.inference_ms!r}")
249
249
  _assert_wire_safe(body["detections"])
250
+ # Which engine answered (plan v7 W8), when one other than plain ORT may have.
251
+ engine = getattr(batch, "engine", None)
252
+ if isinstance(engine, str) and engine:
253
+ body["engine"] = engine
250
254
  hashes = getattr(batch, "model_hashes", {})
251
255
  if hashes:
252
256
  if (not isinstance(hashes, dict) or set(hashes) != set(models)
@@ -440,6 +444,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
440
444
  self._vehicle_end()
441
445
  elif self._path() == "/ready":
442
446
  self._ready()
447
+ elif self._path() == "/backend":
448
+ self._backend()
443
449
  elif self._path() == "/health":
444
450
  self._json(405, {"error": "/health is a GET"})
445
451
  else:
@@ -493,6 +499,37 @@ class DetectorHandler(BaseHTTPRequestHandler):
493
499
  # client's receipt; they must not make its next preparation look busy.
494
500
  self._json(status, payload)
495
501
 
502
+ # ── /backend (plan v7 W8) ────────────────────────────────────────────────
503
+
504
+ def _backend(self) -> None:
505
+ """Set the engine this service was asked for: `{"version": 1, "mode": ...}`.
506
+
507
+ The connector relays the server's runtime_config CCTV_DETECTOR_BACKEND
508
+ here when it changes. Answers with the engines' view; an OpenVINO engine
509
+ serves only after its own parity probe, so this never switches an engine
510
+ on by itself.
511
+ """
512
+ from .execution import BACKEND_MODES
513
+ try:
514
+ raw = json.loads(self._read_body().decode("utf-8"))
515
+ except WireError as err:
516
+ self._json(err.status, {"error": err.message})
517
+ return
518
+ except (ValueError, UnicodeDecodeError):
519
+ self._json(400, {"error": "body must be JSON"})
520
+ return
521
+ if (not isinstance(raw, dict) or set(raw) != {"version", "mode"} or raw["version"] != 1
522
+ or raw["mode"] not in BACKEND_MODES):
523
+ self._json(400, {"error": f"expected {{version: 1, mode: one of {list(BACKEND_MODES)}}}"})
524
+ return
525
+ policy = getattr(self.server.loader, "backend_policy", None)
526
+ if policy is None:
527
+ self._json(503, {"error": "backend selection is unavailable in this detector build"})
528
+ return
529
+ policy.set(raw["mode"])
530
+ info = self.server.loader.info()
531
+ self._json(200, {"version": 1, "mode": policy.mode, "backend": info.get("backend")})
532
+
496
533
  # ── /health ──────────────────────────────────────────────────────────────
497
534
 
498
535
  def _health(self) -> None:
@@ -623,7 +660,12 @@ class DetectorHandler(BaseHTTPRequestHandler):
623
660
  result = loader.estimate_appearance(jpeg, boxes, appearance_model)
624
661
  appearance = build_appearance_evidence(
625
662
  result, hashlib.sha256(jpeg).hexdigest(), frame_size, batch.detections)
626
- body["tracking"] = self.server.actor_tracker.track(
663
+ # The camera's per-track records (track_records.py) as of THIS frame, beside the
664
+ # signed tracking proof and outside its digest. They come from the same lock and
665
+ # transaction as the proof: a second request on the same camera can no longer
666
+ # commit in between and hand this frame its records, and a replayed frame gets
667
+ # the records it had then, not the live ones.
668
+ body["tracking"], body["tracks"] = self.server.actor_tracker.track_and_records(
627
669
  tracking, batch.detections, image_sha256=hashlib.sha256(jpeg).hexdigest(),
628
670
  model_hashes=body.get("model_hashes"), frame_size=frame_size, appearance=appearance)
629
671
  if detection_pose is not None: