@trillboards/edge-sdk 0.2.245 → 0.2.246

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/deploy/detector/bench/detect_frames.py +88 -0
  2. package/deploy/detector/bench/openvino_parity.py +152 -0
  3. package/deploy/detector/pyproject.toml +10 -1
  4. package/deploy/detector/tests/test_actor_tracking.py +32 -0
  5. package/deploy/detector/tests/test_actor_tracking_wire.py +11 -0
  6. package/deploy/detector/tests/test_openvino_engines.py +251 -0
  7. package/deploy/detector/tests/test_server.py +43 -0
  8. package/deploy/detector/tests/test_version_stamp.py +17 -4
  9. package/deploy/detector/trillboards_detector/_version.py +2 -2
  10. package/deploy/detector/trillboards_detector/actor_tracking.py +51 -0
  11. package/deploy/detector/trillboards_detector/cli.py +5 -0
  12. package/deploy/detector/trillboards_detector/detections.py +5 -0
  13. package/deploy/detector/trillboards_detector/execution.py +386 -0
  14. package/deploy/detector/trillboards_detector/loader.py +12 -1
  15. package/deploy/detector/trillboards_detector/registry.py +6 -0
  16. package/deploy/detector/trillboards_detector/server.py +41 -0
  17. package/deploy/detector/trillboards_detector/session.py +81 -3
  18. package/deploy/detector/trillboards_detector/track_records.py +113 -0
  19. package/deploy/windows/ci/test-detector-reinstall.ps1 +474 -0
  20. package/deploy/windows/setup-detector.ps1 +122 -29
  21. package/dist/ActorTracking.d.ts +38 -0
  22. package/dist/ActorTracking.d.ts.map +1 -1
  23. package/dist/ActorTracking.js +40 -0
  24. package/dist/ActorTracking.js.map +1 -1
  25. package/dist/AlertClips.d.ts +4 -2
  26. package/dist/AlertClips.d.ts.map +1 -1
  27. package/dist/AlertClips.js +24 -9
  28. package/dist/AlertClips.js.map +1 -1
  29. package/dist/CloudSensingManager.d.ts.map +1 -1
  30. package/dist/CloudSensingManager.js +6 -1
  31. package/dist/CloudSensingManager.js.map +1 -1
  32. package/dist/DurableVlmCandidateSpool.d.ts +11 -0
  33. package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
  34. package/dist/DurableVlmCandidateSpool.js +31 -2
  35. package/dist/DurableVlmCandidateSpool.js.map +1 -1
  36. package/dist/EdgeAgent.js +1 -1
  37. package/dist/FrameGroupingShadow.d.ts +57 -0
  38. package/dist/FrameGroupingShadow.d.ts.map +1 -1
  39. package/dist/FrameGroupingShadow.js +123 -5
  40. package/dist/FrameGroupingShadow.js.map +1 -1
  41. package/dist/LocalDetectorClient.d.ts +25 -1
  42. package/dist/LocalDetectorClient.d.ts.map +1 -1
  43. package/dist/LocalDetectorClient.js +60 -0
  44. package/dist/LocalDetectorClient.js.map +1 -1
  45. package/dist/LocalVideoBuffer.d.ts +14 -2
  46. package/dist/LocalVideoBuffer.d.ts.map +1 -1
  47. package/dist/LocalVideoBuffer.js +23 -3
  48. package/dist/LocalVideoBuffer.js.map +1 -1
  49. package/dist/SensingProfileClient.d.ts.map +1 -1
  50. package/dist/SensingProfileClient.js +3 -0
  51. package/dist/SensingProfileClient.js.map +1 -1
  52. package/dist/VideoSegmentUploader.d.ts +14 -7
  53. package/dist/VideoSegmentUploader.d.ts.map +1 -1
  54. package/dist/VideoSegmentUploader.js +34 -38
  55. package/dist/VideoSegmentUploader.js.map +1 -1
  56. package/dist/VlmCandidateBuffer.d.ts +13 -0
  57. package/dist/VlmCandidateBuffer.d.ts.map +1 -1
  58. package/dist/VlmCandidateBuffer.js.map +1 -1
  59. package/dist/autoSense.d.ts +4 -0
  60. package/dist/autoSense.d.ts.map +1 -1
  61. package/dist/autoSense.js +4 -0
  62. package/dist/autoSense.js.map +1 -1
  63. package/dist/continuousReview.d.ts +6 -2
  64. package/dist/continuousReview.d.ts.map +1 -1
  65. package/dist/continuousReview.js.map +1 -1
  66. package/dist/detectorBackendRelay.d.ts +24 -0
  67. package/dist/detectorBackendRelay.d.ts.map +1 -0
  68. package/dist/detectorBackendRelay.js +70 -0
  69. package/dist/detectorBackendRelay.js.map +1 -0
  70. package/dist/diagnose.d.ts +1 -1
  71. package/dist/diagnose.js +1 -1
  72. package/dist/localSensing.d.ts +3 -0
  73. package/dist/localSensing.d.ts.map +1 -1
  74. package/dist/localSensing.js +69 -3
  75. package/dist/localSensing.js.map +1 -1
  76. package/dist/pipelineSwitches.d.ts +59 -0
  77. package/dist/pipelineSwitches.d.ts.map +1 -0
  78. package/dist/pipelineSwitches.js +240 -0
  79. package/dist/pipelineSwitches.js.map +1 -0
  80. package/package.json +4 -4
  81. package/retained/RetainedCaptureClient.cjs +37 -2
  82. package/retained/WebsiteAdmissionCaptureClient.cjs +57 -2
  83. package/retained/__tests__/look-reuse.test.cjs +167 -0
  84. package/retained/manager.cjs +28 -2
@@ -13,5 +13,5 @@ human sets.
13
13
  """
14
14
 
15
15
  BASE_VERSION = "0.2.0"
16
- SOURCE_FINGERPRINT = "befcd33e8e35"
17
- VERSION = "0.2.0+src.befcd33e8e35"
16
+ SOURCE_FINGERPRINT = "740c359ba4b7"
17
+ VERSION = "0.2.0+src.740c359ba4b7"
@@ -24,6 +24,7 @@ from .actor_identity import (
24
24
  ActorIdentitySegments, POLICY as IDENTITY_POLICY, checked_appearance,
25
25
  MODEL_ID as IDENTITY_MODEL_ID,
26
26
  )
27
+ from .track_records import MAX_RECORDS_PER_CAMERA, FaceSlot, TrackRecord
27
28
 
28
29
  ALGORITHM = "bytetrack_2.6.1"
29
30
  _ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
@@ -108,6 +109,11 @@ class ActorTrackerJournal:
108
109
  camera_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, payload_json TEXT NOT NULL);
109
110
  CREATE TABLE IF NOT EXISTS actor_checkpoint (
110
111
  camera_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, payload_json TEXT NOT NULL);
112
+ CREATE TABLE IF NOT EXISTS actor_track_record (
113
+ camera_id TEXT NOT NULL, actor_id TEXT NOT NULL, epoch TEXT NOT NULL, kind TEXT NOT NULL,
114
+ first_seen_ms INTEGER NOT NULL, last_seen_ms INTEGER NOT NULL, frames INTEGER NOT NULL,
115
+ box_json TEXT NOT NULL, confidence REAL NOT NULL, face_json TEXT NULL,
116
+ PRIMARY KEY(camera_id, actor_id));
111
117
  """)
112
118
 
113
119
  def close(self):
@@ -120,6 +126,50 @@ class ActorTrackerJournal:
120
126
  with self._lock:
121
127
  return self._db.execute("SELECT count(*) FROM actor_frame WHERE camera_id=?", (camera_id,)).fetchone()[0]
122
128
 
129
+ # ── per-track records (track_records.py) ─────────────────────────────────
130
+
131
+ def _write_track_records(self, camera, result, values, captured_at):
132
+ """Inside track()'s transaction: observed actors refresh, ended actors drop."""
133
+ self._db.execute("DELETE FROM actor_track_record WHERE camera_id=? AND epoch<>?", (camera, result["epoch"]))
134
+ for ended in result["endedActors"]:
135
+ self._db.execute("DELETE FROM actor_track_record WHERE camera_id=? AND actor_id=?", (camera, ended["id"]))
136
+ for association in result["actors"]:
137
+ actor, box = association["actor"], values[association["detectionIndex"]]
138
+ self._db.execute(
139
+ "INSERT INTO actor_track_record VALUES (?,?,?,?,?,?,1,?,?,NULL) "
140
+ "ON CONFLICT(camera_id, actor_id) DO UPDATE SET last_seen_ms=excluded.last_seen_ms, "
141
+ "frames=actor_track_record.frames+1, box_json=excluded.box_json, confidence=excluded.confidence",
142
+ (camera, actor["id"], actor["epoch"], actor["kind"], captured_at, captured_at,
143
+ _json([box["x"], box["y"], box["width"], box["height"]]), float(box["confidence"])))
144
+ self._db.execute(
145
+ "DELETE FROM actor_track_record WHERE camera_id=? AND actor_id NOT IN (SELECT actor_id FROM "
146
+ "actor_track_record WHERE camera_id=? ORDER BY last_seen_ms DESC, actor_id LIMIT ?)",
147
+ (camera, camera, MAX_RECORDS_PER_CAMERA))
148
+
149
+ def track_records(self, camera_id):
150
+ """The camera's live actors' records, oldest first (list of TrackRecord)."""
151
+ with self._lock:
152
+ rows = self._db.execute(
153
+ "SELECT actor_id,kind,epoch,first_seen_ms,last_seen_ms,frames,box_json,confidence,face_json "
154
+ "FROM actor_track_record WHERE camera_id=? ORDER BY first_seen_ms, actor_id", (camera_id,)).fetchall()
155
+ return [TrackRecord(r[0], r[1], r[2], r[3], r[4], r[5], tuple(json.loads(r[6])), r[7],
156
+ None if r[8] is None else FaceSlot.from_wire(json.loads(r[8]))) for r in rows]
157
+
158
+ def record_face(self, camera_id, actor_id, face):
159
+ """Store one FaceSlot on a LIVE actor's record; False when the actor is not live.
160
+
161
+ The face watchlist (plan v7 T1) is the only intended writer. The slot is
162
+ deleted with the record when the actor ends, never kept past its track.
163
+ """
164
+ if not isinstance(face, FaceSlot):
165
+ raise TrackingError("record_face takes a FaceSlot")
166
+ payload = _json(FaceSlot.from_wire(face.to_wire()).to_wire())
167
+ with self._lock:
168
+ cursor = self._db.execute(
169
+ "UPDATE actor_track_record SET face_json=? WHERE camera_id=? AND actor_id=?",
170
+ (payload, camera_id, actor_id))
171
+ return cursor.rowcount == 1
172
+
123
173
  def _new_tracker(self, config=None):
124
174
  config = self._config if config is None else config
125
175
  return self._tracker_class(
@@ -473,6 +523,7 @@ class ActorTrackerJournal:
473
523
  "seq=excluded.seq,active_json=excluded.active_json",
474
524
  (camera, context, epoch, captured_at, seq, _json(active)))
475
525
  self._save_checkpoint(camera,context,epoch,captured_at,seq,fingerprint,tracker,identities)
526
+ self._write_track_records(camera, result, values, captured_at)
476
527
  if appearance is not None:
477
528
  self._save_identity_checkpoint(
478
529
  camera, request, context, epoch, seq, fingerprint,
@@ -22,6 +22,7 @@ from .worker_budget import configure_numeric_workers
22
22
 
23
23
  # This must run before loader/server imports initialize NumPy or BLAS.
24
24
  configure_numeric_workers()
25
+ from .execution import BackendPolicy
25
26
  from .loader import DEFAULT_POLL_INTERVAL_S, DetectorLoader
26
27
  from .modelpath import DEFAULT_MODEL_NAME, resolve_model_path, resolve_ram_budget_mb, resolve_data_dir
27
28
  from .actor_tracking import ActorTrackerJournal
@@ -110,6 +111,9 @@ def build_parser() -> argparse.ArgumentParser:
110
111
  s.add_argument("--poll-interval", type=float, default=DEFAULT_POLL_INTERVAL_S)
111
112
  s.add_argument("--accelerator", choices=("auto", "cpu"), default="auto",
112
113
  help="prefer compatible local NVIDIA CUDA, otherwise CPU; cpu is an explicit rollback")
114
+ s.add_argument("--backend", choices=("ort", "openvino_probe", "openvino", "openvino_gpu"), default="ort",
115
+ help=("the inference engine to start with (plan v7 W8); the connector relays the server's "
116
+ "choice with POST /backend. OpenVINO serves only after its parity probe on this host."))
113
117
  s.add_argument("--log-file", default=None, help="also write to this file (Windows has no task stdout)")
114
118
  s.add_argument("--verbose", action="store_true")
115
119
  s.add_argument("--tracking-journal", default=None, help="durable actor tracker SQLite path; defaults to the connector data directory")
@@ -166,6 +170,7 @@ def main(argv: "list[str] | None" = None) -> int:
166
170
  ram_budget_bytes=ram_budget_bytes,
167
171
  log=LOG.info,
168
172
  accelerator=args.accelerator,
173
+ backend_policy=BackendPolicy(args.backend),
169
174
  )
170
175
  actor_tracker = ActorTrackerJournal(args.tracking_journal or resolve_data_dir() / "actor-tracking" / "tracks.sqlite3")
171
176
  loader.start()
@@ -60,6 +60,9 @@ class DetectionBatch:
60
60
  # Derived from the exact bytes used to create each ONNX session, not a
61
61
  # request pin, a catalogue entry, or a later read of the mutable model path.
62
62
  model_hashes: dict[str, str] = field(default_factory=dict)
63
+ #: Which engine answered (plan v7 W8): `ort`, `openvino_cpu`, `openvino_gpu`;
64
+ #: None for a model that runs on ORT only. Diagnostic, never a contract input.
65
+ engine: "str | None" = None
63
66
 
64
67
 
65
68
  def merge_batches(batches: "list[DetectionBatch]") -> DetectionBatch:
@@ -95,6 +98,8 @@ def merge_batches(batches: "list[DetectionBatch]") -> DetectionBatch:
95
98
  merged.degenerate_dropped += b.degenerate_dropped
96
99
  merged.unnamed_class_dropped += b.unnamed_class_dropped
97
100
  merged.models.extend(b.models)
101
+ if b.engine is not None:
102
+ merged.engine = b.engine if merged.engine in (None, b.engine) else f"{merged.engine}+{b.engine}"
98
103
  for model, digest in b.model_hashes.items():
99
104
  if model in merged.model_hashes and merged.model_hashes[model] != digest:
100
105
  raise ValueError("one frame cannot merge different artifacts under one model id")
@@ -6,6 +6,8 @@ so provider policy can be tested without downloading models or CUDA libraries.
6
6
  from __future__ import annotations
7
7
 
8
8
  import logging
9
+ import threading
10
+ import time
9
11
 
10
12
  LOG = logging.getLogger(__name__)
11
13
  CPU = "CPUExecutionProvider"
@@ -108,3 +110,387 @@ class LocalOrtSession:
108
110
 
109
111
  def __getattr__(self, name):
110
112
  return getattr(self._session, name)
113
+
114
+
115
+ # ── OpenVINO engines (plan v7 W8) ──────────────────────────────────────────
116
+ #
117
+ # The Texas store PC is an i7-7700 (AVX2, no AVX-512) with an HD 630 iGPU and no
118
+ # NVIDIA card, so every RF-DETR call ran on ORT's CPU kernels with the iGPU idle.
119
+ # Measured 2026-11-16 on 4 pinned devbox cores with the shipped artifact: OpenVINO
120
+ # 2026.4.1 on the CPU 2.32 frames/s against ORT 1.70 (+36%), person scores within
121
+ # 0.0004 of ORT on all 252 persons of 360 Texas stills. The ONNX Runtime OpenVINO
122
+ # plug-in on the pinned ORT 1.29.0 does not compile this artifact ("Input names
123
+ # mismatch"), so OpenVINO is used directly.
124
+ #
125
+ # ORT stays loaded as the reference and the fallback. An OpenVINO engine serves
126
+ # only after its parity probe passed on THIS host, on live frames, against ORT:
127
+ # a different kernel library, and on the iGPU a different precision, is a
128
+ # different numeric path, and the bar is identical detections at the operating
129
+ # thresholds with person scores within PROBE_PERSON_TOLERANCE.
130
+
131
+ BACKEND_MODES = ("ort", "openvino_probe", "openvino", "openvino_gpu")
132
+ #: Frames each OpenVINO engine is compared on before it may serve.
133
+ PROBE_FRAMES = 24
134
+ #: The parity bar on person scores (sigmoid probabilities), as in plan v7 W8.
135
+ PROBE_PERSON_TOLERANCE = 1e-3
136
+ #: A box both engines keep may move this much (fraction of the frame, ~6 px at 640) and still be the
137
+ #: same detection. Measured 2026-11-16: the largest shift on 494 Texas and corpus frames was 0.0063.
138
+ PROBE_BOX_TOLERANCE = 0.01
139
+ #: OpenVINO answers only a request whose detection floor is at least this. Below it the two engines
140
+ #: disagree on rare queries (9 of 5,535 kept pairs at a 0.05 floor on 60 Texas frames), so a box
141
+ #: method that asks at 0.05 (the uniform rule's screens and fixtures) keeps ONNX Runtime's answers.
142
+ OPENVINO_MIN_CONFIDENCE = 0.2
143
+ #: The iGPU is tried in FP16 first (its fast path), then FP32 if FP16 misses parity.
144
+ GPU_PRECISIONS = ("f16", "f32")
145
+
146
+
147
+ class EngineUnavailable(RuntimeError):
148
+ """This host cannot build the engine (no OpenVINO, no such device)."""
149
+
150
+
151
+ class BackendPolicy:
152
+ """The engine this service was asked for, shared by every detector session.
153
+
154
+ Set by `POST /backend` (the connector relays the server's runtime_config
155
+ CCTV_DETECTOR_BACKEND) or `serve --backend`. `ort` is today's behaviour.
156
+ """
157
+
158
+ def __init__(self, mode="ort"):
159
+ if mode not in BACKEND_MODES:
160
+ raise ValueError(f"backend must be one of {BACKEND_MODES}")
161
+ self._mode = mode
162
+ self._lock = threading.Lock()
163
+
164
+ @property
165
+ def mode(self):
166
+ with self._lock:
167
+ return self._mode
168
+
169
+ def set(self, mode):
170
+ if mode not in BACKEND_MODES:
171
+ raise ValueError(f"backend must be one of {BACKEND_MODES}")
172
+ with self._lock:
173
+ changed = mode != self._mode
174
+ self._mode = mode
175
+ if changed:
176
+ LOG.info("[detector] backend requested: %s", mode)
177
+ return changed
178
+
179
+
180
+ def openvino_version():
181
+ try:
182
+ import openvino # noqa: F401
183
+ return getattr(openvino, "__version__", None) or "unknown"
184
+ except Exception: # not installed on this host: ORT only
185
+ return None
186
+
187
+
188
+ class _NodeArg:
189
+ """The two NodeArg fields `session._read_graph_spec` reads."""
190
+
191
+ def __init__(self, name, shape):
192
+ self.name, self.shape = name, shape
193
+
194
+
195
+ class OpenVinoSession:
196
+ """One compiled OpenVINO model behind the ORT session surface the detector reads.
197
+
198
+ `get_inputs`/`get_outputs` (name, static shape), `run(output_names, feeds)`
199
+ and `get_providers`. The model is read from the SAME bytes ORT loaded,
200
+ re-verified by hash, so an OTA that replaced the file cannot slip a different
201
+ artifact under the engine. CPU runs FP32 explicitly: a host with BF16/AMX
202
+ would otherwise default to bf16, which is a different numeric path.
203
+ """
204
+
205
+ def __init__(self, model_bytes, device, precision, *, core=None, threads=None):
206
+ try:
207
+ import openvino as ov
208
+ except Exception as error:
209
+ raise EngineUnavailable(f"openvino is not installed: {error}") from None
210
+ core = core or ov.Core()
211
+ if device not in core.available_devices:
212
+ raise EngineUnavailable(f"{device} is not an OpenVINO device on this host ({core.available_devices})")
213
+ config = {"PERFORMANCE_HINT": "LATENCY", "INFERENCE_PRECISION_HINT": precision}
214
+ if device == "CPU" and threads:
215
+ config["INFERENCE_NUM_THREADS"] = int(threads)
216
+ model = core.read_model(model=model_bytes)
217
+ self._compiled = core.compile_model(model, device, config)
218
+ self._request = self._compiled.create_infer_request()
219
+ self.device, self.precision = device, precision
220
+ try:
221
+ self.device_name = str(core.get_property(device, "FULL_DEVICE_NAME")).strip()
222
+ except Exception:
223
+ self.device_name = device
224
+ shape = lambda port: [int(d.get_length()) if d.is_static else None for d in port.get_partial_shape()] # noqa: E731
225
+ self._inputs = [_NodeArg(p.get_any_name(), shape(p)) for p in self._compiled.inputs]
226
+ self._outputs = [_NodeArg(p.get_any_name(), shape(p)) for p in self._compiled.outputs]
227
+ self._ports = {p.get_any_name(): p for p in self._compiled.outputs}
228
+
229
+ def get_inputs(self):
230
+ return list(self._inputs)
231
+
232
+ def get_outputs(self):
233
+ return list(self._outputs)
234
+
235
+ def get_providers(self):
236
+ return [f"OpenVINO:{self.device}:{self.precision}"]
237
+
238
+ def run(self, output_names, feeds):
239
+ if len(feeds) != 1:
240
+ raise ValueError("the detector graph has exactly one input")
241
+ (blob,) = feeds.values()
242
+ self._request.infer({0: blob})
243
+ return [self._request.get_tensor(self._ports[name]).data.copy() for name in output_names]
244
+
245
+
246
+ class _Engine:
247
+ """One engine, its own lock, and what it has done (`busy_ms` is wall time inside run)."""
248
+
249
+ def __init__(self, name, session, state):
250
+ self.name, self.session, self.state = name, session, state
251
+ self.lock = threading.Lock()
252
+ self.calls = 0
253
+ self.busy_ms = 0.0
254
+ self.recent_ms = []
255
+ self.reason = None
256
+ self.probe = None
257
+ self.precision = getattr(session, "precision", None) if session is not None else None
258
+
259
+ def run(self, output_names, feeds):
260
+ started = time.perf_counter()
261
+ try:
262
+ return self.session.run(output_names, feeds)
263
+ finally:
264
+ elapsed = (time.perf_counter() - started) * 1000.0
265
+ self.calls += 1
266
+ self.busy_ms += elapsed
267
+ self.recent_ms = (self.recent_ms + [elapsed])[-64:]
268
+
269
+ def snapshot(self):
270
+ recent = sorted(self.recent_ms)
271
+ row = {"state": self.state, "calls": self.calls, "busy_ms": round(self.busy_ms, 1),
272
+ "ms_p50": round(recent[len(recent) // 2], 1) if recent else None}
273
+ if self.precision:
274
+ row["precision"] = self.precision
275
+ if self.session is not None and getattr(self.session, "device_name", None):
276
+ row["device"] = self.session.device_name
277
+ if self.reason:
278
+ row["reason"] = self.reason[:500]
279
+ if self.probe is not None:
280
+ row["probe"] = dict(self.probe)
281
+ return row
282
+
283
+
284
+ class DetectorEngines:
285
+ """Which engine answers each frame of one detector model (plan v7 W8).
286
+
287
+ ORT is always loaded: the reference every probe compares against, and the
288
+ fallback. Under a policy other than `ort`, OpenVINO engines (CPU, and the
289
+ iGPU when OpenVINO lists one) are built in the background and probed on the
290
+ next PROBE_FRAMES live frames: each frame is answered by ORT and ALSO run
291
+ through the probing engine, and `compare(reference, candidate)` grades the
292
+ pair. An engine that passed serves under `openvino` (CPU) or `openvino_gpu`
293
+ (CPU first, the iGPU as a second engine for a frame that arrives while the
294
+ first is busy). `openvino_probe` probes and reports, and never serves.
295
+
296
+ Never a negative observation: an OpenVINO failure at run time marks that
297
+ engine failed and the same input is answered by ORT.
298
+ """
299
+
300
+ def __init__(self, ort_session, model_reader, policy, compare, *, log=LOG.info,
301
+ session_factory=OpenVinoSession, probe_frames=PROBE_FRAMES, threads=None):
302
+ self._ort = _Engine("ort", ort_session, "reference")
303
+ self._reader = model_reader
304
+ self._policy = policy or BackendPolicy("ort")
305
+ self._compare = compare
306
+ self._log = log
307
+ self._factory = session_factory
308
+ self._probe_frames = probe_frames
309
+ self._threads = threads
310
+ self._lock = threading.Lock()
311
+ self._ov = {}
312
+ self._building = False
313
+ self._openvino = openvino_version()
314
+
315
+ # ── building ──────────────────────────────────────────────────────────
316
+ def _wanted(self, mode):
317
+ if mode == "ort":
318
+ return ()
319
+ return ("openvino_cpu", "openvino_gpu")
320
+
321
+ def _ensure_built(self, mode):
322
+ wanted = [name for name in self._wanted(mode) if name not in self._ov]
323
+ if not wanted or self._building:
324
+ return
325
+ if self._openvino is None:
326
+ with self._lock:
327
+ for name in wanted:
328
+ engine = _Engine(name, None, "unavailable")
329
+ engine.reason = "openvino is not installed on this host"
330
+ self._ov[name] = engine
331
+ return
332
+ self._building = True
333
+ threading.Thread(target=self._build, args=(wanted,), name="detector-openvino-build", daemon=True).start()
334
+
335
+ def _build(self, names):
336
+ try:
337
+ model_bytes = self._reader()
338
+ for name in names:
339
+ device = "CPU" if name == "openvino_cpu" else "GPU"
340
+ precisions = ("f32",) if device == "CPU" else GPU_PRECISIONS
341
+ engine = None
342
+ for precision in precisions:
343
+ try:
344
+ started = time.perf_counter()
345
+ session = self._factory(model_bytes, device, precision, threads=self._threads)
346
+ engine = _Engine(name, session, "probing")
347
+ engine.probe = {"frames": 0, "max_person_diff": 0.0, "max_score_diff": 0.0,
348
+ "mismatches": 0, "max_box_shift": 0.0,
349
+ "compile_ms": round((time.perf_counter() - started) * 1000.0)}
350
+ engine.next_precisions = [p for p in precisions if p != precision][:1]
351
+ self._log(f"[detector] {name} built ({session.get_providers()[0]}, "
352
+ f"{engine.probe['compile_ms']} ms); probing on {self._probe_frames} live frames")
353
+ break
354
+ except EngineUnavailable as error:
355
+ engine = _Engine(name, None, "unavailable")
356
+ engine.reason = str(error)
357
+ break
358
+ except Exception as error:
359
+ engine = _Engine(name, None, "failed")
360
+ engine.reason = f"compile {precision}: {type(error).__name__}: {error}"
361
+ with self._lock:
362
+ self._ov[name] = engine
363
+ except Exception as error:
364
+ with self._lock:
365
+ for name in names:
366
+ if name not in self._ov:
367
+ engine = _Engine(name, None, "failed")
368
+ engine.reason = f"model unavailable for OpenVINO: {error}"
369
+ self._ov[name] = engine
370
+ finally:
371
+ self._building = False
372
+
373
+ def _rebuild_gpu_precision(self, engine):
374
+ """FP16 missed parity on the iGPU: try the next precision once, in the background."""
375
+ rest = getattr(engine, "next_precisions", [])
376
+ if not rest:
377
+ return
378
+ precision = rest[0]
379
+
380
+ def build():
381
+ try:
382
+ session = self._factory(self._reader(), "GPU", precision, threads=None)
383
+ fresh = _Engine(engine.name, session, "probing")
384
+ fresh.probe = {"frames": 0, "max_person_diff": 0.0, "max_score_diff": 0.0, "mismatches": 0,
385
+ "max_box_shift": 0.0, "previous": {"precision": engine.precision, **(engine.probe or {})}}
386
+ fresh.next_precisions = []
387
+ with self._lock:
388
+ self._ov[engine.name] = fresh
389
+ self._log(f"[detector] {engine.name} rebuilt in {precision} after {engine.precision} missed parity")
390
+ except Exception as error:
391
+ engine.reason = f"{engine.reason}; {precision} rebuild failed: {error}"
392
+
393
+ threading.Thread(target=build, name="detector-openvino-rebuild", daemon=True).start()
394
+
395
+ # ── running ───────────────────────────────────────────────────────────
396
+ def _serving(self, mode):
397
+ """(primary, secondary): the engines that answer frames under `mode` now."""
398
+ with self._lock:
399
+ cpu, gpu = self._ov.get("openvino_cpu"), self._ov.get("openvino_gpu")
400
+ primary = cpu if mode in ("openvino", "openvino_gpu") and cpu is not None and cpu.state == "passed" else self._ort
401
+ secondary = gpu if mode == "openvino_gpu" and gpu is not None and gpu.state == "passed" else None
402
+ return primary, secondary
403
+
404
+ def run(self, output_names, feeds, min_confidence=None):
405
+ """Answer one frame; returns (outputs, engine name). `min_confidence` is the request's floor."""
406
+ mode = self._policy.mode
407
+ self._ensure_built(mode)
408
+ with self._lock:
409
+ probing = [e for e in self._ov.values() if e.state == "probing"]
410
+ if probing:
411
+ # While any engine is on probation ORT answers, and is the reference.
412
+ outputs = self._run_on(self._ort, output_names, feeds)
413
+ for engine in probing:
414
+ self._probe_one(engine, output_names, feeds, outputs)
415
+ return outputs, self._ort.name
416
+ primary, secondary = self._serving(mode)
417
+ if min_confidence is not None and min_confidence < OPENVINO_MIN_CONFIDENCE:
418
+ primary, secondary = self._ort, None
419
+ if secondary is None:
420
+ return self._run_on(primary, output_names, feeds), primary.name
421
+ # Two engines: whichever is free; both busy waits on the primary.
422
+ if primary.lock.acquire(blocking=False):
423
+ engine = primary
424
+ elif secondary.lock.acquire(blocking=False):
425
+ engine = secondary
426
+ else:
427
+ primary.lock.acquire()
428
+ engine = primary
429
+ return self._run_locked(engine, output_names, feeds), engine.name
430
+
431
+ def _run_locked(self, engine, output_names, feeds):
432
+ try:
433
+ return self._guarded(engine, output_names, feeds)
434
+ finally:
435
+ engine.lock.release()
436
+
437
+ def _run_on(self, engine, output_names, feeds):
438
+ with engine.lock:
439
+ return self._guarded(engine, output_names, feeds)
440
+
441
+ def _guarded(self, engine, output_names, feeds):
442
+ if engine is self._ort:
443
+ return engine.run(output_names, feeds)
444
+ try:
445
+ return engine.run(output_names, feeds)
446
+ except Exception as error:
447
+ engine.state = "failed"
448
+ engine.reason = f"run: {type(error).__name__}: {error}"
449
+ self._log(f"[detector] {engine.name} failed at run time; ONNX Runtime answers: {engine.reason}")
450
+ with self._ort.lock:
451
+ return self._ort.run(output_names, feeds)
452
+
453
+ def _probe_one(self, engine, output_names, feeds, reference):
454
+ try:
455
+ with engine.lock:
456
+ candidate = engine.run(output_names, feeds)
457
+ except Exception as error:
458
+ engine.state = "failed"
459
+ engine.reason = f"probe run: {type(error).__name__}: {error}"
460
+ return
461
+ graded = self._compare(reference, candidate)
462
+ probe = engine.probe
463
+ probe["frames"] += 1
464
+ probe["max_person_diff"] = max(probe["max_person_diff"], float(graded["person_diff"]))
465
+ probe["max_score_diff"] = max(probe["max_score_diff"], float(graded["score_diff"]))
466
+ probe["mismatches"] += int(graded["mismatches"])
467
+ probe["max_box_shift"] = max(probe.get("max_box_shift", 0.0), float(graded.get("box_shift", 0.0)))
468
+ probe["ort_ms_p50"] = self._ort.snapshot()["ms_p50"]
469
+ probe["ms_p50"] = engine.snapshot()["ms_p50"]
470
+ if probe["frames"] < self._probe_frames:
471
+ return
472
+ passed = (probe["max_person_diff"] <= PROBE_PERSON_TOLERANCE and probe["mismatches"] == 0
473
+ and probe.get("max_box_shift", 0.0) <= PROBE_BOX_TOLERANCE)
474
+ engine.state = "passed" if passed else "failed"
475
+ if not passed:
476
+ engine.reason = (f"parity: person diff {probe['max_person_diff']:.2e} (bar {PROBE_PERSON_TOLERANCE:.0e}), "
477
+ f"{probe['mismatches']} detection mismatch(es), box shift {probe.get('max_box_shift', 0.0):.4f} "
478
+ f"(bar {PROBE_BOX_TOLERANCE}) over {probe['frames']} frames")
479
+ self._log(f"[detector] {engine.name} parity probe {'PASSED' if passed else 'FAILED'}: {probe}")
480
+ if not passed and engine.name == "openvino_gpu":
481
+ self._rebuild_gpu_precision(engine)
482
+
483
+ # ── reporting ─────────────────────────────────────────────────────────
484
+ def snapshot(self):
485
+ """The `/health` view: what was asked, which engines answer now, and each engine's record."""
486
+ mode = self._policy.mode
487
+ with self._lock:
488
+ probing = any(e.state == "probing" for e in self._ov.values())
489
+ engines = {"ort": self._ort.snapshot(), **{name: e.snapshot() for name, e in self._ov.items()}}
490
+ primary, secondary = (self._ort, None) if probing else self._serving(mode)
491
+ return {"requested": mode, "openvino": self._openvino,
492
+ "serving": [primary.name] + ([secondary.name] if secondary is not None else []),
493
+ "engines": engines}
494
+
495
+ def get_providers(self):
496
+ return self._ort.session.get_providers()
@@ -137,6 +137,7 @@ class DetectorLoader:
137
137
  log=print,
138
138
  accelerator: str = "auto",
139
139
  task_models: tuple = BUILTIN_TASK_MODELS,
140
+ backend_policy=None,
140
141
  ):
141
142
  # `ram_budget_bytes=None` is NOT a default policy — it is the absence of
142
143
  # one, and the registry refuses every portfolio decision in that state
@@ -147,6 +148,10 @@ class DetectorLoader:
147
148
  self.model_name = model_name
148
149
  self._intra_op_threads = intra_op_threads
149
150
  self._accelerator = accelerator
151
+ # Plan v7 W8: the engine this service was asked for (POST /backend, serve
152
+ # --backend). Shared by every RF-DETR session; "ort" builds nothing new.
153
+ from .execution import BackendPolicy
154
+ self.backend_policy = backend_policy if backend_policy is not None else BackendPolicy("ort")
150
155
  self._poll_interval_s = poll_interval_s
151
156
  self._log = log
152
157
 
@@ -177,7 +182,8 @@ class DetectorLoader:
177
182
  from .pose import PoseEstimator
178
183
  return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
179
184
  from .session import OnnxDetector
180
- return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator)
185
+ return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator,
186
+ backend_policy=self.backend_policy)
181
187
 
182
188
  def _probe(self, session: OnnxDetector) -> None:
183
189
  """PROVE IT. A session that constructs is not a session that infers.
@@ -307,9 +313,14 @@ class DetectorLoader:
307
313
  "acceleration",
308
314
  "output_layout",
309
315
  "preprocessing",
316
+ "backend",
310
317
  ):
311
318
  if key in pinned:
312
319
  payload[key] = pinned[key]
320
+ policy = getattr(self, "backend_policy", None)
321
+ if "backend" not in payload and policy is not None:
322
+ # Not loaded yet, or a pinned model on ORT only: still say what was asked.
323
+ payload["backend"] = {"requested": policy.mode, "serving": None, "engines": {}}
313
324
  return payload
314
325
 
315
326
  def prepare_models(self, models: list) -> list:
@@ -912,6 +912,12 @@ class ModelRegistry:
912
912
  if isinstance(acceleration, dict):
913
913
  row["acceleration"] = acceleration
914
914
  row["providers"] = acceleration["providers"]
915
+ # Which engines answer this model, and each one's parity probe and
916
+ # busy time (plan v7 W8). Absent for a model that runs on ORT only.
917
+ backend = getattr(s.session, "backend_snapshot", None)
918
+ backend = backend() if callable(backend) else None
919
+ if isinstance(backend, dict):
920
+ row["backend"] = backend
915
921
  # WHICH DECODE AND WHICH PREPROCESSING PRODUCED THIS MODEL'S
916
922
  # NUMBERS. Three layouts now reach this service and their
917
923
  # preprocessors are not interchangeable, so a confidence with
@@ -247,6 +247,10 @@ def build_detect_response(batch) -> dict:
247
247
  if not isinstance(batch.inference_ms, (int, float)) or not math.isfinite(batch.inference_ms):
248
248
  raise ValueError(f"inference_ms is not a finite number: {batch.inference_ms!r}")
249
249
  _assert_wire_safe(body["detections"])
250
+ # Which engine answered (plan v7 W8), when one other than plain ORT may have.
251
+ engine = getattr(batch, "engine", None)
252
+ if isinstance(engine, str) and engine:
253
+ body["engine"] = engine
250
254
  hashes = getattr(batch, "model_hashes", {})
251
255
  if hashes:
252
256
  if (not isinstance(hashes, dict) or set(hashes) != set(models)
@@ -440,6 +444,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
440
444
  self._vehicle_end()
441
445
  elif self._path() == "/ready":
442
446
  self._ready()
447
+ elif self._path() == "/backend":
448
+ self._backend()
443
449
  elif self._path() == "/health":
444
450
  self._json(405, {"error": "/health is a GET"})
445
451
  else:
@@ -493,6 +499,37 @@ class DetectorHandler(BaseHTTPRequestHandler):
493
499
  # client's receipt; they must not make its next preparation look busy.
494
500
  self._json(status, payload)
495
501
 
502
+ # ── /backend (plan v7 W8) ────────────────────────────────────────────────
503
+
504
+ def _backend(self) -> None:
505
+ """Set the engine this service was asked for: `{"version": 1, "mode": ...}`.
506
+
507
+ The connector relays the server's runtime_config CCTV_DETECTOR_BACKEND
508
+ here when it changes. Answers with the engines' view; an OpenVINO engine
509
+ serves only after its own parity probe, so this never switches an engine
510
+ on by itself.
511
+ """
512
+ from .execution import BACKEND_MODES
513
+ try:
514
+ raw = json.loads(self._read_body().decode("utf-8"))
515
+ except WireError as err:
516
+ self._json(err.status, {"error": err.message})
517
+ return
518
+ except (ValueError, UnicodeDecodeError):
519
+ self._json(400, {"error": "body must be JSON"})
520
+ return
521
+ if (not isinstance(raw, dict) or set(raw) != {"version", "mode"} or raw["version"] != 1
522
+ or raw["mode"] not in BACKEND_MODES):
523
+ self._json(400, {"error": f"expected {{version: 1, mode: one of {list(BACKEND_MODES)}}}"})
524
+ return
525
+ policy = getattr(self.server.loader, "backend_policy", None)
526
+ if policy is None:
527
+ self._json(503, {"error": "backend selection is unavailable in this detector build"})
528
+ return
529
+ policy.set(raw["mode"])
530
+ info = self.server.loader.info()
531
+ self._json(200, {"version": 1, "mode": policy.mode, "backend": info.get("backend")})
532
+
496
533
  # ── /health ──────────────────────────────────────────────────────────────
497
534
 
498
535
  def _health(self) -> None:
@@ -626,6 +663,10 @@ class DetectorHandler(BaseHTTPRequestHandler):
626
663
  body["tracking"] = self.server.actor_tracker.track(
627
664
  tracking, batch.detections, image_sha256=hashlib.sha256(jpeg).hexdigest(),
628
665
  model_hashes=body.get("model_hashes"), frame_size=frame_size, appearance=appearance)
666
+ # The camera's live per-track records (track_records.py), beside the
667
+ # signed tracking proof and outside its digest.
668
+ body["tracks"] = [record.to_wire() for record in
669
+ self.server.actor_tracker.track_records(tracking["cameraId"])]
629
670
  if detection_pose is not None:
630
671
  indices = detection_pose_indices(batch.detections, detection_pose)
631
672
  boxes = [[d.x, d.y, d.x + d.width, d.y + d.height]