@trillboards/edge-sdk 0.2.175 → 0.2.178

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/deploy/detector/README.md +6 -0
  2. package/deploy/detector/pyproject.toml +4 -1
  3. package/deploy/detector/tests/test_actor_action_wire.py +103 -0
  4. package/deploy/detector/tests/test_actor_actions.py +94 -0
  5. package/deploy/detector/tests/test_actor_checkpoint.py +99 -0
  6. package/deploy/detector/tests/test_actor_tracking.py +191 -0
  7. package/deploy/detector/tests/test_actor_tracking_wire.py +120 -0
  8. package/deploy/detector/tests/test_actor_views.py +108 -0
  9. package/deploy/detector/tests/test_compute_budget.py +28 -0
  10. package/deploy/detector/tests/test_detector_request.py +52 -0
  11. package/deploy/detector/tests/test_input_recipe.py +77 -0
  12. package/deploy/detector/tests/test_loader.py +20 -1
  13. package/deploy/detector/tests/test_pose.py +66 -0
  14. package/deploy/detector/tests/test_server.py +83 -0
  15. package/deploy/detector/tests/test_task_registry.py +74 -0
  16. package/deploy/detector/trillboards_detector/_version.py +2 -2
  17. package/deploy/detector/trillboards_detector/actor_action_contract.py +65 -0
  18. package/deploy/detector/trillboards_detector/actor_action_wire.py +189 -0
  19. package/deploy/detector/trillboards_detector/actor_actions.py +115 -0
  20. package/deploy/detector/trillboards_detector/actor_checkpoint.py +91 -0
  21. package/deploy/detector/trillboards_detector/actor_tracking.py +337 -0
  22. package/deploy/detector/trillboards_detector/actor_views.py +102 -0
  23. package/deploy/detector/trillboards_detector/cli.py +12 -2
  24. package/deploy/detector/trillboards_detector/labels.py +32 -0
  25. package/deploy/detector/trillboards_detector/loader.py +30 -4
  26. package/deploy/detector/trillboards_detector/pose.py +129 -0
  27. package/deploy/detector/trillboards_detector/registry.py +92 -7
  28. package/deploy/detector/trillboards_detector/server.py +131 -3
  29. package/deploy/detector/trillboards_detector/session.py +83 -49
  30. package/deploy/detector/trillboards_detector/tasks.py +27 -0
  31. package/deploy/detector/trillboards_detector/worker_budget.py +14 -0
  32. package/deploy/windows/README.md +1 -1
  33. package/deploy/windows/ci/test-published-install.ps1 +16 -3
  34. package/dist/ActorActionClient.d.ts +47 -0
  35. package/dist/ActorActionClient.d.ts.map +1 -0
  36. package/dist/ActorActionClient.js +187 -0
  37. package/dist/ActorActionClient.js.map +1 -0
  38. package/dist/ActorActionProducer.d.ts +17 -0
  39. package/dist/ActorActionProducer.d.ts.map +1 -0
  40. package/dist/ActorActionProducer.js +92 -0
  41. package/dist/ActorActionProducer.js.map +1 -0
  42. package/dist/ActorPerception.d.ts +65 -0
  43. package/dist/ActorPerception.d.ts.map +1 -0
  44. package/dist/ActorPerception.js +124 -0
  45. package/dist/ActorPerception.js.map +1 -0
  46. package/dist/ActorPose.d.ts +42 -0
  47. package/dist/ActorPose.d.ts.map +1 -0
  48. package/dist/ActorPose.js +62 -0
  49. package/dist/ActorPose.js.map +1 -0
  50. package/dist/ActorReview.d.ts +51 -0
  51. package/dist/ActorReview.d.ts.map +1 -0
  52. package/dist/ActorReview.js +84 -0
  53. package/dist/ActorReview.js.map +1 -0
  54. package/dist/ActorTracking.d.ts +33 -0
  55. package/dist/ActorTracking.d.ts.map +1 -0
  56. package/dist/ActorTracking.js +90 -0
  57. package/dist/ActorTracking.js.map +1 -0
  58. package/dist/ActorWindow.d.ts +56 -0
  59. package/dist/ActorWindow.d.ts.map +1 -0
  60. package/dist/ActorWindow.js +86 -0
  61. package/dist/ActorWindow.js.map +1 -0
  62. package/dist/ActorWindowContract.d.ts +5 -0
  63. package/dist/ActorWindowContract.d.ts.map +1 -0
  64. package/dist/ActorWindowContract.js +23 -0
  65. package/dist/ActorWindowContract.js.map +1 -0
  66. package/dist/AudienceAnalyzeClient.d.ts +4 -0
  67. package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
  68. package/dist/AudienceAnalyzeClient.js.map +1 -1
  69. package/dist/ChannelProber.d.ts.map +1 -1
  70. package/dist/ChannelProber.js +8 -0
  71. package/dist/ChannelProber.js.map +1 -1
  72. package/dist/CloudSensingManager.d.ts +7 -0
  73. package/dist/CloudSensingManager.d.ts.map +1 -1
  74. package/dist/CloudSensingManager.js +31 -0
  75. package/dist/CloudSensingManager.js.map +1 -1
  76. package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
  77. package/dist/DurableVlmCandidateSpool.js +31 -11
  78. package/dist/DurableVlmCandidateSpool.js.map +1 -1
  79. package/dist/EdgeAgent.js +1 -1
  80. package/dist/LocalDetectorClient.d.ts +5 -1
  81. package/dist/LocalDetectorClient.d.ts.map +1 -1
  82. package/dist/LocalDetectorClient.js +48 -3
  83. package/dist/LocalDetectorClient.js.map +1 -1
  84. package/dist/RetainedFieldContract.d.ts +2 -1
  85. package/dist/RetainedFieldContract.d.ts.map +1 -1
  86. package/dist/RetainedFieldContract.js +17 -5
  87. package/dist/RetainedFieldContract.js.map +1 -1
  88. package/dist/RuntimeModelSelection.d.ts.map +1 -1
  89. package/dist/RuntimeModelSelection.js +63 -2
  90. package/dist/RuntimeModelSelection.js.map +1 -1
  91. package/dist/TemporalReviewClient.d.ts.map +1 -1
  92. package/dist/TemporalReviewClient.js +20 -1
  93. package/dist/TemporalReviewClient.js.map +1 -1
  94. package/dist/TemporalReviewJournal.d.ts +10 -1
  95. package/dist/TemporalReviewJournal.d.ts.map +1 -1
  96. package/dist/TemporalReviewJournal.js +42 -10
  97. package/dist/TemporalReviewJournal.js.map +1 -1
  98. package/dist/connectorUpdate.d.ts +2 -0
  99. package/dist/connectorUpdate.d.ts.map +1 -1
  100. package/dist/connectorUpdate.js +33 -7
  101. package/dist/connectorUpdate.js.map +1 -1
  102. package/dist/diagnose.d.ts +1 -1
  103. package/dist/diagnose.js +1 -1
  104. package/dist/index.d.ts +6 -0
  105. package/dist/index.d.ts.map +1 -1
  106. package/dist/index.js +12 -1
  107. package/dist/index.js.map +1 -1
  108. package/dist/localSensing.d.ts +4 -0
  109. package/dist/localSensing.d.ts.map +1 -1
  110. package/dist/localSensing.js +43 -8
  111. package/dist/localSensing.js.map +1 -1
  112. package/package.json +5 -5
  113. package/retained/RetainedCaptureClient.cjs +55 -9
  114. package/retained/__tests__/actor-perception.test.cjs +125 -0
  115. package/retained/__tests__/actor-window.test.cjs +78 -0
  116. package/retained/__tests__/runtime.test.cjs +44 -4
  117. package/retained/actor-window.cjs +62 -0
  118. package/retained/manager.cjs +14 -2
@@ -0,0 +1,337 @@
1
+ """Camera-bound actor identities from the pinned ByteTrack implementation.
2
+
3
+ SQLite commits frame inputs and identities together. In-memory Kalman state is a
4
+ cache: restart recovery replays the bounded detection journal through the same
5
+ tracker, restoring external actor IDs from exact saved detection associations.
6
+ Predicted boxes are never returned as new visual observations.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import importlib.metadata
12
+ import json
13
+ import math
14
+ import re
15
+ import sqlite3
16
+ import threading
17
+ import uuid
18
+ from dataclasses import asdict
19
+ from pathlib import Path
20
+
21
+ from .detections import Detection
22
+ from .actor_checkpoint import encode_tracker, restore_tracker
23
+
24
+ ALGORITHM = "bytetrack_2.6.1"
25
+ _ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
26
+ _SHA = re.compile(r"[a-f0-9]{64}\Z")
27
+ _SAFE_INTEGER = 9007199254740991
28
+
29
+
30
+ class TrackingError(ValueError):
31
+ """Invalid or unprovable tracking input; never an empty-scene result."""
32
+
33
+
34
+ class _RecoveryConflict(TrackingError):
35
+ pass
36
+
37
+
38
+ def _json(value):
39
+ return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
40
+
41
+
42
+ def _hash(value):
43
+ return hashlib.sha256(_json(value).encode()).hexdigest()
44
+
45
+
46
+ def validate_tracking_request(value):
47
+ if (not isinstance(value, dict)
48
+ or set(value) != {"version", "cameraId", "sourceId", "frameId", "capturedAtMs"}
49
+ or type(value["version"]) is not int or value["version"] != 1
50
+ or any(not isinstance(value[k], str) or not _ID.fullmatch(value[k])
51
+ for k in ("cameraId", "sourceId", "frameId"))
52
+ or type(value["capturedAtMs"]) is not int
53
+ or not 0 <= value["capturedAtMs"] <= _SAFE_INTEGER):
54
+ raise TrackingError("Invalid captured actor tracking identity")
55
+ return dict(value)
56
+
57
+
58
+ class ActorTrackerJournal:
59
+ """Bounded, serialized tracking store. A camera is never silently evicted.
60
+
61
+ Capture timestamps, not inference time, drive motion and track expiration.
62
+ A stream/view/model discontinuity starts a new epoch and explicitly ends
63
+ prior actors. A normal service restart preserves actors through replay.
64
+ """
65
+ def __init__(self, path, *, max_cameras=64, retained_frames=256,
66
+ frame_rate=5.0, max_lost_ms=3000, max_gap_ms=5000):
67
+ if (type(max_cameras) is not int or not 1 <= max_cameras <= 256
68
+ or type(retained_frames) is not int or not 32 <= retained_frames <= 512
69
+ or not isinstance(frame_rate, (int, float)) or isinstance(frame_rate, bool)
70
+ or not math.isfinite(frame_rate) or not 1 <= frame_rate <= 30
71
+ or type(max_lost_ms) is not int or not 200 <= max_lost_ms <= 5000
72
+ or type(max_gap_ms) is not int or not max_lost_ms <= max_gap_ms <= 30000
73
+ or retained_frames < math.ceil(frame_rate * max_lost_ms / 1000) + 3):
74
+ raise TrackingError("Invalid bounded tracker configuration")
75
+ if importlib.metadata.version("trackers") != "2.6.1":
76
+ raise TrackingError("Actor tracking requires trackers==2.6.1")
77
+ from trackers import ByteTrackTracker
78
+ self._tracker_class = ByteTrackTracker
79
+ self._config = {"algorithm": ALGORITHM, "frameRate": frame_rate, "maxLostMs": max_lost_ms,
80
+ "maxGapMs": max_gap_ms, "minimumConsecutiveFrames": 2,
81
+ "activationThreshold": .5, "highConfidenceThreshold": .5, "minimumIou": .1}
82
+ self._max_cameras, self._retained_frames = max_cameras, retained_frames
83
+ self._lock = threading.RLock()
84
+ self._cache = {}
85
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
86
+ self._db = sqlite3.connect(str(path), timeout=30, check_same_thread=False, isolation_level=None)
87
+ self._db.execute("PRAGMA journal_mode=WAL")
88
+ self._db.execute("PRAGMA synchronous=FULL")
89
+ self._db.executescript("""
90
+ CREATE TABLE IF NOT EXISTS actor_camera (
91
+ camera_id TEXT PRIMARY KEY, context_hash TEXT NOT NULL, epoch TEXT NOT NULL,
92
+ captured_at INTEGER NOT NULL, seq INTEGER NOT NULL, active_json TEXT NOT NULL);
93
+ CREATE TABLE IF NOT EXISTS actor_frame (
94
+ camera_id TEXT NOT NULL, frame_id TEXT NOT NULL, seq INTEGER NOT NULL,
95
+ epoch TEXT NOT NULL, fingerprint TEXT NOT NULL,
96
+ input_json TEXT NOT NULL, result_json TEXT NOT NULL,
97
+ PRIMARY KEY(camera_id,frame_id), UNIQUE(camera_id,seq));
98
+ CREATE INDEX IF NOT EXISTS actor_frame_replay ON actor_frame(camera_id,epoch,seq);
99
+ CREATE TABLE IF NOT EXISTS actor_checkpoint (
100
+ camera_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, payload_json TEXT NOT NULL);
101
+ """)
102
+
103
+ def close(self):
104
+ with self._lock:
105
+ self._cache.clear()
106
+ self._db.close()
107
+
108
+ def retained_frames(self, camera_id):
109
+ with self._lock:
110
+ return self._db.execute("SELECT count(*) FROM actor_frame WHERE camera_id=?", (camera_id,)).fetchone()[0]
111
+
112
+ def _new_tracker(self, config=None):
113
+ config = self._config if config is None else config
114
+ return self._tracker_class(
115
+ lost_track_buffer=math.ceil(config["maxLostMs"] * 30 / 1000),
116
+ frame_rate=config["frameRate"], track_activation_threshold=config["activationThreshold"],
117
+ minimum_consecutive_frames=config["minimumConsecutiveFrames"],
118
+ high_conf_det_threshold=config["highConfidenceThreshold"], minimum_iou_threshold=config["minimumIou"])
119
+
120
+ def _update(self, tracker, values, captured_at):
121
+ import numpy as np
122
+ import supervision as sv
123
+ indices = [i for i, d in enumerate(values) if d["label"] == "person"]
124
+ boxes = np.array([[values[i]["x"], values[i]["y"], values[i]["x"] + values[i]["width"],
125
+ values[i]["y"] + values[i]["height"]] for i in indices],
126
+ dtype=np.float32).reshape(-1, 4)
127
+ ds = sv.Detections(xyxy=boxes,
128
+ confidence=np.array([values[i]["confidence"] for i in indices], dtype=np.float32),
129
+ class_id=np.zeros(len(indices), dtype=int),
130
+ data={"detection_index": np.array(indices, dtype=int)})
131
+ result = tracker.update(ds, timestamp=captured_at / 1000)
132
+ return [(int(i), int(t)) for i, t in zip(result.data.get("detection_index", []), result.tracker_id)]
133
+
134
+
135
+ def _checked_frame(self, camera, frame_id, epoch, fingerprint, input_json, result_json):
136
+ try:
137
+ frame, result = json.loads(input_json), json.loads(result_json)
138
+ unsigned = {k: v for k, v in result.items() if k != "sha256"}
139
+ if (fingerprint != _hash(frame) or result.get("sha256") != _hash(unsigned)
140
+ or result.get("cameraId") != camera or result.get("frameId") != frame_id
141
+ or result.get("epoch") != epoch or frame.get("request") != {
142
+ k: result.get(k) for k in ("version", "cameraId", "sourceId", "frameId", "capturedAtMs")}
143
+ or result.get("imageSha256") != frame.get("imageSha256")
144
+ or result.get("contextSha256") != frame.get("contextSha256")):
145
+ raise ValueError("journal identity or digest changed")
146
+ return frame, result
147
+ except (ValueError, TypeError, KeyError) as error:
148
+ raise TrackingError("Corrupt actor frame journal") from error
149
+
150
+ @staticmethod
151
+ def _checked_active(value, camera, epoch):
152
+ try:
153
+ actors = json.loads(value)
154
+ if (not isinstance(actors, list) or len(actors) > 256
155
+ or any(not isinstance(a, dict) or set(a) != {"kind", "cameraId", "id", "epoch"}
156
+ or a["kind"] != "person" or a["cameraId"] != camera or a["epoch"] != epoch
157
+ or not isinstance(a["id"], str) or not _ID.fullmatch(a["id"]) for a in actors)
158
+ or len({a["id"] for a in actors}) != len(actors)):
159
+ raise ValueError("camera head actor identity changed")
160
+ return actors
161
+ except (ValueError, TypeError, KeyError) as error:
162
+ raise TrackingError("Corrupt actor camera journal") from error
163
+
164
+
165
+
166
+ def _save_checkpoint(self, camera, context, epoch, captured_at, seq, fingerprint, tracker, identities):
167
+ try:
168
+ payload={"version":1,"cameraId":camera,"contextSha256":context,"epoch":epoch,
169
+ "capturedAtMs":captured_at,"seq":seq,"frameFingerprint":fingerprint,"state":encode_tracker(tracker,identities)}
170
+ payload["sha256"]=_hash(payload)
171
+ raw=_json(payload)
172
+ if len(raw.encode())>8*1024**2:raise ValueError("Tracker checkpoint exceeds capacity")
173
+ self._db.execute("INSERT INTO actor_checkpoint VALUES (?,?,?) ON CONFLICT(camera_id) DO UPDATE SET "
174
+ "seq=excluded.seq,payload_json=excluded.payload_json",(camera,seq,raw))
175
+ except (ValueError,TypeError,KeyError) as error:
176
+ raise TrackingError("Cannot checkpoint physical tracker state") from error
177
+
178
+ def _load_checkpoint(self,camera,head,config):
179
+ record=self._db.execute("SELECT seq,payload_json FROM actor_checkpoint WHERE camera_id=?",(camera,)).fetchone()
180
+ if record is None:return None
181
+ try:
182
+ if len(record[1].encode())>8*1024**2:raise ValueError("oversized checkpoint")
183
+ payload=json.loads(record[1]);unsigned={k:v for k,v in payload.items() if k!="sha256"}
184
+ if (set(payload)!={"version","cameraId","contextSha256","epoch","capturedAtMs","seq","frameFingerprint","state","sha256"}
185
+ or type(payload["version"]) is not int or payload["version"]!=1 or payload["sha256"]!=_hash(unsigned)
186
+ or payload["cameraId"]!=camera or payload["contextSha256"]!=head[0]
187
+ or payload["epoch"]!=head[1] or payload["capturedAtMs"]!=head[2]
188
+ or payload["seq"]!=head[3] or record[0]!=head[3]):
189
+ raise ValueError("checkpoint identity changed")
190
+ # Preserve the existing journal's integrity checks. A checkpoint
191
+ # cannot make a corrupt retained receipt look healthy.
192
+ verified_head=False
193
+ for frame_id,fingerprint,input_json,result_json,seq in self._db.execute(
194
+ "SELECT frame_id,fingerprint,input_json,result_json,seq FROM actor_frame WHERE camera_id=? AND epoch=? ORDER BY seq",
195
+ (camera,head[1])):
196
+ _,result=self._checked_frame(camera,frame_id,head[1],fingerprint,input_json,result_json)
197
+ if seq==head[3]:
198
+ verified_head=(fingerprint==payload["frameFingerprint"]
199
+ and result["capturedAtMs"]==head[2] and result["contextSha256"]==head[0])
200
+ if not verified_head:raise ValueError("checkpoint lost its committed frame proof")
201
+ tracker,identities=restore_tracker(self._new_tracker(config),payload["state"],
202
+ lambda actors:self._checked_active(_json(actors),camera,head[1]))
203
+ if tracker._last_timestamp!=head[2]/1000:raise ValueError("checkpoint capture time changed")
204
+ active=self._checked_active(head[4],camera,head[1])
205
+ if sorted(_json(a) for a in identities.values())!=sorted(_json(a) for a in active):
206
+ raise ValueError("checkpoint actor bindings changed")
207
+ return tracker,identities
208
+ except (ValueError,TypeError,KeyError) as error:
209
+ raise TrackingError("Corrupt physical tracker checkpoint") from error
210
+
211
+ def _recover(self, camera, epoch, config=None, expected_active=None):
212
+ tracker, identities = self._new_tracker(config), {}
213
+ reverse = {}
214
+ for frame_id, fingerprint, input_json, result_json in self._db.execute(
215
+ "SELECT frame_id,fingerprint,input_json,result_json FROM actor_frame WHERE camera_id=? AND epoch=? ORDER BY seq",
216
+ (camera, epoch)):
217
+ frame, saved = self._checked_frame(camera, frame_id, epoch, fingerprint, input_json, result_json)
218
+ expected = {a["detectionIndex"]: a["actor"] for a in saved["actors"]}
219
+ for index, tid in self._update(tracker, frame["detections"], frame["request"]["capturedAtMs"]):
220
+ if tid < 0 or index not in expected:
221
+ continue
222
+ actor = expected[index]
223
+ if (tid in identities and identities[tid] != actor
224
+ or actor["id"] in reverse and reverse[actor["id"]] != tid):
225
+ raise _RecoveryConflict("Actor history cannot prove a unique tracker recovery")
226
+ identities[tid], reverse[actor["id"]] = actor, tid
227
+ alive_ids = set(int(t) for t in tracker.tracked_objects.tracker_id)
228
+ identities = {tid: actor for tid, actor in identities.items() if tid in alive_ids}
229
+ if expected_active is not None and (
230
+ set(identities) != alive_ids
231
+ or sorted(_json(a) for a in identities.values()) != sorted(_json(a) for a in expected_active)):
232
+ raise _RecoveryConflict("Bounded actor history cannot prove the committed active identities")
233
+ return tracker, identities
234
+
235
+ def track(self, request, detections, *, image_sha256, model_hashes, frame_size):
236
+ request = validate_tracking_request(request)
237
+ if (not isinstance(image_sha256, str) or not _SHA.fullmatch(image_sha256)
238
+ or not isinstance(model_hashes, dict) or not 1 <= len(model_hashes) <= 16
239
+ or any(not isinstance(k, str) or not _ID.fullmatch(k)
240
+ or not isinstance(v, str) or not _SHA.fullmatch(v) for k, v in model_hashes.items())
241
+ or not isinstance(frame_size, (list, tuple)) or len(frame_size) != 2
242
+ or any(type(v) is not int or v < 2 or v > 8192 for v in frame_size)
243
+ or frame_size[0] * frame_size[1] > 16777216
244
+ or not isinstance(detections, list) or len(detections) > 256):
245
+ raise TrackingError("Invalid actor frame or loaded-model proof")
246
+ values = []
247
+ for d in detections:
248
+ if (not isinstance(d, Detection) or not isinstance(d.label, str) or not d.label
249
+ or any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v)
250
+ for v in (d.confidence, d.x, d.y, d.width, d.height))
251
+ or not 0 <= d.confidence <= 1 or min(d.x, d.y) < 0 or min(d.width, d.height) <= 0
252
+ or d.x + d.width > frame_size[0] + .01 or d.y + d.height > frame_size[1] + .01):
253
+ raise TrackingError("Invalid detection in actor frame")
254
+ values.append(asdict(d))
255
+ config = self._config
256
+ context = _hash({"sourceId": request["sourceId"], "modelHashes": model_hashes,
257
+ "frameSize": list(frame_size), "config": config})
258
+ inputs = {"request": request, "detections": values, "imageSha256": image_sha256,
259
+ "contextSha256": context}
260
+ fingerprint = _hash(inputs)
261
+ camera, captured_at = request["cameraId"], request["capturedAtMs"]
262
+ with self._lock:
263
+ self._db.execute("BEGIN IMMEDIATE")
264
+ try:
265
+ own = self._db.execute(
266
+ "SELECT fingerprint,input_json,result_json,epoch FROM actor_frame WHERE camera_id=? AND frame_id=?",
267
+ (camera, request["frameId"])).fetchone()
268
+ if own:
269
+ if own[0] != fingerprint:
270
+ raise TrackingError("Retained actor frame identity changed")
271
+ _, saved = self._checked_frame(camera, request["frameId"], own[3], own[0], own[1], own[2])
272
+ self._db.execute("COMMIT")
273
+ return saved
274
+ head = self._db.execute(
275
+ "SELECT context_hash,epoch,captured_at,seq,active_json FROM actor_camera WHERE camera_id=?",
276
+ (camera,)).fetchone()
277
+ if head and captured_at <= head[2]:
278
+ raise TrackingError("Actor frames must preserve strict capture order")
279
+ if not head and self._db.execute("SELECT count(*) FROM actor_camera").fetchone()[0] >= self._max_cameras:
280
+ raise TrackingError("Actor camera capacity exhausted; retire a disconnected camera explicitly")
281
+ continuity = ("start" if not head else "context_reset" if head[0] != context
282
+ else "gap_reset" if captured_at - head[2] > self._config["maxGapMs"] else "continued")
283
+ old_active = self._checked_active(head[4], camera, head[1]) if head else []
284
+ epoch = head[1] if continuity == "continued" else "te_" + uuid.uuid4().hex
285
+ seq = head[3]+1 if head else 1
286
+ cached = self._cache.get(camera)
287
+ if continuity != "continued":
288
+ tracker, identities = self._new_tracker(config), {}
289
+ elif cached and cached[0] == head[3]:
290
+ _, tracker, identities = cached
291
+ else:
292
+ try:
293
+ restored = self._load_checkpoint(camera,head,config)
294
+ tracker, identities = restored if restored is not None else self._recover(camera, epoch, config, old_active)
295
+ except _RecoveryConflict:
296
+ # Preserve the old receipts; never quietly attach another person
297
+ # to the old duration/latch when replay cannot establish identity.
298
+ continuity, epoch = "recovery_reset", "te_" + uuid.uuid4().hex
299
+ tracker, identities = self._new_tracker(config), {}
300
+ matches = self._update(tracker, values, captured_at)
301
+ actors, unconfirmed = [], []
302
+ for index, tid in matches:
303
+ if tid < 0:
304
+ unconfirmed.append(index)
305
+ continue
306
+ actor = identities.setdefault(tid, {"kind": "person", "cameraId": camera,
307
+ "id": "pa_" + uuid.uuid4().hex, "epoch": epoch})
308
+ actors.append({"detectionIndex": index, "actor": actor})
309
+ alive_ids = set(int(t) for t in tracker.tracked_objects.tracker_id)
310
+ active = [identities[t] for t in sorted(alive_ids) if t in identities]
311
+ identities = {t: a for t, a in identities.items() if t in alive_ids}
312
+ ended = [a for a in old_active if a not in active]
313
+ result = {"version": 1, "algorithm": ALGORITHM, **{k: request[k] for k in
314
+ ("cameraId", "sourceId", "frameId", "capturedAtMs")},
315
+ "epoch": epoch, "imageSha256": image_sha256,
316
+ "modelHashes": model_hashes, "contextSha256": context, "continuity": continuity,
317
+ "actors": sorted(actors, key=lambda a: a["detectionIndex"]),
318
+ "unconfirmed": sorted(unconfirmed), "endedActors": ended}
319
+ result["sha256"] = _hash(result)
320
+ self._db.execute("INSERT INTO actor_frame VALUES (?,?,?,?,?,?,?)",
321
+ (camera, request["frameId"], seq, epoch, fingerprint, _json(inputs), _json(result)))
322
+ self._db.execute(
323
+ "INSERT INTO actor_camera VALUES (?,?,?,?,?,?) ON CONFLICT(camera_id) DO UPDATE SET "
324
+ "context_hash=excluded.context_hash,epoch=excluded.epoch,captured_at=excluded.captured_at,"
325
+ "seq=excluded.seq,active_json=excluded.active_json",
326
+ (camera, context, epoch, captured_at, seq, _json(active)))
327
+ self._save_checkpoint(camera,context,epoch,captured_at,seq,fingerprint,tracker,identities)
328
+ self._db.execute("DELETE FROM actor_frame WHERE camera_id=? AND seq<=?",
329
+ (camera, seq-self._retained_frames))
330
+ self._db.execute("COMMIT")
331
+ self._cache[camera] = (seq, tracker, identities)
332
+ return json.loads(_json(result))
333
+ except BaseException:
334
+ if self._db.in_transaction:
335
+ self._db.execute("ROLLBACK")
336
+ self._cache.pop(camera, None)
337
+ raise
@@ -0,0 +1,102 @@
1
+ """RGB actor views shared by task inference and training extraction."""
2
+ from __future__ import annotations
3
+ import math
4
+ from PIL import Image
5
+
6
+ ACTOR_VIEW_PREPROCESSING="actor_body_upper_rgb_pad1.25_letterbox182_v2"
7
+ ACTOR_VIEW_SIZE=182
8
+
9
+ def actor_view_bounds(box,frame_size,*,upper=False):
10
+ if (not isinstance(box,(tuple,list)) or len(box)!=4
11
+ or any(type(v) not in (int,float) or not math.isfinite(v) for v in box)
12
+ or len(frame_size)!=2 or any(type(v) is not int or v<2 for v in frame_size)
13
+ or type(upper) is not bool):
14
+ raise ValueError("invalid actor view geometry")
15
+ x1,y1,x2,y2=box
16
+ if x2<=x1 or y2<=y1:raise ValueError("empty actor view")
17
+ if upper:y2=y1+(y2-y1)*.68
18
+ width=(x2-x1)*1.25;height=(y2-y1)*1.25
19
+ cx=(x1+x2)/2;cy=(y1+y2)/2
20
+ result=(max(0,math.floor(cx-width/2)),max(0,math.floor(cy-height/2)),
21
+ min(frame_size[0],math.ceil(cx+width/2)),min(frame_size[1],math.ceil(cy+height/2)))
22
+ if result[2]<=result[0] or result[3]<=result[1]:raise ValueError("actor outside original frame")
23
+ return result
24
+
25
+ def fit_actor_view(tile,*,size=ACTOR_VIEW_SIZE):
26
+ if type(size) is not int or not 16<=size<=1024:
27
+ raise ValueError("invalid actor model input size")
28
+ tile=tile.convert("RGB")
29
+ if tile.width<=size and tile.height<=size:
30
+ scale=min(size/tile.width,size/tile.height)
31
+ tile=tile.resize((max(1,round(tile.width*scale)),max(1,round(tile.height*scale))),Image.Resampling.BILINEAR)
32
+ else:
33
+ tile.thumbnail((size,size),Image.Resampling.BILINEAR)
34
+ result=Image.new("RGB",(size,size),(0,0,0))
35
+ result.paste(tile,((size-tile.width)//2,(size-tile.height)//2))
36
+ return result
37
+
38
+ def actor_view(image,box,*,upper=False,size=ACTOR_VIEW_SIZE):
39
+ if not isinstance(image,Image.Image):raise ValueError("actor view requires original RGB image")
40
+ return fit_actor_view(image.crop(actor_view_bounds(box,image.size,upper=upper)),size=size)
41
+
42
+ # Detail views retain source-resolution head/hand evidence. Localization means
43
+ # a crop can be placed; it never means a phone is present or absent.
44
+ ACTOR_DETAIL_PREPROCESSING="actor_head_wrists_rgb_h032_w028_letterbox224_v1"
45
+ ACTOR_DETAIL_SIZE=224
46
+
47
+ def actor_detail_views(image,box,joints):
48
+ from .tasks import COCO17
49
+ if not isinstance(image,Image.Image):
50
+ raise ValueError("actor details require the original RGB image")
51
+ actor_view_bounds(box,image.size)
52
+ x1,y1,x2,y2=box
53
+ if not (0<=x1<x2<=image.width and 0<=y1<y2<=image.height):
54
+ raise ValueError("actor details require an original-frame person box")
55
+ if not isinstance(joints,list) or len(joints)!=len(COCO17):
56
+ raise ValueError("actor details require all named COCO17 joints")
57
+ points={}
58
+ for joint,name in zip(joints,COCO17):
59
+ if not isinstance(joint,dict) or set(joint)!={"name","x","y","response"} or joint["name"]!=name:
60
+ raise ValueError("invalid named joint contract")
61
+ response=joint["response"];x=joint["x"];y=joint["y"]
62
+ if (type(response) not in (int,float) or not math.isfinite(response)
63
+ or (x is None)!=(y is None)
64
+ or (x is not None and (type(x) not in (int,float) or type(y) not in (int,float)
65
+ or not math.isfinite(x) or not math.isfinite(y)
66
+ or not 0<=x<image.width or not 0<=y<image.height))):
67
+ raise ValueError("invalid joint coordinates or response")
68
+ points[name]=joint
69
+ height=y2-y1;width=x2-x1
70
+ def located(name):
71
+ p=points[name]
72
+ return (p["x"] is not None and p["response"]>.15
73
+ and x1-width*.15<=p["x"]<=x2+width*.15
74
+ and y1-height*.15<=p["y"]<=y2+height*.15)
75
+ face=[points[n] for n in COCO17[:5] if located(n)]
76
+ head=(sum(p["x"] for p in face)/len(face),sum(p["y"] for p in face)/len(face)) if face else ((x1+x2)/2,y1+height*.12)
77
+ specs=[("head",head,height*.32,"pose" if face else "box_prior")]
78
+ for name in ("left_wrist","right_wrist"):
79
+ p=points[name]
80
+ specs.append((name,(p["x"],p["y"]) if located(name) else None,height*.28,"pose"))
81
+ result=[]
82
+ for name,center,side,localization in specs:
83
+ view={"name":name,"preprocessing":ACTOR_DETAIL_PREPROCESSING,
84
+ "localized":False,"localization":localization,"reason":None,
85
+ "sourceBounds":None,"sourceSize":None,"image":None}
86
+ if center is None:
87
+ view["reason"]="joint_unobserved";result.append(view);continue
88
+ cx,cy=center
89
+ bounds=[max(0,math.floor(cx-side/2)),max(0,math.floor(cy-side/2)),
90
+ min(image.width,math.ceil(cx+side/2)),min(image.height,math.ceil(cy+side/2))]
91
+ size=[bounds[2]-bounds[0],bounds[3]-bounds[1]]
92
+ view.update(sourceBounds=bounds,sourceSize=size)
93
+ if min(size)<8:
94
+ view["reason"]="too_small";result.append(view);continue
95
+ tile=image.crop(bounds).convert("RGB")
96
+ scale=min(ACTOR_DETAIL_SIZE/tile.width,ACTOR_DETAIL_SIZE/tile.height)
97
+ tile=tile.resize((max(1,round(tile.width*scale)),max(1,round(tile.height*scale))),Image.Resampling.BILINEAR)
98
+ fitted=Image.new("RGB",(ACTOR_DETAIL_SIZE,ACTOR_DETAIL_SIZE),(0,0,0))
99
+ fitted.paste(tile,((ACTOR_DETAIL_SIZE-tile.width)//2,(ACTOR_DETAIL_SIZE-tile.height)//2))
100
+ view.update(image=fitted,localized=True)
101
+ result.append(view)
102
+ return result
@@ -18,8 +18,13 @@ import threading
18
18
  from pathlib import Path
19
19
 
20
20
  from ._version import VERSION
21
+ from .worker_budget import configure_numeric_workers
22
+
23
+ # This must run before loader/server imports initialize NumPy or BLAS.
24
+ configure_numeric_workers()
21
25
  from .loader import DEFAULT_POLL_INTERVAL_S, DetectorLoader
22
- from .modelpath import DEFAULT_MODEL_NAME, resolve_model_path, resolve_ram_budget_mb
26
+ from .modelpath import DEFAULT_MODEL_NAME, resolve_model_path, resolve_ram_budget_mb, resolve_data_dir
27
+ from .actor_tracking import ActorTrackerJournal
23
28
  from .server import DEFAULT_HOST, DEFAULT_PORT, serve
24
29
 
25
30
  LOG = logging.getLogger("trillboards_detector")
@@ -107,6 +112,7 @@ def build_parser() -> argparse.ArgumentParser:
107
112
  help="prefer compatible local NVIDIA CUDA, otherwise CPU; cpu is an explicit rollback")
108
113
  s.add_argument("--log-file", default=None, help="also write to this file (Windows has no task stdout)")
109
114
  s.add_argument("--verbose", action="store_true")
115
+ s.add_argument("--tracking-journal", default=None, help="durable actor tracker SQLite path; defaults to the connector data directory")
110
116
  return p
111
117
 
112
118
 
@@ -161,10 +167,11 @@ def main(argv: "list[str] | None" = None) -> int:
161
167
  log=LOG.info,
162
168
  accelerator=args.accelerator,
163
169
  )
170
+ actor_tracker = ActorTrackerJournal(args.tracking_journal or resolve_data_dir() / "actor-tracking" / "tracks.sqlite3")
164
171
  loader.start()
165
172
 
166
173
  try:
167
- server = serve(loader, args.host, args.port, LOG.info)
174
+ server = serve(loader, args.host, args.port, LOG.info, actor_tracker=actor_tracker)
168
175
  except OSError as err:
169
176
  # A bind failure is FATAL and must stay fatal. `register-litertlm-task.ps1`
170
177
  # has the incident this mirrors: a serve left alive keeps ANSWERING, so an
@@ -173,6 +180,7 @@ def main(argv: "list[str] | None" = None) -> int:
173
180
  # restart policy do its job instead of running two half-services.
174
181
  LOG.error("[detector] could not bind %s:%s — %s", args.host, args.port, err)
175
182
  loader.stop()
183
+ actor_tracker.close()
176
184
  return 1
177
185
 
178
186
  stop = threading.Event()
@@ -193,7 +201,9 @@ def main(argv: "list[str] | None" = None) -> int:
193
201
  except KeyboardInterrupt:
194
202
  pass
195
203
  server.shutdown()
204
+ server.server_close()
196
205
  loader.stop()
206
+ actor_tracker.close()
197
207
  return 0
198
208
 
199
209
 
@@ -228,6 +228,32 @@ RFDETR_MEDIUM_ONNX = VariantContract(
228
228
  )
229
229
 
230
230
 
231
+ # Input pixels are independent of the graph's output layout. Existing catalogue
232
+ # entries retain their scored recipes; a new artifact may declare another one.
233
+ PIL_IMAGENET_STRETCH = "imagenet-normalised-stretch"
234
+ RGB_HALF_PIXEL_IMAGENET = "rgb-float-half-pixel-no-antialias-imagenet-v1"
235
+ BGR_BYTES_STRETCH = "bgr-bytes-cv2-linear-stretch-v1"
236
+ YOLO_LETTERBOX = "ultralytics-letterbox-114-centred"
237
+ LEGACY_INPUT_PREPROCESSING = MappingProxyType({
238
+ RFDETR_DETS_LABELS: PIL_IMAGENET_STRETCH,
239
+ ANCHOR_4_PLUS_NC: YOLO_LETTERBOX,
240
+ END2END_N_6: YOLO_LETTERBOX,
241
+ })
242
+ STRETCH_INPUT_RECIPES = frozenset({
243
+ PIL_IMAGENET_STRETCH, RGB_HALF_PIXEL_IMAGENET, BGR_BYTES_STRETCH,
244
+ })
245
+
246
+
247
+ def resolve_input_preprocessing(layout: str, recipe: "str | None" = None) -> str:
248
+ if layout not in LEGACY_INPUT_PREPROCESSING:
249
+ raise ValueError(f"unknown output layout {layout!r} for input preprocessing")
250
+ selected = LEGACY_INPUT_PREPROCESSING[layout] if recipe is None else recipe
251
+ allowed = STRETCH_INPUT_RECIPES if layout == RFDETR_DETS_LABELS else frozenset({YOLO_LETTERBOX})
252
+ if not isinstance(selected, str) or selected not in allowed:
253
+ raise ValueError(f"input preprocessing {selected!r} is not supported with {layout!r}")
254
+ return selected
255
+
256
+
231
257
  @dataclass(frozen=True)
232
258
  class ModelVocabulary:
233
259
  """What ONE model can be asked about, and what one resident copy costs.
@@ -295,8 +321,14 @@ class ModelVocabulary:
295
321
  #: threshold nobody fitted, on plausible boxes, with every health surface
296
322
  #: green. Declaration is the in-band signal position was standing in for.
297
323
  owns: "frozenset[str]" = frozenset()
324
+ #: Exact ONNX pixel recipe. None preserves the scored legacy catalogue.
325
+ input_preprocessing: "str | None" = None
298
326
 
299
327
  def __post_init__(self) -> None:
328
+ if not isinstance(self.onnx_variant, VariantContract):
329
+ raise ValueError("model preprocessing requires a declared ONNX variant")
330
+ object.__setattr__(self, "input_preprocessing", resolve_input_preprocessing(
331
+ self.onnx_variant.output_layout, self.input_preprocessing))
300
332
  # The model name IS A DIRECTORY NAME — `resolve_model_path` puts the
301
333
  # artifact at `models/<model-name>/model.onnx` on every platform — so it
302
334
  # is restricted to what is safe in a path on all three, and NOT folded
@@ -44,19 +44,23 @@ from __future__ import annotations
44
44
  import io
45
45
  import threading
46
46
  from pathlib import Path
47
+ from typing import TYPE_CHECKING
47
48
 
48
49
  from .detections import DetectionBatch
49
50
  from .errors import DetectorNotReady, UnknownClasses
50
51
  from .labels import BUILTIN_VOCABULARIES, ClassIndex, ModelVocabulary
51
- from .modelpath import resolve_model_path
52
+ from .modelpath import resolve_model_path, resolve_data_dir, MODEL_FILE_NAME
52
53
  from .registry import (
53
54
  STATE_LOADING,
54
55
  STATE_READY,
55
56
  STATE_UNAVAILABLE,
56
57
  ModelEntry,
57
58
  ModelRegistry,
59
+ TaskModelContract,
58
60
  )
59
- from .session import OnnxDetector
61
+ from .tasks import BUILTIN_TASK_MODELS
62
+ if TYPE_CHECKING:
63
+ from .session import OnnxDetector
60
64
 
61
65
  DEFAULT_POLL_INTERVAL_S = 30.0
62
66
 
@@ -80,6 +84,7 @@ def build_catalogue(
80
84
  default_model_path: "Path | None" = None,
81
85
  vocabularies: "tuple[ModelVocabulary, ...]" = BUILTIN_VOCABULARIES,
82
86
  env=None,
87
+ task_models: tuple = (),
83
88
  ) -> "list[ModelEntry]":
84
89
  """Every model this build can label, bound to where it would live on THIS box.
85
90
 
@@ -109,6 +114,8 @@ def build_catalogue(
109
114
  else:
110
115
  path = resolve_model_path(vocabulary.model_name, env)
111
116
  entries.append(ModelEntry(vocabulary, path, pinned=pinned))
117
+ for contract in task_models:
118
+ entries.append(ModelEntry(contract, resolve_data_dir(env) / "models" / contract.model_name / MODEL_FILE_NAME))
112
119
  # Pinned first, so catalogue order — which is what breaks a class-ownership
113
120
  # tie — puts the general detector ahead of every specialist. `person` must
114
121
  # keep coming from the model the 98.8/98.8 was measured on.
@@ -129,6 +136,7 @@ class DetectorLoader:
129
136
  vocabularies: "tuple[ModelVocabulary, ...]" = BUILTIN_VOCABULARIES,
130
137
  log=print,
131
138
  accelerator: str = "auto",
139
+ task_models: tuple = BUILTIN_TASK_MODELS,
132
140
  ):
133
141
  # `ram_budget_bytes=None` is NOT a default policy — it is the absence of
134
142
  # one, and the registry refuses every portfolio decision in that state
@@ -143,7 +151,7 @@ class DetectorLoader:
143
151
  self._log = log
144
152
 
145
153
  self.registry = ModelRegistry(
146
- entries=build_catalogue(model_name, model_path, vocabularies),
154
+ entries=build_catalogue(model_name, model_path, vocabularies, task_models=task_models),
147
155
  session_factory=self._build_session,
148
156
  probe=self._probe,
149
157
  ram_budget_bytes=ram_budget_bytes,
@@ -156,6 +164,13 @@ class DetectorLoader:
156
164
  # ── the two halves that need a real runtime ──────────────────────────────
157
165
 
158
166
  def _build_session(self, entry: ModelEntry) -> OnnxDetector:
167
+ if isinstance(entry.vocabulary, TaskModelContract):
168
+ if entry.vocabulary.task == "actor_actions":
169
+ from .actor_actions import ActorActionEstimator
170
+ return ActorActionEstimator(entry.path,entry.vocabulary,intra_op_threads=self._intra_op_threads or 4,accelerator=self._accelerator)
171
+ from .pose import PoseEstimator
172
+ return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
173
+ from .session import OnnxDetector
159
174
  return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator)
160
175
 
161
176
  def _probe(self, session: OnnxDetector) -> None:
@@ -164,7 +179,12 @@ class DetectorLoader:
164
179
  `0.99` keeps the probe's own post-processing trivial on a grey frame;
165
180
  what is being established is that a forward pass completes at all.
166
181
  """
167
- session.detect(_warmup_jpeg(), 0.99, None)
182
+ if callable(getattr(session,"probe",None)):
183
+ session.probe()
184
+ elif hasattr(session, "estimate"):
185
+ session.estimate(_warmup_jpeg(), [[0., 0., 64., 48.]])
186
+ else:
187
+ session.detect(_warmup_jpeg(), 0.99, None)
168
188
 
169
189
  def start(self) -> None:
170
190
  self._thread.start()
@@ -289,6 +309,12 @@ class DetectorLoader:
289
309
  def detect(self, jpeg: bytes, min_confidence: float, wanted: "set[str] | None", expected_model=None) -> DetectionBatch:
290
310
  return self.registry.detect(jpeg, min_confidence, wanted, expected_model=expected_model)
291
311
 
312
+ def estimate_pose(self, jpeg: bytes, boxes: list, expected_model: dict) -> dict:
313
+ return self.registry.estimate_pose(jpeg, boxes, expected_model)
314
+
315
+ def classify_actor_actions(self,batch):
316
+ return self.registry.classify_actor_actions(batch)
317
+
292
318
  # ── the loop ─────────────────────────────────────────────────────────────
293
319
 
294
320
  def _run(self) -> None: