@trillboards/edge-sdk 0.2.175 → 0.2.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/README.md +6 -0
- package/deploy/detector/pyproject.toml +4 -1
- package/deploy/detector/tests/test_actor_action_wire.py +103 -0
- package/deploy/detector/tests/test_actor_actions.py +94 -0
- package/deploy/detector/tests/test_actor_checkpoint.py +99 -0
- package/deploy/detector/tests/test_actor_tracking.py +191 -0
- package/deploy/detector/tests/test_actor_tracking_wire.py +120 -0
- package/deploy/detector/tests/test_actor_views.py +108 -0
- package/deploy/detector/tests/test_compute_budget.py +28 -0
- package/deploy/detector/tests/test_detector_request.py +52 -0
- package/deploy/detector/tests/test_input_recipe.py +77 -0
- package/deploy/detector/tests/test_loader.py +20 -1
- package/deploy/detector/tests/test_pose.py +66 -0
- package/deploy/detector/tests/test_server.py +83 -0
- package/deploy/detector/tests/test_task_registry.py +74 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/actor_action_contract.py +65 -0
- package/deploy/detector/trillboards_detector/actor_action_wire.py +189 -0
- package/deploy/detector/trillboards_detector/actor_actions.py +115 -0
- package/deploy/detector/trillboards_detector/actor_checkpoint.py +91 -0
- package/deploy/detector/trillboards_detector/actor_tracking.py +337 -0
- package/deploy/detector/trillboards_detector/actor_views.py +102 -0
- package/deploy/detector/trillboards_detector/cli.py +12 -2
- package/deploy/detector/trillboards_detector/labels.py +32 -0
- package/deploy/detector/trillboards_detector/loader.py +30 -4
- package/deploy/detector/trillboards_detector/pose.py +129 -0
- package/deploy/detector/trillboards_detector/registry.py +92 -7
- package/deploy/detector/trillboards_detector/server.py +131 -3
- package/deploy/detector/trillboards_detector/session.py +83 -49
- package/deploy/detector/trillboards_detector/tasks.py +27 -0
- package/deploy/detector/trillboards_detector/worker_budget.py +14 -0
- package/deploy/windows/README.md +1 -1
- package/deploy/windows/ci/test-published-install.ps1 +16 -3
- package/dist/ActorActionClient.d.ts +47 -0
- package/dist/ActorActionClient.d.ts.map +1 -0
- package/dist/ActorActionClient.js +187 -0
- package/dist/ActorActionClient.js.map +1 -0
- package/dist/ActorActionProducer.d.ts +17 -0
- package/dist/ActorActionProducer.d.ts.map +1 -0
- package/dist/ActorActionProducer.js +92 -0
- package/dist/ActorActionProducer.js.map +1 -0
- package/dist/ActorPerception.d.ts +65 -0
- package/dist/ActorPerception.d.ts.map +1 -0
- package/dist/ActorPerception.js +124 -0
- package/dist/ActorPerception.js.map +1 -0
- package/dist/ActorPose.d.ts +42 -0
- package/dist/ActorPose.d.ts.map +1 -0
- package/dist/ActorPose.js +62 -0
- package/dist/ActorPose.js.map +1 -0
- package/dist/ActorReview.d.ts +51 -0
- package/dist/ActorReview.d.ts.map +1 -0
- package/dist/ActorReview.js +84 -0
- package/dist/ActorReview.js.map +1 -0
- package/dist/ActorTracking.d.ts +33 -0
- package/dist/ActorTracking.d.ts.map +1 -0
- package/dist/ActorTracking.js +90 -0
- package/dist/ActorTracking.js.map +1 -0
- package/dist/ActorWindow.d.ts +56 -0
- package/dist/ActorWindow.d.ts.map +1 -0
- package/dist/ActorWindow.js +86 -0
- package/dist/ActorWindow.js.map +1 -0
- package/dist/ActorWindowContract.d.ts +5 -0
- package/dist/ActorWindowContract.d.ts.map +1 -0
- package/dist/ActorWindowContract.js +23 -0
- package/dist/ActorWindowContract.js.map +1 -0
- package/dist/AudienceAnalyzeClient.d.ts +4 -0
- package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
- package/dist/AudienceAnalyzeClient.js.map +1 -1
- package/dist/ChannelProber.d.ts.map +1 -1
- package/dist/ChannelProber.js +8 -0
- package/dist/ChannelProber.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +7 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +31 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/LocalDetectorClient.d.ts +5 -1
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +48 -3
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/RetainedFieldContract.d.ts +2 -1
- package/dist/RetainedFieldContract.d.ts.map +1 -1
- package/dist/RetainedFieldContract.js +17 -5
- package/dist/RetainedFieldContract.js.map +1 -1
- package/dist/RuntimeModelSelection.d.ts.map +1 -1
- package/dist/RuntimeModelSelection.js +63 -2
- package/dist/RuntimeModelSelection.js.map +1 -1
- package/dist/TemporalReviewClient.d.ts.map +1 -1
- package/dist/TemporalReviewClient.js +20 -1
- package/dist/TemporalReviewClient.js.map +1 -1
- package/dist/TemporalReviewJournal.d.ts +10 -1
- package/dist/TemporalReviewJournal.d.ts.map +1 -1
- package/dist/TemporalReviewJournal.js +42 -10
- package/dist/TemporalReviewJournal.js.map +1 -1
- package/dist/connectorUpdate.d.ts +2 -0
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +33 -7
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -1
- package/dist/index.js.map +1 -1
- package/dist/localSensing.d.ts +4 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +43 -8
- package/dist/localSensing.js.map +1 -1
- package/package.json +5 -5
- package/retained/RetainedCaptureClient.cjs +55 -9
- package/retained/__tests__/actor-perception.test.cjs +125 -0
- package/retained/__tests__/actor-window.test.cjs +78 -0
- package/retained/__tests__/runtime.test.cjs +44 -4
- package/retained/actor-window.cjs +62 -0
- package/retained/manager.cjs +14 -2
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""Camera-bound actor identities from the pinned ByteTrack implementation.
|
|
2
|
+
|
|
3
|
+
SQLite commits frame inputs and identities together. In-memory Kalman state is a
|
|
4
|
+
cache: restart recovery replays the bounded detection journal through the same
|
|
5
|
+
tracker, restoring external actor IDs from exact saved detection associations.
|
|
6
|
+
Predicted boxes are never returned as new visual observations.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import importlib.metadata
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import re
|
|
15
|
+
import sqlite3
|
|
16
|
+
import threading
|
|
17
|
+
import uuid
|
|
18
|
+
from dataclasses import asdict
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from .detections import Detection
|
|
22
|
+
from .actor_checkpoint import encode_tracker, restore_tracker
|
|
23
|
+
|
|
24
|
+
ALGORITHM = "bytetrack_2.6.1"
|
|
25
|
+
_ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
|
|
26
|
+
_SHA = re.compile(r"[a-f0-9]{64}\Z")
|
|
27
|
+
_SAFE_INTEGER = 9007199254740991
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TrackingError(ValueError):
|
|
31
|
+
"""Invalid or unprovable tracking input; never an empty-scene result."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _RecoveryConflict(TrackingError):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _json(value):
|
|
39
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _hash(value):
|
|
43
|
+
return hashlib.sha256(_json(value).encode()).hexdigest()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def validate_tracking_request(value):
|
|
47
|
+
if (not isinstance(value, dict)
|
|
48
|
+
or set(value) != {"version", "cameraId", "sourceId", "frameId", "capturedAtMs"}
|
|
49
|
+
or type(value["version"]) is not int or value["version"] != 1
|
|
50
|
+
or any(not isinstance(value[k], str) or not _ID.fullmatch(value[k])
|
|
51
|
+
for k in ("cameraId", "sourceId", "frameId"))
|
|
52
|
+
or type(value["capturedAtMs"]) is not int
|
|
53
|
+
or not 0 <= value["capturedAtMs"] <= _SAFE_INTEGER):
|
|
54
|
+
raise TrackingError("Invalid captured actor tracking identity")
|
|
55
|
+
return dict(value)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ActorTrackerJournal:
|
|
59
|
+
"""Bounded, serialized tracking store. A camera is never silently evicted.
|
|
60
|
+
|
|
61
|
+
Capture timestamps, not inference time, drive motion and track expiration.
|
|
62
|
+
A stream/view/model discontinuity starts a new epoch and explicitly ends
|
|
63
|
+
prior actors. A normal service restart preserves actors through replay.
|
|
64
|
+
"""
|
|
65
|
+
def __init__(self, path, *, max_cameras=64, retained_frames=256,
|
|
66
|
+
frame_rate=5.0, max_lost_ms=3000, max_gap_ms=5000):
|
|
67
|
+
if (type(max_cameras) is not int or not 1 <= max_cameras <= 256
|
|
68
|
+
or type(retained_frames) is not int or not 32 <= retained_frames <= 512
|
|
69
|
+
or not isinstance(frame_rate, (int, float)) or isinstance(frame_rate, bool)
|
|
70
|
+
or not math.isfinite(frame_rate) or not 1 <= frame_rate <= 30
|
|
71
|
+
or type(max_lost_ms) is not int or not 200 <= max_lost_ms <= 5000
|
|
72
|
+
or type(max_gap_ms) is not int or not max_lost_ms <= max_gap_ms <= 30000
|
|
73
|
+
or retained_frames < math.ceil(frame_rate * max_lost_ms / 1000) + 3):
|
|
74
|
+
raise TrackingError("Invalid bounded tracker configuration")
|
|
75
|
+
if importlib.metadata.version("trackers") != "2.6.1":
|
|
76
|
+
raise TrackingError("Actor tracking requires trackers==2.6.1")
|
|
77
|
+
from trackers import ByteTrackTracker
|
|
78
|
+
self._tracker_class = ByteTrackTracker
|
|
79
|
+
self._config = {"algorithm": ALGORITHM, "frameRate": frame_rate, "maxLostMs": max_lost_ms,
|
|
80
|
+
"maxGapMs": max_gap_ms, "minimumConsecutiveFrames": 2,
|
|
81
|
+
"activationThreshold": .5, "highConfidenceThreshold": .5, "minimumIou": .1}
|
|
82
|
+
self._max_cameras, self._retained_frames = max_cameras, retained_frames
|
|
83
|
+
self._lock = threading.RLock()
|
|
84
|
+
self._cache = {}
|
|
85
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
self._db = sqlite3.connect(str(path), timeout=30, check_same_thread=False, isolation_level=None)
|
|
87
|
+
self._db.execute("PRAGMA journal_mode=WAL")
|
|
88
|
+
self._db.execute("PRAGMA synchronous=FULL")
|
|
89
|
+
self._db.executescript("""
|
|
90
|
+
CREATE TABLE IF NOT EXISTS actor_camera (
|
|
91
|
+
camera_id TEXT PRIMARY KEY, context_hash TEXT NOT NULL, epoch TEXT NOT NULL,
|
|
92
|
+
captured_at INTEGER NOT NULL, seq INTEGER NOT NULL, active_json TEXT NOT NULL);
|
|
93
|
+
CREATE TABLE IF NOT EXISTS actor_frame (
|
|
94
|
+
camera_id TEXT NOT NULL, frame_id TEXT NOT NULL, seq INTEGER NOT NULL,
|
|
95
|
+
epoch TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
|
96
|
+
input_json TEXT NOT NULL, result_json TEXT NOT NULL,
|
|
97
|
+
PRIMARY KEY(camera_id,frame_id), UNIQUE(camera_id,seq));
|
|
98
|
+
CREATE INDEX IF NOT EXISTS actor_frame_replay ON actor_frame(camera_id,epoch,seq);
|
|
99
|
+
CREATE TABLE IF NOT EXISTS actor_checkpoint (
|
|
100
|
+
camera_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, payload_json TEXT NOT NULL);
|
|
101
|
+
""")
|
|
102
|
+
|
|
103
|
+
def close(self):
|
|
104
|
+
with self._lock:
|
|
105
|
+
self._cache.clear()
|
|
106
|
+
self._db.close()
|
|
107
|
+
|
|
108
|
+
def retained_frames(self, camera_id):
|
|
109
|
+
with self._lock:
|
|
110
|
+
return self._db.execute("SELECT count(*) FROM actor_frame WHERE camera_id=?", (camera_id,)).fetchone()[0]
|
|
111
|
+
|
|
112
|
+
def _new_tracker(self, config=None):
|
|
113
|
+
config = self._config if config is None else config
|
|
114
|
+
return self._tracker_class(
|
|
115
|
+
lost_track_buffer=math.ceil(config["maxLostMs"] * 30 / 1000),
|
|
116
|
+
frame_rate=config["frameRate"], track_activation_threshold=config["activationThreshold"],
|
|
117
|
+
minimum_consecutive_frames=config["minimumConsecutiveFrames"],
|
|
118
|
+
high_conf_det_threshold=config["highConfidenceThreshold"], minimum_iou_threshold=config["minimumIou"])
|
|
119
|
+
|
|
120
|
+
def _update(self, tracker, values, captured_at):
|
|
121
|
+
import numpy as np
|
|
122
|
+
import supervision as sv
|
|
123
|
+
indices = [i for i, d in enumerate(values) if d["label"] == "person"]
|
|
124
|
+
boxes = np.array([[values[i]["x"], values[i]["y"], values[i]["x"] + values[i]["width"],
|
|
125
|
+
values[i]["y"] + values[i]["height"]] for i in indices],
|
|
126
|
+
dtype=np.float32).reshape(-1, 4)
|
|
127
|
+
ds = sv.Detections(xyxy=boxes,
|
|
128
|
+
confidence=np.array([values[i]["confidence"] for i in indices], dtype=np.float32),
|
|
129
|
+
class_id=np.zeros(len(indices), dtype=int),
|
|
130
|
+
data={"detection_index": np.array(indices, dtype=int)})
|
|
131
|
+
result = tracker.update(ds, timestamp=captured_at / 1000)
|
|
132
|
+
return [(int(i), int(t)) for i, t in zip(result.data.get("detection_index", []), result.tracker_id)]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _checked_frame(self, camera, frame_id, epoch, fingerprint, input_json, result_json):
|
|
136
|
+
try:
|
|
137
|
+
frame, result = json.loads(input_json), json.loads(result_json)
|
|
138
|
+
unsigned = {k: v for k, v in result.items() if k != "sha256"}
|
|
139
|
+
if (fingerprint != _hash(frame) or result.get("sha256") != _hash(unsigned)
|
|
140
|
+
or result.get("cameraId") != camera or result.get("frameId") != frame_id
|
|
141
|
+
or result.get("epoch") != epoch or frame.get("request") != {
|
|
142
|
+
k: result.get(k) for k in ("version", "cameraId", "sourceId", "frameId", "capturedAtMs")}
|
|
143
|
+
or result.get("imageSha256") != frame.get("imageSha256")
|
|
144
|
+
or result.get("contextSha256") != frame.get("contextSha256")):
|
|
145
|
+
raise ValueError("journal identity or digest changed")
|
|
146
|
+
return frame, result
|
|
147
|
+
except (ValueError, TypeError, KeyError) as error:
|
|
148
|
+
raise TrackingError("Corrupt actor frame journal") from error
|
|
149
|
+
|
|
150
|
+
@staticmethod
|
|
151
|
+
def _checked_active(value, camera, epoch):
|
|
152
|
+
try:
|
|
153
|
+
actors = json.loads(value)
|
|
154
|
+
if (not isinstance(actors, list) or len(actors) > 256
|
|
155
|
+
or any(not isinstance(a, dict) or set(a) != {"kind", "cameraId", "id", "epoch"}
|
|
156
|
+
or a["kind"] != "person" or a["cameraId"] != camera or a["epoch"] != epoch
|
|
157
|
+
or not isinstance(a["id"], str) or not _ID.fullmatch(a["id"]) for a in actors)
|
|
158
|
+
or len({a["id"] for a in actors}) != len(actors)):
|
|
159
|
+
raise ValueError("camera head actor identity changed")
|
|
160
|
+
return actors
|
|
161
|
+
except (ValueError, TypeError, KeyError) as error:
|
|
162
|
+
raise TrackingError("Corrupt actor camera journal") from error
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _save_checkpoint(self, camera, context, epoch, captured_at, seq, fingerprint, tracker, identities):
|
|
167
|
+
try:
|
|
168
|
+
payload={"version":1,"cameraId":camera,"contextSha256":context,"epoch":epoch,
|
|
169
|
+
"capturedAtMs":captured_at,"seq":seq,"frameFingerprint":fingerprint,"state":encode_tracker(tracker,identities)}
|
|
170
|
+
payload["sha256"]=_hash(payload)
|
|
171
|
+
raw=_json(payload)
|
|
172
|
+
if len(raw.encode())>8*1024**2:raise ValueError("Tracker checkpoint exceeds capacity")
|
|
173
|
+
self._db.execute("INSERT INTO actor_checkpoint VALUES (?,?,?) ON CONFLICT(camera_id) DO UPDATE SET "
|
|
174
|
+
"seq=excluded.seq,payload_json=excluded.payload_json",(camera,seq,raw))
|
|
175
|
+
except (ValueError,TypeError,KeyError) as error:
|
|
176
|
+
raise TrackingError("Cannot checkpoint physical tracker state") from error
|
|
177
|
+
|
|
178
|
+
def _load_checkpoint(self,camera,head,config):
|
|
179
|
+
record=self._db.execute("SELECT seq,payload_json FROM actor_checkpoint WHERE camera_id=?",(camera,)).fetchone()
|
|
180
|
+
if record is None:return None
|
|
181
|
+
try:
|
|
182
|
+
if len(record[1].encode())>8*1024**2:raise ValueError("oversized checkpoint")
|
|
183
|
+
payload=json.loads(record[1]);unsigned={k:v for k,v in payload.items() if k!="sha256"}
|
|
184
|
+
if (set(payload)!={"version","cameraId","contextSha256","epoch","capturedAtMs","seq","frameFingerprint","state","sha256"}
|
|
185
|
+
or type(payload["version"]) is not int or payload["version"]!=1 or payload["sha256"]!=_hash(unsigned)
|
|
186
|
+
or payload["cameraId"]!=camera or payload["contextSha256"]!=head[0]
|
|
187
|
+
or payload["epoch"]!=head[1] or payload["capturedAtMs"]!=head[2]
|
|
188
|
+
or payload["seq"]!=head[3] or record[0]!=head[3]):
|
|
189
|
+
raise ValueError("checkpoint identity changed")
|
|
190
|
+
# Preserve the existing journal's integrity checks. A checkpoint
|
|
191
|
+
# cannot make a corrupt retained receipt look healthy.
|
|
192
|
+
verified_head=False
|
|
193
|
+
for frame_id,fingerprint,input_json,result_json,seq in self._db.execute(
|
|
194
|
+
"SELECT frame_id,fingerprint,input_json,result_json,seq FROM actor_frame WHERE camera_id=? AND epoch=? ORDER BY seq",
|
|
195
|
+
(camera,head[1])):
|
|
196
|
+
_,result=self._checked_frame(camera,frame_id,head[1],fingerprint,input_json,result_json)
|
|
197
|
+
if seq==head[3]:
|
|
198
|
+
verified_head=(fingerprint==payload["frameFingerprint"]
|
|
199
|
+
and result["capturedAtMs"]==head[2] and result["contextSha256"]==head[0])
|
|
200
|
+
if not verified_head:raise ValueError("checkpoint lost its committed frame proof")
|
|
201
|
+
tracker,identities=restore_tracker(self._new_tracker(config),payload["state"],
|
|
202
|
+
lambda actors:self._checked_active(_json(actors),camera,head[1]))
|
|
203
|
+
if tracker._last_timestamp!=head[2]/1000:raise ValueError("checkpoint capture time changed")
|
|
204
|
+
active=self._checked_active(head[4],camera,head[1])
|
|
205
|
+
if sorted(_json(a) for a in identities.values())!=sorted(_json(a) for a in active):
|
|
206
|
+
raise ValueError("checkpoint actor bindings changed")
|
|
207
|
+
return tracker,identities
|
|
208
|
+
except (ValueError,TypeError,KeyError) as error:
|
|
209
|
+
raise TrackingError("Corrupt physical tracker checkpoint") from error
|
|
210
|
+
|
|
211
|
+
def _recover(self, camera, epoch, config=None, expected_active=None):
|
|
212
|
+
tracker, identities = self._new_tracker(config), {}
|
|
213
|
+
reverse = {}
|
|
214
|
+
for frame_id, fingerprint, input_json, result_json in self._db.execute(
|
|
215
|
+
"SELECT frame_id,fingerprint,input_json,result_json FROM actor_frame WHERE camera_id=? AND epoch=? ORDER BY seq",
|
|
216
|
+
(camera, epoch)):
|
|
217
|
+
frame, saved = self._checked_frame(camera, frame_id, epoch, fingerprint, input_json, result_json)
|
|
218
|
+
expected = {a["detectionIndex"]: a["actor"] for a in saved["actors"]}
|
|
219
|
+
for index, tid in self._update(tracker, frame["detections"], frame["request"]["capturedAtMs"]):
|
|
220
|
+
if tid < 0 or index not in expected:
|
|
221
|
+
continue
|
|
222
|
+
actor = expected[index]
|
|
223
|
+
if (tid in identities and identities[tid] != actor
|
|
224
|
+
or actor["id"] in reverse and reverse[actor["id"]] != tid):
|
|
225
|
+
raise _RecoveryConflict("Actor history cannot prove a unique tracker recovery")
|
|
226
|
+
identities[tid], reverse[actor["id"]] = actor, tid
|
|
227
|
+
alive_ids = set(int(t) for t in tracker.tracked_objects.tracker_id)
|
|
228
|
+
identities = {tid: actor for tid, actor in identities.items() if tid in alive_ids}
|
|
229
|
+
if expected_active is not None and (
|
|
230
|
+
set(identities) != alive_ids
|
|
231
|
+
or sorted(_json(a) for a in identities.values()) != sorted(_json(a) for a in expected_active)):
|
|
232
|
+
raise _RecoveryConflict("Bounded actor history cannot prove the committed active identities")
|
|
233
|
+
return tracker, identities
|
|
234
|
+
|
|
235
|
+
def track(self, request, detections, *, image_sha256, model_hashes, frame_size):
|
|
236
|
+
request = validate_tracking_request(request)
|
|
237
|
+
if (not isinstance(image_sha256, str) or not _SHA.fullmatch(image_sha256)
|
|
238
|
+
or not isinstance(model_hashes, dict) or not 1 <= len(model_hashes) <= 16
|
|
239
|
+
or any(not isinstance(k, str) or not _ID.fullmatch(k)
|
|
240
|
+
or not isinstance(v, str) or not _SHA.fullmatch(v) for k, v in model_hashes.items())
|
|
241
|
+
or not isinstance(frame_size, (list, tuple)) or len(frame_size) != 2
|
|
242
|
+
or any(type(v) is not int or v < 2 or v > 8192 for v in frame_size)
|
|
243
|
+
or frame_size[0] * frame_size[1] > 16777216
|
|
244
|
+
or not isinstance(detections, list) or len(detections) > 256):
|
|
245
|
+
raise TrackingError("Invalid actor frame or loaded-model proof")
|
|
246
|
+
values = []
|
|
247
|
+
for d in detections:
|
|
248
|
+
if (not isinstance(d, Detection) or not isinstance(d.label, str) or not d.label
|
|
249
|
+
or any(isinstance(v, bool) or not isinstance(v, (float, int)) or not math.isfinite(v)
|
|
250
|
+
for v in (d.confidence, d.x, d.y, d.width, d.height))
|
|
251
|
+
or not 0 <= d.confidence <= 1 or min(d.x, d.y) < 0 or min(d.width, d.height) <= 0
|
|
252
|
+
or d.x + d.width > frame_size[0] + .01 or d.y + d.height > frame_size[1] + .01):
|
|
253
|
+
raise TrackingError("Invalid detection in actor frame")
|
|
254
|
+
values.append(asdict(d))
|
|
255
|
+
config = self._config
|
|
256
|
+
context = _hash({"sourceId": request["sourceId"], "modelHashes": model_hashes,
|
|
257
|
+
"frameSize": list(frame_size), "config": config})
|
|
258
|
+
inputs = {"request": request, "detections": values, "imageSha256": image_sha256,
|
|
259
|
+
"contextSha256": context}
|
|
260
|
+
fingerprint = _hash(inputs)
|
|
261
|
+
camera, captured_at = request["cameraId"], request["capturedAtMs"]
|
|
262
|
+
with self._lock:
|
|
263
|
+
self._db.execute("BEGIN IMMEDIATE")
|
|
264
|
+
try:
|
|
265
|
+
own = self._db.execute(
|
|
266
|
+
"SELECT fingerprint,input_json,result_json,epoch FROM actor_frame WHERE camera_id=? AND frame_id=?",
|
|
267
|
+
(camera, request["frameId"])).fetchone()
|
|
268
|
+
if own:
|
|
269
|
+
if own[0] != fingerprint:
|
|
270
|
+
raise TrackingError("Retained actor frame identity changed")
|
|
271
|
+
_, saved = self._checked_frame(camera, request["frameId"], own[3], own[0], own[1], own[2])
|
|
272
|
+
self._db.execute("COMMIT")
|
|
273
|
+
return saved
|
|
274
|
+
head = self._db.execute(
|
|
275
|
+
"SELECT context_hash,epoch,captured_at,seq,active_json FROM actor_camera WHERE camera_id=?",
|
|
276
|
+
(camera,)).fetchone()
|
|
277
|
+
if head and captured_at <= head[2]:
|
|
278
|
+
raise TrackingError("Actor frames must preserve strict capture order")
|
|
279
|
+
if not head and self._db.execute("SELECT count(*) FROM actor_camera").fetchone()[0] >= self._max_cameras:
|
|
280
|
+
raise TrackingError("Actor camera capacity exhausted; retire a disconnected camera explicitly")
|
|
281
|
+
continuity = ("start" if not head else "context_reset" if head[0] != context
|
|
282
|
+
else "gap_reset" if captured_at - head[2] > self._config["maxGapMs"] else "continued")
|
|
283
|
+
old_active = self._checked_active(head[4], camera, head[1]) if head else []
|
|
284
|
+
epoch = head[1] if continuity == "continued" else "te_" + uuid.uuid4().hex
|
|
285
|
+
seq = head[3]+1 if head else 1
|
|
286
|
+
cached = self._cache.get(camera)
|
|
287
|
+
if continuity != "continued":
|
|
288
|
+
tracker, identities = self._new_tracker(config), {}
|
|
289
|
+
elif cached and cached[0] == head[3]:
|
|
290
|
+
_, tracker, identities = cached
|
|
291
|
+
else:
|
|
292
|
+
try:
|
|
293
|
+
restored = self._load_checkpoint(camera,head,config)
|
|
294
|
+
tracker, identities = restored if restored is not None else self._recover(camera, epoch, config, old_active)
|
|
295
|
+
except _RecoveryConflict:
|
|
296
|
+
# Preserve the old receipts; never quietly attach another person
|
|
297
|
+
# to the old duration/latch when replay cannot establish identity.
|
|
298
|
+
continuity, epoch = "recovery_reset", "te_" + uuid.uuid4().hex
|
|
299
|
+
tracker, identities = self._new_tracker(config), {}
|
|
300
|
+
matches = self._update(tracker, values, captured_at)
|
|
301
|
+
actors, unconfirmed = [], []
|
|
302
|
+
for index, tid in matches:
|
|
303
|
+
if tid < 0:
|
|
304
|
+
unconfirmed.append(index)
|
|
305
|
+
continue
|
|
306
|
+
actor = identities.setdefault(tid, {"kind": "person", "cameraId": camera,
|
|
307
|
+
"id": "pa_" + uuid.uuid4().hex, "epoch": epoch})
|
|
308
|
+
actors.append({"detectionIndex": index, "actor": actor})
|
|
309
|
+
alive_ids = set(int(t) for t in tracker.tracked_objects.tracker_id)
|
|
310
|
+
active = [identities[t] for t in sorted(alive_ids) if t in identities]
|
|
311
|
+
identities = {t: a for t, a in identities.items() if t in alive_ids}
|
|
312
|
+
ended = [a for a in old_active if a not in active]
|
|
313
|
+
result = {"version": 1, "algorithm": ALGORITHM, **{k: request[k] for k in
|
|
314
|
+
("cameraId", "sourceId", "frameId", "capturedAtMs")},
|
|
315
|
+
"epoch": epoch, "imageSha256": image_sha256,
|
|
316
|
+
"modelHashes": model_hashes, "contextSha256": context, "continuity": continuity,
|
|
317
|
+
"actors": sorted(actors, key=lambda a: a["detectionIndex"]),
|
|
318
|
+
"unconfirmed": sorted(unconfirmed), "endedActors": ended}
|
|
319
|
+
result["sha256"] = _hash(result)
|
|
320
|
+
self._db.execute("INSERT INTO actor_frame VALUES (?,?,?,?,?,?,?)",
|
|
321
|
+
(camera, request["frameId"], seq, epoch, fingerprint, _json(inputs), _json(result)))
|
|
322
|
+
self._db.execute(
|
|
323
|
+
"INSERT INTO actor_camera VALUES (?,?,?,?,?,?) ON CONFLICT(camera_id) DO UPDATE SET "
|
|
324
|
+
"context_hash=excluded.context_hash,epoch=excluded.epoch,captured_at=excluded.captured_at,"
|
|
325
|
+
"seq=excluded.seq,active_json=excluded.active_json",
|
|
326
|
+
(camera, context, epoch, captured_at, seq, _json(active)))
|
|
327
|
+
self._save_checkpoint(camera,context,epoch,captured_at,seq,fingerprint,tracker,identities)
|
|
328
|
+
self._db.execute("DELETE FROM actor_frame WHERE camera_id=? AND seq<=?",
|
|
329
|
+
(camera, seq-self._retained_frames))
|
|
330
|
+
self._db.execute("COMMIT")
|
|
331
|
+
self._cache[camera] = (seq, tracker, identities)
|
|
332
|
+
return json.loads(_json(result))
|
|
333
|
+
except BaseException:
|
|
334
|
+
if self._db.in_transaction:
|
|
335
|
+
self._db.execute("ROLLBACK")
|
|
336
|
+
self._cache.pop(camera, None)
|
|
337
|
+
raise
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""RGB actor views shared by task inference and training extraction."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import math
|
|
4
|
+
from PIL import Image
|
|
5
|
+
|
|
6
|
+
ACTOR_VIEW_PREPROCESSING="actor_body_upper_rgb_pad1.25_letterbox182_v2"
|
|
7
|
+
ACTOR_VIEW_SIZE=182
|
|
8
|
+
|
|
9
|
+
def actor_view_bounds(box,frame_size,*,upper=False):
|
|
10
|
+
if (not isinstance(box,(tuple,list)) or len(box)!=4
|
|
11
|
+
or any(type(v) not in (int,float) or not math.isfinite(v) for v in box)
|
|
12
|
+
or len(frame_size)!=2 or any(type(v) is not int or v<2 for v in frame_size)
|
|
13
|
+
or type(upper) is not bool):
|
|
14
|
+
raise ValueError("invalid actor view geometry")
|
|
15
|
+
x1,y1,x2,y2=box
|
|
16
|
+
if x2<=x1 or y2<=y1:raise ValueError("empty actor view")
|
|
17
|
+
if upper:y2=y1+(y2-y1)*.68
|
|
18
|
+
width=(x2-x1)*1.25;height=(y2-y1)*1.25
|
|
19
|
+
cx=(x1+x2)/2;cy=(y1+y2)/2
|
|
20
|
+
result=(max(0,math.floor(cx-width/2)),max(0,math.floor(cy-height/2)),
|
|
21
|
+
min(frame_size[0],math.ceil(cx+width/2)),min(frame_size[1],math.ceil(cy+height/2)))
|
|
22
|
+
if result[2]<=result[0] or result[3]<=result[1]:raise ValueError("actor outside original frame")
|
|
23
|
+
return result
|
|
24
|
+
|
|
25
|
+
def fit_actor_view(tile,*,size=ACTOR_VIEW_SIZE):
|
|
26
|
+
if type(size) is not int or not 16<=size<=1024:
|
|
27
|
+
raise ValueError("invalid actor model input size")
|
|
28
|
+
tile=tile.convert("RGB")
|
|
29
|
+
if tile.width<=size and tile.height<=size:
|
|
30
|
+
scale=min(size/tile.width,size/tile.height)
|
|
31
|
+
tile=tile.resize((max(1,round(tile.width*scale)),max(1,round(tile.height*scale))),Image.Resampling.BILINEAR)
|
|
32
|
+
else:
|
|
33
|
+
tile.thumbnail((size,size),Image.Resampling.BILINEAR)
|
|
34
|
+
result=Image.new("RGB",(size,size),(0,0,0))
|
|
35
|
+
result.paste(tile,((size-tile.width)//2,(size-tile.height)//2))
|
|
36
|
+
return result
|
|
37
|
+
|
|
38
|
+
def actor_view(image,box,*,upper=False,size=ACTOR_VIEW_SIZE):
|
|
39
|
+
if not isinstance(image,Image.Image):raise ValueError("actor view requires original RGB image")
|
|
40
|
+
return fit_actor_view(image.crop(actor_view_bounds(box,image.size,upper=upper)),size=size)
|
|
41
|
+
|
|
42
|
+
# Detail views retain source-resolution head/hand evidence. Localization means
|
|
43
|
+
# a crop can be placed; it never means a phone is present or absent.
|
|
44
|
+
ACTOR_DETAIL_PREPROCESSING="actor_head_wrists_rgb_h032_w028_letterbox224_v1"
|
|
45
|
+
ACTOR_DETAIL_SIZE=224
|
|
46
|
+
|
|
47
|
+
def actor_detail_views(image,box,joints):
|
|
48
|
+
from .tasks import COCO17
|
|
49
|
+
if not isinstance(image,Image.Image):
|
|
50
|
+
raise ValueError("actor details require the original RGB image")
|
|
51
|
+
actor_view_bounds(box,image.size)
|
|
52
|
+
x1,y1,x2,y2=box
|
|
53
|
+
if not (0<=x1<x2<=image.width and 0<=y1<y2<=image.height):
|
|
54
|
+
raise ValueError("actor details require an original-frame person box")
|
|
55
|
+
if not isinstance(joints,list) or len(joints)!=len(COCO17):
|
|
56
|
+
raise ValueError("actor details require all named COCO17 joints")
|
|
57
|
+
points={}
|
|
58
|
+
for joint,name in zip(joints,COCO17):
|
|
59
|
+
if not isinstance(joint,dict) or set(joint)!={"name","x","y","response"} or joint["name"]!=name:
|
|
60
|
+
raise ValueError("invalid named joint contract")
|
|
61
|
+
response=joint["response"];x=joint["x"];y=joint["y"]
|
|
62
|
+
if (type(response) not in (int,float) or not math.isfinite(response)
|
|
63
|
+
or (x is None)!=(y is None)
|
|
64
|
+
or (x is not None and (type(x) not in (int,float) or type(y) not in (int,float)
|
|
65
|
+
or not math.isfinite(x) or not math.isfinite(y)
|
|
66
|
+
or not 0<=x<image.width or not 0<=y<image.height))):
|
|
67
|
+
raise ValueError("invalid joint coordinates or response")
|
|
68
|
+
points[name]=joint
|
|
69
|
+
height=y2-y1;width=x2-x1
|
|
70
|
+
def located(name):
|
|
71
|
+
p=points[name]
|
|
72
|
+
return (p["x"] is not None and p["response"]>.15
|
|
73
|
+
and x1-width*.15<=p["x"]<=x2+width*.15
|
|
74
|
+
and y1-height*.15<=p["y"]<=y2+height*.15)
|
|
75
|
+
face=[points[n] for n in COCO17[:5] if located(n)]
|
|
76
|
+
head=(sum(p["x"] for p in face)/len(face),sum(p["y"] for p in face)/len(face)) if face else ((x1+x2)/2,y1+height*.12)
|
|
77
|
+
specs=[("head",head,height*.32,"pose" if face else "box_prior")]
|
|
78
|
+
for name in ("left_wrist","right_wrist"):
|
|
79
|
+
p=points[name]
|
|
80
|
+
specs.append((name,(p["x"],p["y"]) if located(name) else None,height*.28,"pose"))
|
|
81
|
+
result=[]
|
|
82
|
+
for name,center,side,localization in specs:
|
|
83
|
+
view={"name":name,"preprocessing":ACTOR_DETAIL_PREPROCESSING,
|
|
84
|
+
"localized":False,"localization":localization,"reason":None,
|
|
85
|
+
"sourceBounds":None,"sourceSize":None,"image":None}
|
|
86
|
+
if center is None:
|
|
87
|
+
view["reason"]="joint_unobserved";result.append(view);continue
|
|
88
|
+
cx,cy=center
|
|
89
|
+
bounds=[max(0,math.floor(cx-side/2)),max(0,math.floor(cy-side/2)),
|
|
90
|
+
min(image.width,math.ceil(cx+side/2)),min(image.height,math.ceil(cy+side/2))]
|
|
91
|
+
size=[bounds[2]-bounds[0],bounds[3]-bounds[1]]
|
|
92
|
+
view.update(sourceBounds=bounds,sourceSize=size)
|
|
93
|
+
if min(size)<8:
|
|
94
|
+
view["reason"]="too_small";result.append(view);continue
|
|
95
|
+
tile=image.crop(bounds).convert("RGB")
|
|
96
|
+
scale=min(ACTOR_DETAIL_SIZE/tile.width,ACTOR_DETAIL_SIZE/tile.height)
|
|
97
|
+
tile=tile.resize((max(1,round(tile.width*scale)),max(1,round(tile.height*scale))),Image.Resampling.BILINEAR)
|
|
98
|
+
fitted=Image.new("RGB",(ACTOR_DETAIL_SIZE,ACTOR_DETAIL_SIZE),(0,0,0))
|
|
99
|
+
fitted.paste(tile,((ACTOR_DETAIL_SIZE-tile.width)//2,(ACTOR_DETAIL_SIZE-tile.height)//2))
|
|
100
|
+
view.update(image=fitted,localized=True)
|
|
101
|
+
result.append(view)
|
|
102
|
+
return result
|
|
@@ -18,8 +18,13 @@ import threading
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
|
|
20
20
|
from ._version import VERSION
|
|
21
|
+
from .worker_budget import configure_numeric_workers
|
|
22
|
+
|
|
23
|
+
# This must run before loader/server imports initialize NumPy or BLAS.
|
|
24
|
+
configure_numeric_workers()
|
|
21
25
|
from .loader import DEFAULT_POLL_INTERVAL_S, DetectorLoader
|
|
22
|
-
from .modelpath import DEFAULT_MODEL_NAME, resolve_model_path, resolve_ram_budget_mb
|
|
26
|
+
from .modelpath import DEFAULT_MODEL_NAME, resolve_model_path, resolve_ram_budget_mb, resolve_data_dir
|
|
27
|
+
from .actor_tracking import ActorTrackerJournal
|
|
23
28
|
from .server import DEFAULT_HOST, DEFAULT_PORT, serve
|
|
24
29
|
|
|
25
30
|
LOG = logging.getLogger("trillboards_detector")
|
|
@@ -107,6 +112,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
107
112
|
help="prefer compatible local NVIDIA CUDA, otherwise CPU; cpu is an explicit rollback")
|
|
108
113
|
s.add_argument("--log-file", default=None, help="also write to this file (Windows has no task stdout)")
|
|
109
114
|
s.add_argument("--verbose", action="store_true")
|
|
115
|
+
s.add_argument("--tracking-journal", default=None, help="durable actor tracker SQLite path; defaults to the connector data directory")
|
|
110
116
|
return p
|
|
111
117
|
|
|
112
118
|
|
|
@@ -161,10 +167,11 @@ def main(argv: "list[str] | None" = None) -> int:
|
|
|
161
167
|
log=LOG.info,
|
|
162
168
|
accelerator=args.accelerator,
|
|
163
169
|
)
|
|
170
|
+
actor_tracker = ActorTrackerJournal(args.tracking_journal or resolve_data_dir() / "actor-tracking" / "tracks.sqlite3")
|
|
164
171
|
loader.start()
|
|
165
172
|
|
|
166
173
|
try:
|
|
167
|
-
server = serve(loader, args.host, args.port, LOG.info)
|
|
174
|
+
server = serve(loader, args.host, args.port, LOG.info, actor_tracker=actor_tracker)
|
|
168
175
|
except OSError as err:
|
|
169
176
|
# A bind failure is FATAL and must stay fatal. `register-litertlm-task.ps1`
|
|
170
177
|
# has the incident this mirrors: a serve left alive keeps ANSWERING, so an
|
|
@@ -173,6 +180,7 @@ def main(argv: "list[str] | None" = None) -> int:
|
|
|
173
180
|
# restart policy do its job instead of running two half-services.
|
|
174
181
|
LOG.error("[detector] could not bind %s:%s — %s", args.host, args.port, err)
|
|
175
182
|
loader.stop()
|
|
183
|
+
actor_tracker.close()
|
|
176
184
|
return 1
|
|
177
185
|
|
|
178
186
|
stop = threading.Event()
|
|
@@ -193,7 +201,9 @@ def main(argv: "list[str] | None" = None) -> int:
|
|
|
193
201
|
except KeyboardInterrupt:
|
|
194
202
|
pass
|
|
195
203
|
server.shutdown()
|
|
204
|
+
server.server_close()
|
|
196
205
|
loader.stop()
|
|
206
|
+
actor_tracker.close()
|
|
197
207
|
return 0
|
|
198
208
|
|
|
199
209
|
|
|
@@ -228,6 +228,32 @@ RFDETR_MEDIUM_ONNX = VariantContract(
|
|
|
228
228
|
)
|
|
229
229
|
|
|
230
230
|
|
|
231
|
+
# Input pixels are independent of the graph's output layout. Existing catalogue
|
|
232
|
+
# entries retain their scored recipes; a new artifact may declare another one.
|
|
233
|
+
PIL_IMAGENET_STRETCH = "imagenet-normalised-stretch"
|
|
234
|
+
RGB_HALF_PIXEL_IMAGENET = "rgb-float-half-pixel-no-antialias-imagenet-v1"
|
|
235
|
+
BGR_BYTES_STRETCH = "bgr-bytes-cv2-linear-stretch-v1"
|
|
236
|
+
YOLO_LETTERBOX = "ultralytics-letterbox-114-centred"
|
|
237
|
+
LEGACY_INPUT_PREPROCESSING = MappingProxyType({
|
|
238
|
+
RFDETR_DETS_LABELS: PIL_IMAGENET_STRETCH,
|
|
239
|
+
ANCHOR_4_PLUS_NC: YOLO_LETTERBOX,
|
|
240
|
+
END2END_N_6: YOLO_LETTERBOX,
|
|
241
|
+
})
|
|
242
|
+
STRETCH_INPUT_RECIPES = frozenset({
|
|
243
|
+
PIL_IMAGENET_STRETCH, RGB_HALF_PIXEL_IMAGENET, BGR_BYTES_STRETCH,
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def resolve_input_preprocessing(layout: str, recipe: "str | None" = None) -> str:
|
|
248
|
+
if layout not in LEGACY_INPUT_PREPROCESSING:
|
|
249
|
+
raise ValueError(f"unknown output layout {layout!r} for input preprocessing")
|
|
250
|
+
selected = LEGACY_INPUT_PREPROCESSING[layout] if recipe is None else recipe
|
|
251
|
+
allowed = STRETCH_INPUT_RECIPES if layout == RFDETR_DETS_LABELS else frozenset({YOLO_LETTERBOX})
|
|
252
|
+
if not isinstance(selected, str) or selected not in allowed:
|
|
253
|
+
raise ValueError(f"input preprocessing {selected!r} is not supported with {layout!r}")
|
|
254
|
+
return selected
|
|
255
|
+
|
|
256
|
+
|
|
231
257
|
@dataclass(frozen=True)
|
|
232
258
|
class ModelVocabulary:
|
|
233
259
|
"""What ONE model can be asked about, and what one resident copy costs.
|
|
@@ -295,8 +321,14 @@ class ModelVocabulary:
|
|
|
295
321
|
#: threshold nobody fitted, on plausible boxes, with every health surface
|
|
296
322
|
#: green. Declaration is the in-band signal position was standing in for.
|
|
297
323
|
owns: "frozenset[str]" = frozenset()
|
|
324
|
+
#: Exact ONNX pixel recipe. None preserves the scored legacy catalogue.
|
|
325
|
+
input_preprocessing: "str | None" = None
|
|
298
326
|
|
|
299
327
|
def __post_init__(self) -> None:
|
|
328
|
+
if not isinstance(self.onnx_variant, VariantContract):
|
|
329
|
+
raise ValueError("model preprocessing requires a declared ONNX variant")
|
|
330
|
+
object.__setattr__(self, "input_preprocessing", resolve_input_preprocessing(
|
|
331
|
+
self.onnx_variant.output_layout, self.input_preprocessing))
|
|
300
332
|
# The model name IS A DIRECTORY NAME — `resolve_model_path` puts the
|
|
301
333
|
# artifact at `models/<model-name>/model.onnx` on every platform — so it
|
|
302
334
|
# is restricted to what is safe in a path on all three, and NOT folded
|
|
@@ -44,19 +44,23 @@ from __future__ import annotations
|
|
|
44
44
|
import io
|
|
45
45
|
import threading
|
|
46
46
|
from pathlib import Path
|
|
47
|
+
from typing import TYPE_CHECKING
|
|
47
48
|
|
|
48
49
|
from .detections import DetectionBatch
|
|
49
50
|
from .errors import DetectorNotReady, UnknownClasses
|
|
50
51
|
from .labels import BUILTIN_VOCABULARIES, ClassIndex, ModelVocabulary
|
|
51
|
-
from .modelpath import resolve_model_path
|
|
52
|
+
from .modelpath import resolve_model_path, resolve_data_dir, MODEL_FILE_NAME
|
|
52
53
|
from .registry import (
|
|
53
54
|
STATE_LOADING,
|
|
54
55
|
STATE_READY,
|
|
55
56
|
STATE_UNAVAILABLE,
|
|
56
57
|
ModelEntry,
|
|
57
58
|
ModelRegistry,
|
|
59
|
+
TaskModelContract,
|
|
58
60
|
)
|
|
59
|
-
from .
|
|
61
|
+
from .tasks import BUILTIN_TASK_MODELS
|
|
62
|
+
if TYPE_CHECKING:
|
|
63
|
+
from .session import OnnxDetector
|
|
60
64
|
|
|
61
65
|
DEFAULT_POLL_INTERVAL_S = 30.0
|
|
62
66
|
|
|
@@ -80,6 +84,7 @@ def build_catalogue(
|
|
|
80
84
|
default_model_path: "Path | None" = None,
|
|
81
85
|
vocabularies: "tuple[ModelVocabulary, ...]" = BUILTIN_VOCABULARIES,
|
|
82
86
|
env=None,
|
|
87
|
+
task_models: tuple = (),
|
|
83
88
|
) -> "list[ModelEntry]":
|
|
84
89
|
"""Every model this build can label, bound to where it would live on THIS box.
|
|
85
90
|
|
|
@@ -109,6 +114,8 @@ def build_catalogue(
|
|
|
109
114
|
else:
|
|
110
115
|
path = resolve_model_path(vocabulary.model_name, env)
|
|
111
116
|
entries.append(ModelEntry(vocabulary, path, pinned=pinned))
|
|
117
|
+
for contract in task_models:
|
|
118
|
+
entries.append(ModelEntry(contract, resolve_data_dir(env) / "models" / contract.model_name / MODEL_FILE_NAME))
|
|
112
119
|
# Pinned first, so catalogue order — which is what breaks a class-ownership
|
|
113
120
|
# tie — puts the general detector ahead of every specialist. `person` must
|
|
114
121
|
# keep coming from the model the 98.8/98.8 was measured on.
|
|
@@ -129,6 +136,7 @@ class DetectorLoader:
|
|
|
129
136
|
vocabularies: "tuple[ModelVocabulary, ...]" = BUILTIN_VOCABULARIES,
|
|
130
137
|
log=print,
|
|
131
138
|
accelerator: str = "auto",
|
|
139
|
+
task_models: tuple = BUILTIN_TASK_MODELS,
|
|
132
140
|
):
|
|
133
141
|
# `ram_budget_bytes=None` is NOT a default policy — it is the absence of
|
|
134
142
|
# one, and the registry refuses every portfolio decision in that state
|
|
@@ -143,7 +151,7 @@ class DetectorLoader:
|
|
|
143
151
|
self._log = log
|
|
144
152
|
|
|
145
153
|
self.registry = ModelRegistry(
|
|
146
|
-
entries=build_catalogue(model_name, model_path, vocabularies),
|
|
154
|
+
entries=build_catalogue(model_name, model_path, vocabularies, task_models=task_models),
|
|
147
155
|
session_factory=self._build_session,
|
|
148
156
|
probe=self._probe,
|
|
149
157
|
ram_budget_bytes=ram_budget_bytes,
|
|
@@ -156,6 +164,13 @@ class DetectorLoader:
|
|
|
156
164
|
# ── the two halves that need a real runtime ──────────────────────────────
|
|
157
165
|
|
|
158
166
|
def _build_session(self, entry: ModelEntry) -> OnnxDetector:
|
|
167
|
+
if isinstance(entry.vocabulary, TaskModelContract):
|
|
168
|
+
if entry.vocabulary.task == "actor_actions":
|
|
169
|
+
from .actor_actions import ActorActionEstimator
|
|
170
|
+
return ActorActionEstimator(entry.path,entry.vocabulary,intra_op_threads=self._intra_op_threads or 4,accelerator=self._accelerator)
|
|
171
|
+
from .pose import PoseEstimator
|
|
172
|
+
return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
|
|
173
|
+
from .session import OnnxDetector
|
|
159
174
|
return OnnxDetector(entry.path, entry.vocabulary, self._intra_op_threads, self._accelerator)
|
|
160
175
|
|
|
161
176
|
def _probe(self, session: OnnxDetector) -> None:
|
|
@@ -164,7 +179,12 @@ class DetectorLoader:
|
|
|
164
179
|
`0.99` keeps the probe's own post-processing trivial on a grey frame;
|
|
165
180
|
what is being established is that a forward pass completes at all.
|
|
166
181
|
"""
|
|
167
|
-
|
|
182
|
+
if callable(getattr(session,"probe",None)):
|
|
183
|
+
session.probe()
|
|
184
|
+
elif hasattr(session, "estimate"):
|
|
185
|
+
session.estimate(_warmup_jpeg(), [[0., 0., 64., 48.]])
|
|
186
|
+
else:
|
|
187
|
+
session.detect(_warmup_jpeg(), 0.99, None)
|
|
168
188
|
|
|
169
189
|
def start(self) -> None:
|
|
170
190
|
self._thread.start()
|
|
@@ -289,6 +309,12 @@ class DetectorLoader:
|
|
|
289
309
|
def detect(self, jpeg: bytes, min_confidence: float, wanted: "set[str] | None", expected_model=None) -> DetectionBatch:
|
|
290
310
|
return self.registry.detect(jpeg, min_confidence, wanted, expected_model=expected_model)
|
|
291
311
|
|
|
312
|
+
def estimate_pose(self, jpeg: bytes, boxes: list, expected_model: dict) -> dict:
|
|
313
|
+
return self.registry.estimate_pose(jpeg, boxes, expected_model)
|
|
314
|
+
|
|
315
|
+
def classify_actor_actions(self,batch):
|
|
316
|
+
return self.registry.classify_actor_actions(batch)
|
|
317
|
+
|
|
292
318
|
# ── the loop ─────────────────────────────────────────────────────────────
|
|
293
319
|
|
|
294
320
|
def _run(self) -> None:
|