@trillboards/edge-sdk 0.2.175 → 0.2.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/README.md +6 -0
- package/deploy/detector/pyproject.toml +4 -1
- package/deploy/detector/tests/test_actor_action_wire.py +103 -0
- package/deploy/detector/tests/test_actor_actions.py +94 -0
- package/deploy/detector/tests/test_actor_checkpoint.py +99 -0
- package/deploy/detector/tests/test_actor_tracking.py +191 -0
- package/deploy/detector/tests/test_actor_tracking_wire.py +120 -0
- package/deploy/detector/tests/test_actor_views.py +108 -0
- package/deploy/detector/tests/test_compute_budget.py +28 -0
- package/deploy/detector/tests/test_detector_request.py +52 -0
- package/deploy/detector/tests/test_input_recipe.py +77 -0
- package/deploy/detector/tests/test_loader.py +20 -1
- package/deploy/detector/tests/test_pose.py +66 -0
- package/deploy/detector/tests/test_server.py +83 -0
- package/deploy/detector/tests/test_task_registry.py +74 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/actor_action_contract.py +65 -0
- package/deploy/detector/trillboards_detector/actor_action_wire.py +189 -0
- package/deploy/detector/trillboards_detector/actor_actions.py +115 -0
- package/deploy/detector/trillboards_detector/actor_checkpoint.py +91 -0
- package/deploy/detector/trillboards_detector/actor_tracking.py +337 -0
- package/deploy/detector/trillboards_detector/actor_views.py +102 -0
- package/deploy/detector/trillboards_detector/cli.py +12 -2
- package/deploy/detector/trillboards_detector/labels.py +32 -0
- package/deploy/detector/trillboards_detector/loader.py +30 -4
- package/deploy/detector/trillboards_detector/pose.py +129 -0
- package/deploy/detector/trillboards_detector/registry.py +92 -7
- package/deploy/detector/trillboards_detector/server.py +131 -3
- package/deploy/detector/trillboards_detector/session.py +83 -49
- package/deploy/detector/trillboards_detector/tasks.py +27 -0
- package/deploy/detector/trillboards_detector/worker_budget.py +14 -0
- package/deploy/windows/README.md +1 -1
- package/deploy/windows/ci/test-published-install.ps1 +16 -3
- package/dist/ActorActionClient.d.ts +47 -0
- package/dist/ActorActionClient.d.ts.map +1 -0
- package/dist/ActorActionClient.js +187 -0
- package/dist/ActorActionClient.js.map +1 -0
- package/dist/ActorActionProducer.d.ts +17 -0
- package/dist/ActorActionProducer.d.ts.map +1 -0
- package/dist/ActorActionProducer.js +92 -0
- package/dist/ActorActionProducer.js.map +1 -0
- package/dist/ActorPerception.d.ts +65 -0
- package/dist/ActorPerception.d.ts.map +1 -0
- package/dist/ActorPerception.js +124 -0
- package/dist/ActorPerception.js.map +1 -0
- package/dist/ActorPose.d.ts +42 -0
- package/dist/ActorPose.d.ts.map +1 -0
- package/dist/ActorPose.js +62 -0
- package/dist/ActorPose.js.map +1 -0
- package/dist/ActorReview.d.ts +51 -0
- package/dist/ActorReview.d.ts.map +1 -0
- package/dist/ActorReview.js +84 -0
- package/dist/ActorReview.js.map +1 -0
- package/dist/ActorTracking.d.ts +33 -0
- package/dist/ActorTracking.d.ts.map +1 -0
- package/dist/ActorTracking.js +90 -0
- package/dist/ActorTracking.js.map +1 -0
- package/dist/ActorWindow.d.ts +56 -0
- package/dist/ActorWindow.d.ts.map +1 -0
- package/dist/ActorWindow.js +86 -0
- package/dist/ActorWindow.js.map +1 -0
- package/dist/ActorWindowContract.d.ts +5 -0
- package/dist/ActorWindowContract.d.ts.map +1 -0
- package/dist/ActorWindowContract.js +23 -0
- package/dist/ActorWindowContract.js.map +1 -0
- package/dist/AudienceAnalyzeClient.d.ts +4 -0
- package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
- package/dist/AudienceAnalyzeClient.js.map +1 -1
- package/dist/ChannelProber.d.ts.map +1 -1
- package/dist/ChannelProber.js +8 -0
- package/dist/ChannelProber.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +7 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +31 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
- package/dist/DurableVlmCandidateSpool.js +31 -11
- package/dist/DurableVlmCandidateSpool.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/LocalDetectorClient.d.ts +5 -1
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +48 -3
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/RetainedFieldContract.d.ts +2 -1
- package/dist/RetainedFieldContract.d.ts.map +1 -1
- package/dist/RetainedFieldContract.js +17 -5
- package/dist/RetainedFieldContract.js.map +1 -1
- package/dist/RuntimeModelSelection.d.ts.map +1 -1
- package/dist/RuntimeModelSelection.js +63 -2
- package/dist/RuntimeModelSelection.js.map +1 -1
- package/dist/TemporalReviewClient.d.ts.map +1 -1
- package/dist/TemporalReviewClient.js +20 -1
- package/dist/TemporalReviewClient.js.map +1 -1
- package/dist/TemporalReviewJournal.d.ts +10 -1
- package/dist/TemporalReviewJournal.d.ts.map +1 -1
- package/dist/TemporalReviewJournal.js +42 -10
- package/dist/TemporalReviewJournal.js.map +1 -1
- package/dist/connectorUpdate.d.ts +2 -0
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +33 -7
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -1
- package/dist/index.js.map +1 -1
- package/dist/localSensing.d.ts +4 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +43 -8
- package/dist/localSensing.js.map +1 -1
- package/package.json +5 -5
- package/retained/RetainedCaptureClient.cjs +55 -9
- package/retained/__tests__/actor-perception.test.cjs +125 -0
- package/retained/__tests__/actor-window.test.cjs +78 -0
- package/retained/__tests__/runtime.test.cjs +44 -4
- package/retained/actor-window.cjs +62 -0
- package/retained/manager.cjs +14 -2
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""RTMPose body7 export: RGB affine input and COCO17 SimCC outputs.
|
|
2
|
+
|
|
3
|
+
The artifact's pipeline.json declares RGB, 1.25 bbox padding, 192x256 and
|
|
4
|
+
split_ratio=2, without DARK or flip testing. MMPose v1.3.2 SimCC decoding uses
|
|
5
|
+
the minimum axis response. These responses are not calibrated probabilities.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
import hashlib
|
|
10
|
+
import io
|
|
11
|
+
import math
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
import cv2
|
|
17
|
+
import numpy as np
|
|
18
|
+
import onnxruntime as ort
|
|
19
|
+
from PIL import Image
|
|
20
|
+
|
|
21
|
+
from .errors import ModelContractError
|
|
22
|
+
from .execution import LocalOrtSession
|
|
23
|
+
|
|
24
|
+
# OpenCV otherwise creates a worker per host core, outside the model budget.
|
|
25
|
+
cv2.setNumThreads(1)
|
|
26
|
+
|
|
27
|
+
from .tasks import RTMPOSE_MODEL_ID as MODEL_ID, RTMPOSE_SHA256 as MODEL_SHA256, RTMPOSE_PREPROCESSING as PREPROCESSING, COCO17
|
|
28
|
+
MEAN = np.array([123.675,116.28,103.53],np.float32)
|
|
29
|
+
STD = np.array([58.395,57.12,57.375],np.float32)
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class PoseGeometry:
|
|
33
|
+
center: tuple[float,float]
|
|
34
|
+
scale: tuple[float,float]
|
|
35
|
+
frame_size: tuple[int,int]
|
|
36
|
+
|
|
37
|
+
def prepare_pose(rgb, box):
|
|
38
|
+
if (not isinstance(rgb,np.ndarray) or rgb.dtype!=np.uint8
|
|
39
|
+
or rgb.ndim!=3 or rgb.shape[2]!=3 or min(rgb.shape[:2])<2
|
|
40
|
+
or rgb.shape[0]*rgb.shape[1]>16777216
|
|
41
|
+
or not isinstance(box,(tuple,list)) or len(box)!=4
|
|
42
|
+
or any(isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) for v in box)):
|
|
43
|
+
raise ValueError("Invalid RGB pose frame or person box")
|
|
44
|
+
x1,y1,x2,y2=box
|
|
45
|
+
h,w=rgb.shape[:2]
|
|
46
|
+
if not (0<=x1<x2<=w and 0<=y1<y2<=h):
|
|
47
|
+
raise ValueError("Person box lies outside original frame")
|
|
48
|
+
center=((x1+x2)/2,(y1+y2)/2)
|
|
49
|
+
sx,sy=(x2-x1)*1.25,(y2-y1)*1.25
|
|
50
|
+
if sx>sy*.75:sy=sx/.75
|
|
51
|
+
else:sx=sy*.75
|
|
52
|
+
# Same three-point zero-rotation affine as OpenMMLab top_down_affine.
|
|
53
|
+
cx,cy=center
|
|
54
|
+
source=np.zeros((3,2),np.float32)
|
|
55
|
+
source[0]=[cx,cy]
|
|
56
|
+
source[1]=[cx,cy-sx/2]
|
|
57
|
+
direction=source[0]-source[1]
|
|
58
|
+
source[2]=source[1]+np.array([-direction[1],direction[0]],np.float32)
|
|
59
|
+
target=np.array([[96,128],[96,32],[0,32]],np.float32)
|
|
60
|
+
matrix=cv2.getAffineTransform(source,target)
|
|
61
|
+
crop=cv2.warpAffine(rgb,matrix,(192,256),flags=cv2.INTER_LINEAR)
|
|
62
|
+
tensor=np.ascontiguousarray(((crop.astype(np.float32)-MEAN)/STD).transpose(2,0,1)[None])
|
|
63
|
+
return tensor,PoseGeometry(center,(sx,sy),(w,h))
|
|
64
|
+
|
|
65
|
+
def decode_pose(simcc_x,simcc_y,geometry):
|
|
66
|
+
for value,shape in ((simcc_x,(1,17,384)),(simcc_y,(1,17,512))):
|
|
67
|
+
if not isinstance(value,np.ndarray) or value.shape!=shape or not np.isfinite(value).all():
|
|
68
|
+
raise ValueError("Invalid RTMPose SimCC output")
|
|
69
|
+
if (not isinstance(geometry,PoseGeometry)
|
|
70
|
+
or len(geometry.center)!=2 or len(geometry.scale)!=2 or len(geometry.frame_size)!=2
|
|
71
|
+
or any(not math.isfinite(v) for v in (*geometry.center,*geometry.scale))
|
|
72
|
+
or min(geometry.scale)<=0 or min(geometry.frame_size)<=0):
|
|
73
|
+
raise ValueError("Invalid pose geometry")
|
|
74
|
+
x=np.argmax(simcc_x[0],axis=1).astype(np.float32)/2
|
|
75
|
+
y=np.argmax(simcc_y[0],axis=1).astype(np.float32)/2
|
|
76
|
+
responses=np.minimum(np.max(simcc_x[0],axis=1),np.max(simcc_y[0],axis=1))
|
|
77
|
+
x=x/192*geometry.scale[0]+geometry.center[0]-geometry.scale[0]/2
|
|
78
|
+
y=y/256*geometry.scale[1]+geometry.center[1]-geometry.scale[1]/2
|
|
79
|
+
result=[]
|
|
80
|
+
for i,name in enumerate(COCO17):
|
|
81
|
+
observed=responses[i]>0 and 0<=x[i]<geometry.frame_size[0] and 0<=y[i]<geometry.frame_size[1]
|
|
82
|
+
result.append({"name":name,"x":float(x[i]) if observed else None,
|
|
83
|
+
"y":float(y[i]) if observed else None,"response":float(responses[i])})
|
|
84
|
+
return result
|
|
85
|
+
|
|
86
|
+
class PoseEstimator:
|
|
87
|
+
"""One immutable artifact, bounded CPU pools and explicit model proof."""
|
|
88
|
+
model_name = MODEL_ID
|
|
89
|
+
model_sha256 = MODEL_SHA256
|
|
90
|
+
|
|
91
|
+
def __init__(self,model_path,*,intra_op_threads=4,accelerator="auto"):
|
|
92
|
+
if type(intra_op_threads) is not int or not 1<=intra_op_threads<=8:
|
|
93
|
+
raise ValueError("Invalid bounded pose thread count")
|
|
94
|
+
data=Path(model_path).read_bytes()
|
|
95
|
+
if hashlib.sha256(data).hexdigest()!=MODEL_SHA256:
|
|
96
|
+
raise ModelContractError("RTMPose artifact differs from pinned export")
|
|
97
|
+
options=ort.SessionOptions()
|
|
98
|
+
options.intra_op_num_threads=intra_op_threads
|
|
99
|
+
options.inter_op_num_threads=1
|
|
100
|
+
self._session=LocalOrtSession(ort,data,options,accelerator)
|
|
101
|
+
inputs=self._session.get_inputs();outputs={o.name:o for o in self._session.get_outputs()}
|
|
102
|
+
if (len(inputs)!=1 or inputs[0].name!="input" or inputs[0].shape!=["batch",3,256,192]
|
|
103
|
+
or inputs[0].type!="tensor(float)" or set(outputs)!={"simcc_x","simcc_y"}
|
|
104
|
+
or outputs["simcc_x"].shape!=["batch","MatMulsimcc_x_dim_1",384]
|
|
105
|
+
or outputs["simcc_y"].shape!=["batch","MatMulsimcc_x_dim_1",512]
|
|
106
|
+
or any(o.type!="tensor(float)" for o in outputs.values())):
|
|
107
|
+
raise ModelContractError("RTMPose graph differs from declared RGB/COCO17 contract")
|
|
108
|
+
self._lock=threading.Lock()
|
|
109
|
+
|
|
110
|
+
def estimate(self,jpeg,boxes):
|
|
111
|
+
if not isinstance(boxes,list) or len(boxes)>256:
|
|
112
|
+
raise ValueError("Invalid pose person budget")
|
|
113
|
+
if not isinstance(jpeg,bytes) or len(jpeg)>16*1024**2:
|
|
114
|
+
raise ValueError("Invalid pose JPEG")
|
|
115
|
+
with Image.open(io.BytesIO(jpeg)) as image:
|
|
116
|
+
if image.width*image.height>16777216:
|
|
117
|
+
raise ValueError("Oversized pose frame")
|
|
118
|
+
rgb=np.asarray(image.convert("RGB"))
|
|
119
|
+
started=time.monotonic()
|
|
120
|
+
poses=[]
|
|
121
|
+
with self._lock:
|
|
122
|
+
for box in boxes:
|
|
123
|
+
tensor,geometry=prepare_pose(rgb,box)
|
|
124
|
+
x,y=self._session.run(["simcc_x","simcc_y"],{"input":tensor})
|
|
125
|
+
poses.append(decode_pose(x,y,geometry))
|
|
126
|
+
return {"version":1,"modelId":MODEL_ID,"modelSha256":MODEL_SHA256,
|
|
127
|
+
"preprocessing":PREPROCESSING,"imageSha256":hashlib.sha256(jpeg).hexdigest(),
|
|
128
|
+
"frameSize":{"width":rgb.shape[1],"height":rgb.shape[0]},"poses":poses,
|
|
129
|
+
"inferenceMs":(time.monotonic()-started)*1000}
|
|
@@ -60,6 +60,8 @@ import sys
|
|
|
60
60
|
import threading
|
|
61
61
|
import time
|
|
62
62
|
from pathlib import Path
|
|
63
|
+
from dataclasses import dataclass
|
|
64
|
+
import re
|
|
63
65
|
|
|
64
66
|
from .detections import DetectionBatch, merge_batches
|
|
65
67
|
from .errors import DetectorNotReady, ModelContractError, ModelVerificationError, UnknownClasses
|
|
@@ -103,6 +105,27 @@ STATE_UNAVAILABLE = "unavailable"
|
|
|
103
105
|
#: MEASURING it per load so the authority above can eventually stop guessing.
|
|
104
106
|
|
|
105
107
|
|
|
108
|
+
@dataclass(frozen=True)
|
|
109
|
+
class TaskModelContract:
|
|
110
|
+
"""A non-box model's task, immutable bytes and measured residency cost."""
|
|
111
|
+
model_name: str
|
|
112
|
+
task: str
|
|
113
|
+
sha256: str
|
|
114
|
+
approx_ram_bytes: int
|
|
115
|
+
preprocessing: str
|
|
116
|
+
|
|
117
|
+
def __post_init__(self):
|
|
118
|
+
if (not isinstance(self.model_name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,127}", self.model_name)
|
|
119
|
+
or not isinstance(self.sha256, str) or not re.fullmatch(r"[a-f0-9]{64}", self.sha256)
|
|
120
|
+
or type(self.approx_ram_bytes) is not int or self.approx_ram_bytes <= 0
|
|
121
|
+
or not isinstance(self.preprocessing, str) or not self.preprocessing):
|
|
122
|
+
raise ValueError("invalid task model contract")
|
|
123
|
+
self._validate_task()
|
|
124
|
+
|
|
125
|
+
def _validate_task(self):
|
|
126
|
+
if self.task != "pose_coco17":
|
|
127
|
+
raise ValueError("invalid non-box task kind")
|
|
128
|
+
|
|
106
129
|
class ModelEntry:
|
|
107
130
|
"""One catalogue row: a committed vocabulary bound to a path on this box."""
|
|
108
131
|
|
|
@@ -187,6 +210,8 @@ class ModelRegistry:
|
|
|
187
210
|
"exactly one catalogue entry must be pinned — it is what `/health` reports and "
|
|
188
211
|
f"what the cascade gates on; got {[e.name for e in pinned]}"
|
|
189
212
|
)
|
|
213
|
+
if not isinstance(pinned[0].vocabulary, ModelVocabulary):
|
|
214
|
+
raise ValueError("the pinned cascade gate must be an object detector")
|
|
190
215
|
if ram_budget_bytes is not None:
|
|
191
216
|
if ram_budget_bytes <= 0:
|
|
192
217
|
raise ValueError(f"an injected RAM ceiling must be positive, got {ram_budget_bytes}")
|
|
@@ -251,7 +276,8 @@ class ModelRegistry:
|
|
|
251
276
|
with self._lock:
|
|
252
277
|
if self._index is None:
|
|
253
278
|
self._index = ClassIndex(
|
|
254
|
-
[s.entry.vocabulary for s in self._states.values()
|
|
279
|
+
[s.entry.vocabulary for s in self._states.values()
|
|
280
|
+
if s.present and isinstance(s.entry.vocabulary, ModelVocabulary)]
|
|
255
281
|
)
|
|
256
282
|
return self._index
|
|
257
283
|
|
|
@@ -282,7 +308,7 @@ class ModelRegistry:
|
|
|
282
308
|
pending: "dict[str, str]" = {}
|
|
283
309
|
with self._lock:
|
|
284
310
|
for state in self._states.values():
|
|
285
|
-
if state.present:
|
|
311
|
+
if state.present or not isinstance(state.entry.vocabulary, ModelVocabulary):
|
|
286
312
|
continue
|
|
287
313
|
for raw in unknown:
|
|
288
314
|
if normalize_class_name(raw) in state.entry.vocabulary.class_names:
|
|
@@ -395,8 +421,8 @@ class ModelRegistry:
|
|
|
395
421
|
plan = self.class_index().assign(wanted)
|
|
396
422
|
else:
|
|
397
423
|
selected = self._states.get(expected_model["modelId"])
|
|
398
|
-
if selected is None:
|
|
399
|
-
raise ModelVerificationError("selected
|
|
424
|
+
if selected is None or not isinstance(selected.entry.vocabulary, ModelVocabulary):
|
|
425
|
+
raise ModelVerificationError("selected object detector is not in this box's catalogue")
|
|
400
426
|
vocabulary = selected.entry.vocabulary
|
|
401
427
|
if wanted is not None and not wanted.issubset(vocabulary.class_names):
|
|
402
428
|
raise ModelVerificationError("selected model cannot answer all requested classes")
|
|
@@ -445,6 +471,60 @@ class ModelRegistry:
|
|
|
445
471
|
)
|
|
446
472
|
return merged
|
|
447
473
|
|
|
474
|
+
def estimate_pose(self, jpeg: bytes, boxes: list, expected_model: dict) -> dict:
|
|
475
|
+
"""Acquire pose through the same portfolio as all detector sessions."""
|
|
476
|
+
state = self._states.get(expected_model.get("modelId"))
|
|
477
|
+
if (state is None or not isinstance(state.entry.vocabulary, TaskModelContract)
|
|
478
|
+
or state.entry.vocabulary.task != "pose_coco17"
|
|
479
|
+
or state.entry.vocabulary.sha256 != expected_model.get("sha256")):
|
|
480
|
+
raise ModelVerificationError("selected pose task does not match its immutable contract")
|
|
481
|
+
contract = state.entry.vocabulary
|
|
482
|
+
sessions = self._acquire([state.entry.name])
|
|
483
|
+
try:
|
|
484
|
+
session = sessions[0]
|
|
485
|
+
if getattr(session, "model_sha256", None) != contract.sha256:
|
|
486
|
+
raise ModelVerificationError("loaded pose session does not match the selected artifact")
|
|
487
|
+
try:
|
|
488
|
+
result = session.estimate(jpeg, boxes)
|
|
489
|
+
if (result.get("modelId") != contract.model_name
|
|
490
|
+
or result.get("modelSha256") != contract.sha256
|
|
491
|
+
or result.get("preprocessing") != contract.preprocessing):
|
|
492
|
+
raise ModelVerificationError("pose output does not match its task contract")
|
|
493
|
+
except Exception:
|
|
494
|
+
with self._lock:
|
|
495
|
+
state.detect_failures += 1
|
|
496
|
+
raise
|
|
497
|
+
with self._lock:
|
|
498
|
+
state.detect_count += 1
|
|
499
|
+
return result
|
|
500
|
+
finally:
|
|
501
|
+
self._release([state.entry.name])
|
|
502
|
+
|
|
503
|
+
def classify_actor_actions(self,batch):
|
|
504
|
+
"""Acquire and prove the selected non-box action model through the shared portfolio."""
|
|
505
|
+
from .actor_action_contract import ActorActionModelContract
|
|
506
|
+
state=self._states.get(batch.model.get("modelId"))
|
|
507
|
+
if (state is None or not isinstance(state.entry.vocabulary,ActorActionModelContract)
|
|
508
|
+
or state.entry.vocabulary.sha256!=batch.model.get("sha256")):
|
|
509
|
+
raise ModelVerificationError("selected actor action task differs from its immutable contract")
|
|
510
|
+
contract=state.entry.vocabulary
|
|
511
|
+
sessions=self._acquire([state.entry.name])
|
|
512
|
+
try:
|
|
513
|
+
session=sessions[0]
|
|
514
|
+
if getattr(session,"model_sha256",None)!=contract.sha256:
|
|
515
|
+
raise ModelVerificationError("loaded actor action session differs from selected artifact")
|
|
516
|
+
try:
|
|
517
|
+
result=session.classify(batch)
|
|
518
|
+
if (result.get("modelId")!=contract.model_name or result.get("modelSha256")!=contract.sha256
|
|
519
|
+
or result.get("preprocessing")!=contract.preprocessing):
|
|
520
|
+
raise ModelVerificationError("actor action result differs from selected task")
|
|
521
|
+
except Exception:
|
|
522
|
+
with self._lock:state.detect_failures+=1
|
|
523
|
+
raise
|
|
524
|
+
with self._lock:state.detect_count+=1
|
|
525
|
+
return result
|
|
526
|
+
finally:self._release([state.entry.name])
|
|
527
|
+
|
|
448
528
|
def _acquire(self, needed: "list[str]") -> list:
|
|
449
529
|
"""Make every named model resident under the budget, then pin it in flight.
|
|
450
530
|
|
|
@@ -645,7 +725,7 @@ class ModelRegistry:
|
|
|
645
725
|
measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
|
|
646
726
|
self._log(
|
|
647
727
|
f"[detector] ready — {entry.name} at {entry.path} "
|
|
648
|
-
f"(probe {warmup_ms:.0f} ms, {
|
|
728
|
+
f"(probe {warmup_ms:.0f} ms, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
|
|
649
729
|
)
|
|
650
730
|
|
|
651
731
|
def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
|
|
@@ -692,7 +772,8 @@ class ModelRegistry:
|
|
|
692
772
|
"present": s.present,
|
|
693
773
|
"resident": s.resident,
|
|
694
774
|
"pinned": s.entry.pinned,
|
|
695
|
-
"
|
|
775
|
+
"task": getattr(s.entry.vocabulary, "task", "object_detection"),
|
|
776
|
+
"named_classes": len(s.entry.vocabulary.slot_names) if isinstance(s.entry.vocabulary, ModelVocabulary) else 0,
|
|
696
777
|
"approx_ram_bytes": s.entry.vocabulary.approx_ram_bytes,
|
|
697
778
|
"detect_count": s.detect_count,
|
|
698
779
|
"detect_failures": s.detect_failures,
|
|
@@ -733,6 +814,9 @@ class ModelRegistry:
|
|
|
733
814
|
# strings already on the spec, read under the same lock.
|
|
734
815
|
row["output_layout"] = getattr(spec, "output_layout", None)
|
|
735
816
|
row["preprocessing"] = getattr(spec, "preprocessing", None)
|
|
817
|
+
if isinstance(s.entry.vocabulary, TaskModelContract):
|
|
818
|
+
row["model_sha256"] = s.entry.vocabulary.sha256
|
|
819
|
+
row["preprocessing"] = s.entry.vocabulary.preprocessing
|
|
736
820
|
models[name] = row
|
|
737
821
|
|
|
738
822
|
# Counted from `present` directly rather than through
|
|
@@ -742,7 +826,8 @@ class ModelRegistry:
|
|
|
742
826
|
# have to wait for.
|
|
743
827
|
known = len(
|
|
744
828
|
frozenset().union(
|
|
745
|
-
*[s.entry.vocabulary.class_names for s in self._states.values()
|
|
829
|
+
*[s.entry.vocabulary.class_names for s in self._states.values()
|
|
830
|
+
if s.present and isinstance(s.entry.vocabulary, ModelVocabulary)]
|
|
746
831
|
) if any(s.present for s in self._states.values()) else frozenset()
|
|
747
832
|
)
|
|
748
833
|
|
|
@@ -27,11 +27,16 @@ from __future__ import annotations
|
|
|
27
27
|
import base64
|
|
28
28
|
import binascii
|
|
29
29
|
import json
|
|
30
|
+
import hashlib
|
|
31
|
+
import io
|
|
30
32
|
import math
|
|
31
33
|
import sys as _sys
|
|
32
34
|
import threading
|
|
33
35
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
34
36
|
|
|
37
|
+
from .actor_tracking import TrackingError, validate_tracking_request
|
|
38
|
+
from .tasks import RTMPOSE_MODEL_ID, RTMPOSE_SHA256, RTMPOSE_PREPROCESSING, COCO17
|
|
39
|
+
from .actor_action_wire import parse_actor_action_request, validate_actor_action_response
|
|
35
40
|
from ._version import VERSION
|
|
36
41
|
from .errors import DetectorNotReady, ModelVerificationError, UnknownClasses
|
|
37
42
|
|
|
@@ -226,6 +231,49 @@ def build_detect_response(batch) -> dict:
|
|
|
226
231
|
return body
|
|
227
232
|
|
|
228
233
|
|
|
234
|
+
def parse_pose_model(raw: bytes):
|
|
235
|
+
value = json.loads(raw.decode("utf-8")).get("pose_model")
|
|
236
|
+
if value is None:
|
|
237
|
+
return None
|
|
238
|
+
if (not isinstance(value, dict) or set(value) != {"modelId", "sha256"}
|
|
239
|
+
or value.get("modelId") != RTMPOSE_MODEL_ID or value.get("sha256") != RTMPOSE_SHA256):
|
|
240
|
+
raise WireError(409, "pose task differs from its pinned artifact contract")
|
|
241
|
+
return value
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def build_pose_response(result, tracking, frame_size):
|
|
245
|
+
if (result.get("version") != 1 or result.get("modelId") != RTMPOSE_MODEL_ID
|
|
246
|
+
or result.get("modelSha256") != RTMPOSE_SHA256
|
|
247
|
+
or result.get("preprocessing") != RTMPOSE_PREPROCESSING
|
|
248
|
+
or result.get("imageSha256") != tracking["imageSha256"]
|
|
249
|
+
or result.get("frameSize") != {"width": frame_size[0], "height": frame_size[1]}
|
|
250
|
+
or not isinstance(result.get("poses"), list)
|
|
251
|
+
or len(result["poses"]) != len(tracking["actors"])
|
|
252
|
+
or not isinstance(result.get("inferenceMs"), (int, float))
|
|
253
|
+
or not math.isfinite(result["inferenceMs"]) or result["inferenceMs"] < 0):
|
|
254
|
+
raise ValueError("pose result differs from the original tracked frame")
|
|
255
|
+
actors = []
|
|
256
|
+
for association, joints in zip(tracking["actors"], result["poses"]):
|
|
257
|
+
if not isinstance(joints, list) or len(joints) != len(COCO17):
|
|
258
|
+
raise ValueError("pose result does not contain COCO17 joints")
|
|
259
|
+
for name, joint in zip(COCO17, joints):
|
|
260
|
+
if (not isinstance(joint, dict) or set(joint) != {"name", "x", "y", "response"}
|
|
261
|
+
or joint.get("name") != name
|
|
262
|
+
or isinstance(joint.get("response"), bool)
|
|
263
|
+
or not isinstance(joint.get("response"), (int, float))
|
|
264
|
+
or not math.isfinite(joint["response"])):
|
|
265
|
+
raise ValueError("invalid named pose joint")
|
|
266
|
+
x, y = joint["x"], joint["y"]
|
|
267
|
+
if x is None and y is None:
|
|
268
|
+
continue
|
|
269
|
+
if (any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) for v in (x, y))
|
|
270
|
+
or not (0 <= x < frame_size[0] and 0 <= y < frame_size[1]) or joint["response"] <= 0):
|
|
271
|
+
raise ValueError("pose joint is outside the original camera frame")
|
|
272
|
+
actors.append({**association, "joints": joints})
|
|
273
|
+
return {key: value for key, value in result.items() if key != "poses"} | {
|
|
274
|
+
"trackingSha256": tracking["sha256"], "actors": actors}
|
|
275
|
+
|
|
276
|
+
|
|
229
277
|
class DetectorHandler(BaseHTTPRequestHandler):
|
|
230
278
|
"""Routes. The server instance carries `loader` and `log_line`."""
|
|
231
279
|
|
|
@@ -260,6 +308,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
|
|
|
260
308
|
def do_POST(self) -> None: # noqa: N802
|
|
261
309
|
if self._path() == "/detect":
|
|
262
310
|
self._detect()
|
|
311
|
+
elif self._path() == "/actor-actions":
|
|
312
|
+
self._actor_actions()
|
|
263
313
|
elif self._path() == "/health":
|
|
264
314
|
self._json(405, {"error": "/health is a GET"})
|
|
265
315
|
else:
|
|
@@ -293,6 +343,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
|
|
|
293
343
|
# answered 400 for its classes. Reported on the LOADER's payload rather
|
|
294
344
|
# than only in a log because the reader that needs it is a script.
|
|
295
345
|
info["version"] = VERSION
|
|
346
|
+
info["actor_tracking"] = {"algorithm": "bytetrack_2.6.1",
|
|
347
|
+
"state": "ready" if self.server.actor_tracker is not None else "unavailable"}
|
|
296
348
|
self._json(200 if ready else 503, info)
|
|
297
349
|
|
|
298
350
|
# ── /detect ──────────────────────────────────────────────────────────────
|
|
@@ -303,6 +355,33 @@ class DetectorHandler(BaseHTTPRequestHandler):
|
|
|
303
355
|
raw = self._read_body()
|
|
304
356
|
jpeg, min_confidence, requested = parse_detect_request(raw)
|
|
305
357
|
expected_model = parse_expected_model(raw)
|
|
358
|
+
pose_model = parse_pose_model(raw)
|
|
359
|
+
tracking = json.loads(raw.decode("utf-8")).get("tracking")
|
|
360
|
+
if pose_model is not None and tracking is None:
|
|
361
|
+
raise WireError(400, "pose requires original-frame actor tracking")
|
|
362
|
+
if pose_model is not None and not hasattr(loader, "estimate_pose"):
|
|
363
|
+
raise WireError(503, "pose task is unavailable in this detector build")
|
|
364
|
+
if tracking is not None:
|
|
365
|
+
try:
|
|
366
|
+
tracking = validate_tracking_request(tracking)
|
|
367
|
+
except TrackingError as err:
|
|
368
|
+
raise WireError(400, str(err)) from None
|
|
369
|
+
if min_confidence > .25:
|
|
370
|
+
raise WireError(400, "actor tracking requires min_confidence <= 0.25")
|
|
371
|
+
if expected_model is None:
|
|
372
|
+
raise WireError(400, "actor tracking requires a verified selected detector")
|
|
373
|
+
if requested is not None and "person" not in requested:
|
|
374
|
+
raise WireError(400, "actor tracking requires person detections")
|
|
375
|
+
if self.server.actor_tracker is None:
|
|
376
|
+
raise WireError(503, "actor tracking is unavailable")
|
|
377
|
+
from PIL import Image
|
|
378
|
+
try:
|
|
379
|
+
with Image.open(io.BytesIO(jpeg)) as image:
|
|
380
|
+
frame_size = image.size
|
|
381
|
+
if image.format != "JPEG" or image.width * image.height > 16777216:
|
|
382
|
+
raise ValueError("actor frames must be bounded original JPEGs")
|
|
383
|
+
except Exception:
|
|
384
|
+
raise WireError(400, "unreadable actor tracking JPEG") from None
|
|
306
385
|
except WireError as err:
|
|
307
386
|
self._json(err.status, {"error": err.message})
|
|
308
387
|
return
|
|
@@ -347,13 +426,61 @@ class DetectorHandler(BaseHTTPRequestHandler):
|
|
|
347
426
|
if expected_model is not None and (body.get("model_hashes") != {expected_model["modelId"]: expected_model["sha256"]}
|
|
348
427
|
or body["model"] != expected_model["modelId"]):
|
|
349
428
|
raise ValueError("selected detector did not return its loaded artifact proof")
|
|
429
|
+
if tracking is not None:
|
|
430
|
+
body["tracking"] = self.server.actor_tracker.track(
|
|
431
|
+
tracking, batch.detections, image_sha256=hashlib.sha256(jpeg).hexdigest(),
|
|
432
|
+
model_hashes=body.get("model_hashes"), frame_size=frame_size)
|
|
433
|
+
if pose_model is not None:
|
|
434
|
+
boxes = []
|
|
435
|
+
for association in body["tracking"]["actors"]:
|
|
436
|
+
detection = batch.detections[association["detectionIndex"]]
|
|
437
|
+
boxes.append([detection.x, detection.y, detection.x + detection.width, detection.y + detection.height])
|
|
438
|
+
result = loader.estimate_pose(jpeg, boxes, pose_model)
|
|
439
|
+
body["pose"] = build_pose_response(result, body["tracking"], frame_size)
|
|
440
|
+
except ModelVerificationError as err:
|
|
441
|
+
self._json(409, {"error": str(err)})
|
|
442
|
+
return
|
|
443
|
+
except DetectorNotReady as err:
|
|
444
|
+
self._json(503, {"error": str(err)})
|
|
445
|
+
return
|
|
350
446
|
except ValueError as err:
|
|
351
447
|
self._log(f"[detector] REFUSING to emit a malformed payload: {err}")
|
|
352
448
|
self._json(500, {"error": f"refused to emit a malformed payload: {err}"})
|
|
353
449
|
return
|
|
450
|
+
except Exception as err:
|
|
451
|
+
self._log(f"[detector] actor perception failed: {type(err).__name__}: {err}")
|
|
452
|
+
self._json(500, {"error": f"actor perception failed: {type(err).__name__}"})
|
|
453
|
+
return
|
|
354
454
|
|
|
355
455
|
self._json(200, body)
|
|
356
456
|
|
|
457
|
+
def _actor_actions(self):
|
|
458
|
+
try:
|
|
459
|
+
raw=self._read_body()
|
|
460
|
+
try:batch=parse_actor_action_request(raw)
|
|
461
|
+
except ValueError as error:raise WireError(400,str(error)) from error
|
|
462
|
+
except WireError as error:
|
|
463
|
+
self._json(error.status,{"error":error.message});return
|
|
464
|
+
classify=getattr(self.server.loader,"classify_actor_actions",None)
|
|
465
|
+
if not callable(classify):
|
|
466
|
+
self._json(503,{"error":"selected native actor action task unavailable"});return
|
|
467
|
+
try:
|
|
468
|
+
result=classify(batch)
|
|
469
|
+
except ModelVerificationError as error:
|
|
470
|
+
self._json(409,{"error":str(error)});return
|
|
471
|
+
except DetectorNotReady as error:
|
|
472
|
+
self._json(503,{"error":str(error)});return
|
|
473
|
+
except ValueError as error:
|
|
474
|
+
self._json(400,{"error":str(error)});return
|
|
475
|
+
except Exception as error:
|
|
476
|
+
self._log(f"[detector] actor action inference failed: {type(error).__name__}: {error}")
|
|
477
|
+
self._json(500,{"error":"actor action inference failed"});return
|
|
478
|
+
try:validate_actor_action_response(result,batch)
|
|
479
|
+
except ValueError as error:
|
|
480
|
+
self._log(f"[detector] REFUSING malformed actor action output: {error}")
|
|
481
|
+
self._json(500,{"error":"refused malformed actor action output"});return
|
|
482
|
+
self._json(200,result)
|
|
483
|
+
|
|
357
484
|
# ── plumbing ─────────────────────────────────────────────────────────────
|
|
358
485
|
|
|
359
486
|
def _read_body(self) -> bytes:
|
|
@@ -526,14 +653,15 @@ class DetectorServer(ThreadingHTTPServer):
|
|
|
526
653
|
# bind is that incident with no way to even notice.
|
|
527
654
|
allow_reuse_address = _sys.platform != "win32"
|
|
528
655
|
|
|
529
|
-
def __init__(self, address, loader, log_line=print):
|
|
656
|
+
def __init__(self, address, loader, log_line=print, actor_tracker=None):
|
|
530
657
|
super().__init__(address, DetectorHandler)
|
|
531
658
|
self.loader = loader
|
|
659
|
+
self.actor_tracker = actor_tracker
|
|
532
660
|
self.log_line = log_line
|
|
533
661
|
|
|
534
662
|
|
|
535
|
-
def serve(loader, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, log_line=print) -> DetectorServer:
|
|
663
|
+
def serve(loader, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, log_line=print, actor_tracker=None) -> DetectorServer:
|
|
536
664
|
"""Start the server on a background thread and return it."""
|
|
537
|
-
server = DetectorServer((host, port), loader, log_line)
|
|
665
|
+
server = DetectorServer((host, port), loader, log_line, actor_tracker)
|
|
538
666
|
threading.Thread(target=server.serve_forever, name="detector-http", daemon=True).start()
|
|
539
667
|
return server
|