@trillboards/edge-sdk 0.2.175 → 0.2.176

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/deploy/detector/README.md +6 -0
  2. package/deploy/detector/pyproject.toml +4 -1
  3. package/deploy/detector/tests/test_actor_action_wire.py +103 -0
  4. package/deploy/detector/tests/test_actor_actions.py +94 -0
  5. package/deploy/detector/tests/test_actor_checkpoint.py +99 -0
  6. package/deploy/detector/tests/test_actor_tracking.py +191 -0
  7. package/deploy/detector/tests/test_actor_tracking_wire.py +120 -0
  8. package/deploy/detector/tests/test_actor_views.py +108 -0
  9. package/deploy/detector/tests/test_compute_budget.py +28 -0
  10. package/deploy/detector/tests/test_detector_request.py +52 -0
  11. package/deploy/detector/tests/test_input_recipe.py +77 -0
  12. package/deploy/detector/tests/test_loader.py +20 -1
  13. package/deploy/detector/tests/test_pose.py +66 -0
  14. package/deploy/detector/tests/test_server.py +83 -0
  15. package/deploy/detector/tests/test_task_registry.py +74 -0
  16. package/deploy/detector/trillboards_detector/_version.py +2 -2
  17. package/deploy/detector/trillboards_detector/actor_action_contract.py +65 -0
  18. package/deploy/detector/trillboards_detector/actor_action_wire.py +189 -0
  19. package/deploy/detector/trillboards_detector/actor_actions.py +115 -0
  20. package/deploy/detector/trillboards_detector/actor_checkpoint.py +91 -0
  21. package/deploy/detector/trillboards_detector/actor_tracking.py +337 -0
  22. package/deploy/detector/trillboards_detector/actor_views.py +102 -0
  23. package/deploy/detector/trillboards_detector/cli.py +12 -2
  24. package/deploy/detector/trillboards_detector/labels.py +32 -0
  25. package/deploy/detector/trillboards_detector/loader.py +30 -4
  26. package/deploy/detector/trillboards_detector/pose.py +129 -0
  27. package/deploy/detector/trillboards_detector/registry.py +92 -7
  28. package/deploy/detector/trillboards_detector/server.py +131 -3
  29. package/deploy/detector/trillboards_detector/session.py +83 -49
  30. package/deploy/detector/trillboards_detector/tasks.py +27 -0
  31. package/deploy/detector/trillboards_detector/worker_budget.py +14 -0
  32. package/deploy/windows/README.md +1 -1
  33. package/deploy/windows/ci/test-published-install.ps1 +16 -3
  34. package/dist/ActorActionClient.d.ts +47 -0
  35. package/dist/ActorActionClient.d.ts.map +1 -0
  36. package/dist/ActorActionClient.js +187 -0
  37. package/dist/ActorActionClient.js.map +1 -0
  38. package/dist/ActorActionProducer.d.ts +17 -0
  39. package/dist/ActorActionProducer.d.ts.map +1 -0
  40. package/dist/ActorActionProducer.js +92 -0
  41. package/dist/ActorActionProducer.js.map +1 -0
  42. package/dist/ActorPerception.d.ts +65 -0
  43. package/dist/ActorPerception.d.ts.map +1 -0
  44. package/dist/ActorPerception.js +124 -0
  45. package/dist/ActorPerception.js.map +1 -0
  46. package/dist/ActorPose.d.ts +42 -0
  47. package/dist/ActorPose.d.ts.map +1 -0
  48. package/dist/ActorPose.js +62 -0
  49. package/dist/ActorPose.js.map +1 -0
  50. package/dist/ActorReview.d.ts +51 -0
  51. package/dist/ActorReview.d.ts.map +1 -0
  52. package/dist/ActorReview.js +84 -0
  53. package/dist/ActorReview.js.map +1 -0
  54. package/dist/ActorTracking.d.ts +33 -0
  55. package/dist/ActorTracking.d.ts.map +1 -0
  56. package/dist/ActorTracking.js +90 -0
  57. package/dist/ActorTracking.js.map +1 -0
  58. package/dist/ActorWindow.d.ts +56 -0
  59. package/dist/ActorWindow.d.ts.map +1 -0
  60. package/dist/ActorWindow.js +86 -0
  61. package/dist/ActorWindow.js.map +1 -0
  62. package/dist/ActorWindowContract.d.ts +5 -0
  63. package/dist/ActorWindowContract.d.ts.map +1 -0
  64. package/dist/ActorWindowContract.js +23 -0
  65. package/dist/ActorWindowContract.js.map +1 -0
  66. package/dist/AudienceAnalyzeClient.d.ts +4 -0
  67. package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
  68. package/dist/AudienceAnalyzeClient.js.map +1 -1
  69. package/dist/ChannelProber.d.ts.map +1 -1
  70. package/dist/ChannelProber.js +8 -0
  71. package/dist/ChannelProber.js.map +1 -1
  72. package/dist/CloudSensingManager.d.ts +7 -0
  73. package/dist/CloudSensingManager.d.ts.map +1 -1
  74. package/dist/CloudSensingManager.js +31 -0
  75. package/dist/CloudSensingManager.js.map +1 -1
  76. package/dist/EdgeAgent.js +1 -1
  77. package/dist/LocalDetectorClient.d.ts +5 -1
  78. package/dist/LocalDetectorClient.d.ts.map +1 -1
  79. package/dist/LocalDetectorClient.js +48 -3
  80. package/dist/LocalDetectorClient.js.map +1 -1
  81. package/dist/RetainedFieldContract.d.ts +2 -1
  82. package/dist/RetainedFieldContract.d.ts.map +1 -1
  83. package/dist/RetainedFieldContract.js +17 -5
  84. package/dist/RetainedFieldContract.js.map +1 -1
  85. package/dist/RuntimeModelSelection.d.ts.map +1 -1
  86. package/dist/RuntimeModelSelection.js +63 -2
  87. package/dist/RuntimeModelSelection.js.map +1 -1
  88. package/dist/TemporalReviewClient.d.ts.map +1 -1
  89. package/dist/TemporalReviewClient.js +20 -1
  90. package/dist/TemporalReviewClient.js.map +1 -1
  91. package/dist/TemporalReviewJournal.d.ts +10 -1
  92. package/dist/TemporalReviewJournal.d.ts.map +1 -1
  93. package/dist/TemporalReviewJournal.js +42 -10
  94. package/dist/TemporalReviewJournal.js.map +1 -1
  95. package/dist/connectorUpdate.d.ts +2 -0
  96. package/dist/connectorUpdate.d.ts.map +1 -1
  97. package/dist/connectorUpdate.js +33 -7
  98. package/dist/connectorUpdate.js.map +1 -1
  99. package/dist/diagnose.d.ts +1 -1
  100. package/dist/diagnose.js +1 -1
  101. package/dist/index.d.ts +6 -0
  102. package/dist/index.d.ts.map +1 -1
  103. package/dist/index.js +12 -1
  104. package/dist/index.js.map +1 -1
  105. package/dist/localSensing.d.ts +4 -0
  106. package/dist/localSensing.d.ts.map +1 -1
  107. package/dist/localSensing.js +43 -8
  108. package/dist/localSensing.js.map +1 -1
  109. package/package.json +5 -5
  110. package/retained/RetainedCaptureClient.cjs +55 -9
  111. package/retained/__tests__/actor-perception.test.cjs +125 -0
  112. package/retained/__tests__/actor-window.test.cjs +78 -0
  113. package/retained/__tests__/runtime.test.cjs +44 -4
  114. package/retained/actor-window.cjs +62 -0
  115. package/retained/manager.cjs +14 -2
@@ -0,0 +1,129 @@
1
+ """RTMPose body7 export: RGB affine input and COCO17 SimCC outputs.
2
+
3
+ The artifact's pipeline.json declares RGB, 1.25 bbox padding, 192x256 and
4
+ split_ratio=2, without DARK or flip testing. MMPose v1.3.2 SimCC decoding uses
5
+ the minimum axis response. These responses are not calibrated probabilities.
6
+ """
7
+ from __future__ import annotations
8
+ from dataclasses import dataclass
9
+ import hashlib
10
+ import io
11
+ import math
12
+ from pathlib import Path
13
+ import threading
14
+ import time
15
+
16
+ import cv2
17
+ import numpy as np
18
+ import onnxruntime as ort
19
+ from PIL import Image
20
+
21
+ from .errors import ModelContractError
22
+ from .execution import LocalOrtSession
23
+
24
+ # OpenCV otherwise creates a worker per host core, outside the model budget.
25
+ cv2.setNumThreads(1)
26
+
27
+ from .tasks import RTMPOSE_MODEL_ID as MODEL_ID, RTMPOSE_SHA256 as MODEL_SHA256, RTMPOSE_PREPROCESSING as PREPROCESSING, COCO17
28
+ MEAN = np.array([123.675,116.28,103.53],np.float32)
29
+ STD = np.array([58.395,57.12,57.375],np.float32)
30
+
31
+ @dataclass(frozen=True)
32
+ class PoseGeometry:
33
+ center: tuple[float,float]
34
+ scale: tuple[float,float]
35
+ frame_size: tuple[int,int]
36
+
37
+ def prepare_pose(rgb, box):
38
+ if (not isinstance(rgb,np.ndarray) or rgb.dtype!=np.uint8
39
+ or rgb.ndim!=3 or rgb.shape[2]!=3 or min(rgb.shape[:2])<2
40
+ or rgb.shape[0]*rgb.shape[1]>16777216
41
+ or not isinstance(box,(tuple,list)) or len(box)!=4
42
+ or any(isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) for v in box)):
43
+ raise ValueError("Invalid RGB pose frame or person box")
44
+ x1,y1,x2,y2=box
45
+ h,w=rgb.shape[:2]
46
+ if not (0<=x1<x2<=w and 0<=y1<y2<=h):
47
+ raise ValueError("Person box lies outside original frame")
48
+ center=((x1+x2)/2,(y1+y2)/2)
49
+ sx,sy=(x2-x1)*1.25,(y2-y1)*1.25
50
+ if sx>sy*.75:sy=sx/.75
51
+ else:sx=sy*.75
52
+ # Same three-point zero-rotation affine as OpenMMLab top_down_affine.
53
+ cx,cy=center
54
+ source=np.zeros((3,2),np.float32)
55
+ source[0]=[cx,cy]
56
+ source[1]=[cx,cy-sx/2]
57
+ direction=source[0]-source[1]
58
+ source[2]=source[1]+np.array([-direction[1],direction[0]],np.float32)
59
+ target=np.array([[96,128],[96,32],[0,32]],np.float32)
60
+ matrix=cv2.getAffineTransform(source,target)
61
+ crop=cv2.warpAffine(rgb,matrix,(192,256),flags=cv2.INTER_LINEAR)
62
+ tensor=np.ascontiguousarray(((crop.astype(np.float32)-MEAN)/STD).transpose(2,0,1)[None])
63
+ return tensor,PoseGeometry(center,(sx,sy),(w,h))
64
+
65
+ def decode_pose(simcc_x,simcc_y,geometry):
66
+ for value,shape in ((simcc_x,(1,17,384)),(simcc_y,(1,17,512))):
67
+ if not isinstance(value,np.ndarray) or value.shape!=shape or not np.isfinite(value).all():
68
+ raise ValueError("Invalid RTMPose SimCC output")
69
+ if (not isinstance(geometry,PoseGeometry)
70
+ or len(geometry.center)!=2 or len(geometry.scale)!=2 or len(geometry.frame_size)!=2
71
+ or any(not math.isfinite(v) for v in (*geometry.center,*geometry.scale))
72
+ or min(geometry.scale)<=0 or min(geometry.frame_size)<=0):
73
+ raise ValueError("Invalid pose geometry")
74
+ x=np.argmax(simcc_x[0],axis=1).astype(np.float32)/2
75
+ y=np.argmax(simcc_y[0],axis=1).astype(np.float32)/2
76
+ responses=np.minimum(np.max(simcc_x[0],axis=1),np.max(simcc_y[0],axis=1))
77
+ x=x/192*geometry.scale[0]+geometry.center[0]-geometry.scale[0]/2
78
+ y=y/256*geometry.scale[1]+geometry.center[1]-geometry.scale[1]/2
79
+ result=[]
80
+ for i,name in enumerate(COCO17):
81
+ observed=responses[i]>0 and 0<=x[i]<geometry.frame_size[0] and 0<=y[i]<geometry.frame_size[1]
82
+ result.append({"name":name,"x":float(x[i]) if observed else None,
83
+ "y":float(y[i]) if observed else None,"response":float(responses[i])})
84
+ return result
85
+
86
+ class PoseEstimator:
87
+ """One immutable artifact, bounded CPU pools and explicit model proof."""
88
+ model_name = MODEL_ID
89
+ model_sha256 = MODEL_SHA256
90
+
91
+ def __init__(self,model_path,*,intra_op_threads=4,accelerator="auto"):
92
+ if type(intra_op_threads) is not int or not 1<=intra_op_threads<=8:
93
+ raise ValueError("Invalid bounded pose thread count")
94
+ data=Path(model_path).read_bytes()
95
+ if hashlib.sha256(data).hexdigest()!=MODEL_SHA256:
96
+ raise ModelContractError("RTMPose artifact differs from pinned export")
97
+ options=ort.SessionOptions()
98
+ options.intra_op_num_threads=intra_op_threads
99
+ options.inter_op_num_threads=1
100
+ self._session=LocalOrtSession(ort,data,options,accelerator)
101
+ inputs=self._session.get_inputs();outputs={o.name:o for o in self._session.get_outputs()}
102
+ if (len(inputs)!=1 or inputs[0].name!="input" or inputs[0].shape!=["batch",3,256,192]
103
+ or inputs[0].type!="tensor(float)" or set(outputs)!={"simcc_x","simcc_y"}
104
+ or outputs["simcc_x"].shape!=["batch","MatMulsimcc_x_dim_1",384]
105
+ or outputs["simcc_y"].shape!=["batch","MatMulsimcc_x_dim_1",512]
106
+ or any(o.type!="tensor(float)" for o in outputs.values())):
107
+ raise ModelContractError("RTMPose graph differs from declared RGB/COCO17 contract")
108
+ self._lock=threading.Lock()
109
+
110
+ def estimate(self,jpeg,boxes):
111
+ if not isinstance(boxes,list) or len(boxes)>256:
112
+ raise ValueError("Invalid pose person budget")
113
+ if not isinstance(jpeg,bytes) or len(jpeg)>16*1024**2:
114
+ raise ValueError("Invalid pose JPEG")
115
+ with Image.open(io.BytesIO(jpeg)) as image:
116
+ if image.width*image.height>16777216:
117
+ raise ValueError("Oversized pose frame")
118
+ rgb=np.asarray(image.convert("RGB"))
119
+ started=time.monotonic()
120
+ poses=[]
121
+ with self._lock:
122
+ for box in boxes:
123
+ tensor,geometry=prepare_pose(rgb,box)
124
+ x,y=self._session.run(["simcc_x","simcc_y"],{"input":tensor})
125
+ poses.append(decode_pose(x,y,geometry))
126
+ return {"version":1,"modelId":MODEL_ID,"modelSha256":MODEL_SHA256,
127
+ "preprocessing":PREPROCESSING,"imageSha256":hashlib.sha256(jpeg).hexdigest(),
128
+ "frameSize":{"width":rgb.shape[1],"height":rgb.shape[0]},"poses":poses,
129
+ "inferenceMs":(time.monotonic()-started)*1000}
@@ -60,6 +60,8 @@ import sys
60
60
  import threading
61
61
  import time
62
62
  from pathlib import Path
63
+ from dataclasses import dataclass
64
+ import re
63
65
 
64
66
  from .detections import DetectionBatch, merge_batches
65
67
  from .errors import DetectorNotReady, ModelContractError, ModelVerificationError, UnknownClasses
@@ -103,6 +105,27 @@ STATE_UNAVAILABLE = "unavailable"
103
105
  #: MEASURING it per load so the authority above can eventually stop guessing.
104
106
 
105
107
 
108
+ @dataclass(frozen=True)
109
+ class TaskModelContract:
110
+ """A non-box model's task, immutable bytes and measured residency cost."""
111
+ model_name: str
112
+ task: str
113
+ sha256: str
114
+ approx_ram_bytes: int
115
+ preprocessing: str
116
+
117
+ def __post_init__(self):
118
+ if (not isinstance(self.model_name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,127}", self.model_name)
119
+ or not isinstance(self.sha256, str) or not re.fullmatch(r"[a-f0-9]{64}", self.sha256)
120
+ or type(self.approx_ram_bytes) is not int or self.approx_ram_bytes <= 0
121
+ or not isinstance(self.preprocessing, str) or not self.preprocessing):
122
+ raise ValueError("invalid task model contract")
123
+ self._validate_task()
124
+
125
+ def _validate_task(self):
126
+ if self.task != "pose_coco17":
127
+ raise ValueError("invalid non-box task kind")
128
+
106
129
  class ModelEntry:
107
130
  """One catalogue row: a committed vocabulary bound to a path on this box."""
108
131
 
@@ -187,6 +210,8 @@ class ModelRegistry:
187
210
  "exactly one catalogue entry must be pinned — it is what `/health` reports and "
188
211
  f"what the cascade gates on; got {[e.name for e in pinned]}"
189
212
  )
213
+ if not isinstance(pinned[0].vocabulary, ModelVocabulary):
214
+ raise ValueError("the pinned cascade gate must be an object detector")
190
215
  if ram_budget_bytes is not None:
191
216
  if ram_budget_bytes <= 0:
192
217
  raise ValueError(f"an injected RAM ceiling must be positive, got {ram_budget_bytes}")
@@ -251,7 +276,8 @@ class ModelRegistry:
251
276
  with self._lock:
252
277
  if self._index is None:
253
278
  self._index = ClassIndex(
254
- [s.entry.vocabulary for s in self._states.values() if s.present]
279
+ [s.entry.vocabulary for s in self._states.values()
280
+ if s.present and isinstance(s.entry.vocabulary, ModelVocabulary)]
255
281
  )
256
282
  return self._index
257
283
 
@@ -282,7 +308,7 @@ class ModelRegistry:
282
308
  pending: "dict[str, str]" = {}
283
309
  with self._lock:
284
310
  for state in self._states.values():
285
- if state.present:
311
+ if state.present or not isinstance(state.entry.vocabulary, ModelVocabulary):
286
312
  continue
287
313
  for raw in unknown:
288
314
  if normalize_class_name(raw) in state.entry.vocabulary.class_names:
@@ -395,8 +421,8 @@ class ModelRegistry:
395
421
  plan = self.class_index().assign(wanted)
396
422
  else:
397
423
  selected = self._states.get(expected_model["modelId"])
398
- if selected is None:
399
- raise ModelVerificationError("selected model is not in this box's catalogue")
424
+ if selected is None or not isinstance(selected.entry.vocabulary, ModelVocabulary):
425
+ raise ModelVerificationError("selected object detector is not in this box's catalogue")
400
426
  vocabulary = selected.entry.vocabulary
401
427
  if wanted is not None and not wanted.issubset(vocabulary.class_names):
402
428
  raise ModelVerificationError("selected model cannot answer all requested classes")
@@ -445,6 +471,60 @@ class ModelRegistry:
445
471
  )
446
472
  return merged
447
473
 
474
+ def estimate_pose(self, jpeg: bytes, boxes: list, expected_model: dict) -> dict:
475
+ """Acquire pose through the same portfolio as all detector sessions."""
476
+ state = self._states.get(expected_model.get("modelId"))
477
+ if (state is None or not isinstance(state.entry.vocabulary, TaskModelContract)
478
+ or state.entry.vocabulary.task != "pose_coco17"
479
+ or state.entry.vocabulary.sha256 != expected_model.get("sha256")):
480
+ raise ModelVerificationError("selected pose task does not match its immutable contract")
481
+ contract = state.entry.vocabulary
482
+ sessions = self._acquire([state.entry.name])
483
+ try:
484
+ session = sessions[0]
485
+ if getattr(session, "model_sha256", None) != contract.sha256:
486
+ raise ModelVerificationError("loaded pose session does not match the selected artifact")
487
+ try:
488
+ result = session.estimate(jpeg, boxes)
489
+ if (result.get("modelId") != contract.model_name
490
+ or result.get("modelSha256") != contract.sha256
491
+ or result.get("preprocessing") != contract.preprocessing):
492
+ raise ModelVerificationError("pose output does not match its task contract")
493
+ except Exception:
494
+ with self._lock:
495
+ state.detect_failures += 1
496
+ raise
497
+ with self._lock:
498
+ state.detect_count += 1
499
+ return result
500
+ finally:
501
+ self._release([state.entry.name])
502
+
503
+ def classify_actor_actions(self,batch):
504
+ """Acquire and prove the selected non-box action model through the shared portfolio."""
505
+ from .actor_action_contract import ActorActionModelContract
506
+ state=self._states.get(batch.model.get("modelId"))
507
+ if (state is None or not isinstance(state.entry.vocabulary,ActorActionModelContract)
508
+ or state.entry.vocabulary.sha256!=batch.model.get("sha256")):
509
+ raise ModelVerificationError("selected actor action task differs from its immutable contract")
510
+ contract=state.entry.vocabulary
511
+ sessions=self._acquire([state.entry.name])
512
+ try:
513
+ session=sessions[0]
514
+ if getattr(session,"model_sha256",None)!=contract.sha256:
515
+ raise ModelVerificationError("loaded actor action session differs from selected artifact")
516
+ try:
517
+ result=session.classify(batch)
518
+ if (result.get("modelId")!=contract.model_name or result.get("modelSha256")!=contract.sha256
519
+ or result.get("preprocessing")!=contract.preprocessing):
520
+ raise ModelVerificationError("actor action result differs from selected task")
521
+ except Exception:
522
+ with self._lock:state.detect_failures+=1
523
+ raise
524
+ with self._lock:state.detect_count+=1
525
+ return result
526
+ finally:self._release([state.entry.name])
527
+
448
528
  def _acquire(self, needed: "list[str]") -> list:
449
529
  """Make every named model resident under the budget, then pin it in flight.
450
530
 
@@ -645,7 +725,7 @@ class ModelRegistry:
645
725
  measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
646
726
  self._log(
647
727
  f"[detector] ready — {entry.name} at {entry.path} "
648
- f"(probe {warmup_ms:.0f} ms, {len(entry.vocabulary.slot_names)} named classes{measured})"
728
+ f"(probe {warmup_ms:.0f} ms, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
649
729
  )
650
730
 
651
731
  def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
@@ -692,7 +772,8 @@ class ModelRegistry:
692
772
  "present": s.present,
693
773
  "resident": s.resident,
694
774
  "pinned": s.entry.pinned,
695
- "named_classes": len(s.entry.vocabulary.slot_names),
775
+ "task": getattr(s.entry.vocabulary, "task", "object_detection"),
776
+ "named_classes": len(s.entry.vocabulary.slot_names) if isinstance(s.entry.vocabulary, ModelVocabulary) else 0,
696
777
  "approx_ram_bytes": s.entry.vocabulary.approx_ram_bytes,
697
778
  "detect_count": s.detect_count,
698
779
  "detect_failures": s.detect_failures,
@@ -733,6 +814,9 @@ class ModelRegistry:
733
814
  # strings already on the spec, read under the same lock.
734
815
  row["output_layout"] = getattr(spec, "output_layout", None)
735
816
  row["preprocessing"] = getattr(spec, "preprocessing", None)
817
+ if isinstance(s.entry.vocabulary, TaskModelContract):
818
+ row["model_sha256"] = s.entry.vocabulary.sha256
819
+ row["preprocessing"] = s.entry.vocabulary.preprocessing
736
820
  models[name] = row
737
821
 
738
822
  # Counted from `present` directly rather than through
@@ -742,7 +826,8 @@ class ModelRegistry:
742
826
  # have to wait for.
743
827
  known = len(
744
828
  frozenset().union(
745
- *[s.entry.vocabulary.class_names for s in self._states.values() if s.present]
829
+ *[s.entry.vocabulary.class_names for s in self._states.values()
830
+ if s.present and isinstance(s.entry.vocabulary, ModelVocabulary)]
746
831
  ) if any(s.present for s in self._states.values()) else frozenset()
747
832
  )
748
833
 
@@ -27,11 +27,16 @@ from __future__ import annotations
27
27
  import base64
28
28
  import binascii
29
29
  import json
30
+ import hashlib
31
+ import io
30
32
  import math
31
33
  import sys as _sys
32
34
  import threading
33
35
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
34
36
 
37
+ from .actor_tracking import TrackingError, validate_tracking_request
38
+ from .tasks import RTMPOSE_MODEL_ID, RTMPOSE_SHA256, RTMPOSE_PREPROCESSING, COCO17
39
+ from .actor_action_wire import parse_actor_action_request, validate_actor_action_response
35
40
  from ._version import VERSION
36
41
  from .errors import DetectorNotReady, ModelVerificationError, UnknownClasses
37
42
 
@@ -226,6 +231,49 @@ def build_detect_response(batch) -> dict:
226
231
  return body
227
232
 
228
233
 
234
+ def parse_pose_model(raw: bytes):
235
+ value = json.loads(raw.decode("utf-8")).get("pose_model")
236
+ if value is None:
237
+ return None
238
+ if (not isinstance(value, dict) or set(value) != {"modelId", "sha256"}
239
+ or value.get("modelId") != RTMPOSE_MODEL_ID or value.get("sha256") != RTMPOSE_SHA256):
240
+ raise WireError(409, "pose task differs from its pinned artifact contract")
241
+ return value
242
+
243
+
244
+ def build_pose_response(result, tracking, frame_size):
245
+ if (result.get("version") != 1 or result.get("modelId") != RTMPOSE_MODEL_ID
246
+ or result.get("modelSha256") != RTMPOSE_SHA256
247
+ or result.get("preprocessing") != RTMPOSE_PREPROCESSING
248
+ or result.get("imageSha256") != tracking["imageSha256"]
249
+ or result.get("frameSize") != {"width": frame_size[0], "height": frame_size[1]}
250
+ or not isinstance(result.get("poses"), list)
251
+ or len(result["poses"]) != len(tracking["actors"])
252
+ or not isinstance(result.get("inferenceMs"), (int, float))
253
+ or not math.isfinite(result["inferenceMs"]) or result["inferenceMs"] < 0):
254
+ raise ValueError("pose result differs from the original tracked frame")
255
+ actors = []
256
+ for association, joints in zip(tracking["actors"], result["poses"]):
257
+ if not isinstance(joints, list) or len(joints) != len(COCO17):
258
+ raise ValueError("pose result does not contain COCO17 joints")
259
+ for name, joint in zip(COCO17, joints):
260
+ if (not isinstance(joint, dict) or set(joint) != {"name", "x", "y", "response"}
261
+ or joint.get("name") != name
262
+ or isinstance(joint.get("response"), bool)
263
+ or not isinstance(joint.get("response"), (int, float))
264
+ or not math.isfinite(joint["response"])):
265
+ raise ValueError("invalid named pose joint")
266
+ x, y = joint["x"], joint["y"]
267
+ if x is None and y is None:
268
+ continue
269
+ if (any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) for v in (x, y))
270
+ or not (0 <= x < frame_size[0] and 0 <= y < frame_size[1]) or joint["response"] <= 0):
271
+ raise ValueError("pose joint is outside the original camera frame")
272
+ actors.append({**association, "joints": joints})
273
+ return {key: value for key, value in result.items() if key != "poses"} | {
274
+ "trackingSha256": tracking["sha256"], "actors": actors}
275
+
276
+
229
277
  class DetectorHandler(BaseHTTPRequestHandler):
230
278
  """Routes. The server instance carries `loader` and `log_line`."""
231
279
 
@@ -260,6 +308,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
260
308
  def do_POST(self) -> None: # noqa: N802
261
309
  if self._path() == "/detect":
262
310
  self._detect()
311
+ elif self._path() == "/actor-actions":
312
+ self._actor_actions()
263
313
  elif self._path() == "/health":
264
314
  self._json(405, {"error": "/health is a GET"})
265
315
  else:
@@ -293,6 +343,8 @@ class DetectorHandler(BaseHTTPRequestHandler):
293
343
  # answered 400 for its classes. Reported on the LOADER's payload rather
294
344
  # than only in a log because the reader that needs it is a script.
295
345
  info["version"] = VERSION
346
+ info["actor_tracking"] = {"algorithm": "bytetrack_2.6.1",
347
+ "state": "ready" if self.server.actor_tracker is not None else "unavailable"}
296
348
  self._json(200 if ready else 503, info)
297
349
 
298
350
  # ── /detect ──────────────────────────────────────────────────────────────
@@ -303,6 +355,33 @@ class DetectorHandler(BaseHTTPRequestHandler):
303
355
  raw = self._read_body()
304
356
  jpeg, min_confidence, requested = parse_detect_request(raw)
305
357
  expected_model = parse_expected_model(raw)
358
+ pose_model = parse_pose_model(raw)
359
+ tracking = json.loads(raw.decode("utf-8")).get("tracking")
360
+ if pose_model is not None and tracking is None:
361
+ raise WireError(400, "pose requires original-frame actor tracking")
362
+ if pose_model is not None and not hasattr(loader, "estimate_pose"):
363
+ raise WireError(503, "pose task is unavailable in this detector build")
364
+ if tracking is not None:
365
+ try:
366
+ tracking = validate_tracking_request(tracking)
367
+ except TrackingError as err:
368
+ raise WireError(400, str(err)) from None
369
+ if min_confidence > .25:
370
+ raise WireError(400, "actor tracking requires min_confidence <= 0.25")
371
+ if expected_model is None:
372
+ raise WireError(400, "actor tracking requires a verified selected detector")
373
+ if requested is not None and "person" not in requested:
374
+ raise WireError(400, "actor tracking requires person detections")
375
+ if self.server.actor_tracker is None:
376
+ raise WireError(503, "actor tracking is unavailable")
377
+ from PIL import Image
378
+ try:
379
+ with Image.open(io.BytesIO(jpeg)) as image:
380
+ frame_size = image.size
381
+ if image.format != "JPEG" or image.width * image.height > 16777216:
382
+ raise ValueError("actor frames must be bounded original JPEGs")
383
+ except Exception:
384
+ raise WireError(400, "unreadable actor tracking JPEG") from None
306
385
  except WireError as err:
307
386
  self._json(err.status, {"error": err.message})
308
387
  return
@@ -347,13 +426,61 @@ class DetectorHandler(BaseHTTPRequestHandler):
347
426
  if expected_model is not None and (body.get("model_hashes") != {expected_model["modelId"]: expected_model["sha256"]}
348
427
  or body["model"] != expected_model["modelId"]):
349
428
  raise ValueError("selected detector did not return its loaded artifact proof")
429
+ if tracking is not None:
430
+ body["tracking"] = self.server.actor_tracker.track(
431
+ tracking, batch.detections, image_sha256=hashlib.sha256(jpeg).hexdigest(),
432
+ model_hashes=body.get("model_hashes"), frame_size=frame_size)
433
+ if pose_model is not None:
434
+ boxes = []
435
+ for association in body["tracking"]["actors"]:
436
+ detection = batch.detections[association["detectionIndex"]]
437
+ boxes.append([detection.x, detection.y, detection.x + detection.width, detection.y + detection.height])
438
+ result = loader.estimate_pose(jpeg, boxes, pose_model)
439
+ body["pose"] = build_pose_response(result, body["tracking"], frame_size)
440
+ except ModelVerificationError as err:
441
+ self._json(409, {"error": str(err)})
442
+ return
443
+ except DetectorNotReady as err:
444
+ self._json(503, {"error": str(err)})
445
+ return
350
446
  except ValueError as err:
351
447
  self._log(f"[detector] REFUSING to emit a malformed payload: {err}")
352
448
  self._json(500, {"error": f"refused to emit a malformed payload: {err}"})
353
449
  return
450
+ except Exception as err:
451
+ self._log(f"[detector] actor perception failed: {type(err).__name__}: {err}")
452
+ self._json(500, {"error": f"actor perception failed: {type(err).__name__}"})
453
+ return
354
454
 
355
455
  self._json(200, body)
356
456
 
457
+ def _actor_actions(self):
458
+ try:
459
+ raw=self._read_body()
460
+ try:batch=parse_actor_action_request(raw)
461
+ except ValueError as error:raise WireError(400,str(error)) from error
462
+ except WireError as error:
463
+ self._json(error.status,{"error":error.message});return
464
+ classify=getattr(self.server.loader,"classify_actor_actions",None)
465
+ if not callable(classify):
466
+ self._json(503,{"error":"selected native actor action task unavailable"});return
467
+ try:
468
+ result=classify(batch)
469
+ except ModelVerificationError as error:
470
+ self._json(409,{"error":str(error)});return
471
+ except DetectorNotReady as error:
472
+ self._json(503,{"error":str(error)});return
473
+ except ValueError as error:
474
+ self._json(400,{"error":str(error)});return
475
+ except Exception as error:
476
+ self._log(f"[detector] actor action inference failed: {type(error).__name__}: {error}")
477
+ self._json(500,{"error":"actor action inference failed"});return
478
+ try:validate_actor_action_response(result,batch)
479
+ except ValueError as error:
480
+ self._log(f"[detector] REFUSING malformed actor action output: {error}")
481
+ self._json(500,{"error":"refused malformed actor action output"});return
482
+ self._json(200,result)
483
+
357
484
  # ── plumbing ─────────────────────────────────────────────────────────────
358
485
 
359
486
  def _read_body(self) -> bytes:
@@ -526,14 +653,15 @@ class DetectorServer(ThreadingHTTPServer):
526
653
  # bind is that incident with no way to even notice.
527
654
  allow_reuse_address = _sys.platform != "win32"
528
655
 
529
- def __init__(self, address, loader, log_line=print):
656
+ def __init__(self, address, loader, log_line=print, actor_tracker=None):
530
657
  super().__init__(address, DetectorHandler)
531
658
  self.loader = loader
659
+ self.actor_tracker = actor_tracker
532
660
  self.log_line = log_line
533
661
 
534
662
 
535
- def serve(loader, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, log_line=print) -> DetectorServer:
663
+ def serve(loader, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, log_line=print, actor_tracker=None) -> DetectorServer:
536
664
  """Start the server on a background thread and return it."""
537
- server = DetectorServer((host, port), loader, log_line)
665
+ server = DetectorServer((host, port), loader, log_line, actor_tracker)
538
666
  threading.Thread(target=server.serve_forever, name="detector-http", daemon=True).start()
539
667
  return server