@trillboards/edge-sdk 0.2.175 → 0.2.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/README.md +6 -0
- package/deploy/detector/pyproject.toml +4 -1
- package/deploy/detector/tests/test_actor_action_wire.py +103 -0
- package/deploy/detector/tests/test_actor_actions.py +94 -0
- package/deploy/detector/tests/test_actor_checkpoint.py +99 -0
- package/deploy/detector/tests/test_actor_tracking.py +191 -0
- package/deploy/detector/tests/test_actor_tracking_wire.py +120 -0
- package/deploy/detector/tests/test_actor_views.py +108 -0
- package/deploy/detector/tests/test_compute_budget.py +28 -0
- package/deploy/detector/tests/test_detector_request.py +52 -0
- package/deploy/detector/tests/test_input_recipe.py +77 -0
- package/deploy/detector/tests/test_loader.py +20 -1
- package/deploy/detector/tests/test_pose.py +66 -0
- package/deploy/detector/tests/test_server.py +83 -0
- package/deploy/detector/tests/test_task_registry.py +74 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/actor_action_contract.py +65 -0
- package/deploy/detector/trillboards_detector/actor_action_wire.py +189 -0
- package/deploy/detector/trillboards_detector/actor_actions.py +115 -0
- package/deploy/detector/trillboards_detector/actor_checkpoint.py +91 -0
- package/deploy/detector/trillboards_detector/actor_tracking.py +337 -0
- package/deploy/detector/trillboards_detector/actor_views.py +102 -0
- package/deploy/detector/trillboards_detector/cli.py +12 -2
- package/deploy/detector/trillboards_detector/labels.py +32 -0
- package/deploy/detector/trillboards_detector/loader.py +30 -4
- package/deploy/detector/trillboards_detector/pose.py +129 -0
- package/deploy/detector/trillboards_detector/registry.py +92 -7
- package/deploy/detector/trillboards_detector/server.py +131 -3
- package/deploy/detector/trillboards_detector/session.py +83 -49
- package/deploy/detector/trillboards_detector/tasks.py +27 -0
- package/deploy/detector/trillboards_detector/worker_budget.py +14 -0
- package/deploy/windows/README.md +1 -1
- package/deploy/windows/ci/test-published-install.ps1 +16 -3
- package/dist/ActorActionClient.d.ts +47 -0
- package/dist/ActorActionClient.d.ts.map +1 -0
- package/dist/ActorActionClient.js +187 -0
- package/dist/ActorActionClient.js.map +1 -0
- package/dist/ActorActionProducer.d.ts +17 -0
- package/dist/ActorActionProducer.d.ts.map +1 -0
- package/dist/ActorActionProducer.js +92 -0
- package/dist/ActorActionProducer.js.map +1 -0
- package/dist/ActorPerception.d.ts +65 -0
- package/dist/ActorPerception.d.ts.map +1 -0
- package/dist/ActorPerception.js +124 -0
- package/dist/ActorPerception.js.map +1 -0
- package/dist/ActorPose.d.ts +42 -0
- package/dist/ActorPose.d.ts.map +1 -0
- package/dist/ActorPose.js +62 -0
- package/dist/ActorPose.js.map +1 -0
- package/dist/ActorReview.d.ts +51 -0
- package/dist/ActorReview.d.ts.map +1 -0
- package/dist/ActorReview.js +84 -0
- package/dist/ActorReview.js.map +1 -0
- package/dist/ActorTracking.d.ts +33 -0
- package/dist/ActorTracking.d.ts.map +1 -0
- package/dist/ActorTracking.js +90 -0
- package/dist/ActorTracking.js.map +1 -0
- package/dist/ActorWindow.d.ts +56 -0
- package/dist/ActorWindow.d.ts.map +1 -0
- package/dist/ActorWindow.js +86 -0
- package/dist/ActorWindow.js.map +1 -0
- package/dist/ActorWindowContract.d.ts +5 -0
- package/dist/ActorWindowContract.d.ts.map +1 -0
- package/dist/ActorWindowContract.js +23 -0
- package/dist/ActorWindowContract.js.map +1 -0
- package/dist/AudienceAnalyzeClient.d.ts +4 -0
- package/dist/AudienceAnalyzeClient.d.ts.map +1 -1
- package/dist/AudienceAnalyzeClient.js.map +1 -1
- package/dist/ChannelProber.d.ts.map +1 -1
- package/dist/ChannelProber.js +8 -0
- package/dist/ChannelProber.js.map +1 -1
- package/dist/CloudSensingManager.d.ts +7 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +31 -0
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
- package/dist/DurableVlmCandidateSpool.js +31 -11
- package/dist/DurableVlmCandidateSpool.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/LocalDetectorClient.d.ts +5 -1
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +48 -3
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/RetainedFieldContract.d.ts +2 -1
- package/dist/RetainedFieldContract.d.ts.map +1 -1
- package/dist/RetainedFieldContract.js +17 -5
- package/dist/RetainedFieldContract.js.map +1 -1
- package/dist/RuntimeModelSelection.d.ts.map +1 -1
- package/dist/RuntimeModelSelection.js +63 -2
- package/dist/RuntimeModelSelection.js.map +1 -1
- package/dist/TemporalReviewClient.d.ts.map +1 -1
- package/dist/TemporalReviewClient.js +20 -1
- package/dist/TemporalReviewClient.js.map +1 -1
- package/dist/TemporalReviewJournal.d.ts +10 -1
- package/dist/TemporalReviewJournal.d.ts.map +1 -1
- package/dist/TemporalReviewJournal.js +42 -10
- package/dist/TemporalReviewJournal.js.map +1 -1
- package/dist/connectorUpdate.d.ts +2 -0
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +33 -7
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -1
- package/dist/index.js.map +1 -1
- package/dist/localSensing.d.ts +4 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +43 -8
- package/dist/localSensing.js.map +1 -1
- package/package.json +5 -5
- package/retained/RetainedCaptureClient.cjs +55 -9
- package/retained/__tests__/actor-perception.test.cjs +125 -0
- package/retained/__tests__/actor-window.test.cjs +78 -0
- package/retained/__tests__/runtime.test.cjs +44 -4
- package/retained/actor-window.cjs +62 -0
- package/retained/manager.cjs +14 -2
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Bounded actor-action input over loopback, with exact original-window proofs."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import base64,binascii,hashlib,json,math,re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Mapping
|
|
7
|
+
from .tasks import COCO17
|
|
8
|
+
from .actor_action_contract import ActorActionWindowContract
|
|
9
|
+
|
|
10
|
+
_ID=re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
|
|
11
|
+
_SHA=re.compile(r"[a-f0-9]{64}\Z")
|
|
12
|
+
_FIELD=re.compile(r"[a-z][a-z0-9_]{0,127}\Z")
|
|
13
|
+
MAX_ACTION_BODY_BYTES=32*1024**2
|
|
14
|
+
MAX_ACTION_WINDOWS=32
|
|
15
|
+
MAX_ACTION_IMAGES=256
|
|
16
|
+
|
|
17
|
+
def _exact(value,keys):
|
|
18
|
+
return isinstance(value,dict) and set(value)==set(keys.split(","))
|
|
19
|
+
def _id(value):return isinstance(value,str) and _ID.fullmatch(value) is not None
|
|
20
|
+
def _sha(value):return isinstance(value,str) and _SHA.fullmatch(value) is not None
|
|
21
|
+
def _int(value):return type(value) is int
|
|
22
|
+
def _number(value):return type(value) in (int,float) and math.isfinite(value)
|
|
23
|
+
def _pairs(pairs):
|
|
24
|
+
result={}
|
|
25
|
+
for key,value in pairs:
|
|
26
|
+
if key in result:raise ValueError("duplicate actor action JSON key")
|
|
27
|
+
result[key]=value
|
|
28
|
+
return result
|
|
29
|
+
def _constant(value):raise ValueError("nonfinite actor action JSON number")
|
|
30
|
+
def _json(source):
|
|
31
|
+
try:return json.loads(source,object_pairs_hook=_pairs,parse_constant=_constant)
|
|
32
|
+
except (TypeError,UnicodeError,json.JSONDecodeError,RecursionError) as error:
|
|
33
|
+
raise ValueError("invalid actor action JSON") from error
|
|
34
|
+
|
|
35
|
+
def validate_action_window_contract(value):
|
|
36
|
+
return ActorActionWindowContract.from_dict(value).as_dict()
|
|
37
|
+
|
|
38
|
+
def _actor(value):
|
|
39
|
+
if (not _exact(value,"kind,cameraId,epoch,id") or value["kind"]!="person"
|
|
40
|
+
or any(not _id(value[key]) for key in ("cameraId","epoch","id"))):
|
|
41
|
+
raise ValueError("invalid physical actor action identity")
|
|
42
|
+
return value
|
|
43
|
+
|
|
44
|
+
def _joints(value):
|
|
45
|
+
if not isinstance(value,list) or len(value)!=len(COCO17):
|
|
46
|
+
raise ValueError("invalid actor action COCO17 pose")
|
|
47
|
+
for joint,name in zip(value,COCO17):
|
|
48
|
+
if (not _exact(joint,"name,x,y,response") or joint["name"]!=name
|
|
49
|
+
or not _number(joint["response"]) or (joint["x"] is None)!=(joint["y"] is None)
|
|
50
|
+
or joint["x"] is not None and
|
|
51
|
+
(not _number(joint["x"]) or not _number(joint["y"])
|
|
52
|
+
or min(joint["x"],joint["y"])<0 or joint["response"]<=0)):
|
|
53
|
+
raise ValueError("invalid original actor action joint")
|
|
54
|
+
return value
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class ActorActionWindow:
|
|
58
|
+
sha256:str
|
|
59
|
+
proof_source:str
|
|
60
|
+
data:dict
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class ActorActionBatch:
|
|
64
|
+
model:Mapping[str,str]
|
|
65
|
+
fields:tuple[str,...]
|
|
66
|
+
windows:tuple[ActorActionWindow,...]
|
|
67
|
+
images:Mapping[str,bytes]
|
|
68
|
+
|
|
69
|
+
def _window(value):
|
|
70
|
+
if (not _exact(value,"sha256,proofSource") or not _sha(value["sha256"])
|
|
71
|
+
or not isinstance(value["proofSource"],str)):
|
|
72
|
+
raise ValueError("invalid actor action proof envelope")
|
|
73
|
+
try:encoded=value["proofSource"].encode("utf8")
|
|
74
|
+
except UnicodeError as error:raise ValueError("invalid actor action proof text") from error
|
|
75
|
+
if len(encoded)>1024**2 or hashlib.sha256(encoded).hexdigest()!=value["sha256"]:
|
|
76
|
+
raise ValueError("actor action proof text changed")
|
|
77
|
+
data=_json(value["proofSource"])
|
|
78
|
+
if (not _exact(data,"version,actor,sourceId,contextSha256,policySha256,currentFrameId,currentCapturedAtMs,contract,samples,observedFrames,usable")
|
|
79
|
+
or data["version"]!=1 or type(data["version"]) is not int
|
|
80
|
+
or not _id(data["sourceId"]) or not _sha(data["contextSha256"])
|
|
81
|
+
or not _sha(data["policySha256"]) or not _id(data["currentFrameId"])
|
|
82
|
+
or not _int(data["currentCapturedAtMs"]) or data["currentCapturedAtMs"]<0
|
|
83
|
+
or not _int(data["observedFrames"]) or type(data["usable"]) is not bool):
|
|
84
|
+
raise ValueError("invalid actor action window identity")
|
|
85
|
+
actor=_actor(data["actor"]);contract=validate_action_window_contract(data["contract"])
|
|
86
|
+
samples=data["samples"]
|
|
87
|
+
if not isinstance(samples,list) or len(samples)!=contract["frames"]:
|
|
88
|
+
raise ValueError("incomplete actor action slots")
|
|
89
|
+
frames=set();times=set();images=set();previous=None
|
|
90
|
+
for i,sample in enumerate(samples):
|
|
91
|
+
if not isinstance(sample,dict):raise ValueError("invalid actor action slot")
|
|
92
|
+
at=data["currentCapturedAtMs"]-(contract["frames"]-1-i)*contract["periodMs"]
|
|
93
|
+
if not _int(sample.get("slotAtMs")) or sample["slotAtMs"]!=at:
|
|
94
|
+
raise ValueError("actor action slot clock changed")
|
|
95
|
+
if sample.get("state")=="unobservable":
|
|
96
|
+
if (not _exact(sample,"state,slotAtMs,reason")
|
|
97
|
+
or sample["reason"] not in ("not_captured","actor_unobserved","track_reset","capture_gap")):
|
|
98
|
+
raise ValueError("invalid explicit actor action gap")
|
|
99
|
+
continue
|
|
100
|
+
want="state,slotAtMs,frameId,capturedAtMs,imageSha256,perceptionSha256,trackingSha256,actor,box"
|
|
101
|
+
if "joints" in sample:want+=",joints"
|
|
102
|
+
box=sample.get("box")
|
|
103
|
+
if (not _exact(sample,want) or sample["state"]!="observed" or not _id(sample["frameId"])
|
|
104
|
+
or sample["frameId"] in frames or not _int(sample["capturedAtMs"])
|
|
105
|
+
or sample["capturedAtMs"]<0 or sample["capturedAtMs"]>data["currentCapturedAtMs"]
|
|
106
|
+
or abs(sample["capturedAtMs"]-at)>contract["toleranceMs"]
|
|
107
|
+
or sample["capturedAtMs"] in times
|
|
108
|
+
or previous is not None and sample["capturedAtMs"]<=previous
|
|
109
|
+
or any(not _sha(sample[k]) for k in ("imageSha256","perceptionSha256","trackingSha256"))
|
|
110
|
+
or _actor(sample["actor"])!=actor or not isinstance(box,list) or len(box)!=4
|
|
111
|
+
or any(not _number(v) for v in box) or not (0<=box[0]<box[2] and 0<=box[1]<box[3])):
|
|
112
|
+
raise ValueError("invalid original actor action observation")
|
|
113
|
+
if "joints" in sample:_joints(sample["joints"])
|
|
114
|
+
frames.add(sample["frameId"]);times.add(sample["capturedAtMs"]);images.add(sample["imageSha256"])
|
|
115
|
+
previous=sample["capturedAtMs"]
|
|
116
|
+
last=samples[-1]
|
|
117
|
+
if (last.get("state")!="observed" or last["frameId"]!=data["currentFrameId"]
|
|
118
|
+
or last["capturedAtMs"]!=data["currentCapturedAtMs"] or len(frames)!=data["observedFrames"]
|
|
119
|
+
or data["usable"]!=(len(frames)>=contract["minObservedFrames"])):
|
|
120
|
+
raise ValueError("actor action anchor or observed coverage changed")
|
|
121
|
+
return ActorActionWindow(value["sha256"],value["proofSource"],data),images
|
|
122
|
+
|
|
123
|
+
def parse_actor_action_request(raw):
|
|
124
|
+
if not isinstance(raw,bytes) or not raw or len(raw)>MAX_ACTION_BODY_BYTES:
|
|
125
|
+
raise ValueError("invalid bounded actor action body")
|
|
126
|
+
value=_json(raw)
|
|
127
|
+
if (not _exact(value,"version,model,fields,windows,images") or type(value["version"]) is not int or value["version"]!=1
|
|
128
|
+
or not _exact(value["model"],"modelId,sha256") or not _id(value["model"]["modelId"])
|
|
129
|
+
or not _sha(value["model"]["sha256"]) or not isinstance(value["fields"],list)
|
|
130
|
+
or not 1<=len(value["fields"])<=16 or any(not isinstance(f,str) or not _FIELD.fullmatch(f) for f in value["fields"])
|
|
131
|
+
or len(set(value["fields"]))!=len(value["fields"])
|
|
132
|
+
or not isinstance(value["windows"],list) or len(value["windows"])>MAX_ACTION_WINDOWS
|
|
133
|
+
or not isinstance(value["images"],list) or len(value["images"])>MAX_ACTION_IMAGES):
|
|
134
|
+
raise ValueError("invalid selected actor action batch")
|
|
135
|
+
windows=[];wanted=set();actors=set()
|
|
136
|
+
for row in value["windows"]:
|
|
137
|
+
window,images=_window(row);actor=window.data["actor"]
|
|
138
|
+
key=(actor["cameraId"],actor["epoch"],actor["id"])
|
|
139
|
+
if key in actors:raise ValueError("duplicate actor action window")
|
|
140
|
+
actors.add(key);windows.append(window);wanted.update(images)
|
|
141
|
+
images={}
|
|
142
|
+
for row in value["images"]:
|
|
143
|
+
if (not _exact(row,"sha256,jpeg") or not _sha(row["sha256"]) or row["sha256"] in images
|
|
144
|
+
or not isinstance(row["jpeg"],str) or not 1<=len(row["jpeg"])<=24*1024**2):
|
|
145
|
+
raise ValueError("invalid original actor action image")
|
|
146
|
+
try:image=base64.b64decode(row["jpeg"],validate=True)
|
|
147
|
+
except (ValueError,binascii.Error) as error:raise ValueError("invalid original actor action base64") from error
|
|
148
|
+
if not image or len(image)>16*1024**2 or hashlib.sha256(image).hexdigest()!=row["sha256"]:
|
|
149
|
+
raise ValueError("actor action original pixels changed")
|
|
150
|
+
images[row["sha256"]]=image
|
|
151
|
+
if set(images)!=wanted:raise ValueError("actor action original images incomplete or unreferenced")
|
|
152
|
+
return ActorActionBatch(MappingProxyType(dict(value["model"])),tuple(value["fields"]),tuple(windows),MappingProxyType(images))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
_ACTION_REASONS=frozenset(("occluded","too_small","not_visible","insufficient_history","model_unavailable",
|
|
156
|
+
"tracking_unconfirmed","model_uncertain","pose_unavailable"))
|
|
157
|
+
|
|
158
|
+
def validate_actor_action_response(value,batch:ActorActionBatch):
|
|
159
|
+
"""The last operation before HTTP 200 validates against the immutable input proof text."""
|
|
160
|
+
if (not _exact(value,"version,modelId,modelSha256,preprocessing,fields,results,inferenceMs")
|
|
161
|
+
or type(value["version"]) is not int or value["version"]!=1
|
|
162
|
+
or value["modelId"]!=batch.model["modelId"] or value["modelSha256"]!=batch.model["sha256"]
|
|
163
|
+
or not isinstance(value["preprocessing"],str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}",value["preprocessing"])
|
|
164
|
+
or not isinstance(value["fields"],list) or any(not isinstance(f,str) for f in value["fields"])
|
|
165
|
+
or len(value["fields"])!=len(batch.fields) or set(value["fields"])!=set(batch.fields)
|
|
166
|
+
or not _number(value["inferenceMs"]) or value["inferenceMs"]<0
|
|
167
|
+
or not isinstance(value["results"],list) or len(value["results"])!=len(batch.windows)):
|
|
168
|
+
raise ValueError("invalid selected actor action runtime proof")
|
|
169
|
+
pending={window.sha256:_json(window.proof_source) for window in batch.windows}
|
|
170
|
+
for row in value["results"]:
|
|
171
|
+
if not _exact(row,"windowSha256,actor,fields,scores") or not _sha(row["windowSha256"]):
|
|
172
|
+
raise ValueError("invalid actor action result")
|
|
173
|
+
original=pending.pop(row["windowSha256"],None)
|
|
174
|
+
if (original is None or row["actor"]!=original["actor"]
|
|
175
|
+
or not isinstance(row["fields"],dict) or set(row["fields"])!=set(batch.fields)
|
|
176
|
+
or not isinstance(row["scores"],dict) or set(row["scores"])!=set(batch.fields)):
|
|
177
|
+
raise ValueError("incomplete or changed physical actor action result")
|
|
178
|
+
for field in batch.fields:
|
|
179
|
+
reading=row["fields"][field];score=row["scores"][field]
|
|
180
|
+
if (not _exact(reading,"state,reason") or reading["state"] not in ("present","absent","unobservable")
|
|
181
|
+
or (reading["state"]=="unobservable" and reading["reason"] not in _ACTION_REASONS)
|
|
182
|
+
or (reading["state"]!="unobservable" and reading["reason"] is not None)
|
|
183
|
+
or (score is not None and not _number(score))
|
|
184
|
+
or (reading["state"]!="unobservable" and score is None)
|
|
185
|
+
or (reading["state"]=="unobservable" and reading["reason"]=="model_uncertain" and score is None)
|
|
186
|
+
or (not original["usable"] and reading!={"state":"unobservable","reason":"insufficient_history"})):
|
|
187
|
+
raise ValueError("actor action score or coverage does not support its reading")
|
|
188
|
+
if pending:raise ValueError("omitted physical actor action result")
|
|
189
|
+
return value
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Causal X3D actor actions from original RGB, using the training crop functions."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import hashlib,io,json,threading,time
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import cv2,numpy as np,onnxruntime as ort
|
|
6
|
+
from PIL import Image
|
|
7
|
+
from .actor_action_contract import ActorActionModelContract
|
|
8
|
+
from .actor_action_wire import ActorActionBatch,ActorActionWindow
|
|
9
|
+
from .actor_views import actor_view,actor_detail_views
|
|
10
|
+
from .errors import ModelContractError
|
|
11
|
+
from .execution import LocalOrtSession
|
|
12
|
+
from .tasks import COCO17
|
|
13
|
+
|
|
14
|
+
cv2.setNumThreads(1)
|
|
15
|
+
|
|
16
|
+
def prepare_action_window(window:ActorActionWindow,images:dict,contract:ActorActionModelContract):
|
|
17
|
+
if (not isinstance(window,ActorActionWindow) or not isinstance(contract,ActorActionModelContract)
|
|
18
|
+
or hashlib.sha256(window.proof_source.encode("utf8")).hexdigest()!=window.sha256
|
|
19
|
+
or json.loads(window.proof_source)!=window.data or window.data["contract"]!=contract.window.as_dict()):
|
|
20
|
+
raise ValueError("actor action window differs from its selected model or original proof")
|
|
21
|
+
data=window.data
|
|
22
|
+
if not data["usable"]:return None,"insufficient_history"
|
|
23
|
+
observed=[s for s in data["samples"] if s["state"]=="observed"]
|
|
24
|
+
if contract.requires_pose and any("joints" not in sample for sample in observed):
|
|
25
|
+
return None,"pose_unavailable"
|
|
26
|
+
canvas=np.zeros((9,224,224,3),np.uint8)
|
|
27
|
+
for i,sample in enumerate(data["samples"]):
|
|
28
|
+
if sample["state"]!="observed":continue
|
|
29
|
+
raw=images.get(sample["imageSha256"])
|
|
30
|
+
if (not isinstance(raw,bytes) or not raw or len(raw)>16*1024**2
|
|
31
|
+
or hashlib.sha256(raw).hexdigest()!=sample["imageSha256"]):
|
|
32
|
+
raise ValueError("actor action original pixels changed at inference")
|
|
33
|
+
try:
|
|
34
|
+
with Image.open(io.BytesIO(raw)) as source:
|
|
35
|
+
if source.format!="JPEG" or min(source.size)<2 or source.width*source.height>16777216:
|
|
36
|
+
raise ValueError("invalid bounded original actor action JPEG")
|
|
37
|
+
image=source.convert("RGB")
|
|
38
|
+
except (OSError,Image.DecompressionBombError) as error:
|
|
39
|
+
raise ValueError("invalid original actor action JPEG") from error
|
|
40
|
+
joints=sample.get("joints",[{"name":name,"x":None,"y":None,"response":0.} for name in COCO17])
|
|
41
|
+
details=actor_detail_views(image,sample["box"],joints)
|
|
42
|
+
canvas[i,:112,:112]=cv2.resize(np.asarray(actor_view(image,sample["box"],upper=True)),(112,112),interpolation=cv2.INTER_AREA)
|
|
43
|
+
for detail,(y,x) in zip(details,((0,112),(112,0),(112,112))):
|
|
44
|
+
if detail["localized"]:
|
|
45
|
+
canvas[i,y:y+112,x:x+112]=cv2.resize(np.asarray(detail["image"]),(112,112),interpolation=cv2.INTER_AREA)
|
|
46
|
+
tensor=np.ascontiguousarray(((canvas.transpose(3,0,1,2).astype(np.float32)-114.75)/57.38)[None])
|
|
47
|
+
return tensor,None
|
|
48
|
+
|
|
49
|
+
def action_readings(logits,contract:ActorActionModelContract,fields):
|
|
50
|
+
if (not isinstance(logits,np.ndarray) or logits.shape!=(len(contract.outputs),)
|
|
51
|
+
or not np.issubdtype(logits.dtype,np.floating) or not np.isfinite(logits).all()
|
|
52
|
+
or not isinstance(fields,(tuple,list)) or not fields or len(set(fields))!=len(fields)
|
|
53
|
+
or any(field not in {o.field for o in contract.outputs} for field in fields)):
|
|
54
|
+
raise ValueError("invalid original actor action logits or output ownership")
|
|
55
|
+
readings={};scores={}
|
|
56
|
+
for output,value in zip(contract.outputs,logits):
|
|
57
|
+
if output.field not in fields:continue
|
|
58
|
+
score=float(value)
|
|
59
|
+
state=("present" if score>=output.present_above_logit else
|
|
60
|
+
"absent" if score<=output.absent_below_logit else "unobservable")
|
|
61
|
+
readings[output.field]={"state":state,"reason":"model_uncertain" if state=="unobservable" else None}
|
|
62
|
+
scores[output.field]=score
|
|
63
|
+
return readings,scores
|
|
64
|
+
|
|
65
|
+
class ActorActionEstimator:
|
|
66
|
+
"""One verified ONNX artifact. No identity guessing, repeated images or VLM fallback."""
|
|
67
|
+
def __init__(self,model_path,contract:ActorActionModelContract,*,intra_op_threads=4,accelerator="auto"):
|
|
68
|
+
if (not isinstance(contract,ActorActionModelContract)
|
|
69
|
+
or type(intra_op_threads) is not int or not 1<=intra_op_threads<=8):
|
|
70
|
+
raise ValueError("invalid bounded actor action runtime")
|
|
71
|
+
raw=Path(model_path).read_bytes()
|
|
72
|
+
if hashlib.sha256(raw).hexdigest()!=contract.sha256:
|
|
73
|
+
raise ModelContractError("actor action artifact differs from selected immutable task")
|
|
74
|
+
options=ort.SessionOptions();options.intra_op_num_threads=intra_op_threads;options.inter_op_num_threads=1
|
|
75
|
+
self._session=LocalOrtSession(ort,raw,options,accelerator)
|
|
76
|
+
self.contract=contract;self.model_name=contract.model_name;self.model_sha256=contract.sha256
|
|
77
|
+
inputs=self._session.get_inputs();outputs=self._session.get_outputs()
|
|
78
|
+
if (len(inputs)!=1 or inputs[0].name!="input" or inputs[0].shape!=[1,3,9,224,224]
|
|
79
|
+
or inputs[0].type!="tensor(float)" or len(outputs)!=1 or outputs[0].name!="logits"
|
|
80
|
+
or outputs[0].shape!=[len(contract.outputs)] or outputs[0].type!="tensor(float)"):
|
|
81
|
+
raise ModelContractError("actor action graph differs from its RGB/temporal/output contract")
|
|
82
|
+
# Thresholds and cadence live inside the hashed artifact too.
|
|
83
|
+
metadata=self._session.get_modelmeta().custom_metadata_map
|
|
84
|
+
try:declared=json.loads(metadata["actor_action_contract"])
|
|
85
|
+
except (KeyError,TypeError,ValueError) as error:
|
|
86
|
+
raise ModelContractError("actor action artifact has no immutable semantic contract") from error
|
|
87
|
+
if (not isinstance(declared,dict) or set(declared)!={*contract.as_dict(),"qualification"}
|
|
88
|
+
or {k:v for k,v in declared.items() if k!="qualification"}!=contract.as_dict()
|
|
89
|
+
or not isinstance(declared["qualification"],dict)):
|
|
90
|
+
raise ModelContractError("actor action graph semantic contract differs from selected task")
|
|
91
|
+
self._lock=threading.Lock()
|
|
92
|
+
|
|
93
|
+
def probe(self):
|
|
94
|
+
with self._lock:
|
|
95
|
+
values=self._session.run(["logits"],{"input":np.full((1,3,9,224,224),-114.75/57.38,np.float32)})[0]
|
|
96
|
+
action_readings(values,self.contract,tuple(o.field for o in self.contract.outputs))
|
|
97
|
+
|
|
98
|
+
def classify(self,batch:ActorActionBatch):
|
|
99
|
+
if (not isinstance(batch,ActorActionBatch) or batch.model!={"modelId":self.model_name,"sha256":self.model_sha256}
|
|
100
|
+
or any(field not in {o.field for o in self.contract.outputs} for field in batch.fields)):
|
|
101
|
+
raise ValueError("actor action batch does not match the selected task")
|
|
102
|
+
started=time.monotonic();results=[]
|
|
103
|
+
with self._lock:
|
|
104
|
+
for window in batch.windows:
|
|
105
|
+
tensor,reason=prepare_action_window(window,batch.images,self.contract)
|
|
106
|
+
if reason:
|
|
107
|
+
readings={field:{"state":"unobservable","reason":reason} for field in batch.fields}
|
|
108
|
+
scores={field:None for field in batch.fields}
|
|
109
|
+
else:
|
|
110
|
+
logits=self._session.run(["logits"],{"input":tensor})[0]
|
|
111
|
+
readings,scores=action_readings(logits,self.contract,batch.fields)
|
|
112
|
+
results.append({"windowSha256":window.sha256,"actor":window.data["actor"],"fields":readings,"scores":scores})
|
|
113
|
+
return {"version":1,"modelId":self.model_name,"modelSha256":self.model_sha256,
|
|
114
|
+
"preprocessing":self.contract.preprocessing,"fields":list(batch.fields),"results":results,
|
|
115
|
+
"inferenceMs":(time.monotonic()-started)*1000}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Bounded JSON checkpoints for trackers==2.6.1 and its default XYXY estimator."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
FORMAT="bytetrack_2.6.1_xyxy_state_v1"
|
|
6
|
+
MAX_TRACKLETS=512
|
|
7
|
+
SHAPES={"state":(8,1),"state_covariance":(8,8),"transition_mtx":(8,8),
|
|
8
|
+
"observation_mtx":(4,8),"process_noise":(8,8),"measurement_noise":(4,4)}
|
|
9
|
+
COUNTERS=("age","tracker_id","time_since_update","time_since_update_seconds","number_of_successful_consecutive_updates")
|
|
10
|
+
|
|
11
|
+
def _integer(v,minimum=0):
|
|
12
|
+
return type(v) is int and minimum<=v<=9007199254740991
|
|
13
|
+
|
|
14
|
+
def encode_tracker(tracker,identities):
|
|
15
|
+
from trackers.utils.state_representations import XYXYStateEstimator
|
|
16
|
+
if tracker.state_estimator_class is not XYXYStateEstimator or len(tracker.tracks)>MAX_TRACKLETS:
|
|
17
|
+
raise ValueError("Unsupported or over-capacity physical tracker checkpoint")
|
|
18
|
+
rows=[]
|
|
19
|
+
for track in tracker.tracks:
|
|
20
|
+
if type(track.state_estimator) is not XYXYStateEstimator:
|
|
21
|
+
raise ValueError("Tracker estimator changed")
|
|
22
|
+
matrices=track.state_estimator.get_state()
|
|
23
|
+
if set(matrices)!=set(SHAPES):raise ValueError("Tracker filter state changed")
|
|
24
|
+
for name,shape in SHAPES.items():
|
|
25
|
+
import numpy as np
|
|
26
|
+
if matrices[name].shape!=shape or not np.isfinite(matrices[name]).all():
|
|
27
|
+
raise ValueError("Invalid tracker filter state")
|
|
28
|
+
rows.append({**{key:getattr(track,key) for key in COUNTERS},
|
|
29
|
+
"filter":{name:matrix.tolist() for name,matrix in matrices.items()}})
|
|
30
|
+
result={"format":FORMAT,"nextTrackId":tracker._next_track_id,"lastTimestamp":tracker._last_timestamp,
|
|
31
|
+
"tracks":rows,"identities":[{"trackerId":tid,"actor":actor} for tid,actor in sorted(identities.items())]}
|
|
32
|
+
# The same validator checks our output and a restored checkpoint.
|
|
33
|
+
_validate(result)
|
|
34
|
+
return result
|
|
35
|
+
|
|
36
|
+
def _validate(value):
|
|
37
|
+
if (not isinstance(value,dict) or set(value)!={"format","nextTrackId","lastTimestamp","tracks","identities"}
|
|
38
|
+
or value["format"]!=FORMAT or not _integer(value["nextTrackId"])
|
|
39
|
+
or not isinstance(value["lastTimestamp"],(int,float)) or isinstance(value["lastTimestamp"],bool)
|
|
40
|
+
or not math.isfinite(value["lastTimestamp"]) or value["lastTimestamp"]<0
|
|
41
|
+
or not isinstance(value["tracks"],list) or len(value["tracks"])>MAX_TRACKLETS
|
|
42
|
+
or not isinstance(value["identities"],list) or len(value["identities"])>256):
|
|
43
|
+
raise ValueError("Invalid physical tracker checkpoint")
|
|
44
|
+
track_ids=set()
|
|
45
|
+
for row in value["tracks"]:
|
|
46
|
+
if (not isinstance(row,dict) or set(row)!=set(COUNTERS)|{"filter"}
|
|
47
|
+
or not all(_integer(row[k],-1 if k=="tracker_id" else 0) for k in COUNTERS if k!="time_since_update_seconds")
|
|
48
|
+
or not isinstance(row["time_since_update_seconds"],(int,float))
|
|
49
|
+
or isinstance(row["time_since_update_seconds"],bool)
|
|
50
|
+
or not math.isfinite(row["time_since_update_seconds"]) or row["time_since_update_seconds"]<0
|
|
51
|
+
or not isinstance(row["filter"],dict) or set(row["filter"])!=set(SHAPES)):
|
|
52
|
+
raise ValueError("Invalid physical tracklet checkpoint")
|
|
53
|
+
tid=row["tracker_id"]
|
|
54
|
+
if tid>=0:
|
|
55
|
+
if tid in track_ids or tid>=value["nextTrackId"]:raise ValueError("Ambiguous restored track identity")
|
|
56
|
+
track_ids.add(tid)
|
|
57
|
+
for name,(height,width) in SHAPES.items():
|
|
58
|
+
matrix=row["filter"][name]
|
|
59
|
+
if (not isinstance(matrix,list) or len(matrix)!=height
|
|
60
|
+
or any(not isinstance(line,list) or len(line)!=width
|
|
61
|
+
or any(isinstance(x,bool) or not isinstance(x,(int,float)) or not math.isfinite(x) for x in line)
|
|
62
|
+
for line in matrix)):
|
|
63
|
+
raise ValueError("Invalid tracker matrix shape or number")
|
|
64
|
+
identities={}
|
|
65
|
+
for row in value["identities"]:
|
|
66
|
+
if (not isinstance(row,dict) or set(row)!={"trackerId","actor"}
|
|
67
|
+
or not _integer(row["trackerId"]) or row["trackerId"] not in track_ids
|
|
68
|
+
or row["trackerId"] in identities):
|
|
69
|
+
raise ValueError("Invalid restored actor binding")
|
|
70
|
+
identities[row["trackerId"]]=row["actor"]
|
|
71
|
+
if set(identities)!=track_ids:raise ValueError("Incomplete restored actor bindings")
|
|
72
|
+
return identities
|
|
73
|
+
|
|
74
|
+
def restore_tracker(tracker,value,validate_actors):
|
|
75
|
+
import numpy as np
|
|
76
|
+
from trackers.core.bytetrack.tracklet import ByteTrackTracklet
|
|
77
|
+
identities=_validate(value)
|
|
78
|
+
validate_actors(list(identities.values()))
|
|
79
|
+
tracks=[]
|
|
80
|
+
for row in value["tracks"]:
|
|
81
|
+
matrices={name:np.asarray(matrix,dtype=np.float64) for name,matrix in row["filter"].items()}
|
|
82
|
+
track=ByteTrackTracklet(matrices["state"][:4,0],state_estimator_class=tracker.state_estimator_class)
|
|
83
|
+
# The constructor retains the pinned one-frame motion-noise calibration;
|
|
84
|
+
# set_state restores the current filter and clears its step cache.
|
|
85
|
+
track.state_estimator.set_state(matrices)
|
|
86
|
+
for key in COUNTERS:setattr(track,key,row[key])
|
|
87
|
+
tracks.append(track)
|
|
88
|
+
tracker.tracks=tracks
|
|
89
|
+
tracker._next_track_id=value["nextTrackId"]
|
|
90
|
+
tracker._last_timestamp=float(value["lastTimestamp"])
|
|
91
|
+
return tracker,identities
|