@trillboards/edge-sdk 0.2.246 → 0.2.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/README.md +15 -0
- package/deploy/detector/pyproject.toml +1 -1
- package/deploy/detector/tests/test_actor_tracking.py +136 -0
- package/deploy/detector/tests/test_actor_tracking_wire.py +11 -0
- package/deploy/detector/tests/test_face_identity.py +448 -0
- package/deploy/detector/tests/test_loader.py +28 -1
- package/deploy/detector/tests/test_registry.py +53 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/actor_tracking.py +95 -9
- package/deploy/detector/trillboards_detector/face_contract.py +166 -0
- package/deploy/detector/trillboards_detector/face_identity.py +467 -0
- package/deploy/detector/trillboards_detector/loader.py +40 -0
- package/deploy/detector/trillboards_detector/registry.py +72 -2
- package/deploy/detector/trillboards_detector/server.py +48 -6
- package/deploy/detector/trillboards_detector/tasks.py +12 -1
- package/deploy/windows/build-windows.ps1 +1 -0
- package/deploy/windows/ci/test-detector-looks-field.ps1 +15 -3
- package/deploy/windows/ci/test-detector-reinstall.ps1 +173 -2
- package/deploy/windows/ci/test-main-stream-gpu-decode.ps1 +119 -0
- package/deploy/windows/ci/test-rollout-proofs.ps1 +3 -2
- package/deploy/windows/ensure-detector.ps1 +255 -0
- package/deploy/windows/install-update.ps1 +4 -3
- package/deploy/windows/stop-existing.ps1 +1 -0
- package/deploy/windows/trillboards-edge.cmd +10 -22
- package/dist/CloudSensingManager.d.ts +14 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +51 -5
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/ConnectorStatusClient.d.ts +26 -1
- package/dist/ConnectorStatusClient.d.ts.map +1 -1
- package/dist/ConnectorStatusClient.js +9 -0
- package/dist/ConnectorStatusClient.js.map +1 -1
- package/dist/DetectorFields.d.ts +77 -1
- package/dist/DetectorFields.d.ts.map +1 -1
- package/dist/DetectorFields.js +34 -4
- package/dist/DetectorFields.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/FaceIdentity.d.ts +122 -0
- package/dist/FaceIdentity.d.ts.map +1 -0
- package/dist/FaceIdentity.js +256 -0
- package/dist/FaceIdentity.js.map +1 -0
- package/dist/FaceSightingUploader.d.ts +95 -0
- package/dist/FaceSightingUploader.d.ts.map +1 -0
- package/dist/FaceSightingUploader.js +214 -0
- package/dist/FaceSightingUploader.js.map +1 -0
- package/dist/FaceWatchLane.d.ts +297 -0
- package/dist/FaceWatchLane.d.ts.map +1 -0
- package/dist/FaceWatchLane.js +660 -0
- package/dist/FaceWatchLane.js.map +1 -0
- package/dist/FaceWatchPolicy.d.ts +154 -0
- package/dist/FaceWatchPolicy.d.ts.map +1 -0
- package/dist/FaceWatchPolicy.js +101 -0
- package/dist/FaceWatchPolicy.js.map +1 -0
- package/dist/LocalDetectorClient.d.ts +30 -0
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +72 -4
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/MainStreamRing.d.ts +63 -1
- package/dist/MainStreamRing.d.ts.map +1 -1
- package/dist/MainStreamRing.js +101 -7
- package/dist/MainStreamRing.js.map +1 -1
- package/dist/NativeReadiness.d.ts +21 -2
- package/dist/NativeReadiness.d.ts.map +1 -1
- package/dist/NativeReadiness.js +32 -9
- package/dist/NativeReadiness.js.map +1 -1
- package/dist/OrderedReviewClient.d.ts +3 -0
- package/dist/OrderedReviewClient.d.ts.map +1 -1
- package/dist/OrderedReviewClient.js +3 -0
- package/dist/OrderedReviewClient.js.map +1 -1
- package/dist/ReviewPriority.d.ts +9 -0
- package/dist/ReviewPriority.d.ts.map +1 -1
- package/dist/ReviewPriority.js +6 -1
- package/dist/ReviewPriority.js.map +1 -1
- package/dist/ScopedVlmReview.d.ts +11 -2
- package/dist/ScopedVlmReview.d.ts.map +1 -1
- package/dist/ScopedVlmReview.js +43 -16
- package/dist/ScopedVlmReview.js.map +1 -1
- package/dist/SensingProfileClient.d.ts +14 -0
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +9 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/TemporalReviewClient.d.ts +3 -0
- package/dist/TemporalReviewClient.d.ts.map +1 -1
- package/dist/TemporalReviewClient.js +3 -0
- package/dist/TemporalReviewClient.js.map +1 -1
- package/dist/UniformSceneStore.d.ts +6 -0
- package/dist/UniformSceneStore.d.ts.map +1 -1
- package/dist/UniformSceneStore.js +22 -1
- package/dist/UniformSceneStore.js.map +1 -1
- package/dist/VideoSegmentUploader.d.ts +11 -0
- package/dist/VideoSegmentUploader.d.ts.map +1 -1
- package/dist/VideoSegmentUploader.js +28 -2
- package/dist/VideoSegmentUploader.js.map +1 -1
- package/dist/VlmCandidateBuffer.d.ts +2 -0
- package/dist/VlmCandidateBuffer.d.ts.map +1 -1
- package/dist/VlmCandidateBuffer.js.map +1 -1
- package/dist/WatchlistClient.d.ts +55 -0
- package/dist/WatchlistClient.d.ts.map +1 -0
- package/dist/WatchlistClient.js +161 -0
- package/dist/WatchlistClient.js.map +1 -0
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +6 -0
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +34 -9
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +96 -9
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +20 -1
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +55 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/connectorUpdateApplyMac.d.ts +3 -1
- package/dist/connectorUpdateApplyMac.d.ts.map +1 -1
- package/dist/connectorUpdateApplyMac.js +6 -0
- package/dist/connectorUpdateApplyMac.js.map +1 -1
- package/dist/detectorEndpointState.d.ts +30 -0
- package/dist/detectorEndpointState.d.ts.map +1 -0
- package/dist/detectorEndpointState.js +125 -0
- package/dist/detectorEndpointState.js.map +1 -0
- package/dist/detectorLauncher.d.ts +23 -0
- package/dist/detectorLauncher.d.ts.map +1 -0
- package/dist/detectorLauncher.js +94 -0
- package/dist/detectorLauncher.js.map +1 -0
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/hostCpu.d.ts +56 -0
- package/dist/hostCpu.d.ts.map +1 -0
- package/dist/hostCpu.js +113 -0
- package/dist/hostCpu.js.map +1 -0
- package/dist/localSensing.d.ts +86 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +263 -30
- package/dist/localSensing.js.map +1 -1
- package/package.json +2 -2
- package/retained/ProfileAuthorityClient.cjs +4 -0
- package/retained/RetainedCaptureClient.cjs +6 -0
- package/retained/WebsiteAdmissionCaptureClient.cjs +3 -1
- package/retained/__tests__/website-admission-realtime.test.cjs +27 -2
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
"""Faces at the door: find each face, align it, and measure who it is.
|
|
2
|
+
|
|
3
|
+
THREE STEPS, TWO PINNED ARTIFACTS, ONE ANSWER PER FACE
|
|
4
|
+
------------------------------------------------------
|
|
5
|
+
1. YuNet 2023mar (opencv_zoo, MIT) finds each face and its five landmarks on the
|
|
6
|
+
ORIGINAL face-zone frame the connector cut from the main stream, decoded as
|
|
7
|
+
OpenCV's `FaceDetectorYN` decodes it. T1 face research (Texas camera 1): it finds
|
|
8
|
+
56 of the 58 faces SCRFD-10G finds on a man walking in.
|
|
9
|
+
2. A least-squares 5-point SIMILARITY warp (Umeyama 1991; the closed form
|
|
10
|
+
`FaceEmbeddingProcessor.computeSimilarityTransform` uses on the tablet) puts the
|
|
11
|
+
landmarks on the canonical ArcFace 112x112 template. NEVER EMBED WITHOUT
|
|
12
|
+
LANDMARKS: on the tablet a box crop separated same from different people at
|
|
13
|
+
ROC-AUC 0.66 and the aligned chip at 1.00 (FaceEmbeddingProcessor.kt).
|
|
14
|
+
3. EdgeFace-XS-gamma-0.6 (otroshi/edgeface) as ONNX, the twin of the tablet's
|
|
15
|
+
bundled TFLite: exported from the upstream checkpoint (MD5 ebf343ca...d703, the
|
|
16
|
+
Idiap-published checksum) and measured against the APK's
|
|
17
|
+
`edgeface_xs_gamma_06.tflite` at cosine 1.000000000 (max |diff| 8.1e-08) on
|
|
18
|
+
random inputs and on real aligned chips. So a store's vector and a tablet's are
|
|
19
|
+
the same measurement, and one gallery serves both.
|
|
20
|
+
|
|
21
|
+
Each face is measured before it is embedded: the distance between the eyes in
|
|
22
|
+
ORIGINAL pixels (T1 research: 94.7% of true matches at 12 px, 45.7% at 6 px, at a
|
|
23
|
+
false-match rate of 1e-5), a yaw proxy and the sharpness of the aligned chip. A
|
|
24
|
+
face that fails a bar comes back with its box and landmarks (the connector's
|
|
25
|
+
tracker still follows it) and no vector, and says which bar it failed.
|
|
26
|
+
|
|
27
|
+
Stdlib + numpy + OpenCV + onnxruntime, loaded by the loader only: `server.py` and
|
|
28
|
+
`registry.py` never import this module.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import hashlib
|
|
34
|
+
import io
|
|
35
|
+
import math
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
import threading
|
|
38
|
+
import time
|
|
39
|
+
|
|
40
|
+
import cv2
|
|
41
|
+
import numpy as np
|
|
42
|
+
|
|
43
|
+
from .errors import ModelContractError
|
|
44
|
+
from .face_contract import (EMBEDDING_DIM, FACE_DETECT_PREPROCESSING as DETECT_PREPROCESSING,
|
|
45
|
+
FACE_DETECTOR_MODEL_ID as DETECTOR_MODEL_ID, FACE_DETECTOR_SHA256 as DETECTOR_SHA256,
|
|
46
|
+
FACE_EMBED_PREPROCESSING as EMBED_PREPROCESSING,
|
|
47
|
+
FACE_EMBEDDER_MODEL_ID as EMBEDDER_MODEL_ID, FACE_EMBEDDER_SHA256 as EMBEDDER_SHA256)
|
|
48
|
+
|
|
49
|
+
# OpenCV otherwise creates a worker per host core, outside the model budget.
|
|
50
|
+
cv2.setNumThreads(1)
|
|
51
|
+
|
|
52
|
+
#: insightface `arcface_src`, default_square=True: image-left eye, image-right eye,
|
|
53
|
+
#: nose tip, image-left mouth corner, image-right mouth corner. Bit-identical to
|
|
54
|
+
#: `FaceEmbeddingProcessor.ARCFACE_REFERENCE_112`.
|
|
55
|
+
ARCFACE_REFERENCE_112 = np.array(
|
|
56
|
+
[[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]],
|
|
57
|
+
dtype=np.float64,
|
|
58
|
+
)
|
|
59
|
+
CHIP_SIZE = 112
|
|
60
|
+
|
|
61
|
+
#: YuNet runs on the original pixels up to this long side; a larger frame is
|
|
62
|
+
#: area-resized for detection only, and the landmarks are mapped back so the
|
|
63
|
+
#: chip is still cut from the original pixels.
|
|
64
|
+
MAX_DETECT_LONG_SIDE = 1920
|
|
65
|
+
#: Faces down to this score are reported (gate "score" below the request's
|
|
66
|
+
#: minScore), so the connector's tracker keeps a face across a weak frame.
|
|
67
|
+
DETECT_SCORE_FLOOR = 0.5
|
|
68
|
+
NMS_THRESHOLD = 0.3
|
|
69
|
+
TOP_K = 5000
|
|
70
|
+
MAX_JPEG_BYTES = 16 * 1024**2
|
|
71
|
+
MAX_PIXELS = 16_777_216
|
|
72
|
+
|
|
73
|
+
#: The eye distance at which a face's size stops lowering its quality: past it
|
|
74
|
+
#: the T1 LFW curve is flat (TAR@1e-5 97.7% at 16 px, 98.3% at >= 20 px).
|
|
75
|
+
EYE_PX_FULL = 24.0
|
|
76
|
+
#: The yaw proxy at which quality reaches zero. yaw = (nose - eye midpoint) along
|
|
77
|
+
#: the eye line / eye distance, about 0.35 * tan(yaw angle): 0.35 is ~45 degrees.
|
|
78
|
+
YAW_ZERO = 0.5
|
|
79
|
+
#: Sharpness = variance of the Laplacian of the aligned 112x112 grey chip. A chip
|
|
80
|
+
#: is upsampled from the original face, so sharpness falls with the eye distance
|
|
81
|
+
#: as much as with blur. MEASURED (2026-11-16, measure_cam1.py, 400 LFW faces
|
|
82
|
+
#: aligned at fixed eye distances): sharp faces at 12 px p1 17.4 / p5 22.2 /
|
|
83
|
+
#: p50 37.4; at 16 px p1 24.9 / p50 56.1; at 20 px p1 30.7 / p50 72.3. The same
|
|
84
|
+
#: 20 px faces smeared along 15 chip px: p50 16.1, 9 px: p50 22.6. Texas camera 1's
|
|
85
|
+
#: SUBSTREAM faces (8 px between the eyes, the eye bar refuses them first): p50 11.9.
|
|
86
|
+
#: The floor rejects only clearly smeared chips (it keeps 99% of sharp 12 px faces);
|
|
87
|
+
#: quality stops rising at the typical sharp 16-20 px face.
|
|
88
|
+
SHARPNESS_REF = 60.0
|
|
89
|
+
MIN_SHARPNESS = 15.0
|
|
90
|
+
|
|
91
|
+
#: The twelve heads, in OpenCV's order: (cls, obj, bbox, kps) x strides (8, 16, 32).
|
|
92
|
+
YUNET_STRIDES = (8, 16, 32)
|
|
93
|
+
YUNET_OUTPUTS = tuple(f"{head}_{stride}" for head in ("cls", "obj", "bbox", "kps") for stride in YUNET_STRIDES)
|
|
94
|
+
#: OpenCV pads the input up to a multiple of this, with zeros on the right and bottom.
|
|
95
|
+
YUNET_DIVISOR = 32
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def decode_yunet(outputs, pad_w, pad_h, score_threshold, nms_threshold=NMS_THRESHOLD, top_k=TOP_K):
|
|
99
|
+
"""OpenCV `FaceDetectorYNImpl::postProcess` (2023mar heads), then `dnn::NMSBoxes`.
|
|
100
|
+
|
|
101
|
+
Rows of (x, y, w, h, 10 landmark coordinates, score) in padded-input pixels, in
|
|
102
|
+
the order NMS keeps them (score descending). Landmarks are YuNet's: right eye,
|
|
103
|
+
left eye (the subject's), nose tip, right and left mouth corners.
|
|
104
|
+
"""
|
|
105
|
+
candidates = []
|
|
106
|
+
for stride in YUNET_STRIDES:
|
|
107
|
+
cols, rows = pad_w // stride, pad_h // stride
|
|
108
|
+
count = cols * rows
|
|
109
|
+
cls = np.asarray(outputs[f"cls_{stride}"], np.float32).reshape(-1)
|
|
110
|
+
obj = np.asarray(outputs[f"obj_{stride}"], np.float32).reshape(-1)
|
|
111
|
+
bbox = np.asarray(outputs[f"bbox_{stride}"], np.float32).reshape(-1, 4)
|
|
112
|
+
kps = np.asarray(outputs[f"kps_{stride}"], np.float32).reshape(-1, 10)
|
|
113
|
+
if cls.shape[0] != count or obj.shape[0] != count or bbox.shape[0] != count or kps.shape[0] != count:
|
|
114
|
+
raise ModelContractError("Face detector heads do not match the padded input")
|
|
115
|
+
score = np.sqrt(np.clip(cls, 0.0, 1.0) * np.clip(obj, 0.0, 1.0))
|
|
116
|
+
keep = np.nonzero(score >= score_threshold)[0]
|
|
117
|
+
if keep.size == 0:
|
|
118
|
+
continue
|
|
119
|
+
c = (keep % cols).astype(np.float32)
|
|
120
|
+
r = (keep // cols).astype(np.float32)
|
|
121
|
+
cx = (c + bbox[keep, 0]) * stride
|
|
122
|
+
cy = (r + bbox[keep, 1]) * stride
|
|
123
|
+
w = np.exp(bbox[keep, 2]) * stride
|
|
124
|
+
h = np.exp(bbox[keep, 3]) * stride
|
|
125
|
+
landmarks = np.empty((keep.size, 10), np.float32)
|
|
126
|
+
landmarks[:, 0::2] = (kps[keep, 0::2] + c[:, None]) * stride
|
|
127
|
+
landmarks[:, 1::2] = (kps[keep, 1::2] + r[:, None]) * stride
|
|
128
|
+
block = np.column_stack([cx - w / 2.0, cy - h / 2.0, w, h, landmarks, score[keep]])
|
|
129
|
+
candidates.append(block)
|
|
130
|
+
if not candidates:
|
|
131
|
+
return np.zeros((0, 15), np.float64)
|
|
132
|
+
faces = np.vstack(candidates).astype(np.float64)
|
|
133
|
+
if not np.isfinite(faces).all():
|
|
134
|
+
raise ModelContractError("Native face detector output is invalid")
|
|
135
|
+
if len(faces) == 1:
|
|
136
|
+
return faces
|
|
137
|
+
return faces[nms_boxes(faces[:, 0:4], faces[:, 14], score_threshold, nms_threshold, top_k)]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def nms_boxes(boxes, scores, score_threshold, nms_threshold, top_k):
|
|
141
|
+
"""`cv::dnn::NMSBoxes` on Rect2i(int(x), int(y), int(w), int(h)), eta 1: kept indices, best first."""
|
|
142
|
+
rects = np.trunc(np.asarray(boxes, np.float64)).astype(np.int64)
|
|
143
|
+
# GetMaxScoreIndex keeps a score strictly above the threshold, then the top_k.
|
|
144
|
+
order = [i for i in np.argsort(-np.asarray(scores), kind="stable") if scores[i] > score_threshold]
|
|
145
|
+
if top_k > 0:
|
|
146
|
+
order = order[:top_k]
|
|
147
|
+
kept = []
|
|
148
|
+
for i in order:
|
|
149
|
+
x, y, w, h = rects[i]
|
|
150
|
+
area = max(0, w) * max(0, h)
|
|
151
|
+
overlapping = False
|
|
152
|
+
for j in kept:
|
|
153
|
+
x2, y2, w2, h2 = rects[j]
|
|
154
|
+
iw = min(x + w, x2 + w2) - max(x, x2)
|
|
155
|
+
ih = min(y + h, y2 + h2) - max(y, y2)
|
|
156
|
+
inter = max(0, iw) * max(0, ih)
|
|
157
|
+
union = area + max(0, w2) * max(0, h2) - inter
|
|
158
|
+
if union > 0 and inter / union > nms_threshold:
|
|
159
|
+
overlapping = True
|
|
160
|
+
break
|
|
161
|
+
if not overlapping:
|
|
162
|
+
kept.append(i)
|
|
163
|
+
return np.asarray(kept, dtype=np.int64)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def umeyama(src, dst):
|
|
167
|
+
"""Least-squares similarity (scale, rotation, translation; no reflection) taking src onto dst.
|
|
168
|
+
|
|
169
|
+
The Umeyama closed form `skimage.SimilarityTransform` and the tablet's
|
|
170
|
+
`computeSimilarityTransform` both use. Returns the 2x3 affine, or None when
|
|
171
|
+
the source points are degenerate (coincident: no stable transform).
|
|
172
|
+
"""
|
|
173
|
+
src = np.asarray(src, dtype=np.float64)
|
|
174
|
+
dst = np.asarray(dst, dtype=np.float64)
|
|
175
|
+
if src.shape != dst.shape or src.ndim != 2 or src.shape[1] != 2 or src.shape[0] < 3:
|
|
176
|
+
return None
|
|
177
|
+
if not (np.isfinite(src).all() and np.isfinite(dst).all()):
|
|
178
|
+
return None
|
|
179
|
+
mu_s, mu_d = src.mean(0), dst.mean(0)
|
|
180
|
+
sc, dc = src - mu_s, dst - mu_d
|
|
181
|
+
var_s = float((sc ** 2).sum() / len(src))
|
|
182
|
+
if var_s < 1e-12:
|
|
183
|
+
return None
|
|
184
|
+
cov = dc.T @ sc / len(src)
|
|
185
|
+
u, s, vt = np.linalg.svd(cov)
|
|
186
|
+
d = np.ones(2)
|
|
187
|
+
if np.linalg.det(u) * np.linalg.det(vt) < 0:
|
|
188
|
+
d[-1] = -1.0
|
|
189
|
+
rotation = u @ np.diag(d) @ vt
|
|
190
|
+
scale = float((s * d).sum() / var_s)
|
|
191
|
+
translation = mu_d - scale * rotation @ mu_s
|
|
192
|
+
matrix = np.hstack([scale * rotation, translation[:, None]])
|
|
193
|
+
return matrix if np.isfinite(matrix).all() else None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def arcface_order(landmarks):
|
|
197
|
+
"""The five points in template order, by IMAGE position (handedness-proof).
|
|
198
|
+
|
|
199
|
+
Template point 0 is the image-LEFT eye and point 3 the image-left mouth corner,
|
|
200
|
+
whatever a detector calls them. Feeding the fit a mirrored correspondence
|
|
201
|
+
collapses the vector (0.53 AUC on the tablet), so the eyes and the mouth
|
|
202
|
+
corners are each ordered by x, exactly as `toArcFaceSourcePoints` does.
|
|
203
|
+
"""
|
|
204
|
+
points = np.asarray(landmarks, dtype=np.float64).reshape(5, 2)
|
|
205
|
+
eyes = points[[0, 1]] if points[0, 0] <= points[1, 0] else points[[1, 0]]
|
|
206
|
+
mouth = points[[3, 4]] if points[3, 0] <= points[4, 0] else points[[4, 3]]
|
|
207
|
+
return np.vstack([eyes, points[2:3], mouth])
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def align_chip(bgr, landmarks):
|
|
211
|
+
"""The 112x112 BGR chip of one face, warped (bilinear) from the ORIGINAL pixels."""
|
|
212
|
+
matrix = umeyama(arcface_order(landmarks), ARCFACE_REFERENCE_112)
|
|
213
|
+
if matrix is None:
|
|
214
|
+
return None
|
|
215
|
+
return cv2.warpAffine(bgr, matrix, (CHIP_SIZE, CHIP_SIZE), flags=cv2.INTER_LINEAR,
|
|
216
|
+
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def face_measures(landmarks, chip):
|
|
220
|
+
"""(eye distance in original px, signed yaw proxy, sharpness of the aligned chip)."""
|
|
221
|
+
points = arcface_order(landmarks)
|
|
222
|
+
left, right, nose = points[0], points[1], points[2]
|
|
223
|
+
axis = right - left
|
|
224
|
+
eye_distance = float(math.hypot(axis[0], axis[1]))
|
|
225
|
+
if eye_distance <= 0:
|
|
226
|
+
yaw = 0.0
|
|
227
|
+
else:
|
|
228
|
+
# Along the eye line, so a tilted (rolled) head does not read as turned.
|
|
229
|
+
yaw = float(np.dot(nose - (left + right) / 2.0, axis / eye_distance) / eye_distance)
|
|
230
|
+
grey = cv2.cvtColor(chip, cv2.COLOR_BGR2GRAY)
|
|
231
|
+
sharpness = float(cv2.Laplacian(grey, cv2.CV_64F).var())
|
|
232
|
+
return eye_distance, yaw, sharpness
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def face_quality(score, eye_distance, yaw, sharpness):
|
|
236
|
+
"""0..1: detector score x size x frontality x sharpness, each saturating at its reference."""
|
|
237
|
+
size = min(1.0, eye_distance / EYE_PX_FULL)
|
|
238
|
+
frontal = max(0.0, 1.0 - abs(yaw) / YAW_ZERO)
|
|
239
|
+
sharp = min(1.0, max(0.0, sharpness) / SHARPNESS_REF)
|
|
240
|
+
value = float(score) * size * frontal * sharp
|
|
241
|
+
return min(1.0, max(0.0, value))
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def face_gate(score, eye_distance, yaw, sharpness, request):
|
|
245
|
+
"""The first bar a face fails, in a fixed order, or "passed"."""
|
|
246
|
+
if score < request["minScore"]:
|
|
247
|
+
return "score"
|
|
248
|
+
if eye_distance < request["minEyePx"]:
|
|
249
|
+
return "eye_distance"
|
|
250
|
+
if abs(yaw) > request["maxAbsYaw"]:
|
|
251
|
+
return "yaw"
|
|
252
|
+
if sharpness < request["minSharpness"]:
|
|
253
|
+
return "sharpness"
|
|
254
|
+
return "passed"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def decode_original_jpeg(jpeg):
|
|
258
|
+
"""(BGR array, width, height) of a bounded original JPEG, or ValueError."""
|
|
259
|
+
from PIL import Image
|
|
260
|
+
|
|
261
|
+
if not isinstance(jpeg, bytes) or not 0 < len(jpeg) <= MAX_JPEG_BYTES:
|
|
262
|
+
raise ValueError("Faces need a bounded original JPEG")
|
|
263
|
+
try:
|
|
264
|
+
with Image.open(io.BytesIO(jpeg)) as header:
|
|
265
|
+
width, height = header.size
|
|
266
|
+
if (header.format != "JPEG" or not 2 <= width <= 8192 or not 2 <= height <= 8192
|
|
267
|
+
or width * height > MAX_PIXELS):
|
|
268
|
+
raise ValueError("Faces need a bounded original JPEG")
|
|
269
|
+
except ValueError:
|
|
270
|
+
raise
|
|
271
|
+
except Exception as error:
|
|
272
|
+
raise ValueError("Invalid original face JPEG") from error
|
|
273
|
+
bgr = cv2.imdecode(np.frombuffer(jpeg, np.uint8), cv2.IMREAD_COLOR)
|
|
274
|
+
if bgr is None or bgr.shape != (height, width, 3):
|
|
275
|
+
raise ValueError("Original face JPEG could not be decoded")
|
|
276
|
+
return bgr, width, height
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class EdgeFaceEmbedder:
|
|
280
|
+
"""EdgeFace-XS: one immutable CPU artifact, the tablet model's ONNX twin."""
|
|
281
|
+
|
|
282
|
+
model_name = EMBEDDER_MODEL_ID
|
|
283
|
+
model_sha256 = EMBEDDER_SHA256
|
|
284
|
+
|
|
285
|
+
def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
|
|
286
|
+
if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
|
|
287
|
+
raise ValueError("Invalid bounded face embedder thread count")
|
|
288
|
+
if accelerator != "cpu":
|
|
289
|
+
raise ModelContractError("This face embedder is qualified for native CPU execution")
|
|
290
|
+
raw = Path(model_path).read_bytes()
|
|
291
|
+
if hashlib.sha256(raw).hexdigest() != EMBEDDER_SHA256:
|
|
292
|
+
raise ModelContractError("Face embedder artifact differs from the pinned export")
|
|
293
|
+
import onnxruntime as ort
|
|
294
|
+
|
|
295
|
+
from .execution import LocalOrtSession
|
|
296
|
+
|
|
297
|
+
options = ort.SessionOptions()
|
|
298
|
+
options.intra_op_num_threads = intra_op_threads
|
|
299
|
+
options.inter_op_num_threads = 1
|
|
300
|
+
options.log_severity_level = 3
|
|
301
|
+
self._session = LocalOrtSession(ort, raw, options, "cpu")
|
|
302
|
+
inputs, outputs = self._session.get_inputs(), self._session.get_outputs()
|
|
303
|
+
if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].shape != [1, 3, CHIP_SIZE, CHIP_SIZE]
|
|
304
|
+
or inputs[0].type != "tensor(float)" or len(outputs) != 1 or outputs[0].name != "embedding"
|
|
305
|
+
or outputs[0].shape != [1, EMBEDDING_DIM] or outputs[0].type != "tensor(float)"):
|
|
306
|
+
raise ModelContractError("Face embedder graph differs from its 112x112 RGB -> 512 contract")
|
|
307
|
+
self._lock = threading.Lock()
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def acceleration(self):
|
|
311
|
+
return self._session.acceleration
|
|
312
|
+
|
|
313
|
+
def embed(self, chip_bgr):
|
|
314
|
+
"""One aligned 112x112 BGR chip -> its L2-normalised 512-d vector (float64)."""
|
|
315
|
+
if (not isinstance(chip_bgr, np.ndarray) or chip_bgr.dtype != np.uint8
|
|
316
|
+
or chip_bgr.shape != (CHIP_SIZE, CHIP_SIZE, 3)):
|
|
317
|
+
raise ValueError("A face vector needs one aligned 112x112 chip")
|
|
318
|
+
rgb = chip_bgr[:, :, ::-1].astype(np.float32)
|
|
319
|
+
tensor = np.ascontiguousarray(((rgb / 127.5) - 1.0).transpose(2, 0, 1)[None], dtype=np.float32)
|
|
320
|
+
with self._lock:
|
|
321
|
+
value = self._session.run(["embedding"], {"input": tensor})[0]
|
|
322
|
+
if value.shape != (1, EMBEDDING_DIM) or not np.isfinite(value).all():
|
|
323
|
+
raise ModelContractError("Native face embedder output is invalid")
|
|
324
|
+
vector = value.reshape(EMBEDDING_DIM).astype(np.float64)
|
|
325
|
+
norm = float(np.linalg.norm(vector))
|
|
326
|
+
if not math.isfinite(norm) or norm <= 1e-12:
|
|
327
|
+
raise ModelContractError("Native face embedder output cannot identify a face")
|
|
328
|
+
return vector / norm
|
|
329
|
+
|
|
330
|
+
def probe(self):
|
|
331
|
+
"""A forward pass completes (the loader's admission check)."""
|
|
332
|
+
self.embed(np.full((CHIP_SIZE, CHIP_SIZE, 3), 128, np.uint8))
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class YunetFaceDetector:
|
|
336
|
+
"""YuNet 2023mar on onnxruntime, decoded exactly as OpenCV's FaceDetectorYN decodes it.
|
|
337
|
+
|
|
338
|
+
WHY NOT `cv2.FaceDetectorYN`, WHICH IS THE REFERENCE: on the pinned
|
|
339
|
+
opencv-python 5.0.0.93 its new DNN graph engine ran this graph 2.5x slower
|
|
340
|
+
than onnxruntime 1.29 on the same 800x448 Texas frame, one thread each,
|
|
341
|
+
interleaved (scratchpad/t1 yunet_ab.py). The decode below is OpenCV's
|
|
342
|
+
`FaceDetectorYNImpl::postProcess` line for line, and `test_face_identity.py`
|
|
343
|
+
holds the two to the same faces on real frames.
|
|
344
|
+
|
|
345
|
+
The artifact is the published file with only its DECLARED shape made symbolic
|
|
346
|
+
(`make_yunet_dynamic.py`): every node and weight is opencv_zoo's
|
|
347
|
+
8f2383e4...2fa4, whose 640x640 declaration would otherwise refuse any other
|
|
348
|
+
frame size in onnxruntime.
|
|
349
|
+
"""
|
|
350
|
+
|
|
351
|
+
model_name = DETECTOR_MODEL_ID
|
|
352
|
+
model_sha256 = DETECTOR_SHA256
|
|
353
|
+
|
|
354
|
+
def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
|
|
355
|
+
if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
|
|
356
|
+
raise ValueError("Invalid bounded face detector thread count")
|
|
357
|
+
if accelerator != "cpu":
|
|
358
|
+
raise ModelContractError("This face detector is qualified for native CPU execution")
|
|
359
|
+
raw = Path(model_path).read_bytes()
|
|
360
|
+
if hashlib.sha256(raw).hexdigest() != DETECTOR_SHA256:
|
|
361
|
+
raise ModelContractError("Face detector artifact differs from the pinned export")
|
|
362
|
+
import onnxruntime as ort
|
|
363
|
+
|
|
364
|
+
from .execution import LocalOrtSession
|
|
365
|
+
|
|
366
|
+
options = ort.SessionOptions()
|
|
367
|
+
options.intra_op_num_threads = intra_op_threads
|
|
368
|
+
options.inter_op_num_threads = 1
|
|
369
|
+
options.log_severity_level = 3
|
|
370
|
+
self._session = LocalOrtSession(ort, raw, options, "cpu")
|
|
371
|
+
inputs = self._session.get_inputs()
|
|
372
|
+
outputs = [o.name for o in self._session.get_outputs()]
|
|
373
|
+
if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].type != "tensor(float)"
|
|
374
|
+
or len(inputs[0].shape) != 4 or inputs[0].shape[:2] != [1, 3]
|
|
375
|
+
or not all(isinstance(v, str) for v in inputs[0].shape[2:])
|
|
376
|
+
or sorted(outputs) != sorted(YUNET_OUTPUTS)):
|
|
377
|
+
raise ModelContractError("Face detector graph differs from the YuNet 2023mar contract")
|
|
378
|
+
self._lock = threading.Lock()
|
|
379
|
+
|
|
380
|
+
@property
|
|
381
|
+
def acceleration(self):
|
|
382
|
+
return self._session.acceleration
|
|
383
|
+
|
|
384
|
+
def detect(self, bgr, min_score):
|
|
385
|
+
"""[(box [x, y, w, h], score, landmarks 5x2)] in the pixels of `bgr`, best score first."""
|
|
386
|
+
height, width = bgr.shape[:2]
|
|
387
|
+
scale = 1.0
|
|
388
|
+
image = bgr
|
|
389
|
+
if max(width, height) > MAX_DETECT_LONG_SIDE:
|
|
390
|
+
scale = MAX_DETECT_LONG_SIDE / float(max(width, height))
|
|
391
|
+
image = cv2.resize(bgr, (max(2, round(width * scale)), max(2, round(height * scale))),
|
|
392
|
+
interpolation=cv2.INTER_AREA)
|
|
393
|
+
pad_h = ((image.shape[0] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
|
|
394
|
+
pad_w = ((image.shape[1] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
|
|
395
|
+
padded = np.zeros((pad_h, pad_w, 3), np.uint8)
|
|
396
|
+
padded[:image.shape[0], :image.shape[1]] = image
|
|
397
|
+
blob = np.ascontiguousarray(padded.transpose(2, 0, 1)[None], dtype=np.float32)
|
|
398
|
+
with self._lock:
|
|
399
|
+
values = self._session.run(list(YUNET_OUTPUTS), {"input": blob})
|
|
400
|
+
rows = decode_yunet(dict(zip(YUNET_OUTPUTS, values)), pad_w, pad_h,
|
|
401
|
+
float(min(DETECT_SCORE_FLOOR, min_score)))
|
|
402
|
+
faces = []
|
|
403
|
+
for row in rows:
|
|
404
|
+
faces.append((row[0:4] / scale, float(row[14]), row[4:14].reshape(5, 2) / scale))
|
|
405
|
+
return faces
|
|
406
|
+
|
|
407
|
+
def estimate(self, jpeg, request, embedder):
|
|
408
|
+
"""The /v1/faces answer for one original JPEG (face_contract.check_faces_response holds its shape)."""
|
|
409
|
+
if not callable(getattr(embedder, "embed", None)):
|
|
410
|
+
raise ModelContractError("Faces need the pinned embedder session")
|
|
411
|
+
bgr, width, height = decode_original_jpeg(jpeg)
|
|
412
|
+
started = time.perf_counter()
|
|
413
|
+
detected = self.detect(bgr, request["minScore"])
|
|
414
|
+
detect_ms = (time.perf_counter() - started) * 1000.0
|
|
415
|
+
embed_ms = 0.0
|
|
416
|
+
faces = []
|
|
417
|
+
for box, score, landmarks in detected[:request["maxFaces"]]:
|
|
418
|
+
# Clipped to the frame, then rounded at the corners, so x + width never passes the edge.
|
|
419
|
+
# Positions keep 3 decimals and scores 6: the regression corpus holds this answer to the
|
|
420
|
+
# reference implementations at 3e-5 of the frame, which coarser rounding would exceed.
|
|
421
|
+
x0 = round(max(0.0, float(box[0])), 3)
|
|
422
|
+
y0 = round(max(0.0, float(box[1])), 3)
|
|
423
|
+
x1 = round(min(float(width), float(box[0] + box[2])), 3)
|
|
424
|
+
y1 = round(min(float(height), float(box[1] + box[3])), 3)
|
|
425
|
+
if x1 - x0 <= 0 or y1 - y0 <= 0:
|
|
426
|
+
continue
|
|
427
|
+
ordered = arcface_order(landmarks)
|
|
428
|
+
chip = align_chip(bgr, ordered)
|
|
429
|
+
if chip is None:
|
|
430
|
+
continue
|
|
431
|
+
# Rounded FIRST, and the gate and quality decided on the rounded numbers, so the
|
|
432
|
+
# answer is self-consistent to the digit (server.py re-derives every gate from them).
|
|
433
|
+
eye_distance, yaw, sharpness = (round(v, d) for v, d in zip(face_measures(ordered, chip), (3, 6, 3)))
|
|
434
|
+
score = round(min(1.0, max(0.0, score)), 6)
|
|
435
|
+
gate = face_gate(score, eye_distance, yaw, sharpness, request)
|
|
436
|
+
embedding = None
|
|
437
|
+
if gate == "passed":
|
|
438
|
+
began = time.perf_counter()
|
|
439
|
+
vector = embedder.embed(chip)
|
|
440
|
+
embed_ms += (time.perf_counter() - began) * 1000.0
|
|
441
|
+
embedding = [round(float(v), 6) for v in vector]
|
|
442
|
+
faces.append({
|
|
443
|
+
"box": {"x": x0, "y": y0, "width": round(x1 - x0, 3), "height": round(y1 - y0, 3)},
|
|
444
|
+
"score": score,
|
|
445
|
+
"landmarks": [[round(float(px), 3), round(float(py), 3)] for px, py in ordered],
|
|
446
|
+
"eyeDistancePx": eye_distance,
|
|
447
|
+
"yaw": yaw,
|
|
448
|
+
"sharpness": sharpness,
|
|
449
|
+
"quality": round(face_quality(score, eye_distance, yaw, sharpness), 6),
|
|
450
|
+
"gate": gate,
|
|
451
|
+
"embedding": embedding,
|
|
452
|
+
})
|
|
453
|
+
return {
|
|
454
|
+
"version": 1,
|
|
455
|
+
"imageSha256": hashlib.sha256(jpeg).hexdigest(),
|
|
456
|
+
"frameSize": {"width": width, "height": height},
|
|
457
|
+
"detector": {"modelId": DETECTOR_MODEL_ID, "sha256": DETECTOR_SHA256,
|
|
458
|
+
"preprocessing": DETECT_PREPROCESSING},
|
|
459
|
+
"embedder": {"modelId": EMBEDDER_MODEL_ID, "sha256": EMBEDDER_SHA256,
|
|
460
|
+
"preprocessing": EMBED_PREPROCESSING},
|
|
461
|
+
"faces": faces,
|
|
462
|
+
"inferenceMs": {"detect": round(detect_ms, 3), "embed": round(embed_ms, 3)},
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
def probe(self):
|
|
466
|
+
"""A forward pass completes on a grey frame (no face is expected or required)."""
|
|
467
|
+
self.detect(np.full((48, 64, 3), 128, np.uint8), DETECT_SCORE_FLOOR)
|
|
@@ -79,6 +79,24 @@ def _warmup_jpeg() -> bytes:
|
|
|
79
79
|
return buf.getvalue()
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
#: The frame size a live look sends (the connector's 640x480 analysis frame).
|
|
83
|
+
FULL_SIZE_WARMUP = (640, 480)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _full_size_warmup_jpeg() -> bytes:
|
|
87
|
+
"""A 640x480 JPEG with real texture, for the pinned model's full-size warm-up.
|
|
88
|
+
|
|
89
|
+
Noise, not a flat fill: a flat frame compresses to almost nothing and decodes
|
|
90
|
+
in no time, so it would warm the forward pass but not the decode a live frame
|
|
91
|
+
costs. See ModelRegistry._ensure_loaded for why this pass exists.
|
|
92
|
+
"""
|
|
93
|
+
from PIL import Image
|
|
94
|
+
|
|
95
|
+
buf = io.BytesIO()
|
|
96
|
+
Image.effect_noise(FULL_SIZE_WARMUP, 48).convert("RGB").save(buf, format="JPEG", quality=80)
|
|
97
|
+
return buf.getvalue()
|
|
98
|
+
|
|
99
|
+
|
|
82
100
|
def build_catalogue(
|
|
83
101
|
default_model_name: str,
|
|
84
102
|
default_model_path: "Path | None" = None,
|
|
@@ -161,6 +179,7 @@ class DetectorLoader:
|
|
|
161
179
|
probe=self._probe,
|
|
162
180
|
ram_budget_bytes=ram_budget_bytes,
|
|
163
181
|
log=log,
|
|
182
|
+
warm=self._warm_full_size,
|
|
164
183
|
)
|
|
165
184
|
|
|
166
185
|
self._stop = threading.Event()
|
|
@@ -179,6 +198,15 @@ class DetectorLoader:
|
|
|
179
198
|
if entry.vocabulary.task == "vehicle_end_orientation":
|
|
180
199
|
from .vehicle_end import VehicleEndEstimator
|
|
181
200
|
return VehicleEndEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator="cpu")
|
|
201
|
+
# The door faces task is called by the connector's face lane at up to 4 frames a second,
|
|
202
|
+
# beside the look loop's object detector: ONE thread by default keeps it to one core and
|
|
203
|
+
# out of RF-DETR's pool (devbox A/B under load: 4 threads ran slower than 1).
|
|
204
|
+
if entry.vocabulary.task == "face_detection_landmarks5":
|
|
205
|
+
from .face_identity import YunetFaceDetector
|
|
206
|
+
return YunetFaceDetector(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
|
|
207
|
+
if entry.vocabulary.task == "face_identity_embedding":
|
|
208
|
+
from .face_identity import EdgeFaceEmbedder
|
|
209
|
+
return EdgeFaceEmbedder(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
|
|
182
210
|
from .pose import PoseEstimator
|
|
183
211
|
return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
|
|
184
212
|
from .session import OnnxDetector
|
|
@@ -198,6 +226,15 @@ class DetectorLoader:
|
|
|
198
226
|
else:
|
|
199
227
|
session.detect(_warmup_jpeg(), 0.99, None)
|
|
200
228
|
|
|
229
|
+
@staticmethod
|
|
230
|
+
def _warm_full_size(session) -> None:
|
|
231
|
+
"""One more forward pass, on a live-sized frame, for the pinned cascade gate.
|
|
232
|
+
|
|
233
|
+
The registry runs this after `_probe` and never as a gate (see
|
|
234
|
+
ModelRegistry._ensure_loaded). `0.99` for the same reason as the probe.
|
|
235
|
+
"""
|
|
236
|
+
session.detect(_full_size_warmup_jpeg(), 0.99, None)
|
|
237
|
+
|
|
201
238
|
def start(self) -> None:
|
|
202
239
|
self._thread.start()
|
|
203
240
|
|
|
@@ -338,6 +375,9 @@ class DetectorLoader:
|
|
|
338
375
|
def estimate_vehicle_end(self, jpeg: bytes, boxes: list, side: str, expected_model: dict) -> dict:
|
|
339
376
|
return self.registry.estimate_vehicle_end(jpeg, boxes, side, expected_model)
|
|
340
377
|
|
|
378
|
+
def estimate_faces(self, jpeg: bytes, request: dict, detector_pin: dict, embedder_pin: dict) -> dict:
|
|
379
|
+
return self.registry.estimate_faces(jpeg, request, detector_pin, embedder_pin)
|
|
380
|
+
|
|
341
381
|
def classify_actor_actions(self,batch):
|
|
342
382
|
return self.registry.classify_actor_actions(batch)
|
|
343
383
|
|
|
@@ -123,7 +123,8 @@ class TaskModelContract:
|
|
|
123
123
|
self._validate_task()
|
|
124
124
|
|
|
125
125
|
def _validate_task(self):
|
|
126
|
-
if self.task not in {"pose_coco17", "actor_appearance", "vehicle_end_orientation"
|
|
126
|
+
if self.task not in {"pose_coco17", "actor_appearance", "vehicle_end_orientation",
|
|
127
|
+
"face_detection_landmarks5", "face_identity_embedding"}:
|
|
127
128
|
raise ValueError("invalid non-box task kind")
|
|
128
129
|
|
|
129
130
|
class ModelEntry:
|
|
@@ -214,6 +215,7 @@ class ModelRegistry:
|
|
|
214
215
|
ram_budget_bytes: "int | None",
|
|
215
216
|
log=print,
|
|
216
217
|
clock=time.monotonic,
|
|
218
|
+
warm=None,
|
|
217
219
|
):
|
|
218
220
|
if not entries:
|
|
219
221
|
raise ValueError("a registry with no models can answer nothing")
|
|
@@ -242,6 +244,8 @@ class ModelRegistry:
|
|
|
242
244
|
self._entries = list(entries)
|
|
243
245
|
self._session_factory = session_factory
|
|
244
246
|
self._probe = probe
|
|
247
|
+
#: The pinned model's full-size warm-up (see `_ensure_loaded`), or None.
|
|
248
|
+
self._warm = warm
|
|
245
249
|
#: `None` means NO CEILING WAS INJECTED — not "unlimited".
|
|
246
250
|
self._budget = None if ram_budget_bytes is None else int(ram_budget_bytes)
|
|
247
251
|
self._log = log
|
|
@@ -607,6 +611,49 @@ class ModelRegistry:
|
|
|
607
611
|
finally:
|
|
608
612
|
self._release([state.entry.name])
|
|
609
613
|
|
|
614
|
+
def estimate_faces(self, jpeg: bytes, request: dict, detector_pin: dict, embedder_pin: dict) -> dict:
|
|
615
|
+
"""The door faces task: BOTH pinned sessions acquired together through the portfolio.
|
|
616
|
+
|
|
617
|
+
All or nothing, like `detect`: a face answer without its vectors would read
|
|
618
|
+
as "nobody we know" to the watchlist, so a missing embedder is a 503 for the
|
|
619
|
+
whole request, never a detector-only 200.
|
|
620
|
+
"""
|
|
621
|
+
pins = (("face_detection_landmarks5", detector_pin), ("face_identity_embedding", embedder_pin))
|
|
622
|
+
states = []
|
|
623
|
+
for task, pin in pins:
|
|
624
|
+
state = self._states.get(pin.get("modelId")) if isinstance(pin, dict) else None
|
|
625
|
+
if (state is None or not isinstance(state.entry.vocabulary, TaskModelContract)
|
|
626
|
+
or state.entry.vocabulary.task != task
|
|
627
|
+
or state.entry.vocabulary.sha256 != pin.get("sha256")):
|
|
628
|
+
raise ModelVerificationError("selected face task does not match its immutable contract")
|
|
629
|
+
states.append(state)
|
|
630
|
+
names = [state.entry.name for state in states]
|
|
631
|
+
sessions = self._acquire(names)
|
|
632
|
+
try:
|
|
633
|
+
for state, session in zip(states, sessions):
|
|
634
|
+
if getattr(session, "model_sha256", None) != state.entry.vocabulary.sha256:
|
|
635
|
+
raise ModelVerificationError("loaded face session does not match the selected artifact")
|
|
636
|
+
detector, embedder = sessions
|
|
637
|
+
try:
|
|
638
|
+
result = detector.estimate(jpeg, request, embedder)
|
|
639
|
+
for key, state in zip(("detector", "embedder"), states):
|
|
640
|
+
contract = state.entry.vocabulary
|
|
641
|
+
if not isinstance(result, dict) or result.get(key) != {
|
|
642
|
+
"modelId": contract.model_name, "sha256": contract.sha256,
|
|
643
|
+
"preprocessing": contract.preprocessing}:
|
|
644
|
+
raise ModelVerificationError("face output does not match its task contracts")
|
|
645
|
+
except Exception:
|
|
646
|
+
with self._lock:
|
|
647
|
+
for state in states:
|
|
648
|
+
state.detect_failures += 1
|
|
649
|
+
raise
|
|
650
|
+
with self._lock:
|
|
651
|
+
for state in states:
|
|
652
|
+
state.detect_count += 1
|
|
653
|
+
return result
|
|
654
|
+
finally:
|
|
655
|
+
self._release(names)
|
|
656
|
+
|
|
610
657
|
def classify_actor_actions(self,batch):
|
|
611
658
|
"""Acquire and prove the selected non-box action model through the shared portfolio."""
|
|
612
659
|
from .actor_action_contract import ActorActionModelContract
|
|
@@ -818,6 +865,26 @@ class ModelRegistry:
|
|
|
818
865
|
rss_delta = max(0, after[0] - before[0])
|
|
819
866
|
rss_source = after[1]
|
|
820
867
|
|
|
868
|
+
# THE PINNED MODEL WARMS AT FULL SIZE BEFORE IT IS READY (plan v7 W8 follow-up).
|
|
869
|
+
# The probe above proves a forward pass completes, on a 64x48 frame, and that
|
|
870
|
+
# frame is too small to warm the path a live look takes: after the 0.2.246
|
|
871
|
+
# detector swap at Texas the first live call took 3.2 s and the next 18 averaged
|
|
872
|
+
# 1.4 s. So the cascade gate runs once more on a full-size frame before READY.
|
|
873
|
+
# Only the pinned model: it loads at start or on a new artifact, ahead of the
|
|
874
|
+
# looks it gates, while a specialist loads inside the first request that needs
|
|
875
|
+
# it, where another pass would only lengthen that request. Never a gate: a
|
|
876
|
+
# failure here is logged and the model is ready on its proof. Not timed into
|
|
877
|
+
# warmup_ms (`warmup_inference_ms`), which stays the proof pass's own time, and
|
|
878
|
+
# after the RSS reading, so the admission number is unchanged.
|
|
879
|
+
warm_ms = None
|
|
880
|
+
if entry.pinned and self._warm is not None:
|
|
881
|
+
try:
|
|
882
|
+
warm_started = time.perf_counter()
|
|
883
|
+
self._warm(session)
|
|
884
|
+
warm_ms = (time.perf_counter() - warm_started) * 1000.0
|
|
885
|
+
except Exception as err:
|
|
886
|
+
self._log(f"[detector] {entry.name}: full-size warm-up failed; ready on its probe: {err}")
|
|
887
|
+
|
|
821
888
|
with self._lock:
|
|
822
889
|
state.session = session
|
|
823
890
|
state.stamp = stamp
|
|
@@ -830,9 +897,10 @@ class ModelRegistry:
|
|
|
830
897
|
state.rss_source = rss_source
|
|
831
898
|
self._index = None
|
|
832
899
|
measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
|
|
900
|
+
warmed = f", full-size warm-up {warm_ms:.0f} ms" if warm_ms is not None else ""
|
|
833
901
|
self._log(
|
|
834
902
|
f"[detector] ready — {entry.name} at {entry.path} "
|
|
835
|
-
f"(probe {warmup_ms:.0f} ms, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
|
|
903
|
+
f"(probe {warmup_ms:.0f} ms{warmed}, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
|
|
836
904
|
)
|
|
837
905
|
|
|
838
906
|
def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
|
|
@@ -892,6 +960,8 @@ class ModelRegistry:
|
|
|
892
960
|
if s.reason:
|
|
893
961
|
row["reason"] = s.reason
|
|
894
962
|
if s.warmup_ms is not None:
|
|
963
|
+
# The proof pass (64x48) only; the pinned model's full-size warm-up
|
|
964
|
+
# is in the ready log line, not here (see `_ensure_loaded`).
|
|
895
965
|
row["warmup_inference_ms"] = round(s.warmup_ms, 1)
|
|
896
966
|
if s.loaded_at is not None:
|
|
897
967
|
row["loaded_at"] = s.loaded_at
|