@trillboards/edge-sdk 0.2.253 → 0.2.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/README.md +15 -0
- package/deploy/detector/pyproject.toml +1 -1
- package/deploy/detector/tests/test_face_identity.py +448 -0
- package/deploy/detector/tests/test_loader.py +3 -1
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/face_contract.py +166 -0
- package/deploy/detector/trillboards_detector/face_identity.py +467 -0
- package/deploy/detector/trillboards_detector/loader.py +12 -0
- package/deploy/detector/trillboards_detector/registry.py +45 -1
- package/deploy/detector/trillboards_detector/server.py +42 -1
- package/deploy/detector/trillboards_detector/tasks.py +12 -1
- package/deploy/windows/ci/test-main-stream-gpu-decode.ps1 +119 -0
- package/dist/CloudSensingManager.d.ts +5 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +5 -2
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/ConnectorStatusClient.d.ts +18 -1
- package/dist/ConnectorStatusClient.d.ts.map +1 -1
- package/dist/ConnectorStatusClient.js +6 -0
- package/dist/ConnectorStatusClient.js.map +1 -1
- package/dist/DetectorFields.d.ts +77 -1
- package/dist/DetectorFields.d.ts.map +1 -1
- package/dist/DetectorFields.js +34 -4
- package/dist/DetectorFields.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/FaceIdentity.d.ts +122 -0
- package/dist/FaceIdentity.d.ts.map +1 -0
- package/dist/FaceIdentity.js +256 -0
- package/dist/FaceIdentity.js.map +1 -0
- package/dist/FaceSightingUploader.d.ts +95 -0
- package/dist/FaceSightingUploader.d.ts.map +1 -0
- package/dist/FaceSightingUploader.js +214 -0
- package/dist/FaceSightingUploader.js.map +1 -0
- package/dist/FaceWatchLane.d.ts +297 -0
- package/dist/FaceWatchLane.d.ts.map +1 -0
- package/dist/FaceWatchLane.js +660 -0
- package/dist/FaceWatchLane.js.map +1 -0
- package/dist/FaceWatchPolicy.d.ts +154 -0
- package/dist/FaceWatchPolicy.d.ts.map +1 -0
- package/dist/FaceWatchPolicy.js +101 -0
- package/dist/FaceWatchPolicy.js.map +1 -0
- package/dist/LocalDetectorClient.d.ts +11 -0
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +44 -0
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/MainStreamRing.d.ts +63 -1
- package/dist/MainStreamRing.d.ts.map +1 -1
- package/dist/MainStreamRing.js +101 -7
- package/dist/MainStreamRing.js.map +1 -1
- package/dist/NativeReadiness.d.ts.map +1 -1
- package/dist/NativeReadiness.js +7 -3
- package/dist/NativeReadiness.js.map +1 -1
- package/dist/ReviewPriority.d.ts +9 -0
- package/dist/ReviewPriority.d.ts.map +1 -1
- package/dist/ReviewPriority.js +6 -1
- package/dist/ReviewPriority.js.map +1 -1
- package/dist/ScopedVlmReview.d.ts +11 -2
- package/dist/ScopedVlmReview.d.ts.map +1 -1
- package/dist/ScopedVlmReview.js +43 -16
- package/dist/ScopedVlmReview.js.map +1 -1
- package/dist/SensingProfileClient.d.ts +14 -0
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +9 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/UniformSceneStore.d.ts +6 -0
- package/dist/UniformSceneStore.d.ts.map +1 -1
- package/dist/UniformSceneStore.js +22 -1
- package/dist/UniformSceneStore.js.map +1 -1
- package/dist/VideoSegmentUploader.d.ts +11 -0
- package/dist/VideoSegmentUploader.d.ts.map +1 -1
- package/dist/VideoSegmentUploader.js +28 -2
- package/dist/VideoSegmentUploader.js.map +1 -1
- package/dist/VlmCandidateBuffer.d.ts +2 -0
- package/dist/VlmCandidateBuffer.d.ts.map +1 -1
- package/dist/VlmCandidateBuffer.js.map +1 -1
- package/dist/WatchlistClient.d.ts +55 -0
- package/dist/WatchlistClient.d.ts.map +1 -0
- package/dist/WatchlistClient.js +161 -0
- package/dist/WatchlistClient.js.map +1 -0
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +3 -0
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3 -0
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +16 -0
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +30 -6
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/hostCpu.d.ts +56 -0
- package/dist/hostCpu.d.ts.map +1 -0
- package/dist/hostCpu.js +113 -0
- package/dist/hostCpu.js.map +1 -0
- package/dist/localSensing.d.ts +17 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +52 -2
- package/dist/localSensing.js.map +1 -1
- package/package.json +2 -2
- package/retained/RetainedCaptureClient.cjs +2 -0
- package/retained/WebsiteAdmissionCaptureClient.cjs +3 -1
- package/retained/__tests__/website-admission-realtime.test.cjs +27 -2
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""The `/v1/faces` contract: the two pinned artifacts, the request and the answer.
|
|
2
|
+
|
|
3
|
+
STDLIB ONLY, like `server.py` and `registry.py`: the wire refusals and the check
|
|
4
|
+
of every answer before it leaves run on a runner with no numpy, no OpenCV and no
|
|
5
|
+
onnxruntime. The arithmetic lives in `face_identity.py`, which only the loader
|
|
6
|
+
imports.
|
|
7
|
+
|
|
8
|
+
The consumer is `LocalDetectorClient.faces` (edge-sdk), which re-checks the same
|
|
9
|
+
things on its side; this module is the service's half of that bargain — a
|
|
10
|
+
malformed answer becomes a 500 here, never a plausible 200.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import base64
|
|
16
|
+
import binascii
|
|
17
|
+
import hashlib
|
|
18
|
+
import math
|
|
19
|
+
import re
|
|
20
|
+
|
|
21
|
+
FACE_DETECT_TASK = "face_detection_landmarks5"
|
|
22
|
+
FACE_EMBED_TASK = "face_identity_embedding"
|
|
23
|
+
|
|
24
|
+
FACE_DETECTOR_MODEL_ID = "yunet_2023mar"
|
|
25
|
+
#: opencv_zoo `face_detection_yunet_2023mar.onnx` (MIT; sha256 8f2383e4...2fa4) with its input
|
|
26
|
+
#: height/width and its outputs' anchor counts declared symbolic, nodes and weights untouched.
|
|
27
|
+
FACE_DETECTOR_SHA256 = "2a62f235515c3a3f80c8c96ee75f5146d3518d5a63d2b3c77d9d1ae50e632f20"
|
|
28
|
+
FACE_DETECT_PREPROCESSING = "yunet_bgr_raw_long1920_pad32_floor0.5_nms0.3_top5000_v1"
|
|
29
|
+
|
|
30
|
+
FACE_EMBEDDER_MODEL_ID = "edgeface_xs_gamma_06"
|
|
31
|
+
#: EdgeFace-XS-gamma-0.6 exported to ONNX from the upstream checkpoint; cosine 1.000000000 against
|
|
32
|
+
#: the tablet's bundled TFLite (trillboard-ctv agent-core assets, sha256 7f912022...a254).
|
|
33
|
+
FACE_EMBEDDER_SHA256 = "1028899e16ca6c756dd1b333f0f2c136c45a049fc2ef098a2b9645e128d38f63"
|
|
34
|
+
FACE_EMBED_PREPROCESSING = "arcface5_umeyama_112_bilinear_rgb_unit127.5_l2_v1"
|
|
35
|
+
|
|
36
|
+
EMBEDDING_DIM = 512
|
|
37
|
+
MAX_FACES = 16
|
|
38
|
+
GATES = ("passed", "score", "eye_distance", "yaw", "sharpness")
|
|
39
|
+
REQUEST_KEYS = frozenset({"image_b64", "detector", "embedder", "minScore", "minEyePx", "maxAbsYaw",
|
|
40
|
+
"minSharpness", "maxFaces"})
|
|
41
|
+
_SHA = re.compile(r"[0-9a-f]{64}\Z")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _number(value):
|
|
45
|
+
return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _pin(value, model_id, label):
|
|
49
|
+
if (not isinstance(value, dict) or set(value) != {"modelId", "sha256"}
|
|
50
|
+
or not isinstance(value["modelId"], str) or not isinstance(value["sha256"], str)
|
|
51
|
+
or not _SHA.fullmatch(value["sha256"])):
|
|
52
|
+
raise ValueError(f"faces {label} must name one modelId and a lowercase sha256")
|
|
53
|
+
if value["modelId"] != model_id:
|
|
54
|
+
raise ValueError(f"faces {label} must be {model_id}")
|
|
55
|
+
return {"modelId": value["modelId"], "sha256": value["sha256"]}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def validate_faces_request(body):
|
|
59
|
+
"""Body -> (jpeg bytes, detector pin, embedder pin, bars). ValueError on anything else.
|
|
60
|
+
|
|
61
|
+
A pin naming the right model with another sha256 is NOT refused here: whether
|
|
62
|
+
those bytes are loaded is the registry's question, and it answers 409.
|
|
63
|
+
"""
|
|
64
|
+
if not isinstance(body, dict) or set(body) != REQUEST_KEYS:
|
|
65
|
+
raise ValueError("faces request needs exactly " + ", ".join(sorted(REQUEST_KEYS)))
|
|
66
|
+
image = body["image_b64"]
|
|
67
|
+
if not isinstance(image, str) or not image:
|
|
68
|
+
raise ValueError("`image_b64` must be a non-empty base64 string")
|
|
69
|
+
try:
|
|
70
|
+
jpeg = base64.b64decode(image, validate=True)
|
|
71
|
+
except (binascii.Error, ValueError):
|
|
72
|
+
raise ValueError("`image_b64` is not valid base64") from None
|
|
73
|
+
if not jpeg:
|
|
74
|
+
raise ValueError("`image_b64` decoded to zero bytes")
|
|
75
|
+
detector = _pin(body["detector"], FACE_DETECTOR_MODEL_ID, "detector")
|
|
76
|
+
embedder = _pin(body["embedder"], FACE_EMBEDDER_MODEL_ID, "embedder")
|
|
77
|
+
bars = {key: body[key] for key in ("minScore", "minEyePx", "maxAbsYaw", "minSharpness", "maxFaces")}
|
|
78
|
+
if not _number(bars["minScore"]) or not 0 < bars["minScore"] <= 1:
|
|
79
|
+
raise ValueError("minScore must be in (0, 1]")
|
|
80
|
+
if not _number(bars["minEyePx"]) or not 1 <= bars["minEyePx"] <= 512:
|
|
81
|
+
raise ValueError("minEyePx must be in [1, 512]")
|
|
82
|
+
if not _number(bars["maxAbsYaw"]) or not 0 < bars["maxAbsYaw"] <= 2:
|
|
83
|
+
raise ValueError("maxAbsYaw must be in (0, 2]")
|
|
84
|
+
if not _number(bars["minSharpness"]) or not 0 <= bars["minSharpness"] <= 100000:
|
|
85
|
+
raise ValueError("minSharpness must be in [0, 100000]")
|
|
86
|
+
if type(bars["maxFaces"]) is not int or not 1 <= bars["maxFaces"] <= MAX_FACES:
|
|
87
|
+
raise ValueError("maxFaces must be a whole number 1..%d" % MAX_FACES)
|
|
88
|
+
bars = {key: float(value) if key != "maxFaces" else value for key, value in bars.items()}
|
|
89
|
+
return jpeg, detector, embedder, bars
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def check_faces_response(result, jpeg, detector, embedder, bars):
|
|
93
|
+
"""Raise ValueError unless `result` is exactly the answer to this request.
|
|
94
|
+
|
|
95
|
+
Every face: a box inside the frame with extent, a score in [0,1], five finite
|
|
96
|
+
landmarks, finite measures, a quality in [0,1], a known gate, and a vector
|
|
97
|
+
(512 finite values of unit length) exactly when the gate passed. The faces
|
|
98
|
+
are at most maxFaces, best score first.
|
|
99
|
+
"""
|
|
100
|
+
if not isinstance(result, dict) or set(result) != {"version", "imageSha256", "frameSize", "detector",
|
|
101
|
+
"embedder", "faces", "inferenceMs"}:
|
|
102
|
+
raise ValueError("faces answer has unknown or missing keys")
|
|
103
|
+
if result["version"] != 1 or result["imageSha256"] != hashlib.sha256(jpeg).hexdigest():
|
|
104
|
+
raise ValueError("faces answer is not about this image")
|
|
105
|
+
size = result["frameSize"]
|
|
106
|
+
if (not isinstance(size, dict) or set(size) != {"width", "height"}
|
|
107
|
+
or type(size["width"]) is not int or type(size["height"]) is not int
|
|
108
|
+
or not 2 <= size["width"] <= 8192 or not 2 <= size["height"] <= 8192):
|
|
109
|
+
raise ValueError("faces answer has no original frame size")
|
|
110
|
+
width, height = size["width"], size["height"]
|
|
111
|
+
for label, pin, preprocessing in (("detector", detector, FACE_DETECT_PREPROCESSING),
|
|
112
|
+
("embedder", embedder, FACE_EMBED_PREPROCESSING)):
|
|
113
|
+
if result[label] != {**pin, "preprocessing": preprocessing}:
|
|
114
|
+
raise ValueError(f"faces answer's {label} is not the selected artifact")
|
|
115
|
+
timing = result["inferenceMs"]
|
|
116
|
+
if (not isinstance(timing, dict) or set(timing) != {"detect", "embed"}
|
|
117
|
+
or any(not _number(timing[key]) or timing[key] < 0 for key in timing)):
|
|
118
|
+
raise ValueError("faces answer has no timing")
|
|
119
|
+
faces = result["faces"]
|
|
120
|
+
if not isinstance(faces, list) or len(faces) > bars["maxFaces"]:
|
|
121
|
+
raise ValueError("faces answer has more faces than were asked for")
|
|
122
|
+
previous = None
|
|
123
|
+
for index, face in enumerate(faces):
|
|
124
|
+
if not isinstance(face, dict) or set(face) != {"box", "score", "landmarks", "eyeDistancePx", "yaw",
|
|
125
|
+
"sharpness", "quality", "gate", "embedding"}:
|
|
126
|
+
raise ValueError(f"face {index} has unknown or missing keys")
|
|
127
|
+
box = face["box"]
|
|
128
|
+
if (not isinstance(box, dict) or set(box) != {"x", "y", "width", "height"}
|
|
129
|
+
or any(not _number(box[key]) for key in box)
|
|
130
|
+
or box["width"] <= 0 or box["height"] <= 0 or box["x"] < 0 or box["y"] < 0
|
|
131
|
+
or box["x"] + box["width"] > width + 0.01 or box["y"] + box["height"] > height + 0.01):
|
|
132
|
+
raise ValueError(f"face {index} box is outside the original frame")
|
|
133
|
+
if not _number(face["score"]) or not 0 <= face["score"] <= 1:
|
|
134
|
+
raise ValueError(f"face {index} score is outside [0, 1]")
|
|
135
|
+
if previous is not None and face["score"] > previous:
|
|
136
|
+
raise ValueError("faces are not best score first")
|
|
137
|
+
previous = face["score"]
|
|
138
|
+
points = face["landmarks"]
|
|
139
|
+
if (not isinstance(points, list) or len(points) != 5
|
|
140
|
+
or any(not isinstance(p, list) or len(p) != 2 or not all(_number(v) for v in p) for p in points)
|
|
141
|
+
or points[0][0] > points[1][0] or points[3][0] > points[4][0]):
|
|
142
|
+
raise ValueError(f"face {index} landmarks are not five points in template order")
|
|
143
|
+
if (not _number(face["eyeDistancePx"]) or face["eyeDistancePx"] < 0 or not _number(face["yaw"])
|
|
144
|
+
or not _number(face["sharpness"]) or face["sharpness"] < 0):
|
|
145
|
+
raise ValueError(f"face {index} measures are not finite")
|
|
146
|
+
if not _number(face["quality"]) or not 0 <= face["quality"] <= 1:
|
|
147
|
+
raise ValueError(f"face {index} quality is outside [0, 1]")
|
|
148
|
+
if face["gate"] not in GATES:
|
|
149
|
+
raise ValueError(f"face {index} gate is unknown")
|
|
150
|
+
expected = ("score" if face["score"] < bars["minScore"]
|
|
151
|
+
else "eye_distance" if face["eyeDistancePx"] < bars["minEyePx"]
|
|
152
|
+
else "yaw" if abs(face["yaw"]) > bars["maxAbsYaw"]
|
|
153
|
+
else "sharpness" if face["sharpness"] < bars["minSharpness"] else "passed")
|
|
154
|
+
if face["gate"] != expected:
|
|
155
|
+
raise ValueError(f"face {index} gate does not follow from its measures")
|
|
156
|
+
vector = face["embedding"]
|
|
157
|
+
if face["gate"] != "passed":
|
|
158
|
+
if vector is not None:
|
|
159
|
+
raise ValueError(f"face {index} has a vector it did not earn")
|
|
160
|
+
continue
|
|
161
|
+
if (not isinstance(vector, list) or len(vector) != EMBEDDING_DIM
|
|
162
|
+
or not all(_number(v) for v in vector)):
|
|
163
|
+
raise ValueError(f"face {index} vector is not {EMBEDDING_DIM} finite values")
|
|
164
|
+
norm = math.sqrt(math.fsum(v * v for v in vector))
|
|
165
|
+
if not 0.999 <= norm <= 1.001:
|
|
166
|
+
raise ValueError(f"face {index} vector is not unit length")
|
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
"""Faces at the door: find each face, align it, and measure who it is.
|
|
2
|
+
|
|
3
|
+
THREE STEPS, TWO PINNED ARTIFACTS, ONE ANSWER PER FACE
|
|
4
|
+
------------------------------------------------------
|
|
5
|
+
1. YuNet 2023mar (opencv_zoo, MIT) finds each face and its five landmarks on the
|
|
6
|
+
ORIGINAL face-zone frame the connector cut from the main stream, decoded as
|
|
7
|
+
OpenCV's `FaceDetectorYN` decodes it. T1 face research (Texas camera 1): it finds
|
|
8
|
+
56 of the 58 faces SCRFD-10G finds on a man walking in.
|
|
9
|
+
2. A least-squares 5-point SIMILARITY warp (Umeyama 1991; the closed form
|
|
10
|
+
`FaceEmbeddingProcessor.computeSimilarityTransform` uses on the tablet) puts the
|
|
11
|
+
landmarks on the canonical ArcFace 112x112 template. NEVER EMBED WITHOUT
|
|
12
|
+
LANDMARKS: on the tablet a box crop separated same from different people at
|
|
13
|
+
ROC-AUC 0.66 and the aligned chip at 1.00 (FaceEmbeddingProcessor.kt).
|
|
14
|
+
3. EdgeFace-XS-gamma-0.6 (otroshi/edgeface) as ONNX, the twin of the tablet's
|
|
15
|
+
bundled TFLite: exported from the upstream checkpoint (MD5 ebf343ca...d703, the
|
|
16
|
+
Idiap-published checksum) and measured against the APK's
|
|
17
|
+
`edgeface_xs_gamma_06.tflite` at cosine 1.000000000 (max |diff| 8.1e-08) on
|
|
18
|
+
random inputs and on real aligned chips. So a store's vector and a tablet's are
|
|
19
|
+
the same measurement, and one gallery serves both.
|
|
20
|
+
|
|
21
|
+
Each face is measured before it is embedded: the distance between the eyes in
|
|
22
|
+
ORIGINAL pixels (T1 research: 94.7% of true matches at 12 px, 45.7% at 6 px, at a
|
|
23
|
+
false-match rate of 1e-5), a yaw proxy and the sharpness of the aligned chip. A
|
|
24
|
+
face that fails a bar comes back with its box and landmarks (the connector's
|
|
25
|
+
tracker still follows it) and no vector, and says which bar it failed.
|
|
26
|
+
|
|
27
|
+
Stdlib + numpy + OpenCV + onnxruntime, loaded by the loader only: `server.py` and
|
|
28
|
+
`registry.py` never import this module.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import hashlib
|
|
34
|
+
import io
|
|
35
|
+
import math
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
import threading
|
|
38
|
+
import time
|
|
39
|
+
|
|
40
|
+
import cv2
|
|
41
|
+
import numpy as np
|
|
42
|
+
|
|
43
|
+
from .errors import ModelContractError
|
|
44
|
+
from .face_contract import (EMBEDDING_DIM, FACE_DETECT_PREPROCESSING as DETECT_PREPROCESSING,
|
|
45
|
+
FACE_DETECTOR_MODEL_ID as DETECTOR_MODEL_ID, FACE_DETECTOR_SHA256 as DETECTOR_SHA256,
|
|
46
|
+
FACE_EMBED_PREPROCESSING as EMBED_PREPROCESSING,
|
|
47
|
+
FACE_EMBEDDER_MODEL_ID as EMBEDDER_MODEL_ID, FACE_EMBEDDER_SHA256 as EMBEDDER_SHA256)
|
|
48
|
+
|
|
49
|
+
# OpenCV otherwise creates a worker per host core, outside the model budget.
|
|
50
|
+
cv2.setNumThreads(1)
|
|
51
|
+
|
|
52
|
+
#: insightface `arcface_src`, default_square=True: image-left eye, image-right eye,
|
|
53
|
+
#: nose tip, image-left mouth corner, image-right mouth corner. Bit-identical to
|
|
54
|
+
#: `FaceEmbeddingProcessor.ARCFACE_REFERENCE_112`.
|
|
55
|
+
ARCFACE_REFERENCE_112 = np.array(
|
|
56
|
+
[[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]],
|
|
57
|
+
dtype=np.float64,
|
|
58
|
+
)
|
|
59
|
+
CHIP_SIZE = 112
|
|
60
|
+
|
|
61
|
+
#: YuNet runs on the original pixels up to this long side; a larger frame is
|
|
62
|
+
#: area-resized for detection only, and the landmarks are mapped back so the
|
|
63
|
+
#: chip is still cut from the original pixels.
|
|
64
|
+
MAX_DETECT_LONG_SIDE = 1920
|
|
65
|
+
#: Faces down to this score are reported (gate "score" below the request's
|
|
66
|
+
#: minScore), so the connector's tracker keeps a face across a weak frame.
|
|
67
|
+
DETECT_SCORE_FLOOR = 0.5
|
|
68
|
+
NMS_THRESHOLD = 0.3
|
|
69
|
+
TOP_K = 5000
|
|
70
|
+
MAX_JPEG_BYTES = 16 * 1024**2
|
|
71
|
+
MAX_PIXELS = 16_777_216
|
|
72
|
+
|
|
73
|
+
#: The eye distance at which a face's size stops lowering its quality: past it
|
|
74
|
+
#: the T1 LFW curve is flat (TAR@1e-5 97.7% at 16 px, 98.3% at >= 20 px).
|
|
75
|
+
EYE_PX_FULL = 24.0
|
|
76
|
+
#: The yaw proxy at which quality reaches zero. yaw = (nose - eye midpoint) along
|
|
77
|
+
#: the eye line / eye distance, about 0.35 * tan(yaw angle): 0.35 is ~45 degrees.
|
|
78
|
+
YAW_ZERO = 0.5
|
|
79
|
+
#: Sharpness = variance of the Laplacian of the aligned 112x112 grey chip. A chip
|
|
80
|
+
#: is upsampled from the original face, so sharpness falls with the eye distance
|
|
81
|
+
#: as much as with blur. MEASURED (2026-11-16, measure_cam1.py, 400 LFW faces
|
|
82
|
+
#: aligned at fixed eye distances): sharp faces at 12 px p1 17.4 / p5 22.2 /
|
|
83
|
+
#: p50 37.4; at 16 px p1 24.9 / p50 56.1; at 20 px p1 30.7 / p50 72.3. The same
|
|
84
|
+
#: 20 px faces smeared along 15 chip px: p50 16.1, 9 px: p50 22.6. Texas camera 1's
|
|
85
|
+
#: SUBSTREAM faces (8 px between the eyes, the eye bar refuses them first): p50 11.9.
|
|
86
|
+
#: The floor rejects only clearly smeared chips (it keeps 99% of sharp 12 px faces);
|
|
87
|
+
#: quality stops rising at the typical sharp 16-20 px face.
|
|
88
|
+
SHARPNESS_REF = 60.0
|
|
89
|
+
MIN_SHARPNESS = 15.0
|
|
90
|
+
|
|
91
|
+
#: The twelve heads, in OpenCV's order: (cls, obj, bbox, kps) x strides (8, 16, 32).
|
|
92
|
+
YUNET_STRIDES = (8, 16, 32)
|
|
93
|
+
YUNET_OUTPUTS = tuple(f"{head}_{stride}" for head in ("cls", "obj", "bbox", "kps") for stride in YUNET_STRIDES)
|
|
94
|
+
#: OpenCV pads the input up to a multiple of this, with zeros on the right and bottom.
|
|
95
|
+
YUNET_DIVISOR = 32
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def decode_yunet(outputs, pad_w, pad_h, score_threshold, nms_threshold=NMS_THRESHOLD, top_k=TOP_K):
|
|
99
|
+
"""OpenCV `FaceDetectorYNImpl::postProcess` (2023mar heads), then `dnn::NMSBoxes`.
|
|
100
|
+
|
|
101
|
+
Rows of (x, y, w, h, 10 landmark coordinates, score) in padded-input pixels, in
|
|
102
|
+
the order NMS keeps them (score descending). Landmarks are YuNet's: right eye,
|
|
103
|
+
left eye (the subject's), nose tip, right and left mouth corners.
|
|
104
|
+
"""
|
|
105
|
+
candidates = []
|
|
106
|
+
for stride in YUNET_STRIDES:
|
|
107
|
+
cols, rows = pad_w // stride, pad_h // stride
|
|
108
|
+
count = cols * rows
|
|
109
|
+
cls = np.asarray(outputs[f"cls_{stride}"], np.float32).reshape(-1)
|
|
110
|
+
obj = np.asarray(outputs[f"obj_{stride}"], np.float32).reshape(-1)
|
|
111
|
+
bbox = np.asarray(outputs[f"bbox_{stride}"], np.float32).reshape(-1, 4)
|
|
112
|
+
kps = np.asarray(outputs[f"kps_{stride}"], np.float32).reshape(-1, 10)
|
|
113
|
+
if cls.shape[0] != count or obj.shape[0] != count or bbox.shape[0] != count or kps.shape[0] != count:
|
|
114
|
+
raise ModelContractError("Face detector heads do not match the padded input")
|
|
115
|
+
score = np.sqrt(np.clip(cls, 0.0, 1.0) * np.clip(obj, 0.0, 1.0))
|
|
116
|
+
keep = np.nonzero(score >= score_threshold)[0]
|
|
117
|
+
if keep.size == 0:
|
|
118
|
+
continue
|
|
119
|
+
c = (keep % cols).astype(np.float32)
|
|
120
|
+
r = (keep // cols).astype(np.float32)
|
|
121
|
+
cx = (c + bbox[keep, 0]) * stride
|
|
122
|
+
cy = (r + bbox[keep, 1]) * stride
|
|
123
|
+
w = np.exp(bbox[keep, 2]) * stride
|
|
124
|
+
h = np.exp(bbox[keep, 3]) * stride
|
|
125
|
+
landmarks = np.empty((keep.size, 10), np.float32)
|
|
126
|
+
landmarks[:, 0::2] = (kps[keep, 0::2] + c[:, None]) * stride
|
|
127
|
+
landmarks[:, 1::2] = (kps[keep, 1::2] + r[:, None]) * stride
|
|
128
|
+
block = np.column_stack([cx - w / 2.0, cy - h / 2.0, w, h, landmarks, score[keep]])
|
|
129
|
+
candidates.append(block)
|
|
130
|
+
if not candidates:
|
|
131
|
+
return np.zeros((0, 15), np.float64)
|
|
132
|
+
faces = np.vstack(candidates).astype(np.float64)
|
|
133
|
+
if not np.isfinite(faces).all():
|
|
134
|
+
raise ModelContractError("Native face detector output is invalid")
|
|
135
|
+
if len(faces) == 1:
|
|
136
|
+
return faces
|
|
137
|
+
return faces[nms_boxes(faces[:, 0:4], faces[:, 14], score_threshold, nms_threshold, top_k)]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def nms_boxes(boxes, scores, score_threshold, nms_threshold, top_k):
|
|
141
|
+
"""`cv::dnn::NMSBoxes` on Rect2i(int(x), int(y), int(w), int(h)), eta 1: kept indices, best first."""
|
|
142
|
+
rects = np.trunc(np.asarray(boxes, np.float64)).astype(np.int64)
|
|
143
|
+
# GetMaxScoreIndex keeps a score strictly above the threshold, then the top_k.
|
|
144
|
+
order = [i for i in np.argsort(-np.asarray(scores), kind="stable") if scores[i] > score_threshold]
|
|
145
|
+
if top_k > 0:
|
|
146
|
+
order = order[:top_k]
|
|
147
|
+
kept = []
|
|
148
|
+
for i in order:
|
|
149
|
+
x, y, w, h = rects[i]
|
|
150
|
+
area = max(0, w) * max(0, h)
|
|
151
|
+
overlapping = False
|
|
152
|
+
for j in kept:
|
|
153
|
+
x2, y2, w2, h2 = rects[j]
|
|
154
|
+
iw = min(x + w, x2 + w2) - max(x, x2)
|
|
155
|
+
ih = min(y + h, y2 + h2) - max(y, y2)
|
|
156
|
+
inter = max(0, iw) * max(0, ih)
|
|
157
|
+
union = area + max(0, w2) * max(0, h2) - inter
|
|
158
|
+
if union > 0 and inter / union > nms_threshold:
|
|
159
|
+
overlapping = True
|
|
160
|
+
break
|
|
161
|
+
if not overlapping:
|
|
162
|
+
kept.append(i)
|
|
163
|
+
return np.asarray(kept, dtype=np.int64)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def umeyama(src, dst):
|
|
167
|
+
"""Least-squares similarity (scale, rotation, translation; no reflection) taking src onto dst.
|
|
168
|
+
|
|
169
|
+
The Umeyama closed form `skimage.SimilarityTransform` and the tablet's
|
|
170
|
+
`computeSimilarityTransform` both use. Returns the 2x3 affine, or None when
|
|
171
|
+
the source points are degenerate (coincident: no stable transform).
|
|
172
|
+
"""
|
|
173
|
+
src = np.asarray(src, dtype=np.float64)
|
|
174
|
+
dst = np.asarray(dst, dtype=np.float64)
|
|
175
|
+
if src.shape != dst.shape or src.ndim != 2 or src.shape[1] != 2 or src.shape[0] < 3:
|
|
176
|
+
return None
|
|
177
|
+
if not (np.isfinite(src).all() and np.isfinite(dst).all()):
|
|
178
|
+
return None
|
|
179
|
+
mu_s, mu_d = src.mean(0), dst.mean(0)
|
|
180
|
+
sc, dc = src - mu_s, dst - mu_d
|
|
181
|
+
var_s = float((sc ** 2).sum() / len(src))
|
|
182
|
+
if var_s < 1e-12:
|
|
183
|
+
return None
|
|
184
|
+
cov = dc.T @ sc / len(src)
|
|
185
|
+
u, s, vt = np.linalg.svd(cov)
|
|
186
|
+
d = np.ones(2)
|
|
187
|
+
if np.linalg.det(u) * np.linalg.det(vt) < 0:
|
|
188
|
+
d[-1] = -1.0
|
|
189
|
+
rotation = u @ np.diag(d) @ vt
|
|
190
|
+
scale = float((s * d).sum() / var_s)
|
|
191
|
+
translation = mu_d - scale * rotation @ mu_s
|
|
192
|
+
matrix = np.hstack([scale * rotation, translation[:, None]])
|
|
193
|
+
return matrix if np.isfinite(matrix).all() else None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def arcface_order(landmarks):
|
|
197
|
+
"""The five points in template order, by IMAGE position (handedness-proof).
|
|
198
|
+
|
|
199
|
+
Template point 0 is the image-LEFT eye and point 3 the image-left mouth corner,
|
|
200
|
+
whatever a detector calls them. Feeding the fit a mirrored correspondence
|
|
201
|
+
collapses the vector (0.53 AUC on the tablet), so the eyes and the mouth
|
|
202
|
+
corners are each ordered by x, exactly as `toArcFaceSourcePoints` does.
|
|
203
|
+
"""
|
|
204
|
+
points = np.asarray(landmarks, dtype=np.float64).reshape(5, 2)
|
|
205
|
+
eyes = points[[0, 1]] if points[0, 0] <= points[1, 0] else points[[1, 0]]
|
|
206
|
+
mouth = points[[3, 4]] if points[3, 0] <= points[4, 0] else points[[4, 3]]
|
|
207
|
+
return np.vstack([eyes, points[2:3], mouth])
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def align_chip(bgr, landmarks):
|
|
211
|
+
"""The 112x112 BGR chip of one face, warped (bilinear) from the ORIGINAL pixels."""
|
|
212
|
+
matrix = umeyama(arcface_order(landmarks), ARCFACE_REFERENCE_112)
|
|
213
|
+
if matrix is None:
|
|
214
|
+
return None
|
|
215
|
+
return cv2.warpAffine(bgr, matrix, (CHIP_SIZE, CHIP_SIZE), flags=cv2.INTER_LINEAR,
|
|
216
|
+
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def face_measures(landmarks, chip):
|
|
220
|
+
"""(eye distance in original px, signed yaw proxy, sharpness of the aligned chip)."""
|
|
221
|
+
points = arcface_order(landmarks)
|
|
222
|
+
left, right, nose = points[0], points[1], points[2]
|
|
223
|
+
axis = right - left
|
|
224
|
+
eye_distance = float(math.hypot(axis[0], axis[1]))
|
|
225
|
+
if eye_distance <= 0:
|
|
226
|
+
yaw = 0.0
|
|
227
|
+
else:
|
|
228
|
+
# Along the eye line, so a tilted (rolled) head does not read as turned.
|
|
229
|
+
yaw = float(np.dot(nose - (left + right) / 2.0, axis / eye_distance) / eye_distance)
|
|
230
|
+
grey = cv2.cvtColor(chip, cv2.COLOR_BGR2GRAY)
|
|
231
|
+
sharpness = float(cv2.Laplacian(grey, cv2.CV_64F).var())
|
|
232
|
+
return eye_distance, yaw, sharpness
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def face_quality(score, eye_distance, yaw, sharpness):
|
|
236
|
+
"""0..1: detector score x size x frontality x sharpness, each saturating at its reference."""
|
|
237
|
+
size = min(1.0, eye_distance / EYE_PX_FULL)
|
|
238
|
+
frontal = max(0.0, 1.0 - abs(yaw) / YAW_ZERO)
|
|
239
|
+
sharp = min(1.0, max(0.0, sharpness) / SHARPNESS_REF)
|
|
240
|
+
value = float(score) * size * frontal * sharp
|
|
241
|
+
return min(1.0, max(0.0, value))
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def face_gate(score, eye_distance, yaw, sharpness, request):
|
|
245
|
+
"""The first bar a face fails, in a fixed order, or "passed"."""
|
|
246
|
+
if score < request["minScore"]:
|
|
247
|
+
return "score"
|
|
248
|
+
if eye_distance < request["minEyePx"]:
|
|
249
|
+
return "eye_distance"
|
|
250
|
+
if abs(yaw) > request["maxAbsYaw"]:
|
|
251
|
+
return "yaw"
|
|
252
|
+
if sharpness < request["minSharpness"]:
|
|
253
|
+
return "sharpness"
|
|
254
|
+
return "passed"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def decode_original_jpeg(jpeg):
|
|
258
|
+
"""(BGR array, width, height) of a bounded original JPEG, or ValueError."""
|
|
259
|
+
from PIL import Image
|
|
260
|
+
|
|
261
|
+
if not isinstance(jpeg, bytes) or not 0 < len(jpeg) <= MAX_JPEG_BYTES:
|
|
262
|
+
raise ValueError("Faces need a bounded original JPEG")
|
|
263
|
+
try:
|
|
264
|
+
with Image.open(io.BytesIO(jpeg)) as header:
|
|
265
|
+
width, height = header.size
|
|
266
|
+
if (header.format != "JPEG" or not 2 <= width <= 8192 or not 2 <= height <= 8192
|
|
267
|
+
or width * height > MAX_PIXELS):
|
|
268
|
+
raise ValueError("Faces need a bounded original JPEG")
|
|
269
|
+
except ValueError:
|
|
270
|
+
raise
|
|
271
|
+
except Exception as error:
|
|
272
|
+
raise ValueError("Invalid original face JPEG") from error
|
|
273
|
+
bgr = cv2.imdecode(np.frombuffer(jpeg, np.uint8), cv2.IMREAD_COLOR)
|
|
274
|
+
if bgr is None or bgr.shape != (height, width, 3):
|
|
275
|
+
raise ValueError("Original face JPEG could not be decoded")
|
|
276
|
+
return bgr, width, height
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class EdgeFaceEmbedder:
|
|
280
|
+
"""EdgeFace-XS: one immutable CPU artifact, the tablet model's ONNX twin."""
|
|
281
|
+
|
|
282
|
+
model_name = EMBEDDER_MODEL_ID
|
|
283
|
+
model_sha256 = EMBEDDER_SHA256
|
|
284
|
+
|
|
285
|
+
def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
|
|
286
|
+
if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
|
|
287
|
+
raise ValueError("Invalid bounded face embedder thread count")
|
|
288
|
+
if accelerator != "cpu":
|
|
289
|
+
raise ModelContractError("This face embedder is qualified for native CPU execution")
|
|
290
|
+
raw = Path(model_path).read_bytes()
|
|
291
|
+
if hashlib.sha256(raw).hexdigest() != EMBEDDER_SHA256:
|
|
292
|
+
raise ModelContractError("Face embedder artifact differs from the pinned export")
|
|
293
|
+
import onnxruntime as ort
|
|
294
|
+
|
|
295
|
+
from .execution import LocalOrtSession
|
|
296
|
+
|
|
297
|
+
options = ort.SessionOptions()
|
|
298
|
+
options.intra_op_num_threads = intra_op_threads
|
|
299
|
+
options.inter_op_num_threads = 1
|
|
300
|
+
options.log_severity_level = 3
|
|
301
|
+
self._session = LocalOrtSession(ort, raw, options, "cpu")
|
|
302
|
+
inputs, outputs = self._session.get_inputs(), self._session.get_outputs()
|
|
303
|
+
if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].shape != [1, 3, CHIP_SIZE, CHIP_SIZE]
|
|
304
|
+
or inputs[0].type != "tensor(float)" or len(outputs) != 1 or outputs[0].name != "embedding"
|
|
305
|
+
or outputs[0].shape != [1, EMBEDDING_DIM] or outputs[0].type != "tensor(float)"):
|
|
306
|
+
raise ModelContractError("Face embedder graph differs from its 112x112 RGB -> 512 contract")
|
|
307
|
+
self._lock = threading.Lock()
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def acceleration(self):
|
|
311
|
+
return self._session.acceleration
|
|
312
|
+
|
|
313
|
+
def embed(self, chip_bgr):
|
|
314
|
+
"""One aligned 112x112 BGR chip -> its L2-normalised 512-d vector (float64)."""
|
|
315
|
+
if (not isinstance(chip_bgr, np.ndarray) or chip_bgr.dtype != np.uint8
|
|
316
|
+
or chip_bgr.shape != (CHIP_SIZE, CHIP_SIZE, 3)):
|
|
317
|
+
raise ValueError("A face vector needs one aligned 112x112 chip")
|
|
318
|
+
rgb = chip_bgr[:, :, ::-1].astype(np.float32)
|
|
319
|
+
tensor = np.ascontiguousarray(((rgb / 127.5) - 1.0).transpose(2, 0, 1)[None], dtype=np.float32)
|
|
320
|
+
with self._lock:
|
|
321
|
+
value = self._session.run(["embedding"], {"input": tensor})[0]
|
|
322
|
+
if value.shape != (1, EMBEDDING_DIM) or not np.isfinite(value).all():
|
|
323
|
+
raise ModelContractError("Native face embedder output is invalid")
|
|
324
|
+
vector = value.reshape(EMBEDDING_DIM).astype(np.float64)
|
|
325
|
+
norm = float(np.linalg.norm(vector))
|
|
326
|
+
if not math.isfinite(norm) or norm <= 1e-12:
|
|
327
|
+
raise ModelContractError("Native face embedder output cannot identify a face")
|
|
328
|
+
return vector / norm
|
|
329
|
+
|
|
330
|
+
def probe(self):
|
|
331
|
+
"""A forward pass completes (the loader's admission check)."""
|
|
332
|
+
self.embed(np.full((CHIP_SIZE, CHIP_SIZE, 3), 128, np.uint8))
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class YunetFaceDetector:
|
|
336
|
+
"""YuNet 2023mar on onnxruntime, decoded exactly as OpenCV's FaceDetectorYN decodes it.
|
|
337
|
+
|
|
338
|
+
WHY NOT `cv2.FaceDetectorYN`, WHICH IS THE REFERENCE: on the pinned
|
|
339
|
+
opencv-python 5.0.0.93 its new DNN graph engine ran this graph 2.5x slower
|
|
340
|
+
than onnxruntime 1.29 on the same 800x448 Texas frame, one thread each,
|
|
341
|
+
interleaved (scratchpad/t1 yunet_ab.py). The decode below is OpenCV's
|
|
342
|
+
`FaceDetectorYNImpl::postProcess` line for line, and `test_face_identity.py`
|
|
343
|
+
holds the two to the same faces on real frames.
|
|
344
|
+
|
|
345
|
+
The artifact is the published file with only its DECLARED shape made symbolic
|
|
346
|
+
(`make_yunet_dynamic.py`): every node and weight is opencv_zoo's
|
|
347
|
+
8f2383e4...2fa4, whose 640x640 declaration would otherwise refuse any other
|
|
348
|
+
frame size in onnxruntime.
|
|
349
|
+
"""
|
|
350
|
+
|
|
351
|
+
model_name = DETECTOR_MODEL_ID
|
|
352
|
+
model_sha256 = DETECTOR_SHA256
|
|
353
|
+
|
|
354
|
+
def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
|
|
355
|
+
if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
|
|
356
|
+
raise ValueError("Invalid bounded face detector thread count")
|
|
357
|
+
if accelerator != "cpu":
|
|
358
|
+
raise ModelContractError("This face detector is qualified for native CPU execution")
|
|
359
|
+
raw = Path(model_path).read_bytes()
|
|
360
|
+
if hashlib.sha256(raw).hexdigest() != DETECTOR_SHA256:
|
|
361
|
+
raise ModelContractError("Face detector artifact differs from the pinned export")
|
|
362
|
+
import onnxruntime as ort
|
|
363
|
+
|
|
364
|
+
from .execution import LocalOrtSession
|
|
365
|
+
|
|
366
|
+
options = ort.SessionOptions()
|
|
367
|
+
options.intra_op_num_threads = intra_op_threads
|
|
368
|
+
options.inter_op_num_threads = 1
|
|
369
|
+
options.log_severity_level = 3
|
|
370
|
+
self._session = LocalOrtSession(ort, raw, options, "cpu")
|
|
371
|
+
inputs = self._session.get_inputs()
|
|
372
|
+
outputs = [o.name for o in self._session.get_outputs()]
|
|
373
|
+
if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].type != "tensor(float)"
|
|
374
|
+
or len(inputs[0].shape) != 4 or inputs[0].shape[:2] != [1, 3]
|
|
375
|
+
or not all(isinstance(v, str) for v in inputs[0].shape[2:])
|
|
376
|
+
or sorted(outputs) != sorted(YUNET_OUTPUTS)):
|
|
377
|
+
raise ModelContractError("Face detector graph differs from the YuNet 2023mar contract")
|
|
378
|
+
self._lock = threading.Lock()
|
|
379
|
+
|
|
380
|
+
@property
|
|
381
|
+
def acceleration(self):
|
|
382
|
+
return self._session.acceleration
|
|
383
|
+
|
|
384
|
+
def detect(self, bgr, min_score):
|
|
385
|
+
"""[(box [x, y, w, h], score, landmarks 5x2)] in the pixels of `bgr`, best score first."""
|
|
386
|
+
height, width = bgr.shape[:2]
|
|
387
|
+
scale = 1.0
|
|
388
|
+
image = bgr
|
|
389
|
+
if max(width, height) > MAX_DETECT_LONG_SIDE:
|
|
390
|
+
scale = MAX_DETECT_LONG_SIDE / float(max(width, height))
|
|
391
|
+
image = cv2.resize(bgr, (max(2, round(width * scale)), max(2, round(height * scale))),
|
|
392
|
+
interpolation=cv2.INTER_AREA)
|
|
393
|
+
pad_h = ((image.shape[0] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
|
|
394
|
+
pad_w = ((image.shape[1] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
|
|
395
|
+
padded = np.zeros((pad_h, pad_w, 3), np.uint8)
|
|
396
|
+
padded[:image.shape[0], :image.shape[1]] = image
|
|
397
|
+
blob = np.ascontiguousarray(padded.transpose(2, 0, 1)[None], dtype=np.float32)
|
|
398
|
+
with self._lock:
|
|
399
|
+
values = self._session.run(list(YUNET_OUTPUTS), {"input": blob})
|
|
400
|
+
rows = decode_yunet(dict(zip(YUNET_OUTPUTS, values)), pad_w, pad_h,
|
|
401
|
+
float(min(DETECT_SCORE_FLOOR, min_score)))
|
|
402
|
+
faces = []
|
|
403
|
+
for row in rows:
|
|
404
|
+
faces.append((row[0:4] / scale, float(row[14]), row[4:14].reshape(5, 2) / scale))
|
|
405
|
+
return faces
|
|
406
|
+
|
|
407
|
+
def estimate(self, jpeg, request, embedder):
|
|
408
|
+
"""The /v1/faces answer for one original JPEG (face_contract.check_faces_response holds its shape)."""
|
|
409
|
+
if not callable(getattr(embedder, "embed", None)):
|
|
410
|
+
raise ModelContractError("Faces need the pinned embedder session")
|
|
411
|
+
bgr, width, height = decode_original_jpeg(jpeg)
|
|
412
|
+
started = time.perf_counter()
|
|
413
|
+
detected = self.detect(bgr, request["minScore"])
|
|
414
|
+
detect_ms = (time.perf_counter() - started) * 1000.0
|
|
415
|
+
embed_ms = 0.0
|
|
416
|
+
faces = []
|
|
417
|
+
for box, score, landmarks in detected[:request["maxFaces"]]:
|
|
418
|
+
# Clipped to the frame, then rounded at the corners, so x + width never passes the edge.
|
|
419
|
+
# Positions keep 3 decimals and scores 6: the regression corpus holds this answer to the
|
|
420
|
+
# reference implementations at 3e-5 of the frame, which coarser rounding would exceed.
|
|
421
|
+
x0 = round(max(0.0, float(box[0])), 3)
|
|
422
|
+
y0 = round(max(0.0, float(box[1])), 3)
|
|
423
|
+
x1 = round(min(float(width), float(box[0] + box[2])), 3)
|
|
424
|
+
y1 = round(min(float(height), float(box[1] + box[3])), 3)
|
|
425
|
+
if x1 - x0 <= 0 or y1 - y0 <= 0:
|
|
426
|
+
continue
|
|
427
|
+
ordered = arcface_order(landmarks)
|
|
428
|
+
chip = align_chip(bgr, ordered)
|
|
429
|
+
if chip is None:
|
|
430
|
+
continue
|
|
431
|
+
# Rounded FIRST, and the gate and quality decided on the rounded numbers, so the
|
|
432
|
+
# answer is self-consistent to the digit (server.py re-derives every gate from them).
|
|
433
|
+
eye_distance, yaw, sharpness = (round(v, d) for v, d in zip(face_measures(ordered, chip), (3, 6, 3)))
|
|
434
|
+
score = round(min(1.0, max(0.0, score)), 6)
|
|
435
|
+
gate = face_gate(score, eye_distance, yaw, sharpness, request)
|
|
436
|
+
embedding = None
|
|
437
|
+
if gate == "passed":
|
|
438
|
+
began = time.perf_counter()
|
|
439
|
+
vector = embedder.embed(chip)
|
|
440
|
+
embed_ms += (time.perf_counter() - began) * 1000.0
|
|
441
|
+
embedding = [round(float(v), 6) for v in vector]
|
|
442
|
+
faces.append({
|
|
443
|
+
"box": {"x": x0, "y": y0, "width": round(x1 - x0, 3), "height": round(y1 - y0, 3)},
|
|
444
|
+
"score": score,
|
|
445
|
+
"landmarks": [[round(float(px), 3), round(float(py), 3)] for px, py in ordered],
|
|
446
|
+
"eyeDistancePx": eye_distance,
|
|
447
|
+
"yaw": yaw,
|
|
448
|
+
"sharpness": sharpness,
|
|
449
|
+
"quality": round(face_quality(score, eye_distance, yaw, sharpness), 6),
|
|
450
|
+
"gate": gate,
|
|
451
|
+
"embedding": embedding,
|
|
452
|
+
})
|
|
453
|
+
return {
|
|
454
|
+
"version": 1,
|
|
455
|
+
"imageSha256": hashlib.sha256(jpeg).hexdigest(),
|
|
456
|
+
"frameSize": {"width": width, "height": height},
|
|
457
|
+
"detector": {"modelId": DETECTOR_MODEL_ID, "sha256": DETECTOR_SHA256,
|
|
458
|
+
"preprocessing": DETECT_PREPROCESSING},
|
|
459
|
+
"embedder": {"modelId": EMBEDDER_MODEL_ID, "sha256": EMBEDDER_SHA256,
|
|
460
|
+
"preprocessing": EMBED_PREPROCESSING},
|
|
461
|
+
"faces": faces,
|
|
462
|
+
"inferenceMs": {"detect": round(detect_ms, 3), "embed": round(embed_ms, 3)},
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
def probe(self):
|
|
466
|
+
"""A forward pass completes on a grey frame (no face is expected or required)."""
|
|
467
|
+
self.detect(np.full((48, 64, 3), 128, np.uint8), DETECT_SCORE_FLOOR)
|
|
@@ -198,6 +198,15 @@ class DetectorLoader:
|
|
|
198
198
|
if entry.vocabulary.task == "vehicle_end_orientation":
|
|
199
199
|
from .vehicle_end import VehicleEndEstimator
|
|
200
200
|
return VehicleEndEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator="cpu")
|
|
201
|
+
# The door faces task is called by the connector's face lane at up to 4 frames a second,
|
|
202
|
+
# beside the look loop's object detector: ONE thread by default keeps it to one core and
|
|
203
|
+
# out of RF-DETR's pool (devbox A/B under load: 4 threads ran slower than 1).
|
|
204
|
+
if entry.vocabulary.task == "face_detection_landmarks5":
|
|
205
|
+
from .face_identity import YunetFaceDetector
|
|
206
|
+
return YunetFaceDetector(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
|
|
207
|
+
if entry.vocabulary.task == "face_identity_embedding":
|
|
208
|
+
from .face_identity import EdgeFaceEmbedder
|
|
209
|
+
return EdgeFaceEmbedder(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
|
|
201
210
|
from .pose import PoseEstimator
|
|
202
211
|
return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
|
|
203
212
|
from .session import OnnxDetector
|
|
@@ -366,6 +375,9 @@ class DetectorLoader:
|
|
|
366
375
|
def estimate_vehicle_end(self, jpeg: bytes, boxes: list, side: str, expected_model: dict) -> dict:
|
|
367
376
|
return self.registry.estimate_vehicle_end(jpeg, boxes, side, expected_model)
|
|
368
377
|
|
|
378
|
+
def estimate_faces(self, jpeg: bytes, request: dict, detector_pin: dict, embedder_pin: dict) -> dict:
|
|
379
|
+
return self.registry.estimate_faces(jpeg, request, detector_pin, embedder_pin)
|
|
380
|
+
|
|
369
381
|
def classify_actor_actions(self,batch):
|
|
370
382
|
return self.registry.classify_actor_actions(batch)
|
|
371
383
|
|