@trillboards/edge-sdk 0.2.253 → 0.2.255

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/deploy/detector/README.md +15 -0
  2. package/deploy/detector/pyproject.toml +1 -1
  3. package/deploy/detector/tests/test_face_identity.py +448 -0
  4. package/deploy/detector/tests/test_loader.py +3 -1
  5. package/deploy/detector/trillboards_detector/_version.py +2 -2
  6. package/deploy/detector/trillboards_detector/face_contract.py +166 -0
  7. package/deploy/detector/trillboards_detector/face_identity.py +467 -0
  8. package/deploy/detector/trillboards_detector/loader.py +12 -0
  9. package/deploy/detector/trillboards_detector/registry.py +45 -1
  10. package/deploy/detector/trillboards_detector/server.py +42 -1
  11. package/deploy/detector/trillboards_detector/tasks.py +12 -1
  12. package/deploy/windows/ci/test-main-stream-gpu-decode.ps1 +119 -0
  13. package/dist/CctvLiveViewCoordinator.d.ts +34 -2
  14. package/dist/CctvLiveViewCoordinator.d.ts.map +1 -1
  15. package/dist/CctvLiveViewCoordinator.js +106 -25
  16. package/dist/CctvLiveViewCoordinator.js.map +1 -1
  17. package/dist/CloudSensingManager.d.ts +17 -0
  18. package/dist/CloudSensingManager.d.ts.map +1 -1
  19. package/dist/CloudSensingManager.js +11 -2
  20. package/dist/CloudSensingManager.js.map +1 -1
  21. package/dist/ConnectorStatusClient.d.ts +18 -1
  22. package/dist/ConnectorStatusClient.d.ts.map +1 -1
  23. package/dist/ConnectorStatusClient.js +6 -0
  24. package/dist/ConnectorStatusClient.js.map +1 -1
  25. package/dist/DetectorFields.d.ts +77 -1
  26. package/dist/DetectorFields.d.ts.map +1 -1
  27. package/dist/DetectorFields.js +34 -4
  28. package/dist/DetectorFields.js.map +1 -1
  29. package/dist/EdgeAgent.js +1 -1
  30. package/dist/FaceIdentity.d.ts +122 -0
  31. package/dist/FaceIdentity.d.ts.map +1 -0
  32. package/dist/FaceIdentity.js +256 -0
  33. package/dist/FaceIdentity.js.map +1 -0
  34. package/dist/FaceSightingUploader.d.ts +95 -0
  35. package/dist/FaceSightingUploader.d.ts.map +1 -0
  36. package/dist/FaceSightingUploader.js +214 -0
  37. package/dist/FaceSightingUploader.js.map +1 -0
  38. package/dist/FaceWatchLane.d.ts +297 -0
  39. package/dist/FaceWatchLane.d.ts.map +1 -0
  40. package/dist/FaceWatchLane.js +660 -0
  41. package/dist/FaceWatchLane.js.map +1 -0
  42. package/dist/FaceWatchPolicy.d.ts +154 -0
  43. package/dist/FaceWatchPolicy.d.ts.map +1 -0
  44. package/dist/FaceWatchPolicy.js +101 -0
  45. package/dist/FaceWatchPolicy.js.map +1 -0
  46. package/dist/LiveViewControlHttp.d.ts +11 -2
  47. package/dist/LiveViewControlHttp.d.ts.map +1 -1
  48. package/dist/LiveViewControlHttp.js +14 -5
  49. package/dist/LiveViewControlHttp.js.map +1 -1
  50. package/dist/LiveViewCpuMeter.d.ts +26 -0
  51. package/dist/LiveViewCpuMeter.d.ts.map +1 -0
  52. package/dist/LiveViewCpuMeter.js +73 -0
  53. package/dist/LiveViewCpuMeter.js.map +1 -0
  54. package/dist/LiveViewProducerSignaling.d.ts +7 -0
  55. package/dist/LiveViewProducerSignaling.d.ts.map +1 -1
  56. package/dist/LiveViewProducerSignaling.js +12 -2
  57. package/dist/LiveViewProducerSignaling.js.map +1 -1
  58. package/dist/LocalDetectorClient.d.ts +11 -0
  59. package/dist/LocalDetectorClient.d.ts.map +1 -1
  60. package/dist/LocalDetectorClient.js +44 -0
  61. package/dist/LocalDetectorClient.js.map +1 -1
  62. package/dist/MainStreamRing.d.ts +63 -1
  63. package/dist/MainStreamRing.d.ts.map +1 -1
  64. package/dist/MainStreamRing.js +101 -7
  65. package/dist/MainStreamRing.js.map +1 -1
  66. package/dist/NativeReadiness.d.ts.map +1 -1
  67. package/dist/NativeReadiness.js +7 -3
  68. package/dist/NativeReadiness.js.map +1 -1
  69. package/dist/ReviewPriority.d.ts +9 -0
  70. package/dist/ReviewPriority.d.ts.map +1 -1
  71. package/dist/ReviewPriority.js +6 -1
  72. package/dist/ReviewPriority.js.map +1 -1
  73. package/dist/ScopedVlmReview.d.ts +11 -2
  74. package/dist/ScopedVlmReview.d.ts.map +1 -1
  75. package/dist/ScopedVlmReview.js +43 -16
  76. package/dist/ScopedVlmReview.js.map +1 -1
  77. package/dist/SensingProfileClient.d.ts +14 -0
  78. package/dist/SensingProfileClient.d.ts.map +1 -1
  79. package/dist/SensingProfileClient.js +9 -0
  80. package/dist/SensingProfileClient.js.map +1 -1
  81. package/dist/UniformSceneStore.d.ts +6 -0
  82. package/dist/UniformSceneStore.d.ts.map +1 -1
  83. package/dist/UniformSceneStore.js +22 -1
  84. package/dist/UniformSceneStore.js.map +1 -1
  85. package/dist/VideoSegmentUploader.d.ts +11 -0
  86. package/dist/VideoSegmentUploader.d.ts.map +1 -1
  87. package/dist/VideoSegmentUploader.js +28 -2
  88. package/dist/VideoSegmentUploader.js.map +1 -1
  89. package/dist/VlmCandidateBuffer.d.ts +2 -0
  90. package/dist/VlmCandidateBuffer.d.ts.map +1 -1
  91. package/dist/VlmCandidateBuffer.js.map +1 -1
  92. package/dist/WatchlistClient.d.ts +55 -0
  93. package/dist/WatchlistClient.d.ts.map +1 -0
  94. package/dist/WatchlistClient.js +161 -0
  95. package/dist/WatchlistClient.js.map +1 -0
  96. package/dist/autoSense.d.ts +2 -0
  97. package/dist/autoSense.d.ts.map +1 -1
  98. package/dist/autoSense.js +23 -1
  99. package/dist/autoSense.js.map +1 -1
  100. package/dist/cli.d.ts.map +1 -1
  101. package/dist/cli.js +3 -0
  102. package/dist/cli.js.map +1 -1
  103. package/dist/connectorUpdate.d.ts +16 -0
  104. package/dist/connectorUpdate.d.ts.map +1 -1
  105. package/dist/connectorUpdate.js +30 -6
  106. package/dist/connectorUpdate.js.map +1 -1
  107. package/dist/diagnose.d.ts +1 -1
  108. package/dist/diagnose.js +1 -1
  109. package/dist/hostCpu.d.ts +56 -0
  110. package/dist/hostCpu.d.ts.map +1 -0
  111. package/dist/hostCpu.js +113 -0
  112. package/dist/hostCpu.js.map +1 -0
  113. package/dist/localSensing.d.ts +17 -0
  114. package/dist/localSensing.d.ts.map +1 -1
  115. package/dist/localSensing.js +53 -2
  116. package/dist/localSensing.js.map +1 -1
  117. package/dist/windowsLiveView.d.ts +19 -2
  118. package/dist/windowsLiveView.d.ts.map +1 -1
  119. package/dist/windowsLiveView.js +72 -5
  120. package/dist/windowsLiveView.js.map +1 -1
  121. package/package.json +2 -2
  122. package/retained/RetainedCaptureClient.cjs +2 -0
  123. package/retained/WebsiteAdmissionCaptureClient.cjs +3 -1
  124. package/retained/__tests__/website-admission-realtime.test.cjs +27 -2
@@ -0,0 +1,166 @@
1
+ """The `/v1/faces` contract: the two pinned artifacts, the request and the answer.
2
+
3
+ STDLIB ONLY, like `server.py` and `registry.py`: the wire refusals and the check
4
+ of every answer before it leaves run on a runner with no numpy, no OpenCV and no
5
+ onnxruntime. The arithmetic lives in `face_identity.py`, which only the loader
6
+ imports.
7
+
8
+ The consumer is `LocalDetectorClient.faces` (edge-sdk), which re-checks the same
9
+ things on its side; this module is the service's half of that bargain — a
10
+ malformed answer becomes a 500 here, never a plausible 200.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import binascii
17
+ import hashlib
18
+ import math
19
+ import re
20
+
21
+ FACE_DETECT_TASK = "face_detection_landmarks5"
22
+ FACE_EMBED_TASK = "face_identity_embedding"
23
+
24
+ FACE_DETECTOR_MODEL_ID = "yunet_2023mar"
25
+ #: opencv_zoo `face_detection_yunet_2023mar.onnx` (MIT; sha256 8f2383e4...2fa4) with its input
26
+ #: height/width and its outputs' anchor counts declared symbolic, nodes and weights untouched.
27
+ FACE_DETECTOR_SHA256 = "2a62f235515c3a3f80c8c96ee75f5146d3518d5a63d2b3c77d9d1ae50e632f20"
28
+ FACE_DETECT_PREPROCESSING = "yunet_bgr_raw_long1920_pad32_floor0.5_nms0.3_top5000_v1"
29
+
30
+ FACE_EMBEDDER_MODEL_ID = "edgeface_xs_gamma_06"
31
+ #: EdgeFace-XS-gamma-0.6 exported to ONNX from the upstream checkpoint; cosine 1.000000000 against
32
+ #: the tablet's bundled TFLite (trillboard-ctv agent-core assets, sha256 7f912022...a254).
33
+ FACE_EMBEDDER_SHA256 = "1028899e16ca6c756dd1b333f0f2c136c45a049fc2ef098a2b9645e128d38f63"
34
+ FACE_EMBED_PREPROCESSING = "arcface5_umeyama_112_bilinear_rgb_unit127.5_l2_v1"
35
+
36
+ EMBEDDING_DIM = 512
37
+ MAX_FACES = 16
38
+ GATES = ("passed", "score", "eye_distance", "yaw", "sharpness")
39
+ REQUEST_KEYS = frozenset({"image_b64", "detector", "embedder", "minScore", "minEyePx", "maxAbsYaw",
40
+ "minSharpness", "maxFaces"})
41
+ _SHA = re.compile(r"[0-9a-f]{64}\Z")
42
+
43
+
44
+ def _number(value):
45
+ return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value)
46
+
47
+
48
+ def _pin(value, model_id, label):
49
+ if (not isinstance(value, dict) or set(value) != {"modelId", "sha256"}
50
+ or not isinstance(value["modelId"], str) or not isinstance(value["sha256"], str)
51
+ or not _SHA.fullmatch(value["sha256"])):
52
+ raise ValueError(f"faces {label} must name one modelId and a lowercase sha256")
53
+ if value["modelId"] != model_id:
54
+ raise ValueError(f"faces {label} must be {model_id}")
55
+ return {"modelId": value["modelId"], "sha256": value["sha256"]}
56
+
57
+
58
+ def validate_faces_request(body):
59
+ """Body -> (jpeg bytes, detector pin, embedder pin, bars). ValueError on anything else.
60
+
61
+ A pin naming the right model with another sha256 is NOT refused here: whether
62
+ those bytes are loaded is the registry's question, and it answers 409.
63
+ """
64
+ if not isinstance(body, dict) or set(body) != REQUEST_KEYS:
65
+ raise ValueError("faces request needs exactly " + ", ".join(sorted(REQUEST_KEYS)))
66
+ image = body["image_b64"]
67
+ if not isinstance(image, str) or not image:
68
+ raise ValueError("`image_b64` must be a non-empty base64 string")
69
+ try:
70
+ jpeg = base64.b64decode(image, validate=True)
71
+ except (binascii.Error, ValueError):
72
+ raise ValueError("`image_b64` is not valid base64") from None
73
+ if not jpeg:
74
+ raise ValueError("`image_b64` decoded to zero bytes")
75
+ detector = _pin(body["detector"], FACE_DETECTOR_MODEL_ID, "detector")
76
+ embedder = _pin(body["embedder"], FACE_EMBEDDER_MODEL_ID, "embedder")
77
+ bars = {key: body[key] for key in ("minScore", "minEyePx", "maxAbsYaw", "minSharpness", "maxFaces")}
78
+ if not _number(bars["minScore"]) or not 0 < bars["minScore"] <= 1:
79
+ raise ValueError("minScore must be in (0, 1]")
80
+ if not _number(bars["minEyePx"]) or not 1 <= bars["minEyePx"] <= 512:
81
+ raise ValueError("minEyePx must be in [1, 512]")
82
+ if not _number(bars["maxAbsYaw"]) or not 0 < bars["maxAbsYaw"] <= 2:
83
+ raise ValueError("maxAbsYaw must be in (0, 2]")
84
+ if not _number(bars["minSharpness"]) or not 0 <= bars["minSharpness"] <= 100000:
85
+ raise ValueError("minSharpness must be in [0, 100000]")
86
+ if type(bars["maxFaces"]) is not int or not 1 <= bars["maxFaces"] <= MAX_FACES:
87
+ raise ValueError("maxFaces must be a whole number 1..%d" % MAX_FACES)
88
+ bars = {key: float(value) if key != "maxFaces" else value for key, value in bars.items()}
89
+ return jpeg, detector, embedder, bars
90
+
91
+
92
+ def check_faces_response(result, jpeg, detector, embedder, bars):
93
+ """Raise ValueError unless `result` is exactly the answer to this request.
94
+
95
+ Every face: a box inside the frame with extent, a score in [0,1], five finite
96
+ landmarks, finite measures, a quality in [0,1], a known gate, and a vector
97
+ (512 finite values of unit length) exactly when the gate passed. The faces
98
+ are at most maxFaces, best score first.
99
+ """
100
+ if not isinstance(result, dict) or set(result) != {"version", "imageSha256", "frameSize", "detector",
101
+ "embedder", "faces", "inferenceMs"}:
102
+ raise ValueError("faces answer has unknown or missing keys")
103
+ if result["version"] != 1 or result["imageSha256"] != hashlib.sha256(jpeg).hexdigest():
104
+ raise ValueError("faces answer is not about this image")
105
+ size = result["frameSize"]
106
+ if (not isinstance(size, dict) or set(size) != {"width", "height"}
107
+ or type(size["width"]) is not int or type(size["height"]) is not int
108
+ or not 2 <= size["width"] <= 8192 or not 2 <= size["height"] <= 8192):
109
+ raise ValueError("faces answer has no original frame size")
110
+ width, height = size["width"], size["height"]
111
+ for label, pin, preprocessing in (("detector", detector, FACE_DETECT_PREPROCESSING),
112
+ ("embedder", embedder, FACE_EMBED_PREPROCESSING)):
113
+ if result[label] != {**pin, "preprocessing": preprocessing}:
114
+ raise ValueError(f"faces answer's {label} is not the selected artifact")
115
+ timing = result["inferenceMs"]
116
+ if (not isinstance(timing, dict) or set(timing) != {"detect", "embed"}
117
+ or any(not _number(timing[key]) or timing[key] < 0 for key in timing)):
118
+ raise ValueError("faces answer has no timing")
119
+ faces = result["faces"]
120
+ if not isinstance(faces, list) or len(faces) > bars["maxFaces"]:
121
+ raise ValueError("faces answer has more faces than were asked for")
122
+ previous = None
123
+ for index, face in enumerate(faces):
124
+ if not isinstance(face, dict) or set(face) != {"box", "score", "landmarks", "eyeDistancePx", "yaw",
125
+ "sharpness", "quality", "gate", "embedding"}:
126
+ raise ValueError(f"face {index} has unknown or missing keys")
127
+ box = face["box"]
128
+ if (not isinstance(box, dict) or set(box) != {"x", "y", "width", "height"}
129
+ or any(not _number(box[key]) for key in box)
130
+ or box["width"] <= 0 or box["height"] <= 0 or box["x"] < 0 or box["y"] < 0
131
+ or box["x"] + box["width"] > width + 0.01 or box["y"] + box["height"] > height + 0.01):
132
+ raise ValueError(f"face {index} box is outside the original frame")
133
+ if not _number(face["score"]) or not 0 <= face["score"] <= 1:
134
+ raise ValueError(f"face {index} score is outside [0, 1]")
135
+ if previous is not None and face["score"] > previous:
136
+ raise ValueError("faces are not best score first")
137
+ previous = face["score"]
138
+ points = face["landmarks"]
139
+ if (not isinstance(points, list) or len(points) != 5
140
+ or any(not isinstance(p, list) or len(p) != 2 or not all(_number(v) for v in p) for p in points)
141
+ or points[0][0] > points[1][0] or points[3][0] > points[4][0]):
142
+ raise ValueError(f"face {index} landmarks are not five points in template order")
143
+ if (not _number(face["eyeDistancePx"]) or face["eyeDistancePx"] < 0 or not _number(face["yaw"])
144
+ or not _number(face["sharpness"]) or face["sharpness"] < 0):
145
+ raise ValueError(f"face {index} measures are not finite")
146
+ if not _number(face["quality"]) or not 0 <= face["quality"] <= 1:
147
+ raise ValueError(f"face {index} quality is outside [0, 1]")
148
+ if face["gate"] not in GATES:
149
+ raise ValueError(f"face {index} gate is unknown")
150
+ expected = ("score" if face["score"] < bars["minScore"]
151
+ else "eye_distance" if face["eyeDistancePx"] < bars["minEyePx"]
152
+ else "yaw" if abs(face["yaw"]) > bars["maxAbsYaw"]
153
+ else "sharpness" if face["sharpness"] < bars["minSharpness"] else "passed")
154
+ if face["gate"] != expected:
155
+ raise ValueError(f"face {index} gate does not follow from its measures")
156
+ vector = face["embedding"]
157
+ if face["gate"] != "passed":
158
+ if vector is not None:
159
+ raise ValueError(f"face {index} has a vector it did not earn")
160
+ continue
161
+ if (not isinstance(vector, list) or len(vector) != EMBEDDING_DIM
162
+ or not all(_number(v) for v in vector)):
163
+ raise ValueError(f"face {index} vector is not {EMBEDDING_DIM} finite values")
164
+ norm = math.sqrt(math.fsum(v * v for v in vector))
165
+ if not 0.999 <= norm <= 1.001:
166
+ raise ValueError(f"face {index} vector is not unit length")
@@ -0,0 +1,467 @@
1
+ """Faces at the door: find each face, align it, and measure who it is.
2
+
3
+ THREE STEPS, TWO PINNED ARTIFACTS, ONE ANSWER PER FACE
4
+ ------------------------------------------------------
5
+ 1. YuNet 2023mar (opencv_zoo, MIT) finds each face and its five landmarks on the
6
+ ORIGINAL face-zone frame the connector cut from the main stream, decoded as
7
+ OpenCV's `FaceDetectorYN` decodes it. T1 face research (Texas camera 1): it finds
8
+ 56 of the 58 faces SCRFD-10G finds on a man walking in.
9
+ 2. A least-squares 5-point SIMILARITY warp (Umeyama 1991; the closed form
10
+ `FaceEmbeddingProcessor.computeSimilarityTransform` uses on the tablet) puts the
11
+ landmarks on the canonical ArcFace 112x112 template. NEVER EMBED WITHOUT
12
+ LANDMARKS: on the tablet a box crop separated same from different people at
13
+ ROC-AUC 0.66 and the aligned chip at 1.00 (FaceEmbeddingProcessor.kt).
14
+ 3. EdgeFace-XS-gamma-0.6 (otroshi/edgeface) as ONNX, the twin of the tablet's
15
+ bundled TFLite: exported from the upstream checkpoint (MD5 ebf343ca...d703, the
16
+ Idiap-published checksum) and measured against the APK's
17
+ `edgeface_xs_gamma_06.tflite` at cosine 1.000000000 (max |diff| 8.1e-08) on
18
+ random inputs and on real aligned chips. So a store's vector and a tablet's are
19
+ the same measurement, and one gallery serves both.
20
+
21
+ Each face is measured before it is embedded: the distance between the eyes in
22
+ ORIGINAL pixels (T1 research: 94.7% of true matches at 12 px, 45.7% at 6 px, at a
23
+ false-match rate of 1e-5), a yaw proxy and the sharpness of the aligned chip. A
24
+ face that fails a bar comes back with its box and landmarks (the connector's
25
+ tracker still follows it) and no vector, and says which bar it failed.
26
+
27
+ Stdlib + numpy + OpenCV + onnxruntime, loaded by the loader only: `server.py` and
28
+ `registry.py` never import this module.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import hashlib
34
+ import io
35
+ import math
36
+ from pathlib import Path
37
+ import threading
38
+ import time
39
+
40
+ import cv2
41
+ import numpy as np
42
+
43
+ from .errors import ModelContractError
44
+ from .face_contract import (EMBEDDING_DIM, FACE_DETECT_PREPROCESSING as DETECT_PREPROCESSING,
45
+ FACE_DETECTOR_MODEL_ID as DETECTOR_MODEL_ID, FACE_DETECTOR_SHA256 as DETECTOR_SHA256,
46
+ FACE_EMBED_PREPROCESSING as EMBED_PREPROCESSING,
47
+ FACE_EMBEDDER_MODEL_ID as EMBEDDER_MODEL_ID, FACE_EMBEDDER_SHA256 as EMBEDDER_SHA256)
48
+
49
+ # OpenCV otherwise creates a worker per host core, outside the model budget.
50
+ cv2.setNumThreads(1)
51
+
52
+ #: insightface `arcface_src`, default_square=True: image-left eye, image-right eye,
53
+ #: nose tip, image-left mouth corner, image-right mouth corner. Bit-identical to
54
+ #: `FaceEmbeddingProcessor.ARCFACE_REFERENCE_112`.
55
+ ARCFACE_REFERENCE_112 = np.array(
56
+ [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]],
57
+ dtype=np.float64,
58
+ )
59
+ CHIP_SIZE = 112
60
+
61
+ #: YuNet runs on the original pixels up to this long side; a larger frame is
62
+ #: area-resized for detection only, and the landmarks are mapped back so the
63
+ #: chip is still cut from the original pixels.
64
+ MAX_DETECT_LONG_SIDE = 1920
65
+ #: Faces down to this score are reported (gate "score" below the request's
66
+ #: minScore), so the connector's tracker keeps a face across a weak frame.
67
+ DETECT_SCORE_FLOOR = 0.5
68
+ NMS_THRESHOLD = 0.3
69
+ TOP_K = 5000
70
+ MAX_JPEG_BYTES = 16 * 1024**2
71
+ MAX_PIXELS = 16_777_216
72
+
73
+ #: The eye distance at which a face's size stops lowering its quality: past it
74
+ #: the T1 LFW curve is flat (TAR@1e-5 97.7% at 16 px, 98.3% at >= 20 px).
75
+ EYE_PX_FULL = 24.0
76
+ #: The yaw proxy at which quality reaches zero. yaw = (nose - eye midpoint) along
77
+ #: the eye line / eye distance, about 0.35 * tan(yaw angle): 0.35 is ~45 degrees.
78
+ YAW_ZERO = 0.5
79
+ #: Sharpness = variance of the Laplacian of the aligned 112x112 grey chip. A chip
80
+ #: is upsampled from the original face, so sharpness falls with the eye distance
81
+ #: as much as with blur. MEASURED (2026-11-16, measure_cam1.py, 400 LFW faces
82
+ #: aligned at fixed eye distances): sharp faces at 12 px p1 17.4 / p5 22.2 /
83
+ #: p50 37.4; at 16 px p1 24.9 / p50 56.1; at 20 px p1 30.7 / p50 72.3. The same
84
+ #: 20 px faces smeared along 15 chip px: p50 16.1, 9 px: p50 22.6. Texas camera 1's
85
+ #: SUBSTREAM faces (8 px between the eyes, the eye bar refuses them first): p50 11.9.
86
+ #: The floor rejects only clearly smeared chips (it keeps 99% of sharp 12 px faces);
87
+ #: quality stops rising at the typical sharp 16-20 px face.
88
+ SHARPNESS_REF = 60.0
89
+ MIN_SHARPNESS = 15.0
90
+
91
+ #: The twelve heads, in OpenCV's order: (cls, obj, bbox, kps) x strides (8, 16, 32).
92
+ YUNET_STRIDES = (8, 16, 32)
93
+ YUNET_OUTPUTS = tuple(f"{head}_{stride}" for head in ("cls", "obj", "bbox", "kps") for stride in YUNET_STRIDES)
94
+ #: OpenCV pads the input up to a multiple of this, with zeros on the right and bottom.
95
+ YUNET_DIVISOR = 32
96
+
97
+
98
+ def decode_yunet(outputs, pad_w, pad_h, score_threshold, nms_threshold=NMS_THRESHOLD, top_k=TOP_K):
99
+ """OpenCV `FaceDetectorYNImpl::postProcess` (2023mar heads), then `dnn::NMSBoxes`.
100
+
101
+ Rows of (x, y, w, h, 10 landmark coordinates, score) in padded-input pixels, in
102
+ the order NMS keeps them (score descending). Landmarks are YuNet's: right eye,
103
+ left eye (the subject's), nose tip, right and left mouth corners.
104
+ """
105
+ candidates = []
106
+ for stride in YUNET_STRIDES:
107
+ cols, rows = pad_w // stride, pad_h // stride
108
+ count = cols * rows
109
+ cls = np.asarray(outputs[f"cls_{stride}"], np.float32).reshape(-1)
110
+ obj = np.asarray(outputs[f"obj_{stride}"], np.float32).reshape(-1)
111
+ bbox = np.asarray(outputs[f"bbox_{stride}"], np.float32).reshape(-1, 4)
112
+ kps = np.asarray(outputs[f"kps_{stride}"], np.float32).reshape(-1, 10)
113
+ if cls.shape[0] != count or obj.shape[0] != count or bbox.shape[0] != count or kps.shape[0] != count:
114
+ raise ModelContractError("Face detector heads do not match the padded input")
115
+ score = np.sqrt(np.clip(cls, 0.0, 1.0) * np.clip(obj, 0.0, 1.0))
116
+ keep = np.nonzero(score >= score_threshold)[0]
117
+ if keep.size == 0:
118
+ continue
119
+ c = (keep % cols).astype(np.float32)
120
+ r = (keep // cols).astype(np.float32)
121
+ cx = (c + bbox[keep, 0]) * stride
122
+ cy = (r + bbox[keep, 1]) * stride
123
+ w = np.exp(bbox[keep, 2]) * stride
124
+ h = np.exp(bbox[keep, 3]) * stride
125
+ landmarks = np.empty((keep.size, 10), np.float32)
126
+ landmarks[:, 0::2] = (kps[keep, 0::2] + c[:, None]) * stride
127
+ landmarks[:, 1::2] = (kps[keep, 1::2] + r[:, None]) * stride
128
+ block = np.column_stack([cx - w / 2.0, cy - h / 2.0, w, h, landmarks, score[keep]])
129
+ candidates.append(block)
130
+ if not candidates:
131
+ return np.zeros((0, 15), np.float64)
132
+ faces = np.vstack(candidates).astype(np.float64)
133
+ if not np.isfinite(faces).all():
134
+ raise ModelContractError("Native face detector output is invalid")
135
+ if len(faces) == 1:
136
+ return faces
137
+ return faces[nms_boxes(faces[:, 0:4], faces[:, 14], score_threshold, nms_threshold, top_k)]
138
+
139
+
140
+ def nms_boxes(boxes, scores, score_threshold, nms_threshold, top_k):
141
+ """`cv::dnn::NMSBoxes` on Rect2i(int(x), int(y), int(w), int(h)), eta 1: kept indices, best first."""
142
+ rects = np.trunc(np.asarray(boxes, np.float64)).astype(np.int64)
143
+ # GetMaxScoreIndex keeps a score strictly above the threshold, then the top_k.
144
+ order = [i for i in np.argsort(-np.asarray(scores), kind="stable") if scores[i] > score_threshold]
145
+ if top_k > 0:
146
+ order = order[:top_k]
147
+ kept = []
148
+ for i in order:
149
+ x, y, w, h = rects[i]
150
+ area = max(0, w) * max(0, h)
151
+ overlapping = False
152
+ for j in kept:
153
+ x2, y2, w2, h2 = rects[j]
154
+ iw = min(x + w, x2 + w2) - max(x, x2)
155
+ ih = min(y + h, y2 + h2) - max(y, y2)
156
+ inter = max(0, iw) * max(0, ih)
157
+ union = area + max(0, w2) * max(0, h2) - inter
158
+ if union > 0 and inter / union > nms_threshold:
159
+ overlapping = True
160
+ break
161
+ if not overlapping:
162
+ kept.append(i)
163
+ return np.asarray(kept, dtype=np.int64)
164
+
165
+
166
+ def umeyama(src, dst):
167
+ """Least-squares similarity (scale, rotation, translation; no reflection) taking src onto dst.
168
+
169
+ The Umeyama closed form `skimage.SimilarityTransform` and the tablet's
170
+ `computeSimilarityTransform` both use. Returns the 2x3 affine, or None when
171
+ the source points are degenerate (coincident: no stable transform).
172
+ """
173
+ src = np.asarray(src, dtype=np.float64)
174
+ dst = np.asarray(dst, dtype=np.float64)
175
+ if src.shape != dst.shape or src.ndim != 2 or src.shape[1] != 2 or src.shape[0] < 3:
176
+ return None
177
+ if not (np.isfinite(src).all() and np.isfinite(dst).all()):
178
+ return None
179
+ mu_s, mu_d = src.mean(0), dst.mean(0)
180
+ sc, dc = src - mu_s, dst - mu_d
181
+ var_s = float((sc ** 2).sum() / len(src))
182
+ if var_s < 1e-12:
183
+ return None
184
+ cov = dc.T @ sc / len(src)
185
+ u, s, vt = np.linalg.svd(cov)
186
+ d = np.ones(2)
187
+ if np.linalg.det(u) * np.linalg.det(vt) < 0:
188
+ d[-1] = -1.0
189
+ rotation = u @ np.diag(d) @ vt
190
+ scale = float((s * d).sum() / var_s)
191
+ translation = mu_d - scale * rotation @ mu_s
192
+ matrix = np.hstack([scale * rotation, translation[:, None]])
193
+ return matrix if np.isfinite(matrix).all() else None
194
+
195
+
196
+ def arcface_order(landmarks):
197
+ """The five points in template order, by IMAGE position (handedness-proof).
198
+
199
+ Template point 0 is the image-LEFT eye and point 3 the image-left mouth corner,
200
+ whatever a detector calls them. Feeding the fit a mirrored correspondence
201
+ collapses the vector (0.53 AUC on the tablet), so the eyes and the mouth
202
+ corners are each ordered by x, exactly as `toArcFaceSourcePoints` does.
203
+ """
204
+ points = np.asarray(landmarks, dtype=np.float64).reshape(5, 2)
205
+ eyes = points[[0, 1]] if points[0, 0] <= points[1, 0] else points[[1, 0]]
206
+ mouth = points[[3, 4]] if points[3, 0] <= points[4, 0] else points[[4, 3]]
207
+ return np.vstack([eyes, points[2:3], mouth])
208
+
209
+
210
+ def align_chip(bgr, landmarks):
211
+ """The 112x112 BGR chip of one face, warped (bilinear) from the ORIGINAL pixels."""
212
+ matrix = umeyama(arcface_order(landmarks), ARCFACE_REFERENCE_112)
213
+ if matrix is None:
214
+ return None
215
+ return cv2.warpAffine(bgr, matrix, (CHIP_SIZE, CHIP_SIZE), flags=cv2.INTER_LINEAR,
216
+ borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
217
+
218
+
219
+ def face_measures(landmarks, chip):
220
+ """(eye distance in original px, signed yaw proxy, sharpness of the aligned chip)."""
221
+ points = arcface_order(landmarks)
222
+ left, right, nose = points[0], points[1], points[2]
223
+ axis = right - left
224
+ eye_distance = float(math.hypot(axis[0], axis[1]))
225
+ if eye_distance <= 0:
226
+ yaw = 0.0
227
+ else:
228
+ # Along the eye line, so a tilted (rolled) head does not read as turned.
229
+ yaw = float(np.dot(nose - (left + right) / 2.0, axis / eye_distance) / eye_distance)
230
+ grey = cv2.cvtColor(chip, cv2.COLOR_BGR2GRAY)
231
+ sharpness = float(cv2.Laplacian(grey, cv2.CV_64F).var())
232
+ return eye_distance, yaw, sharpness
233
+
234
+
235
+ def face_quality(score, eye_distance, yaw, sharpness):
236
+ """0..1: detector score x size x frontality x sharpness, each saturating at its reference."""
237
+ size = min(1.0, eye_distance / EYE_PX_FULL)
238
+ frontal = max(0.0, 1.0 - abs(yaw) / YAW_ZERO)
239
+ sharp = min(1.0, max(0.0, sharpness) / SHARPNESS_REF)
240
+ value = float(score) * size * frontal * sharp
241
+ return min(1.0, max(0.0, value))
242
+
243
+
244
+ def face_gate(score, eye_distance, yaw, sharpness, request):
245
+ """The first bar a face fails, in a fixed order, or "passed"."""
246
+ if score < request["minScore"]:
247
+ return "score"
248
+ if eye_distance < request["minEyePx"]:
249
+ return "eye_distance"
250
+ if abs(yaw) > request["maxAbsYaw"]:
251
+ return "yaw"
252
+ if sharpness < request["minSharpness"]:
253
+ return "sharpness"
254
+ return "passed"
255
+
256
+
257
+ def decode_original_jpeg(jpeg):
258
+ """(BGR array, width, height) of a bounded original JPEG, or ValueError."""
259
+ from PIL import Image
260
+
261
+ if not isinstance(jpeg, bytes) or not 0 < len(jpeg) <= MAX_JPEG_BYTES:
262
+ raise ValueError("Faces need a bounded original JPEG")
263
+ try:
264
+ with Image.open(io.BytesIO(jpeg)) as header:
265
+ width, height = header.size
266
+ if (header.format != "JPEG" or not 2 <= width <= 8192 or not 2 <= height <= 8192
267
+ or width * height > MAX_PIXELS):
268
+ raise ValueError("Faces need a bounded original JPEG")
269
+ except ValueError:
270
+ raise
271
+ except Exception as error:
272
+ raise ValueError("Invalid original face JPEG") from error
273
+ bgr = cv2.imdecode(np.frombuffer(jpeg, np.uint8), cv2.IMREAD_COLOR)
274
+ if bgr is None or bgr.shape != (height, width, 3):
275
+ raise ValueError("Original face JPEG could not be decoded")
276
+ return bgr, width, height
277
+
278
+
279
+ class EdgeFaceEmbedder:
280
+ """EdgeFace-XS: one immutable CPU artifact, the tablet model's ONNX twin."""
281
+
282
+ model_name = EMBEDDER_MODEL_ID
283
+ model_sha256 = EMBEDDER_SHA256
284
+
285
+ def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
286
+ if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
287
+ raise ValueError("Invalid bounded face embedder thread count")
288
+ if accelerator != "cpu":
289
+ raise ModelContractError("This face embedder is qualified for native CPU execution")
290
+ raw = Path(model_path).read_bytes()
291
+ if hashlib.sha256(raw).hexdigest() != EMBEDDER_SHA256:
292
+ raise ModelContractError("Face embedder artifact differs from the pinned export")
293
+ import onnxruntime as ort
294
+
295
+ from .execution import LocalOrtSession
296
+
297
+ options = ort.SessionOptions()
298
+ options.intra_op_num_threads = intra_op_threads
299
+ options.inter_op_num_threads = 1
300
+ options.log_severity_level = 3
301
+ self._session = LocalOrtSession(ort, raw, options, "cpu")
302
+ inputs, outputs = self._session.get_inputs(), self._session.get_outputs()
303
+ if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].shape != [1, 3, CHIP_SIZE, CHIP_SIZE]
304
+ or inputs[0].type != "tensor(float)" or len(outputs) != 1 or outputs[0].name != "embedding"
305
+ or outputs[0].shape != [1, EMBEDDING_DIM] or outputs[0].type != "tensor(float)"):
306
+ raise ModelContractError("Face embedder graph differs from its 112x112 RGB -> 512 contract")
307
+ self._lock = threading.Lock()
308
+
309
+ @property
310
+ def acceleration(self):
311
+ return self._session.acceleration
312
+
313
+ def embed(self, chip_bgr):
314
+ """One aligned 112x112 BGR chip -> its L2-normalised 512-d vector (float64)."""
315
+ if (not isinstance(chip_bgr, np.ndarray) or chip_bgr.dtype != np.uint8
316
+ or chip_bgr.shape != (CHIP_SIZE, CHIP_SIZE, 3)):
317
+ raise ValueError("A face vector needs one aligned 112x112 chip")
318
+ rgb = chip_bgr[:, :, ::-1].astype(np.float32)
319
+ tensor = np.ascontiguousarray(((rgb / 127.5) - 1.0).transpose(2, 0, 1)[None], dtype=np.float32)
320
+ with self._lock:
321
+ value = self._session.run(["embedding"], {"input": tensor})[0]
322
+ if value.shape != (1, EMBEDDING_DIM) or not np.isfinite(value).all():
323
+ raise ModelContractError("Native face embedder output is invalid")
324
+ vector = value.reshape(EMBEDDING_DIM).astype(np.float64)
325
+ norm = float(np.linalg.norm(vector))
326
+ if not math.isfinite(norm) or norm <= 1e-12:
327
+ raise ModelContractError("Native face embedder output cannot identify a face")
328
+ return vector / norm
329
+
330
+ def probe(self):
331
+ """A forward pass completes (the loader's admission check)."""
332
+ self.embed(np.full((CHIP_SIZE, CHIP_SIZE, 3), 128, np.uint8))
333
+
334
+
335
+ class YunetFaceDetector:
336
+ """YuNet 2023mar on onnxruntime, decoded exactly as OpenCV's FaceDetectorYN decodes it.
337
+
338
+ WHY NOT `cv2.FaceDetectorYN`, WHICH IS THE REFERENCE: on the pinned
339
+ opencv-python 5.0.0.93 its new DNN graph engine ran this graph 2.5x slower
340
+ than onnxruntime 1.29 on the same 800x448 Texas frame, one thread each,
341
+ interleaved (scratchpad/t1 yunet_ab.py). The decode below is OpenCV's
342
+ `FaceDetectorYNImpl::postProcess` line for line, and `test_face_identity.py`
343
+ holds the two to the same faces on real frames.
344
+
345
+ The artifact is the published file with only its DECLARED shape made symbolic
346
+ (`make_yunet_dynamic.py`): every node and weight is opencv_zoo's
347
+ 8f2383e4...2fa4, whose 640x640 declaration would otherwise refuse any other
348
+ frame size in onnxruntime.
349
+ """
350
+
351
+ model_name = DETECTOR_MODEL_ID
352
+ model_sha256 = DETECTOR_SHA256
353
+
354
+ def __init__(self, model_path, *, intra_op_threads=4, accelerator="cpu"):
355
+ if type(intra_op_threads) is not int or not 1 <= intra_op_threads <= 8:
356
+ raise ValueError("Invalid bounded face detector thread count")
357
+ if accelerator != "cpu":
358
+ raise ModelContractError("This face detector is qualified for native CPU execution")
359
+ raw = Path(model_path).read_bytes()
360
+ if hashlib.sha256(raw).hexdigest() != DETECTOR_SHA256:
361
+ raise ModelContractError("Face detector artifact differs from the pinned export")
362
+ import onnxruntime as ort
363
+
364
+ from .execution import LocalOrtSession
365
+
366
+ options = ort.SessionOptions()
367
+ options.intra_op_num_threads = intra_op_threads
368
+ options.inter_op_num_threads = 1
369
+ options.log_severity_level = 3
370
+ self._session = LocalOrtSession(ort, raw, options, "cpu")
371
+ inputs = self._session.get_inputs()
372
+ outputs = [o.name for o in self._session.get_outputs()]
373
+ if (len(inputs) != 1 or inputs[0].name != "input" or inputs[0].type != "tensor(float)"
374
+ or len(inputs[0].shape) != 4 or inputs[0].shape[:2] != [1, 3]
375
+ or not all(isinstance(v, str) for v in inputs[0].shape[2:])
376
+ or sorted(outputs) != sorted(YUNET_OUTPUTS)):
377
+ raise ModelContractError("Face detector graph differs from the YuNet 2023mar contract")
378
+ self._lock = threading.Lock()
379
+
380
+ @property
381
+ def acceleration(self):
382
+ return self._session.acceleration
383
+
384
+ def detect(self, bgr, min_score):
385
+ """[(box [x, y, w, h], score, landmarks 5x2)] in the pixels of `bgr`, best score first."""
386
+ height, width = bgr.shape[:2]
387
+ scale = 1.0
388
+ image = bgr
389
+ if max(width, height) > MAX_DETECT_LONG_SIDE:
390
+ scale = MAX_DETECT_LONG_SIDE / float(max(width, height))
391
+ image = cv2.resize(bgr, (max(2, round(width * scale)), max(2, round(height * scale))),
392
+ interpolation=cv2.INTER_AREA)
393
+ pad_h = ((image.shape[0] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
394
+ pad_w = ((image.shape[1] - 1) // YUNET_DIVISOR + 1) * YUNET_DIVISOR
395
+ padded = np.zeros((pad_h, pad_w, 3), np.uint8)
396
+ padded[:image.shape[0], :image.shape[1]] = image
397
+ blob = np.ascontiguousarray(padded.transpose(2, 0, 1)[None], dtype=np.float32)
398
+ with self._lock:
399
+ values = self._session.run(list(YUNET_OUTPUTS), {"input": blob})
400
+ rows = decode_yunet(dict(zip(YUNET_OUTPUTS, values)), pad_w, pad_h,
401
+ float(min(DETECT_SCORE_FLOOR, min_score)))
402
+ faces = []
403
+ for row in rows:
404
+ faces.append((row[0:4] / scale, float(row[14]), row[4:14].reshape(5, 2) / scale))
405
+ return faces
406
+
407
+ def estimate(self, jpeg, request, embedder):
408
+ """The /v1/faces answer for one original JPEG (face_contract.check_faces_response holds its shape)."""
409
+ if not callable(getattr(embedder, "embed", None)):
410
+ raise ModelContractError("Faces need the pinned embedder session")
411
+ bgr, width, height = decode_original_jpeg(jpeg)
412
+ started = time.perf_counter()
413
+ detected = self.detect(bgr, request["minScore"])
414
+ detect_ms = (time.perf_counter() - started) * 1000.0
415
+ embed_ms = 0.0
416
+ faces = []
417
+ for box, score, landmarks in detected[:request["maxFaces"]]:
418
+ # Clipped to the frame, then rounded at the corners, so x + width never passes the edge.
419
+ # Positions keep 3 decimals and scores 6: the regression corpus holds this answer to the
420
+ # reference implementations at 3e-5 of the frame, which coarser rounding would exceed.
421
+ x0 = round(max(0.0, float(box[0])), 3)
422
+ y0 = round(max(0.0, float(box[1])), 3)
423
+ x1 = round(min(float(width), float(box[0] + box[2])), 3)
424
+ y1 = round(min(float(height), float(box[1] + box[3])), 3)
425
+ if x1 - x0 <= 0 or y1 - y0 <= 0:
426
+ continue
427
+ ordered = arcface_order(landmarks)
428
+ chip = align_chip(bgr, ordered)
429
+ if chip is None:
430
+ continue
431
+ # Rounded FIRST, and the gate and quality decided on the rounded numbers, so the
432
+ # answer is self-consistent to the digit (server.py re-derives every gate from them).
433
+ eye_distance, yaw, sharpness = (round(v, d) for v, d in zip(face_measures(ordered, chip), (3, 6, 3)))
434
+ score = round(min(1.0, max(0.0, score)), 6)
435
+ gate = face_gate(score, eye_distance, yaw, sharpness, request)
436
+ embedding = None
437
+ if gate == "passed":
438
+ began = time.perf_counter()
439
+ vector = embedder.embed(chip)
440
+ embed_ms += (time.perf_counter() - began) * 1000.0
441
+ embedding = [round(float(v), 6) for v in vector]
442
+ faces.append({
443
+ "box": {"x": x0, "y": y0, "width": round(x1 - x0, 3), "height": round(y1 - y0, 3)},
444
+ "score": score,
445
+ "landmarks": [[round(float(px), 3), round(float(py), 3)] for px, py in ordered],
446
+ "eyeDistancePx": eye_distance,
447
+ "yaw": yaw,
448
+ "sharpness": sharpness,
449
+ "quality": round(face_quality(score, eye_distance, yaw, sharpness), 6),
450
+ "gate": gate,
451
+ "embedding": embedding,
452
+ })
453
+ return {
454
+ "version": 1,
455
+ "imageSha256": hashlib.sha256(jpeg).hexdigest(),
456
+ "frameSize": {"width": width, "height": height},
457
+ "detector": {"modelId": DETECTOR_MODEL_ID, "sha256": DETECTOR_SHA256,
458
+ "preprocessing": DETECT_PREPROCESSING},
459
+ "embedder": {"modelId": EMBEDDER_MODEL_ID, "sha256": EMBEDDER_SHA256,
460
+ "preprocessing": EMBED_PREPROCESSING},
461
+ "faces": faces,
462
+ "inferenceMs": {"detect": round(detect_ms, 3), "embed": round(embed_ms, 3)},
463
+ }
464
+
465
+ def probe(self):
466
+ """A forward pass completes on a grey frame (no face is expected or required)."""
467
+ self.detect(np.full((48, 64, 3), 128, np.uint8), DETECT_SCORE_FLOOR)
@@ -198,6 +198,15 @@ class DetectorLoader:
198
198
  if entry.vocabulary.task == "vehicle_end_orientation":
199
199
  from .vehicle_end import VehicleEndEstimator
200
200
  return VehicleEndEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator="cpu")
201
+ # The door faces task is called by the connector's face lane at up to 4 frames a second,
202
+ # beside the look loop's object detector: ONE thread by default keeps it to one core and
203
+ # out of RF-DETR's pool (devbox A/B under load: 4 threads ran slower than 1).
204
+ if entry.vocabulary.task == "face_detection_landmarks5":
205
+ from .face_identity import YunetFaceDetector
206
+ return YunetFaceDetector(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
207
+ if entry.vocabulary.task == "face_identity_embedding":
208
+ from .face_identity import EdgeFaceEmbedder
209
+ return EdgeFaceEmbedder(entry.path, intra_op_threads=self._intra_op_threads or 1, accelerator="cpu")
201
210
  from .pose import PoseEstimator
202
211
  return PoseEstimator(entry.path, intra_op_threads=self._intra_op_threads or 4, accelerator=self._accelerator)
203
212
  from .session import OnnxDetector
@@ -366,6 +375,9 @@ class DetectorLoader:
366
375
  def estimate_vehicle_end(self, jpeg: bytes, boxes: list, side: str, expected_model: dict) -> dict:
367
376
  return self.registry.estimate_vehicle_end(jpeg, boxes, side, expected_model)
368
377
 
378
+ def estimate_faces(self, jpeg: bytes, request: dict, detector_pin: dict, embedder_pin: dict) -> dict:
379
+ return self.registry.estimate_faces(jpeg, request, detector_pin, embedder_pin)
380
+
369
381
  def classify_actor_actions(self,batch):
370
382
  return self.registry.classify_actor_actions(batch)
371
383