@camstack/addon-pipeline 1.2.333 → 1.2.334
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_MODELS.md +3 -8
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +2 -2
- package/dist/{default-detection-model-Biwf5-ML.js → custom-model-catalog-CVeosmYm.js} +236 -165
- package/dist/{default-detection-model-CDffuH2m.mjs → custom-model-catalog-DoOVE7kz.mjs} +225 -166
- package/dist/detection-pipeline/index.js +70 -104
- package/dist/detection-pipeline/index.mjs +18 -52
- package/dist/{dist-DJV7Z6Bb.js → dist-C7v30DMP.js} +69 -22
- package/dist/{dist-BsT15i7i.mjs → dist-Dn3knu_K.mjs} +69 -22
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-CkmucLYG.js → node-DWdekhW3.js} +1 -1
- package/dist/{node-Crsi_8ea.mjs → node-_BruMF4L.mjs} +1 -1
- package/dist/pipeline-runner/index.js +228 -42
- package/dist/pipeline-runner/index.mjs +206 -20
- package/dist/{process-memory-BJb7Ihdp.js → process-memory-CaDpfUdM.js} +1 -1
- package/dist/{process-memory-1mAUV8S1.mjs → process-memory-Dp-4bFOp.mjs} +1 -1
- package/dist/recorder/index.js +2 -2
- package/dist/recorder/index.mjs +2 -2
- package/dist/{segment-demux-js-B_tboBcJ.mjs → segment-demux-js-B_-zBECg.mjs} +1 -1
- package/dist/{segment-demux-js-mK9IBGLf.js → segment-demux-js-BqPTLFZG.js} +1 -1
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-a3HMFtd5.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BP8FFU2i.mjs} +2 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CpjKK9II.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-DChPnZ5C.mjs → hostInit-BEh-hm1x.mjs} +2 -2
- package/dist/stream-broker/index.js +3 -3
- package/dist/stream-broker/index.mjs +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/python/postprocessors/test_yolo_seg_root.py +109 -0
- package/python/postprocessors/yolo.py +52 -13
- package/python/postprocessors/yolo_seg.py +79 -156
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DPbj3ML4.mjs +0 -26
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/addon-pipeline",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.334",
|
|
4
4
|
"description": "Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"camstack",
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""D737 — a YOLO26-seg ROOT decodes its boxes exactly like the plain detector
|
|
2
|
+
and keeps one mask per kept detection.
|
|
3
|
+
|
|
4
|
+
Run like the other postprocessor tests:
|
|
5
|
+
``PYTHONPATH=postprocessors:. python -m pytest postprocessors/`` from
|
|
6
|
+
``packages/addon-pipeline/python``.
|
|
7
|
+
"""
|
|
8
|
+
import base64
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from postprocessors.yolo import postprocess_yolo
|
|
13
|
+
from postprocessors.yolo_seg import postprocess_yolo_seg
|
|
14
|
+
|
|
15
|
+
LABELS = ["person", "car", "dog"]
|
|
16
|
+
CONFIG = {"labels": LABELS, "numClasses": 3, "confidence": 0.3, "inputSize": 64}
|
|
17
|
+
# A 1280x720 frame letterboxed into 64: scale 0.05, vertical pad 14.
|
|
18
|
+
ORIG_W, ORIG_H, SCALE, PAD = 1280, 720, 0.05, (0, 14)
|
|
19
|
+
PROTO = 16 # P x P prototype grid at the 64 input (4 px per cell)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _protos(rng: np.random.Generator) -> np.ndarray:
|
|
23
|
+
return rng.normal(size=(1, 32, PROTO, PROTO)).astype(np.float32)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _post_nms_rows(rng: np.random.Generator) -> np.ndarray:
|
|
27
|
+
"""[1, 6, 38]: two overlapping persons (the end-to-end head keeps both),
|
|
28
|
+
a car, one row under the floor and two padding rows (class -1)."""
|
|
29
|
+
boxes = [
|
|
30
|
+
[8, 20, 24, 44, 0.91, 0],
|
|
31
|
+
[10, 22, 26, 46, 0.80, 0],
|
|
32
|
+
[30, 30, 56, 48, 0.66, 1],
|
|
33
|
+
[40, 18, 50, 28, 0.10, 2],
|
|
34
|
+
[0, 0, 0, 0, 0.0, -1],
|
|
35
|
+
[0, 0, 0, 0, 0.0, -1],
|
|
36
|
+
]
|
|
37
|
+
coeffs = rng.normal(size=(len(boxes), 32))
|
|
38
|
+
return np.concatenate([np.array(boxes, dtype=np.float32), coeffs], axis=1)[None].astype(np.float32)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _raw_head(rng: np.random.Generator) -> np.ndarray:
|
|
42
|
+
"""[1, 4 + 3 + 32, 5] with LOGIT scores (YOLO26 raw), two overlapping."""
|
|
43
|
+
cxcywh = np.array(
|
|
44
|
+
[[16, 32, 16, 24], [18, 34, 16, 24], [43, 39, 26, 18], [45, 23, 10, 10], [5, 5, 4, 4]],
|
|
45
|
+
dtype=np.float32,
|
|
46
|
+
).T
|
|
47
|
+
logits = np.array(
|
|
48
|
+
[[3.0, 2.0, -4, -4, -6], [-4, -4, 1.5, -4, -6], [-4, -4, -4, -2.0, -6]], dtype=np.float32
|
|
49
|
+
)
|
|
50
|
+
coeffs = rng.normal(size=(32, 5)).astype(np.float32)
|
|
51
|
+
return np.concatenate([cxcywh, logits, coeffs], axis=0)[None]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _boxes(out: dict) -> list[tuple]:
|
|
55
|
+
return [(d["class"], d["score"], tuple(d["bbox"])) for d in out["detections"]]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_post_nms_head_boxes_equal_the_plain_detectors():
|
|
59
|
+
rng = np.random.default_rng(1)
|
|
60
|
+
rows = _post_nms_rows(rng)
|
|
61
|
+
plain = postprocess_yolo({"output0": rows[..., :6].copy()}, dict(CONFIG), ORIG_W, ORIG_H, SCALE, PAD)
|
|
62
|
+
seg = postprocess_yolo_seg(
|
|
63
|
+
{"output0": rows, "output1": _protos(rng)}, dict(CONFIG), ORIG_W, ORIG_H, SCALE, PAD
|
|
64
|
+
)
|
|
65
|
+
assert _boxes(seg) == _boxes(plain)
|
|
66
|
+
# The plain detector does not NMS an end-to-end head; neither does the seg root.
|
|
67
|
+
assert len(seg["detections"]) == 3
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_raw_head_boxes_equal_the_plain_detectors():
|
|
71
|
+
rng = np.random.default_rng(2)
|
|
72
|
+
head = _raw_head(rng)
|
|
73
|
+
plain = postprocess_yolo({"output0": head[:, :7, :].copy()}, dict(CONFIG), ORIG_W, ORIG_H, SCALE, PAD)
|
|
74
|
+
seg = postprocess_yolo_seg(
|
|
75
|
+
{"output0": head.copy(), "output1": _protos(rng)}, dict(CONFIG), ORIG_W, ORIG_H, SCALE, PAD
|
|
76
|
+
)
|
|
77
|
+
assert _boxes(seg) == _boxes(plain)
|
|
78
|
+
assert len(seg["detections"]) == 2 # the overlap is suppressed, the floor drops two
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_same_nms_iou_knob_as_the_plain_detector():
|
|
82
|
+
rng = np.random.default_rng(3)
|
|
83
|
+
head = _raw_head(rng)
|
|
84
|
+
loose = dict(CONFIG, nmsIouThreshold=0.95)
|
|
85
|
+
plain = postprocess_yolo({"output0": head[:, :7, :].copy()}, loose, ORIG_W, ORIG_H, SCALE, PAD)
|
|
86
|
+
seg = postprocess_yolo_seg(
|
|
87
|
+
{"output0": head.copy(), "output1": _protos(rng)}, loose, ORIG_W, ORIG_H, SCALE, PAD
|
|
88
|
+
)
|
|
89
|
+
assert _boxes(seg) == _boxes(plain)
|
|
90
|
+
assert len(seg["detections"]) == 3
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_every_kept_detection_carries_its_own_mask_over_its_box():
|
|
94
|
+
rng = np.random.default_rng(4)
|
|
95
|
+
rows = _post_nms_rows(rng)
|
|
96
|
+
protos = _protos(rng)
|
|
97
|
+
out = postprocess_yolo_seg({"output0": rows, "output1": protos}, dict(CONFIG), ORIG_W, ORIG_H, SCALE, PAD)
|
|
98
|
+
first = out["detections"][0]
|
|
99
|
+
# Box [8,20,24,44] in the 64 input → proto cells [2,5,6,11] → 4 x 6.
|
|
100
|
+
assert (first["maskWidth"], first["maskHeight"]) == (4, 6)
|
|
101
|
+
raw = np.frombuffer(base64.b64decode(first["mask"]), dtype=np.uint8)
|
|
102
|
+
assert raw.size == 4 * 6
|
|
103
|
+
assert set(np.unique(raw)) <= {0, 255}
|
|
104
|
+
# The mask IS sigmoid(coeffs @ protos) > 0.5 over that window, for this row.
|
|
105
|
+
logits = rows[0, 0, 6:] @ protos[0][:, 5:11, 2:6].reshape(32, -1)
|
|
106
|
+
expected = ((1 / (1 + np.exp(-logits))) > 0.5).astype(np.uint8) * 255
|
|
107
|
+
assert np.array_equal(raw, expected)
|
|
108
|
+
# Different rows → different coefficient vectors → masks are per instance.
|
|
109
|
+
assert out["detections"][1]["mask"] != first["mask"]
|
|
@@ -180,12 +180,28 @@ def _postprocess_yolo11_nms(
|
|
|
180
180
|
|
|
181
181
|
Coordinates are in input pixel space (0..inputSize). Already post-NMS.
|
|
182
182
|
"""
|
|
183
|
+
rows = decode_yolo11_nms_rows(output, config, orig_w, orig_h, scale, pad)
|
|
184
|
+
return {"kind": "detections", "detections": [d for _, d in rows]}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def decode_yolo11_nms_rows(
|
|
188
|
+
output: np.ndarray,
|
|
189
|
+
config: dict,
|
|
190
|
+
orig_w: int,
|
|
191
|
+
orig_h: int,
|
|
192
|
+
scale: float,
|
|
193
|
+
pad: tuple[int, int],
|
|
194
|
+
) -> list[tuple[int, dict]]:
|
|
195
|
+
"""The post-NMS box decode, each kept detection paired with its ROW in
|
|
196
|
+
``output``. Only the first 6 columns are read, so a seg head's
|
|
197
|
+
``[N, 6 + 32]`` rows decode exactly like the plain detector's ``[N, 6]``
|
|
198
|
+
(D737) and the row index finds the detection's mask coefficients."""
|
|
183
199
|
conf_threshold = float(config.get("confidence", 0))
|
|
184
200
|
labels = config.get("labels", COCO_80)
|
|
185
201
|
|
|
186
|
-
detections = []
|
|
202
|
+
detections: list[tuple[int, dict]] = []
|
|
187
203
|
for i in range(output.shape[0]):
|
|
188
|
-
x1, y1, x2, y2, score, class_id = output[i]
|
|
204
|
+
x1, y1, x2, y2, score, class_id = output[i][:6]
|
|
189
205
|
score = float(score)
|
|
190
206
|
if score < conf_threshold:
|
|
191
207
|
continue
|
|
@@ -200,13 +216,13 @@ def _postprocess_yolo11_nms(
|
|
|
200
216
|
oy2 = max(0, min(orig_h, (float(y2) - pad[1]) / scale))
|
|
201
217
|
|
|
202
218
|
label = labels[class_id] if class_id < len(labels) else str(class_id)
|
|
203
|
-
detections.append({
|
|
219
|
+
detections.append((i, {
|
|
204
220
|
"class": label,
|
|
205
221
|
"score": round(score, 4),
|
|
206
222
|
"bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
|
|
207
|
-
})
|
|
223
|
+
}))
|
|
208
224
|
|
|
209
|
-
return
|
|
225
|
+
return detections
|
|
210
226
|
|
|
211
227
|
|
|
212
228
|
def postprocess_yolo(
|
|
@@ -223,12 +239,8 @@ def postprocess_yolo(
|
|
|
223
239
|
1. Raw tensor (ONNX/OpenVINO): [4+numClasses, numBoxes] — needs NMS
|
|
224
240
|
2. CoreML built-in NMS: 'confidence' + 'coordinates' tensors — already post-NMS
|
|
225
241
|
"""
|
|
226
|
-
conf_threshold = float(config.get("confidence", 0))
|
|
227
242
|
labels = config.get("labels", COCO_80)
|
|
228
243
|
num_classes = config.get("numClasses", len(labels))
|
|
229
|
-
# NMS IoU from the step config (operator slider); the 0.45 literal stays as
|
|
230
|
-
# the fallback so an unset value is byte-identical to today (#4).
|
|
231
|
-
nms_iou = float(config.get("nmsIouThreshold", 0.45))
|
|
232
244
|
|
|
233
245
|
yolox = yolox_head_to_v8(predictions, int(config.get("inputSize", 640)))
|
|
234
246
|
if yolox is not None:
|
|
@@ -272,6 +284,31 @@ def postprocess_yolo(
|
|
|
272
284
|
else:
|
|
273
285
|
output = output.T
|
|
274
286
|
|
|
287
|
+
rows = decode_raw_yolo_columns(output, config, orig_w, orig_h, scale, pad)
|
|
288
|
+
return {"kind": "detections", "detections": [d for _, d in rows]}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def decode_raw_yolo_columns(
|
|
292
|
+
output: np.ndarray,
|
|
293
|
+
config: dict,
|
|
294
|
+
orig_w: int,
|
|
295
|
+
orig_h: int,
|
|
296
|
+
scale: float,
|
|
297
|
+
pad: tuple[int, int],
|
|
298
|
+
) -> list[tuple[int, dict]]:
|
|
299
|
+
"""The raw-head box decode (``[4 + numClasses (+ extra), numBoxes]``, one
|
|
300
|
+
COLUMN per candidate): sigmoid when the scores are logits, confidence
|
|
301
|
+
floor, NMS. Each kept detection is paired with its column in ``output``.
|
|
302
|
+
Rows past ``4 + numClasses`` are never read, so a seg head's mask
|
|
303
|
+
coefficients ride along untouched and decode exactly like the plain
|
|
304
|
+
detector (D737). May apply the sigmoid IN PLACE on the class rows."""
|
|
305
|
+
conf_threshold = float(config.get("confidence", 0))
|
|
306
|
+
labels = config.get("labels", COCO_80)
|
|
307
|
+
num_classes = config.get("numClasses", len(labels))
|
|
308
|
+
# NMS IoU from the step config (operator slider); the 0.45 literal stays as
|
|
309
|
+
# the fallback so an unset value is byte-identical to today (#4).
|
|
310
|
+
nms_iou = float(config.get("nmsIouThreshold", 0.45))
|
|
311
|
+
|
|
275
312
|
num_boxes = output.shape[1]
|
|
276
313
|
|
|
277
314
|
# Detect if scores are raw logits (YOLO11/v26) vs sigmoid-activated (YOLOv8/v9).
|
|
@@ -290,7 +327,7 @@ def postprocess_yolo(
|
|
|
290
327
|
best_scores = class_scores[best_classes, np.arange(num_boxes)] # (N,)
|
|
291
328
|
mask = best_scores >= conf_threshold
|
|
292
329
|
if not np.any(mask):
|
|
293
|
-
return
|
|
330
|
+
return []
|
|
294
331
|
|
|
295
332
|
# Gather only above-threshold boxes
|
|
296
333
|
idxs = np.nonzero(mask)[0]
|
|
@@ -315,11 +352,13 @@ def postprocess_yolo(
|
|
|
315
352
|
"score": round(float(sc[k]), 4),
|
|
316
353
|
"bbox": [round(float(x1[k]), 1), round(float(y1[k]), 1), round(float(x2[k]), 1), round(float(y2[k]), 1)],
|
|
317
354
|
"_xyxy": (float(x1[k]), float(y1[k]), float(x2[k]), float(y2[k])),
|
|
355
|
+
"_col": int(idxs[k]),
|
|
318
356
|
})
|
|
319
357
|
|
|
320
358
|
kept = _nms(candidates, nms_iou)
|
|
321
|
-
# Remove internal
|
|
359
|
+
# Remove the internal fields; the column travels beside the detection.
|
|
360
|
+
out: list[tuple[int, dict]] = []
|
|
322
361
|
for d in kept:
|
|
323
362
|
del d["_xyxy"]
|
|
324
|
-
|
|
325
|
-
return
|
|
363
|
+
out.append((d.pop("_col"), d))
|
|
364
|
+
return out
|
|
@@ -1,95 +1,66 @@
|
|
|
1
1
|
"""YOLO-seg postprocessor (YOLO26-seg instance segmentation).
|
|
2
2
|
|
|
3
3
|
Input: raw predictions dict with two outputs:
|
|
4
|
-
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
4
|
+
- detections: post-NMS ``[1, 300, 6 + 32]`` rows
|
|
5
|
+
``[x1, y1, x2, y2, conf, class_id, coeff_0..coeff_31]``, or a raw head
|
|
6
|
+
``[1, 4 + numClasses + 32, numBoxes]`` (one column per candidate)
|
|
7
|
+
- prototypes: ``[1, 32, P, P]`` (P = 160 at a 640 input)
|
|
8
|
+
|
|
9
|
+
Output: ``{"kind": "detections", "detections": [{..., "mask": "<b64>",
|
|
10
|
+
"maskWidth": N, "maskHeight": N}]}`` — the mask is the detection's box region
|
|
11
|
+
of the prototype grid, thresholded to 0/255 bytes, row-major, base64.
|
|
12
|
+
|
|
13
|
+
The BOXES are decoded by the plain detector's own functions
|
|
14
|
+
(``yolo.decode_yolo11_nms_rows`` / ``yolo.decode_raw_yolo_columns``): same
|
|
15
|
+
confidence floor, same sigmoid detection, same NMS (and none on a post-NMS
|
|
16
|
+
head, exactly like ``yolo``), same letterbox inversion and rounding. A seg
|
|
17
|
+
model chosen as the camera ROOT therefore produces the detections a plain
|
|
18
|
+
detector would from the same tensors, plus a mask on each (D737). Masks are
|
|
19
|
+
computed only for the detections that survive, and only over their box.
|
|
11
20
|
"""
|
|
12
21
|
import base64
|
|
13
22
|
|
|
14
23
|
import numpy as np
|
|
15
24
|
|
|
16
|
-
|
|
17
|
-
COCO_80 = [
|
|
18
|
-
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck",
|
|
19
|
-
"boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
|
|
20
|
-
"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
|
|
21
|
-
"giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
|
|
22
|
-
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
|
|
23
|
-
"skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup",
|
|
24
|
-
"fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
|
|
25
|
-
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
|
|
26
|
-
"potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
|
|
27
|
-
"remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
|
|
28
|
-
"refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier",
|
|
29
|
-
"toothbrush",
|
|
30
|
-
]
|
|
25
|
+
from .yolo import COCO_80, decode_raw_yolo_columns, decode_yolo11_nms_rows
|
|
31
26
|
|
|
32
27
|
NUM_MASK_COEFFS = 32
|
|
33
28
|
PROTO_SIZE = 160
|
|
34
29
|
|
|
35
30
|
|
|
36
|
-
def _iou(a: tuple, b: tuple) -> float:
|
|
37
|
-
"""IoU between two boxes (x1, y1, x2, y2)."""
|
|
38
|
-
ax1, ay1, ax2, ay2 = a
|
|
39
|
-
bx1, by1, bx2, by2 = b
|
|
40
|
-
ix1 = max(ax1, bx1)
|
|
41
|
-
iy1 = max(ay1, by1)
|
|
42
|
-
ix2 = min(ax2, bx2)
|
|
43
|
-
iy2 = min(ay2, by2)
|
|
44
|
-
inter = max(0, ix2 - ix1) * max(0, iy2 - iy1)
|
|
45
|
-
if inter == 0:
|
|
46
|
-
return 0.0
|
|
47
|
-
area_a = (ax2 - ax1) * (ay2 - ay1)
|
|
48
|
-
area_b = (bx2 - bx1) * (by2 - by1)
|
|
49
|
-
union = area_a + area_b - inter
|
|
50
|
-
return inter / union if union > 0 else 0.0
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]:
|
|
54
|
-
"""Non-maximum suppression. Input: list of dicts with _xyxy, score, class."""
|
|
55
|
-
if not boxes:
|
|
56
|
-
return []
|
|
57
|
-
sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True)
|
|
58
|
-
kept = []
|
|
59
|
-
suppressed = set()
|
|
60
|
-
for i, box in enumerate(sorted_boxes):
|
|
61
|
-
if i in suppressed:
|
|
62
|
-
continue
|
|
63
|
-
kept.append(box)
|
|
64
|
-
for j in range(i + 1, len(sorted_boxes)):
|
|
65
|
-
if j in suppressed:
|
|
66
|
-
continue
|
|
67
|
-
if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold:
|
|
68
|
-
suppressed.add(j)
|
|
69
|
-
return kept
|
|
70
|
-
|
|
71
|
-
|
|
72
31
|
def _sigmoid(x: np.ndarray) -> np.ndarray:
|
|
73
32
|
"""Numerically stable sigmoid."""
|
|
74
33
|
return np.where(x >= 0, 1 / (1 + np.exp(-x)), np.exp(x) / (1 + np.exp(x)))
|
|
75
34
|
|
|
76
35
|
|
|
36
|
+
def _as_prototypes(arr: np.ndarray) -> np.ndarray | None:
|
|
37
|
+
"""``[32, P, P]`` when ``arr`` is a prototype tensor, else ``None``."""
|
|
38
|
+
squeezed = arr
|
|
39
|
+
while squeezed.ndim > 3 and squeezed.shape[0] == 1:
|
|
40
|
+
squeezed = squeezed[0]
|
|
41
|
+
if squeezed.ndim == 3 and squeezed.shape[0] == NUM_MASK_COEFFS and squeezed.shape[1] == squeezed.shape[2]:
|
|
42
|
+
return squeezed
|
|
43
|
+
if arr.size == NUM_MASK_COEFFS * PROTO_SIZE * PROTO_SIZE:
|
|
44
|
+
return arr.reshape(NUM_MASK_COEFFS, PROTO_SIZE, PROTO_SIZE)
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
77
48
|
def _find_tensors(predictions: dict, num_classes: int) -> tuple[np.ndarray, np.ndarray]:
|
|
78
49
|
"""Locate the detection tensor and prototype tensor from model outputs.
|
|
79
50
|
|
|
80
51
|
Returns (detections, prototypes) as numpy arrays.
|
|
81
52
|
detections: [num_dets, channels] row-major
|
|
82
|
-
prototypes: [32,
|
|
53
|
+
prototypes: [32, P, P]
|
|
83
54
|
"""
|
|
84
55
|
det_tensor = None
|
|
85
56
|
proto_tensor = None
|
|
86
57
|
|
|
87
|
-
for
|
|
58
|
+
for _key, val in predictions.items():
|
|
88
59
|
arr = np.array(val, dtype=np.float32)
|
|
89
60
|
|
|
90
|
-
|
|
91
|
-
if
|
|
92
|
-
proto_tensor =
|
|
61
|
+
protos = _as_prototypes(arr)
|
|
62
|
+
if protos is not None and proto_tensor is None:
|
|
63
|
+
proto_tensor = protos
|
|
93
64
|
continue
|
|
94
65
|
|
|
95
66
|
# Skip tiny tensors
|
|
@@ -128,28 +99,36 @@ def _find_tensors(predictions: dict, num_classes: int) -> tuple[np.ndarray, np.n
|
|
|
128
99
|
return det_tensor, proto_tensor
|
|
129
100
|
|
|
130
101
|
|
|
131
|
-
def
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
102
|
+
def _instance_mask(
|
|
103
|
+
coeffs: np.ndarray,
|
|
104
|
+
protos: np.ndarray,
|
|
105
|
+
bbox: list[float],
|
|
106
|
+
scale: float,
|
|
107
|
+
pad: tuple[int, int],
|
|
137
108
|
input_size: int,
|
|
138
|
-
mask_threshold: float
|
|
139
|
-
) -> tuple[
|
|
140
|
-
"""
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
109
|
+
mask_threshold: float,
|
|
110
|
+
) -> tuple[str, int, int]:
|
|
111
|
+
"""The detection's mask over its OWN box: the emitted (original-image)
|
|
112
|
+
bbox mapped back into the letterboxed input, then onto the prototype grid.
|
|
113
|
+
Only that window of ``coeffs @ protos`` is computed."""
|
|
114
|
+
proto_size = protos.shape[-1]
|
|
115
|
+
proto_scale = proto_size / input_size
|
|
116
|
+
ox1, oy1, ox2, oy2 = bbox
|
|
117
|
+
px1 = max(0, int(np.floor((ox1 * scale + pad[0]) * proto_scale)))
|
|
118
|
+
py1 = max(0, int(np.floor((oy1 * scale + pad[1]) * proto_scale)))
|
|
119
|
+
px2 = min(proto_size, int(np.ceil((ox2 * scale + pad[0]) * proto_scale)))
|
|
120
|
+
py2 = min(proto_size, int(np.ceil((oy2 * scale + pad[1]) * proto_scale)))
|
|
147
121
|
crop_w = max(1, px2 - px1)
|
|
148
122
|
crop_h = max(1, py2 - py1)
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
123
|
+
window = protos[:, py1:py1 + crop_h, px1:px1 + crop_w]
|
|
124
|
+
h, w = window.shape[1], window.shape[2]
|
|
125
|
+
if h == 0 or w == 0:
|
|
126
|
+
binary = np.zeros((crop_h, crop_w), dtype=np.uint8)
|
|
127
|
+
else:
|
|
128
|
+
logits = coeffs.astype(np.float32) @ window.reshape(NUM_MASK_COEFFS, -1)
|
|
129
|
+
binary = (_sigmoid(logits.reshape(h, w)) > mask_threshold).astype(np.uint8) * 255
|
|
130
|
+
crop_h, crop_w = h, w
|
|
131
|
+
return base64.b64encode(binary.tobytes()).decode("ascii"), crop_w, crop_h
|
|
153
132
|
|
|
154
133
|
|
|
155
134
|
def postprocess_yolo_seg(
|
|
@@ -166,85 +145,29 @@ def postprocess_yolo_seg(
|
|
|
166
145
|
1. NMS-enabled export: [N, 6+32] with [x1,y1,x2,y2,conf,class_id,...coeffs]
|
|
167
146
|
2. Raw export: [4+numClasses+32, N] standard YOLO transposed format
|
|
168
147
|
"""
|
|
169
|
-
conf_threshold = float(config.get("confidence", 0))
|
|
170
148
|
labels = config.get("labels", COCO_80)
|
|
171
149
|
num_classes = config.get("numClasses", len(labels))
|
|
172
|
-
input_size = config.get("inputSize", 640)
|
|
173
|
-
mask_threshold = config.get("maskThreshold", 0.5)
|
|
174
|
-
# NMS IoU from the step config (operator slider); the 0.45 literal stays as
|
|
175
|
-
# the fallback so an unset value is byte-identical to today (#4).
|
|
176
|
-
nms_iou = float(config.get("nmsIouThreshold", 0.45))
|
|
150
|
+
input_size = int(config.get("inputSize", 640))
|
|
151
|
+
mask_threshold = float(config.get("maskThreshold", 0.5))
|
|
177
152
|
|
|
178
153
|
det_tensor, proto_tensor = _find_tensors(predictions, num_classes)
|
|
179
154
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
if is_nms_format:
|
|
190
|
-
# [x1, y1, x2, y2, confidence, class_id, coeff_0..coeff_31]
|
|
191
|
-
x1, y1, x2, y2 = float(row[0]), float(row[1]), float(row[2]), float(row[3])
|
|
192
|
-
best_score = float(row[4])
|
|
193
|
-
best_class = int(row[5])
|
|
194
|
-
coeffs = row[6:6 + NUM_MASK_COEFFS]
|
|
195
|
-
|
|
196
|
-
if best_score < conf_threshold:
|
|
197
|
-
continue
|
|
198
|
-
if best_class < 0:
|
|
199
|
-
continue # padding row
|
|
200
|
-
else:
|
|
201
|
-
# [cx, cy, w, h, class_scores..., coeff_0..coeff_31]
|
|
202
|
-
cx, cy, w, h = float(row[0]), float(row[1]), float(row[2]), float(row[3])
|
|
203
|
-
class_scores = row[4:4 + num_classes]
|
|
204
|
-
best_class = int(np.argmax(class_scores))
|
|
205
|
-
best_score = float(class_scores[best_class])
|
|
206
|
-
coeffs = row[4 + num_classes:4 + num_classes + NUM_MASK_COEFFS]
|
|
207
|
-
|
|
208
|
-
if best_score < conf_threshold:
|
|
209
|
-
continue
|
|
210
|
-
|
|
211
|
-
x1 = cx - w / 2
|
|
212
|
-
y1 = cy - h / 2
|
|
213
|
-
x2 = cx + w / 2
|
|
214
|
-
y2 = cy + h / 2
|
|
215
|
-
|
|
216
|
-
# Compute instance mask: sigmoid(coefficients @ prototypes)
|
|
217
|
-
# coeffs: [32], proto_tensor: [32, 160, 160]
|
|
218
|
-
mask_raw = _sigmoid(coeffs @ proto_tensor.reshape(NUM_MASK_COEFFS, -1)).reshape(
|
|
219
|
-
PROTO_SIZE, PROTO_SIZE
|
|
155
|
+
if det_tensor.shape[1] == 6 + NUM_MASK_COEFFS:
|
|
156
|
+
kept = decode_yolo11_nms_rows(det_tensor, config, orig_w, orig_h, scale, pad)
|
|
157
|
+
coeffs = det_tensor[:, 6:6 + NUM_MASK_COEFFS]
|
|
158
|
+
else:
|
|
159
|
+
# The raw decode reads [C, N] and may sigmoid its class rows in place:
|
|
160
|
+
# hand it its own copy so the coefficients below stay untouched.
|
|
161
|
+
kept = decode_raw_yolo_columns(
|
|
162
|
+
np.ascontiguousarray(det_tensor.T), config, orig_w, orig_h, scale, pad
|
|
220
163
|
)
|
|
164
|
+
coeffs = det_tensor[:, 4 + num_classes:4 + num_classes + NUM_MASK_COEFFS]
|
|
221
165
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
166
|
+
detections = []
|
|
167
|
+
for index, det in kept:
|
|
168
|
+
mask_b64, mask_w, mask_h = _instance_mask(
|
|
169
|
+
coeffs[index], proto_tensor, det["bbox"], scale, pad, input_size, mask_threshold
|
|
225
170
|
)
|
|
226
|
-
mask_b64
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
ox1 = max(0, min(orig_w, (x1 - pad[0]) / scale))
|
|
230
|
-
oy1 = max(0, min(orig_h, (y1 - pad[1]) / scale))
|
|
231
|
-
ox2 = max(0, min(orig_w, (x2 - pad[0]) / scale))
|
|
232
|
-
oy2 = max(0, min(orig_h, (y2 - pad[1]) / scale))
|
|
233
|
-
|
|
234
|
-
label = labels[best_class] if best_class < len(labels) else str(best_class)
|
|
235
|
-
candidates.append({
|
|
236
|
-
"class": label,
|
|
237
|
-
"score": round(best_score, 4),
|
|
238
|
-
"bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
|
|
239
|
-
"mask": mask_b64,
|
|
240
|
-
"maskWidth": crop_w,
|
|
241
|
-
"maskHeight": crop_h,
|
|
242
|
-
"_xyxy": (ox1, oy1, ox2, oy2),
|
|
243
|
-
})
|
|
244
|
-
|
|
245
|
-
kept = _nms(candidates, nms_iou)
|
|
246
|
-
# Remove internal _xyxy field
|
|
247
|
-
for d in kept:
|
|
248
|
-
del d["_xyxy"]
|
|
249
|
-
|
|
250
|
-
return {"kind": "detections", "detections": kept}
|
|
171
|
+
detections.append({**det, "mask": mask_b64, "maskWidth": mask_w, "maskHeight": mask_h})
|
|
172
|
+
|
|
173
|
+
return {"kind": "detections", "detections": detections}
|