@camstack/addon-detection-pipeline 0.1.1

package/python/postprocessors/__init__.py

@@ -0,0 +1,24 @@
+"""Postprocessor dispatch table for inference_pool.py.
+
+Each postprocessor takes raw model output (dict of numpy arrays) and model config,
+returns a structured dict matching the StepOutput JSON format.
+"""
+from .yolo import postprocess_yolo
+from .yolo_seg import postprocess_yolo_seg
+from .scrfd import postprocess_scrfd
+from .arcface import postprocess_arcface
+from .softmax import postprocess_softmax
+from .ctc import postprocess_ctc
+from .saliency import postprocess_saliency
+from .yamnet import postprocess_yamnet
+
+POSTPROCESSORS = {
+    "yolo": postprocess_yolo,
+    "yolo-seg": postprocess_yolo_seg,
+    "scrfd": postprocess_scrfd,
+    "arcface": postprocess_arcface,
+    "softmax": postprocess_softmax,
+    "ctc": postprocess_ctc,
+    "saliency": postprocess_saliency,
+    "yamnet": postprocess_yamnet,
+}

package/python/postprocessors/arcface.py

@@ -0,0 +1,31 @@
+"""ArcFace face embedding postprocessor.
+
+Input: raw embedding tensor from ArcFace model
+Output: {"kind": "embedding", "embedding": [...], "embeddingNorm": [...]}
+"""
+import numpy as np
+
+
+def postprocess_arcface(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """L2-normalize the embedding vector."""
+    # Get the first output tensor (usually named "fc1" or similar)
+    raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32)
+
+    norm = np.linalg.norm(raw)
+    if norm == 0:
+        normalized = np.zeros_like(raw)
+    else:
+        normalized = raw / norm
+
+    return {
+        "kind": "embedding",
+        "embedding": raw.tolist(),
+        "embeddingNorm": normalized.tolist(),
+    }

package/python/postprocessors/ctc.py

@@ -0,0 +1,68 @@
+"""CTC text recognition postprocessor.
+
+Used for: vgg_english_g2 (plate OCR).
+
+Input: [1, seqLen, numChars] logits
+Output: {"kind": "text", "text": "ABC123", "confidence": 0.95}
+
+Algorithm: argmax per timestep → collapse consecutive duplicates → remove blank (index 0) → join
+"""
+import numpy as np
+
+
+def postprocess_ctc(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """CTC greedy decode."""
+    charset = config.get("charset", [])
+
+    # Get first output tensor
+    raw = np.array(list(predictions.values())[0]).astype(np.float32)
+
+    # Handle shapes: [1, T, C] or [T, C]
+    if raw.ndim == 3:
+        raw = raw[0]  # remove batch
+    # raw is now [T, C] — logits (not probabilities)
+    seq_len, num_chars = raw.shape
+
+    # Apply softmax to get probabilities
+    raw_max = np.max(raw, axis=1, keepdims=True)
+    exp_raw = np.exp(raw - raw_max)
+    probs = exp_raw / np.sum(exp_raw, axis=1, keepdims=True)
+
+    # Argmax per timestep
+    best_indices = np.argmax(probs, axis=1)  # [T]
+    best_scores = probs[np.arange(seq_len), best_indices]  # [T]
+
+    # Collapse consecutive duplicates
+    collapsed = []
+    collapsed_scores = []
+    for i, idx in enumerate(best_indices):
+        if i == 0 or idx != best_indices[i - 1]:
+            collapsed.append(int(idx))
+            collapsed_scores.append(float(best_scores[i]))
+
+    # Remove blank (index 0) and join
+    chars = []
+    scores = []
+    for idx, score in zip(collapsed, collapsed_scores):
+        if idx != 0:
+            if idx < len(charset):
+                chars.append(charset[idx])
+            else:
+                chars.append(chr(idx))
+            scores.append(score)
+
+    text = "".join(chars)
+    confidence = float(np.mean(scores)) if scores else 0.0
+
+    return {
+        "kind": "text",
+        "text": text,
+        "confidence": round(confidence, 4),
+    }

package/python/postprocessors/saliency.py

@@ -0,0 +1,44 @@
+"""Saliency / segmentation refiner postprocessor.
+
+Used for: u2netp.
+
+Input: raw saliency map tensor
+Output: {"kind": "mask", "mask": "<base64>", "maskWidth": N, "maskHeight": N}
+"""
+import base64
+import numpy as np
+
+
+def postprocess_saliency(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Sigmoid → threshold → binary mask → base64."""
+    mask_threshold = config.get("maskThreshold", 0.5)
+
+    # U2-Net outputs multiple saliency maps; use the first (highest resolution)
+    raw = np.array(list(predictions.values())[0]).astype(np.float32)
+
+    # Remove batch/channel dims to get [H, W]
+    while raw.ndim > 2:
+        raw = raw[0]
+
+    # Sigmoid
+    saliency = 1.0 / (1.0 + np.exp(-raw))
+
+    # Threshold to binary uint8 (0 or 255)
+    binary = (saliency > mask_threshold).astype(np.uint8) * 255
+
+    h, w = binary.shape
+    mask_b64 = base64.b64encode(binary.tobytes()).decode("ascii")
+
+    return {
+        "kind": "mask",
+        "mask": mask_b64,
+        "maskWidth": int(w),
+        "maskHeight": int(h),
+    }

package/python/postprocessors/scrfd.py

@@ -0,0 +1,212 @@
+"""SCRFD face detection postprocessor.
+
+Multi-stride anchor-based face detector (strides 8, 16, 32).
+Ported from packages/addon-vision/src/shared/postprocess/scrfd.ts.
+
+Output: {"kind": "detections", "detections": [{"class": "face", "score", "bbox": [x1,y1,x2,y2], "landmarks?"}]}
+"""
+import numpy as np
+
+
+STRIDES = [8, 16, 32]
+NUM_ANCHORS_PER_STRIDE = 2
+
+
+def _generate_anchors(stride: int, input_size: int) -> list[tuple[float, float]]:
+    """Generate anchor centers for a given stride."""
+    feat_size = int(np.ceil(input_size / stride))
+    anchors = []
+    for y in range(feat_size):
+        for x in range(feat_size):
+            for _ in range(NUM_ANCHORS_PER_STRIDE):
+                anchors.append(((x + 0.5) * stride, (y + 0.5) * stride))
+    return anchors
+
+
+def _iou(a: tuple, b: tuple) -> float:
+    ax1, ay1, ax2, ay2 = a
+    bx1, by1, bx2, by2 = b
+    ix1, iy1 = max(ax1, bx1), max(ay1, by1)
+    ix2, iy2 = min(ax2, bx2), min(ay2, by2)
+    inter = max(0, ix2 - ix1) * max(0, iy2 - iy1)
+    if inter == 0:
+        return 0.0
+    union = (ax2 - ax1) * (ay2 - ay1) + (bx2 - bx1) * (by2 - by1) - inter
+    return inter / union if union > 0 else 0.0
+
+
+def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]:
+    if not boxes:
+        return []
+    sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True)
+    kept = []
+    suppressed = set()
+    for i, box in enumerate(sorted_boxes):
+        if i in suppressed:
+            continue
+        kept.append(box)
+        for j in range(i + 1, len(sorted_boxes)):
+            if j in suppressed:
+                continue
+            if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold:
+                suppressed.add(j)
+    return kept
+
+
+def _match_outputs_to_strides(
+    raw_outputs: dict[str, np.ndarray],
+    strides: list[int],
+    input_size: int,
+) -> dict[int, dict]:
+    """Match model output tensors to strides.
+
+    Handles two naming conventions:
+    1. Named keys: 'score_8', 'bbox_8', 'kps_8' (ONNX/OpenVINO)
+    2. Generic keys: 'var_732' etc. — matched by shape (CoreML)
+
+    For CoreML, each stride has 3 tensors with num_anchors matching the stride:
+      stride 8  → ceil(input/8)^2 * 2 anchors → shapes (1, N, 1), (1, N, 4), (1, N, 10)
+      stride 16 → ceil(input/16)^2 * 2
+      stride 32 → ceil(input/32)^2 * 2
+    """
+    result: dict[int, dict] = {}
+
+    # Try named keys first
+    flat = {k: v.flatten() for k, v in raw_outputs.items()}
+    for stride in strides:
+        score_key = bbox_key = kps_key = None
+        for k in flat:
+            if f"score_{stride}" in k or f"_{stride}_score" in k:
+                score_key = k
+            elif f"bbox_{stride}" in k or f"_{stride}_bbox" in k:
+                bbox_key = k
+            elif f"kps_{stride}" in k or f"_{stride}_kps" in k:
+                kps_key = k
+        if score_key and bbox_key:
+            result[stride] = {
+                "scores": flat[score_key],
+                "bboxes": flat[bbox_key],
+                "kps": flat.get(kps_key),
+            }
+
+    if len(result) == len(strides):
+        return result
+
+    # Fallback: match by shape (CoreML generic names)
+    # Group outputs by last dimension: 1=scores, 4=bboxes, 10=landmarks
+    scores_by_n: dict[int, np.ndarray] = {}
+    bboxes_by_n: dict[int, np.ndarray] = {}
+    kps_by_n: dict[int, np.ndarray] = {}
+
+    for _k, arr in raw_outputs.items():
+        if arr.ndim == 3:
+            arr = arr[0]  # remove batch dim → (N, C)
+        elif arr.ndim == 1:
+            continue
+        n, c = arr.shape
+        if c == 1:
+            scores_by_n[n] = arr.flatten()
+        elif c == 4:
+            bboxes_by_n[n] = arr.flatten()
+        elif c == 10:
+            kps_by_n[n] = arr.flatten()
+
+    result = {}
+    for stride in strides:
+        feat = int(np.ceil(input_size / stride))
+        expected_n = feat * feat * NUM_ANCHORS_PER_STRIDE
+        if expected_n in scores_by_n and expected_n in bboxes_by_n:
+            result[stride] = {
+                "scores": scores_by_n[expected_n],
+                "bboxes": bboxes_by_n[expected_n],
+                "kps": kps_by_n.get(expected_n),
+            }
+
+    return result
+
+
+def postprocess_scrfd(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Postprocess SCRFD output to structured face detections."""
+    conf_threshold = float(config.get("confidence", 0.5) or 0.5)
+    input_size = config.get("inputSize", 640)
+    strides = config.get("strides", STRIDES)
+
+    # Letterbox transform: scale and pad are passed from inference_pool
+    # scale = input_size / max(orig_w, orig_h)
+    # pad = ((input_size - orig_w*scale)//2, (input_size - orig_h*scale)//2)
+
+    # Convert all outputs to numpy arrays (keep ndim)
+    raw_outputs = {k: np.array(v) for k, v in predictions.items()}
+
+    # Match outputs to strides — by named key or by shape
+    stride_data = _match_outputs_to_strides(raw_outputs, strides, input_size)
+
+    candidates = []
+
+    for stride in strides:
+        data = stride_data.get(stride)
+        if data is None:
+            continue
+
+        scores = data["scores"]
+        bboxes = data["bboxes"]
+        kps = data.get("kps")
+        anchors = _generate_anchors(stride, input_size)
+
+        n = len(anchors)
+        for i in range(n):
+            if i >= len(scores):
+                break
+            score = float(scores[i])
+            if score < conf_threshold:
+                continue
+
+            cx, cy = anchors[i]
+
+            # Bbox: offsets from anchor center, scaled by stride
+            x1 = cx - float(bboxes[i * 4]) * stride
+            y1 = cy - float(bboxes[i * 4 + 1]) * stride
+            x2 = cx + float(bboxes[i * 4 + 2]) * stride
+            y2 = cy + float(bboxes[i * 4 + 3]) * stride
+
+            # Transform from letterbox coords to original image
+            ox1 = max(0, min(orig_w, (x1 - pad[0]) / scale))
+            oy1 = max(0, min(orig_h, (y1 - pad[1]) / scale))
+            ox2 = max(0, min(orig_w, (x2 - pad[0]) / scale))
+            oy2 = max(0, min(orig_h, (y2 - pad[1]) / scale))
+
+            if (ox2 - ox1) < 1 or (oy2 - oy1) < 1:
+                continue
+
+            det = {
+                "class": "face",
+                "score": round(score, 4),
+                "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
+                "_xyxy": (ox1, oy1, ox2, oy2),
+            }
+
+            # Landmarks (5 points × 2 coords)
+            if kps is not None and (i * 10 + 9) < len(kps):
+                landmarks = []
+                for p in range(5):
+                    raw_lx = cx + float(kps[i * 10 + p * 2]) * stride
+                    raw_ly = cy + float(kps[i * 10 + p * 2 + 1]) * stride
+                    lx = (raw_lx - pad[0]) / scale
+                    ly = (raw_ly - pad[1]) / scale
+                    landmarks.append({"x": round(lx, 1), "y": round(ly, 1)})
+                det["landmarks"] = landmarks
+
+            candidates.append(det)
+
+    kept = _nms(candidates)
+    for d in kept:
+        del d["_xyxy"]
+
+    return {"kind": "detections", "detections": kept}

package/python/postprocessors/softmax.py

@@ -0,0 +1,43 @@
+"""Softmax classifier postprocessor.
+
+Used for: animals-10, bird-nabirds-404, vehicle-type-efficientnet.
+
+Input: raw logits tensor
+Output: {"kind": "classifications", "classifications": [{"class", "score"}]}
+"""
+import numpy as np
+
+
+def postprocess_softmax(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Softmax + argmax + label lookup."""
+    labels = config.get("labels", [])
+
+    # Get first output tensor
+    raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32)
+
+    # Stable softmax
+    shifted = raw - np.max(raw)
+    exps = np.exp(shifted)
+    probs = exps / np.sum(exps)
+
+    # Return top-K classifications — normalization (top-1 + alternates) happens in TypeScript
+    top_k = min(5, len(probs))
+    top_indices = np.argsort(probs)[::-1][:top_k]
+
+    classifications = []
+    for idx in top_indices:
+        idx = int(idx)
+        score = float(probs[idx])
+        if score < 0.01:
+            break
+        label = labels[idx] if idx < len(labels) else str(idx)
+        classifications.append({"class": label, "score": round(score, 4)})
+
+    return {"kind": "classifications", "classifications": classifications}

package/python/postprocessors/yamnet.py

@@ -0,0 +1,41 @@
+"""YAMNet audio classification postprocessor.
+
+Input: [numFrames, 521] score matrix
+Output: {"kind": "classifications", "classifications": [{"class", "score"}]}
+"""
+import numpy as np
+
+
+def postprocess_yamnet(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Average scores across frames, return top classes above threshold."""
+    labels = config.get("labels", [])
+    min_score = config.get("confidence", 0.1)
+
+    # Get output_0 (scores) — shape [numFrames, 521]
+    raw = np.array(list(predictions.values())[0]).astype(np.float32)
+    if raw.ndim == 1:
+        raw = raw.reshape(1, -1)
+
+    num_frames, num_classes = raw.shape
+
+    # Average across frames
+    avg_scores = np.mean(raw, axis=0)
+
+    # Collect classes above threshold, sorted by score
+    results = []
+    for c in range(num_classes):
+        score = float(avg_scores[c])
+        if score >= min_score:
+            label = labels[c] if c < len(labels) else str(c)
+            results.append({"class": label, "score": round(score, 4)})
+
+    results.sort(key=lambda x: x["score"], reverse=True)
+
+    return {"kind": "classifications", "classifications": results}