@camstack/addon-detection-pipeline 0.1.1

package/python/postprocessors/yolo.py

@@ -0,0 +1,278 @@
+"""YOLO v8/v9/v11 postprocessor.
+
+Input: raw predictions dict from CoreML/OpenVINO/ONNX
+Output: {"kind": "detections", "detections": [{"class", "score", "bbox": [x1,y1,x2,y2]}]}
+
+Handles both standard and built-in NMS outputs.
+"""
+import numpy as np
+
+COCO_80 = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck",
+    "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
+    "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
+    "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+    "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
+    "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup",
+    "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
+    "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
+    "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
+    "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
+    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier",
+    "toothbrush",
+]
+
+
+def _iou(a: tuple, b: tuple) -> float:
+    """IoU between two boxes (x1, y1, x2, y2)."""
+    ax1, ay1, ax2, ay2 = a
+    bx1, by1, bx2, by2 = b
+    ix1 = max(ax1, bx1)
+    iy1 = max(ay1, by1)
+    ix2 = min(ax2, bx2)
+    iy2 = min(ay2, by2)
+    inter = max(0, ix2 - ix1) * max(0, iy2 - iy1)
+    if inter == 0:
+        return 0.0
+    area_a = (ax2 - ax1) * (ay2 - ay1)
+    area_b = (bx2 - bx1) * (by2 - by1)
+    union = area_a + area_b - inter
+    return inter / union if union > 0 else 0.0
+
+
+def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]:
+    """Non-maximum suppression. Input: list of {bbox, score, class}."""
+    if not boxes:
+        return []
+    sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True)
+    kept = []
+    suppressed = set()
+    for i, box in enumerate(sorted_boxes):
+        if i in suppressed:
+            continue
+        kept.append(box)
+        for j in range(i + 1, len(sorted_boxes)):
+            if j in suppressed:
+                continue
+            if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold:
+                suppressed.add(j)
+    return kept
+
+
+def _postprocess_coreml_nms(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    input_size: int,
+) -> dict:
+    """Handle CoreML models with built-in NMS.
+
+    CoreML NMS output has two tensors:
+      - 'confidence': (N, numClasses) — class scores post-NMS
+      - 'coordinates': (N, 4) — [cx, cy, w, h] normalized to input image size
+    """
+    conf_threshold = float(config.get("confidence", 0))
+    labels = config.get("labels", COCO_80)
+
+    confidence = np.array(predictions["confidence"])
+    coordinates = np.array(predictions["coordinates"])
+
+    if confidence.ndim == 1:
+        confidence = confidence.reshape(1, -1)
+    if coordinates.ndim == 1:
+        coordinates = coordinates.reshape(1, -1)
+
+    detections = []
+    for i in range(confidence.shape[0]):
+        best_class = int(np.argmax(confidence[i]))
+        best_score = float(confidence[i, best_class])
+
+        if best_score < conf_threshold:
+            continue
+
+        # CoreML coordinates: [cx, cy, w, h] normalized to input_size
+        cx, cy, bw, bh = coordinates[i]
+        # Scale to input pixel coords
+        cx_px = cx * input_size
+        cy_px = cy * input_size
+        bw_px = bw * input_size
+        bh_px = bh * input_size
+
+        x1 = cx_px - bw_px / 2
+        y1 = cy_px - bh_px / 2
+        x2 = cx_px + bw_px / 2
+        y2 = cy_px + bh_px / 2
+
+        # Map back to original image coords (undo letterbox)
+        # CoreML imageType with letterbox: the model sees the padded image
+        # so we need to figure out the letterbox transform
+        scale = input_size / max(orig_w, orig_h)
+        new_w = int(orig_w * scale)
+        new_h = int(orig_h * scale)
+        pad_x = (input_size - new_w) / 2
+        pad_y = (input_size - new_h) / 2
+
+        ox1 = max(0, min(orig_w, (x1 - pad_x) / scale))
+        oy1 = max(0, min(orig_h, (y1 - pad_y) / scale))
+        ox2 = max(0, min(orig_w, (x2 - pad_x) / scale))
+        oy2 = max(0, min(orig_h, (y2 - pad_y) / scale))
+
+        label = labels[best_class] if best_class < len(labels) else str(best_class)
+        detections.append({
+            "class": label,
+            "score": round(best_score, 4),
+            "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
+        })
+
+    return {"kind": "detections", "detections": detections}
+
+
+def _postprocess_yolo11_nms(
+    output: np.ndarray,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Handle YOLO11/v26 post-NMS output: [N, 6] — (x1, y1, x2, y2, score, class_id).
+
+    Coordinates are in input pixel space (0..inputSize). Already post-NMS.
+    """
+    conf_threshold = float(config.get("confidence", 0))
+    labels = config.get("labels", COCO_80)
+
+    detections = []
+    for i in range(output.shape[0]):
+        x1, y1, x2, y2, score, class_id = output[i]
+        score = float(score)
+        if score < conf_threshold:
+            continue
+        class_id = int(class_id)
+        if class_id < 0:
+            continue
+
+        # Transform from letterbox pixel coords to original image
+        ox1 = max(0, min(orig_w, (float(x1) - pad[0]) / scale))
+        oy1 = max(0, min(orig_h, (float(y1) - pad[1]) / scale))
+        ox2 = max(0, min(orig_w, (float(x2) - pad[0]) / scale))
+        oy2 = max(0, min(orig_h, (float(y2) - pad[1]) / scale))
+
+        label = labels[class_id] if class_id < len(labels) else str(class_id)
+        detections.append({
+            "class": label,
+            "score": round(score, 4),
+            "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
+        })
+
+    return {"kind": "detections", "detections": detections}
+
+
+def postprocess_yolo(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Postprocess YOLO output to structured detections.
+
+    Handles two formats:
+    1. Raw tensor (ONNX/OpenVINO): [4+numClasses, numBoxes] — needs NMS
+    2. CoreML built-in NMS: 'confidence' + 'coordinates' tensors — already post-NMS
+    """
+    conf_threshold = float(config.get("confidence", 0))
+    labels = config.get("labels", COCO_80)
+    num_classes = config.get("numClasses", len(labels))
+
+    # Detect CoreML NMS output format
+    if "confidence" in predictions and "coordinates" in predictions:
+        input_size = config.get("inputSize", 640)
+        return _postprocess_coreml_nms(predictions, config, orig_w, orig_h, input_size)
+
+    # Detect YOLO11/v26 post-NMS format: [batch, N, 6] — (x1, y1, x2, y2, score, class_id)
+    first_arr = np.array(list(predictions.values())[0])
+    if first_arr.ndim == 3:
+        first_arr = first_arr[0]  # remove batch dim
+    if first_arr.ndim == 2 and first_arr.shape[1] == 6:
+        return _postprocess_yolo11_nms(first_arr, config, orig_w, orig_h, scale, pad)
+
+    # Raw tensor path (ONNX / OpenVINO / CoreML without NMS)
+    output = None
+    for key, val in predictions.items():
+        arr = np.array(val)
+        if arr.ndim >= 2 and any(d in (4 + num_classes, 84, 116, 144) for d in arr.shape):
+            output = arr
+            break
+    if output is None:
+        output = np.array(list(predictions.values())[0])
+
+    # Handle various shapes
+    if output.ndim == 3:
+        output = output[0]  # remove batch dim
+
+    # YOLO output: [4+numClasses, numBoxes] or [numBoxes, 4+numClasses]
+    if output.shape[0] == 4 + num_classes:
+        pass  # already [C, N]
+    elif output.shape[1] == 4 + num_classes:
+        output = output.T  # transpose to [C, N]
+    else:
+        # Try to infer
+        if output.shape[0] < output.shape[1]:
+            pass  # likely [C, N]
+        else:
+            output = output.T
+
+    num_boxes = output.shape[1]
+
+    # Detect if scores are raw logits (YOLO11/v26) vs sigmoid-activated (YOLOv8/v9).
+    # Sigmoid-activated scores are in [0, 1]. If any class score exceeds 1.0,
+    # the model outputs raw logits and we need to apply sigmoid.
+    sample_scores = output[4:4 + num_classes, :min(100, num_boxes)]
+    needs_sigmoid = float(np.max(sample_scores)) > 1.0 or float(np.min(sample_scores)) < 0.0
+    if needs_sigmoid:
+        # Apply sigmoid to all class scores in-place (rows 4..4+numClasses)
+        output[4:4 + num_classes, :] = 1.0 / (1.0 + np.exp(-np.clip(output[4:4 + num_classes, :], -50, 50)))
+
+    # Vectorised candidate extraction — replaces the per-box Python loop
+    # that took ~20ms on 8400 boxes. Pure numpy: ~0.5ms.
+    class_scores = output[4:4 + num_classes, :]            # (C, N)
+    best_classes = np.argmax(class_scores, axis=0)         # (N,)
+    best_scores = class_scores[best_classes, np.arange(num_boxes)]  # (N,)
+    mask = best_scores >= conf_threshold
+    if not np.any(mask):
+        return {"kind": "detections", "detections": []}
+
+    # Gather only above-threshold boxes
+    idxs = np.nonzero(mask)[0]
+    cx = output[0, idxs]
+    cy = output[1, idxs]
+    bw = output[2, idxs]
+    bh = output[3, idxs]
+    sc = best_scores[idxs]
+    cl = best_classes[idxs]
+
+    # Centre → corner, letterbox → original coords
+    x1 = np.clip((cx - bw / 2 - pad[0]) / scale, 0, orig_w)
+    y1 = np.clip((cy - bh / 2 - pad[1]) / scale, 0, orig_h)
+    x2 = np.clip((cx + bw / 2 - pad[0]) / scale, 0, orig_w)
+    y2 = np.clip((cy + bh / 2 - pad[1]) / scale, 0, orig_h)
+
+    candidates = []
+    for k in range(len(idxs)):
+        ci = int(cl[k])
+        candidates.append({
+            "class": labels[ci] if ci < len(labels) else str(ci),
+            "score": round(float(sc[k]), 4),
+            "bbox": [round(float(x1[k]), 1), round(float(y1[k]), 1), round(float(x2[k]), 1), round(float(y2[k]), 1)],
+            "_xyxy": (float(x1[k]), float(y1[k]), float(x2[k]), float(y2[k])),
+        })
+
+    kept = _nms(candidates)
+    # Remove internal _xyxy field
+    for d in kept:
+        del d["_xyxy"]
+
+    return {"kind": "detections", "detections": kept}

package/python/postprocessors/yolo_seg.py

@@ -0,0 +1,247 @@
+"""YOLO-seg postprocessor (YOLO26-seg instance segmentation).
+
+Input: raw predictions dict with two outputs:
+  - output0: [1, 300, 38] — 300 NMS-filtered detections, each [x1,y1,x2,y2,conf,class_id,coeff_0..coeff_31]
+  - output1: [1, 32, 160, 160] — 32 prototype masks at 160x160
+
+For raw (no NMS) models:
+  - output0: [1, 4+numClasses+32, numBoxes] — standard YOLO format with mask coefficients appended
+
+Output: {"kind": "detections", "detections": [{..., "mask": "<b64>", "maskWidth": N, "maskHeight": N}]}
+"""
+import base64
+
+import numpy as np
+
+
+COCO_80 = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck",
+    "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
+    "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
+    "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+    "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
+    "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup",
+    "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
+    "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
+    "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
+    "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
+    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier",
+    "toothbrush",
+]
+
+NUM_MASK_COEFFS = 32
+PROTO_SIZE = 160
+
+
+def _iou(a: tuple, b: tuple) -> float:
+    """IoU between two boxes (x1, y1, x2, y2)."""
+    ax1, ay1, ax2, ay2 = a
+    bx1, by1, bx2, by2 = b
+    ix1 = max(ax1, bx1)
+    iy1 = max(ay1, by1)
+    ix2 = min(ax2, bx2)
+    iy2 = min(ay2, by2)
+    inter = max(0, ix2 - ix1) * max(0, iy2 - iy1)
+    if inter == 0:
+        return 0.0
+    area_a = (ax2 - ax1) * (ay2 - ay1)
+    area_b = (bx2 - bx1) * (by2 - by1)
+    union = area_a + area_b - inter
+    return inter / union if union > 0 else 0.0
+
+
+def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]:
+    """Non-maximum suppression. Input: list of dicts with _xyxy, score, class."""
+    if not boxes:
+        return []
+    sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True)
+    kept = []
+    suppressed = set()
+    for i, box in enumerate(sorted_boxes):
+        if i in suppressed:
+            continue
+        kept.append(box)
+        for j in range(i + 1, len(sorted_boxes)):
+            if j in suppressed:
+                continue
+            if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold:
+                suppressed.add(j)
+    return kept
+
+
+def _sigmoid(x: np.ndarray) -> np.ndarray:
+    """Numerically stable sigmoid."""
+    return np.where(x >= 0, 1 / (1 + np.exp(-x)), np.exp(x) / (1 + np.exp(x)))
+
+
+def _find_tensors(predictions: dict, num_classes: int) -> tuple[np.ndarray, np.ndarray]:
+    """Locate the detection tensor and prototype tensor from model outputs.
+
+    Returns (detections, prototypes) as numpy arrays.
+    detections: [num_dets, channels] row-major
+    prototypes: [32, 160, 160]
+    """
+    det_tensor = None
+    proto_tensor = None
+
+    for key, val in predictions.items():
+        arr = np.array(val, dtype=np.float32)
+
+        # Prototype: shape containing 32 * 160 * 160 = 819200
+        if arr.size == NUM_MASK_COEFFS * PROTO_SIZE * PROTO_SIZE:
+            proto_tensor = arr.reshape(NUM_MASK_COEFFS, PROTO_SIZE, PROTO_SIZE)
+            continue
+
+        # Skip tiny tensors
+        if arr.size < 10:
+            continue
+
+        # Detection tensor: the other large tensor
+        if det_tensor is None:
+            det_tensor = arr
+
+    if det_tensor is None:
+        raise ValueError("YOLO-seg postprocessor: could not find detection tensor")
+    if proto_tensor is None:
+        raise ValueError("YOLO-seg postprocessor: could not find prototype tensor")
+
+    # Normalize detection tensor to [num_dets, channels]
+    while det_tensor.ndim > 2:
+        det_tensor = det_tensor[0]  # strip batch dim
+
+    # NMS format: [300, 38] — rows are detections
+    # Raw format: [4+nc+32, N] — cols are detections, needs transpose
+    nms_cols = 6 + NUM_MASK_COEFFS  # 38
+    raw_cols = 4 + num_classes + NUM_MASK_COEFFS
+
+    if det_tensor.shape[1] == nms_cols:
+        pass  # already [N, 38]
+    elif det_tensor.shape[0] == nms_cols:
+        det_tensor = det_tensor.T
+    elif det_tensor.shape[1] == raw_cols:
+        pass  # already [N, raw_cols]
+    elif det_tensor.shape[0] == raw_cols:
+        det_tensor = det_tensor.T
+    elif det_tensor.shape[0] < det_tensor.shape[1]:
+        det_tensor = det_tensor.T  # likely [C, N] -> [N, C]
+
+    return det_tensor, proto_tensor
+
+
+def _crop_mask_to_bbox(
+    mask_full: np.ndarray,
+    x1: float,
+    y1: float,
+    x2: float,
+    y2: float,
+    input_size: int,
+    mask_threshold: float = 0.5,
+) -> tuple[np.ndarray, int, int]:
+    """Crop sigmoid mask to bbox region, threshold to binary, return (binary, w, h)."""
+    proto_scale = PROTO_SIZE / input_size
+    px1 = max(0, int(np.floor(x1 * proto_scale)))
+    py1 = max(0, int(np.floor(y1 * proto_scale)))
+    px2 = min(PROTO_SIZE, int(np.ceil(x2 * proto_scale)))
+    py2 = min(PROTO_SIZE, int(np.ceil(y2 * proto_scale)))
+
+    crop_w = max(1, px2 - px1)
+    crop_h = max(1, py2 - py1)
+
+    cropped = mask_full[py1:py1 + crop_h, px1:px1 + crop_w]
+    binary = (cropped > mask_threshold).astype(np.uint8) * 255
+    return binary, crop_w, crop_h
+
+
+def postprocess_yolo_seg(
+    predictions: dict,
+    config: dict,
+    orig_w: int,
+    orig_h: int,
+    scale: float,
+    pad: tuple[int, int],
+) -> dict:
+    """Postprocess YOLO-seg output to detections with per-instance masks.
+
+    Handles:
+    1. NMS-enabled export: [N, 6+32] with [x1,y1,x2,y2,conf,class_id,...coeffs]
+    2. Raw export: [4+numClasses+32, N] standard YOLO transposed format
+    """
+    conf_threshold = float(config.get("confidence", 0))
+    labels = config.get("labels", COCO_80)
+    num_classes = config.get("numClasses", len(labels))
+    input_size = config.get("inputSize", 640)
+    mask_threshold = config.get("maskThreshold", 0.5)
+
+    det_tensor, proto_tensor = _find_tensors(predictions, num_classes)
+
+    num_dets = det_tensor.shape[0]
+    num_cols = det_tensor.shape[1]
+    is_nms_format = num_cols == 6 + NUM_MASK_COEFFS
+
+    candidates = []
+
+    for i in range(num_dets):
+        row = det_tensor[i]
+
+        if is_nms_format:
+            # [x1, y1, x2, y2, confidence, class_id, coeff_0..coeff_31]
+            x1, y1, x2, y2 = float(row[0]), float(row[1]), float(row[2]), float(row[3])
+            best_score = float(row[4])
+            best_class = int(row[5])
+            coeffs = row[6:6 + NUM_MASK_COEFFS]
+
+            if best_score < conf_threshold:
+                continue
+            if best_class < 0:
+                continue  # padding row
+        else:
+            # [cx, cy, w, h, class_scores..., coeff_0..coeff_31]
+            cx, cy, w, h = float(row[0]), float(row[1]), float(row[2]), float(row[3])
+            class_scores = row[4:4 + num_classes]
+            best_class = int(np.argmax(class_scores))
+            best_score = float(class_scores[best_class])
+            coeffs = row[4 + num_classes:4 + num_classes + NUM_MASK_COEFFS]
+
+            if best_score < conf_threshold:
+                continue
+
+            x1 = cx - w / 2
+            y1 = cy - h / 2
+            x2 = cx + w / 2
+            y2 = cy + h / 2
+
+        # Compute instance mask: sigmoid(coefficients @ prototypes)
+        # coeffs: [32], proto_tensor: [32, 160, 160]
+        mask_raw = _sigmoid(coeffs @ proto_tensor.reshape(NUM_MASK_COEFFS, -1)).reshape(
+            PROTO_SIZE, PROTO_SIZE
+        )
+
+        # Crop mask to bbox, threshold to binary
+        binary, crop_w, crop_h = _crop_mask_to_bbox(
+            mask_raw, x1, y1, x2, y2, input_size, mask_threshold
+        )
+        mask_b64 = base64.b64encode(binary.tobytes()).decode("ascii")
+
+        # Transform bbox from model coords to original image coords
+        ox1 = max(0, min(orig_w, (x1 - pad[0]) / scale))
+        oy1 = max(0, min(orig_h, (y1 - pad[1]) / scale))
+        ox2 = max(0, min(orig_w, (x2 - pad[0]) / scale))
+        oy2 = max(0, min(orig_h, (y2 - pad[1]) / scale))
+
+        label = labels[best_class] if best_class < len(labels) else str(best_class)
+        candidates.append({
+            "class": label,
+            "score": round(best_score, 4),
+            "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)],
+            "mask": mask_b64,
+            "maskWidth": crop_w,
+            "maskHeight": crop_h,
+            "_xyxy": (ox1, oy1, ox2, oy2),
+        })
+
+    kept = _nms(candidates)
+    # Remove internal _xyxy field
+    for d in kept:
+        del d["_xyxy"]
+
+    return {"kind": "detections", "detections": kept}

package/python/requirements-coreml.txt

@@ -0,0 +1,4 @@
+# CoreML backend (macOS only). coremltools 8.x supports Python 3.9-3.12
+# and ships universal2 wheels for darwin. Pulls tensorflow as transitive,
+# so first install is ~150 MB on disk.
+coremltools>=8.0,<9

package/python/requirements-onnxruntime.txt

@@ -0,0 +1,3 @@
+# ONNX Runtime backend. Ships wheels for darwin/linux/windows on
+# arm64+x86_64 for Python 3.10-3.13.
+onnxruntime>=1.20,<2

package/python/requirements-openvino.txt

@@ -0,0 +1,3 @@
+# OpenVINO backend (Linux/Windows; macOS support is x86_64 only).
+# Wheels available for Python 3.10-3.12.
+openvino>=2025,<2026

package/python/requirements.txt

@@ -0,0 +1,9 @@
+# Base requirements for inference_pool.py — installed by camstack
+# at addon boot via ctx.deps.installPythonRequirements(...).
+#
+# Backend-specific deps (coremltools / onnxruntime / openvino) are NOT
+# listed here — they are installed lazily from requirements-<backend>.txt
+# the first time the user selects that runtime, to keep first-boot
+# install footprint small.
+numpy>=1.26,<3
+Pillow>=10.0,<12