@camstack/addon-vision 0.1.0

package/dist/index.mjs

@@ -0,0 +1,111 @@
+import {
+  CameraNativeDetectionAddon
+} from "./chunk-DTOAB2CE.mjs";
+import {
+  BirdGlobalClassifierAddon
+} from "./chunk-Q3SQOYG6.mjs";
+import {
+  BirdNABirdsClassifierAddon
+} from "./chunk-3MQFUDRU.mjs";
+import {
+  AnimalClassifierAddon
+} from "./chunk-J4WRYHHY.mjs";
+import {
+  ANIMAL_TYPE_MODELS,
+  BIRD_NABIRDS_MODELS,
+  BIRD_SPECIES_MODELS
+} from "./chunk-ISOIDU4U.mjs";
+import {
+  MotionDetectionAddon,
+  detectMotion
+} from "./chunk-QIMDG34B.mjs";
+import {
+  OBJECT_DETECTION_MODELS,
+  ObjectDetectionAddon,
+  SEGMENTATION_MODELS
+} from "./chunk-AYBFB7ID.mjs";
+import {
+  FACE_DETECTION_MODELS,
+  FaceDetectionAddon,
+  scrfdPostprocess
+} from "./chunk-MEVASN3P.mjs";
+import {
+  FACE_RECOGNITION_MODELS,
+  FaceRecognitionAddon,
+  cosineSimilarity,
+  l2Normalize
+} from "./chunk-B3R66MPF.mjs";
+import {
+  PLATE_DETECTION_MODELS,
+  PlateDetectionAddon
+} from "./chunk-5JJZGKL7.mjs";
+import {
+  iou,
+  nms,
+  yoloPostprocess
+} from "./chunk-KUO2BVFY.mjs";
+import {
+  PLATE_RECOGNITION_MODELS,
+  PlateRecognitionAddon,
+  ctcDecode
+} from "./chunk-PDSHDDPV.mjs";
+import {
+  cropRegion,
+  jpegToRgb,
+  letterbox,
+  resizeAndNormalize,
+  rgbToGrayscale
+} from "./chunk-6OR5TE7A.mjs";
+import {
+  AUDIO_CLASSIFICATION_MODELS,
+  AudioClassificationAddon,
+  yamnetPostprocess
+} from "./chunk-5AIQSN32.mjs";
+import {
+  NodeInferenceEngine,
+  PythonInferenceEngine,
+  probeOnnxBackends,
+  resolveEngine
+} from "./chunk-LPI42WL6.mjs";
+export {
+  ANIMAL_TYPE_MODELS,
+  AUDIO_CLASSIFICATION_MODELS,
+  AnimalClassifierAddon,
+  AudioClassificationAddon,
+  BIRD_NABIRDS_MODELS,
+  BIRD_SPECIES_MODELS,
+  BirdGlobalClassifierAddon,
+  BirdNABirdsClassifierAddon,
+  CameraNativeDetectionAddon,
+  FACE_DETECTION_MODELS,
+  FACE_RECOGNITION_MODELS,
+  FaceDetectionAddon,
+  FaceRecognitionAddon,
+  MotionDetectionAddon,
+  NodeInferenceEngine,
+  OBJECT_DETECTION_MODELS,
+  ObjectDetectionAddon,
+  PLATE_DETECTION_MODELS,
+  PLATE_RECOGNITION_MODELS,
+  PlateDetectionAddon,
+  PlateRecognitionAddon,
+  PythonInferenceEngine,
+  SEGMENTATION_MODELS,
+  cosineSimilarity,
+  cropRegion,
+  ctcDecode,
+  detectMotion,
+  iou,
+  jpegToRgb,
+  l2Normalize,
+  letterbox,
+  nms,
+  probeOnnxBackends,
+  resizeAndNormalize,
+  resolveEngine,
+  rgbToGrayscale,
+  scrfdPostprocess,
+  yamnetPostprocess,
+  yoloPostprocess
+};
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@camstack/addon-vision",
+  "version": "0.1.0",
+  "description": "Builtin detection addons for CamStack — object detection, face, plate, audio, motion",
+  "keywords": ["camstack", "addon", "camstack-addon", "vision", "object-detection", "face-detection", "motion-detection", "camera"],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/camstack/server"
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": { "import": "./dist/index.mjs", "require": "./dist/index.js", "types": "./dist/index.d.ts" },
+    "./addons/*": { "import": "./dist/addons/*/index.mjs", "require": "./dist/addons/*/index.js", "types": "./dist/addons/*/index.d.ts" }
+  },
+  "camstack": {
+    "addons": [
+      { "id": "motion-detection", "entry": "./dist/addons/motion-detection/index.js", "slot": "detector" },
+      { "id": "object-detection", "entry": "./dist/addons/object-detection/index.js", "slot": "detector" },
+      { "id": "face-detection", "entry": "./dist/addons/face-detection/index.js", "slot": "cropper" },
+      { "id": "face-recognition", "entry": "./dist/addons/face-recognition/index.js", "slot": "classifier" },
+      { "id": "plate-detection", "entry": "./dist/addons/plate-detection/index.js", "slot": "cropper" },
+      { "id": "plate-recognition", "entry": "./dist/addons/plate-recognition/index.js", "slot": "classifier" },
+      { "id": "audio-classification", "entry": "./dist/addons/audio-classification/index.js", "slot": "classifier" },
+      { "id": "camera-native-detection", "entry": "./dist/addons/camera-native-detection/index.js", "slot": "detector" },
+      { "id": "bird-global-classifier", "entry": "./dist/addons/bird-global-classifier/index.js", "slot": "classifier" },
+      { "id": "bird-nabirds-classifier", "entry": "./dist/addons/bird-nabirds-classifier/index.js", "slot": "classifier" },
+      { "id": "animal-classifier", "entry": "./dist/addons/animal-classifier/index.js", "slot": "classifier" }
+    ]
+  },
+  "files": ["dist", "python"],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "peerDependencies": { "@camstack/types": "^0.1.0" },
+  "dependencies": { "onnxruntime-node": "^1.24.3", "sharp": "^0.34.0" },
+  "devDependencies": {
+    "@camstack/types": "*",
+    "tsup": "^8.0.0",
+    "typescript": "~5.9.0",
+    "vitest": "^3.0.0"
+  }
+}

package/python/coreml_inference.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""CoreML inference for YOLO object detection.
+
+Binary IPC protocol over stdin/stdout:
+  Input:  [4 bytes LE uint32 length][JPEG bytes]
+  Output: [4 bytes LE uint32 length][JSON bytes]
+
+JSON output format:
+{
+  "detections": [
+    {"className": "person", "score": 0.92, "bbox": [x1, y1, x2, y2]},
+    ...
+  ],
+  "inferenceMs": 12.5
+}
+
+Bounding boxes are NORMALIZED (0-1 range).
+
+Usage:
+  python coreml_inference.py <model_path> [--device cpu|ane|gpu|all] [--input-size 640] [--confidence 0.25]
+"""
+import sys
+import struct
+import json
+import argparse
+import time
+import io
+import numpy as np
+from PIL import Image
+
+
+# ---------------------------------------------------------------------------
+# Binary IPC helpers
+# ---------------------------------------------------------------------------
+
+def read_frame(stream) -> bytes:
+    """Read one JPEG frame from binary IPC stream."""
+    header = stream.read(4)
+    if len(header) < 4:
+        return b""
+    length = struct.unpack("<I", header)[0]
+    return stream.read(length)
+
+
+def write_result(stream, result: dict) -> None:
+    """Write JSON result to binary IPC stream."""
+    payload = json.dumps(result).encode("utf-8")
+    stream.write(struct.pack("<I", len(payload)) + payload)
+    stream.flush()
+
+
+# ---------------------------------------------------------------------------
+# Image preprocessing
+# ---------------------------------------------------------------------------
+
+def letterbox(img: Image.Image, size: int) -> tuple[np.ndarray, float, tuple[int, int]]:
+    """Resize with letterbox padding (maintain aspect ratio).
+
+    Returns (CHW float32 array normalised 0-1, scale, (pad_x, pad_y)).
+    """
+    w, h = img.size
+    scale = min(size / w, size / h)
+    nw, nh = int(w * scale), int(h * scale)
+    img_resized = img.resize((nw, nh), Image.BILINEAR)
+
+    canvas = Image.new("RGB", (size, size), (114, 114, 114))
+    pad_x, pad_y = (size - nw) // 2, (size - nh) // 2
+    canvas.paste(img_resized, (pad_x, pad_y))
+
+    arr = np.array(canvas, dtype=np.float32) / 255.0
+    return arr, scale, (pad_x, pad_y)
+
+
+# ---------------------------------------------------------------------------
+# COCO 80 class names
+# ---------------------------------------------------------------------------
+
+COCO_CLASSES = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat",
+    "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat",
+    "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
+    "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball",
+    "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket",
+    "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
+    "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
+    "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
+    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush",
+]
+
+
+# ---------------------------------------------------------------------------
+# YOLO output parsing
+# ---------------------------------------------------------------------------
+
+def compute_iou(x1a: float, y1a: float, x2a: float, y2a: float,
+                x1b: float, y1b: float, x2b: float, y2b: float) -> float:
+    """Compute IoU between two bounding boxes."""
+    xi1 = max(x1a, x1b)
+    yi1 = max(y1a, y1b)
+    xi2 = min(x2a, x2b)
+    yi2 = min(y2a, y2b)
+    inter = max(0, xi2 - xi1) * max(0, yi2 - yi1)
+    area_a = (x2a - x1a) * (y2a - y1a)
+    area_b = (x2b - x1b) * (y2b - y1b)
+    union = area_a + area_b - inter
+    return inter / union if union > 0 else 0.0
+
+
+def parse_yolo_output(output: np.ndarray, conf_threshold: float,
+                      img_w: int, img_h: int, input_size: int,
+                      scale: float, pad: tuple[int, int]) -> list[dict]:
+    """Parse YOLO output tensor [1, 84, 8400] into normalised detections."""
+    # Squeeze batch dim and transpose to [num_boxes, 84]
+    if output.ndim == 3 and output.shape[0] == 1:
+        output = output[0]
+    if output.shape[0] == 84:
+        output = output.T  # [8400, 84]
+
+    cx, cy, w, h = output[:, 0], output[:, 1], output[:, 2], output[:, 3]
+    class_scores = output[:, 4:]  # [num_boxes, 80]
+
+    class_ids = np.argmax(class_scores, axis=1)
+    scores = class_scores[np.arange(len(class_ids)), class_ids]
+
+    mask = scores > conf_threshold
+    cx, cy, w, h = cx[mask], cy[mask], w[mask], h[mask]
+    scores, class_ids = scores[mask], class_ids[mask]
+
+    # Centre-format -> corner-format
+    x1 = cx - w / 2
+    y1 = cy - h / 2
+    x2 = cx + w / 2
+    y2 = cy + h / 2
+
+    # Undo letterbox padding and scale
+    pad_x, pad_y = pad
+    x1 = (x1 - pad_x) / scale
+    y1 = (y1 - pad_y) / scale
+    x2 = (x2 - pad_x) / scale
+    y2 = (y2 - pad_y) / scale
+
+    # Normalise to 0-1 range
+    x1 = np.clip(x1 / img_w, 0, 1)
+    y1 = np.clip(y1 / img_h, 0, 1)
+    x2 = np.clip(x2 / img_w, 0, 1)
+    y2 = np.clip(y2 / img_h, 0, 1)
+
+    # Greedy NMS
+    detections: list[dict] = []
+    indices = np.argsort(-scores)
+    suppressed: set[int] = set()
+
+    for i in indices[:100]:
+        idx = int(i)
+        if idx in suppressed:
+            continue
+        detections.append({
+            "className": COCO_CLASSES[class_ids[idx]] if class_ids[idx] < len(COCO_CLASSES) else f"class_{class_ids[idx]}",
+            "score": round(float(scores[idx]), 4),
+            "bbox": [round(float(x1[idx]), 4), round(float(y1[idx]), 4),
+                     round(float(x2[idx]), 4), round(float(y2[idx]), 4)],
+        })
+        for j in indices:
+            jdx = int(j)
+            if jdx in suppressed or jdx == idx:
+                continue
+            iou = compute_iou(float(x1[idx]), float(y1[idx]), float(x2[idx]), float(y2[idx]),
+                              float(x1[jdx]), float(y1[jdx]), float(x2[jdx]), float(y2[jdx]))
+            if iou > 0.45:
+                suppressed.add(jdx)
+        suppressed.add(idx)
+
+    return detections
+
+
+# ---------------------------------------------------------------------------
+# CoreML NMS output parser (model with built-in NMS)
+# ---------------------------------------------------------------------------
+
+def _parse_nms_output(coords: np.ndarray, confs: np.ndarray,
+                      conf_threshold: float, img_w: int, img_h: int) -> list[dict]:
+    """Parse CoreML NMS output: (N, 4) coords [cx, cy, w, h] normalized + (N, 80) scores."""
+    if coords.shape[0] == 0:
+        return []
+
+    class_ids = np.argmax(confs, axis=1)
+    scores = confs[np.arange(len(class_ids)), class_ids]
+
+    mask = scores > conf_threshold
+    coords = coords[mask]
+    scores = scores[mask]
+    class_ids = class_ids[mask]
+
+    if len(scores) == 0:
+        return []
+
+    # CoreML coords are [cx, cy, w, h] already normalized 0-1
+    cx, cy, w, h = coords[:, 0], coords[:, 1], coords[:, 2], coords[:, 3]
+    x1 = np.clip(cx - w / 2, 0, 1)
+    y1 = np.clip(cy - h / 2, 0, 1)
+    x2 = np.clip(cx + w / 2, 0, 1)
+    y2 = np.clip(cy + h / 2, 0, 1)
+
+    detections = []
+    for i in range(len(scores)):
+        detections.append({
+            "className": COCO_CLASSES[class_ids[i]] if class_ids[i] < len(COCO_CLASSES) else f"class_{class_ids[i]}",
+            "score": round(float(scores[i]), 4),
+            "bbox": [round(float(x1[i]), 4), round(float(y1[i]), 4),
+                     round(float(x2[i]), 4), round(float(y2[i]), 4)],
+        })
+
+    return sorted(detections, key=lambda d: d["score"], reverse=True)[:100]
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="CoreML inference via binary IPC")
+    parser.add_argument("model_path", help="Path to .mlpackage or .mlmodel")
+    parser.add_argument("--device", default="all", choices=["cpu", "ane", "gpu", "all"],
+                        help="Compute unit: cpu, ane (Apple Neural Engine), gpu, or all")
+    parser.add_argument("--input-size", type=int, default=640,
+                        help="Model input size (square)")
+    parser.add_argument("--confidence", type=float, default=0.25,
+                        help="Confidence threshold")
+    args = parser.parse_args()
+
+    import coremltools as ct
+
+    compute_unit_map = {
+        "cpu": ct.ComputeUnit.CPU_ONLY,
+        "gpu": ct.ComputeUnit.CPU_AND_GPU,
+        "ane": ct.ComputeUnit.CPU_AND_NE,
+        "all": ct.ComputeUnit.ALL,
+    }
+    compute_units = compute_unit_map.get(args.device, ct.ComputeUnit.ALL)
+
+    sys.stderr.write(f"[coreml] Loading model: {args.model_path}\n")
+    sys.stderr.write(f"[coreml] Compute units: {args.device}\n")
+    sys.stderr.flush()
+
+    model = ct.models.MLModel(args.model_path, compute_units=compute_units)
+
+    # Resolve the first input name from the model spec
+    input_spec = model.get_spec().description.input[0]
+    input_name = input_spec.name
+
+    sys.stderr.write(f"[coreml] Model loaded — input: {input_name}, ready for inference\n")
+    sys.stderr.flush()
+
+    stdin_binary = sys.stdin.buffer
+    stdout_binary = sys.stdout.buffer
+
+    while True:
+        jpeg = read_frame(stdin_binary)
+        if not jpeg:
+            break
+
+        try:
+            start = time.perf_counter()
+
+            img = Image.open(io.BytesIO(jpeg)).convert("RGB")
+            orig_w, orig_h = img.size
+
+            arr, scale, pad = letterbox(img, args.input_size)
+
+            # Check if model has NMS built-in (outputs 'confidence' + 'coordinates')
+            output_names = [o.name for o in model.get_spec().description.output]
+            has_builtin_nms = 'confidence' in output_names and 'coordinates' in output_names
+
+            # Check input type — CoreML models may expect PIL Image or numpy array
+            input_type = input_spec.type.WhichOneof("Type")
+            predict_input: dict = {}
+            if input_type == "imageType":
+                letterboxed_img = Image.fromarray((arr * 255).astype(np.uint8))
+                predict_input[input_name] = letterboxed_img
+            else:
+                input_arr = arr.transpose(2, 0, 1)[np.newaxis]
+                predict_input[input_name] = input_arr
+
+            # Pass thresholds if model accepts them
+            input_names = [i.name for i in model.get_spec().description.input]
+            if 'iouThreshold' in input_names:
+                predict_input['iouThreshold'] = 0.45
+            if 'confidenceThreshold' in input_names:
+                predict_input['confidenceThreshold'] = args.confidence
+
+            predictions = model.predict(predict_input)
+
+            if has_builtin_nms:
+                # Model has built-in NMS: outputs are (N, 4) boxes + (N, 80) scores
+                coords = np.array(predictions['coordinates'])   # (N, 4) — cx, cy, w, h normalized
+                confs = np.array(predictions['confidence'])     # (N, 80)
+                detections = _parse_nms_output(coords, confs, args.confidence, orig_w, orig_h)
+            else:
+                # Raw YOLO output: [1, 84, 8400]
+                output_key = list(predictions.keys())[0]
+                output = np.array(predictions[output_key])
+                detections = parse_yolo_output(
+                    output, args.confidence, orig_w, orig_h,
+                    args.input_size, scale, pad,
+                )
+
+            elapsed = (time.perf_counter() - start) * 1000
+            result = {"detections": detections, "inferenceMs": round(elapsed, 2)}
+        except Exception as exc:
+            sys.stderr.write(f"[coreml] Inference error: {exc}\n")
+            sys.stderr.flush()
+            result = {"detections": [], "error": str(exc), "inferenceMs": 0}
+
+        write_result(stdout_binary, result)
+
+
+if __name__ == "__main__":
+    main()