@camstack/addon-vision 0.1.0

package/python/openvino_inference.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""OpenVINO inference for YOLO object detection.
+
+Binary IPC protocol over stdin/stdout:
+  Input:  [4 bytes LE uint32 length][JPEG bytes]
+  Output: [4 bytes LE uint32 length][JSON bytes]
+
+JSON output format:
+{
+  "detections": [
+    {"className": "person", "score": 0.92, "bbox": [x1, y1, x2, y2]},
+    ...
+  ],
+  "inferenceMs": 12.5
+}
+
+Bounding boxes are NORMALIZED (0-1 range).
+
+Usage:
+  python openvino_inference.py <model_path> [--device CPU|GPU|AUTO] [--input-size 640] [--confidence 0.25]
+"""
+import sys
+import struct
+import json
+import argparse
+import time
+import io
+import numpy as np
+from PIL import Image
+
+
+# ---------------------------------------------------------------------------
+# Binary IPC helpers
+# ---------------------------------------------------------------------------
+
+def read_frame(stream) -> bytes:
+    """Read one JPEG frame from binary IPC stream."""
+    header = stream.read(4)
+    if len(header) < 4:
+        return b""
+    length = struct.unpack("<I", header)[0]
+    return stream.read(length)
+
+
+def write_result(stream, result: dict) -> None:
+    """Write JSON result to binary IPC stream."""
+    payload = json.dumps(result).encode("utf-8")
+    stream.write(struct.pack("<I", len(payload)) + payload)
+    stream.flush()
+
+
+# ---------------------------------------------------------------------------
+# Image preprocessing
+# ---------------------------------------------------------------------------
+
+def letterbox(img: Image.Image, size: int) -> tuple[np.ndarray, float, tuple[int, int]]:
+    """Resize with letterbox padding (maintain aspect ratio).
+
+    Returns (CHW float32 array normalised 0-1, scale, (pad_x, pad_y)).
+    """
+    w, h = img.size
+    scale = min(size / w, size / h)
+    nw, nh = int(w * scale), int(h * scale)
+    img_resized = img.resize((nw, nh), Image.BILINEAR)
+
+    canvas = Image.new("RGB", (size, size), (114, 114, 114))
+    pad_x, pad_y = (size - nw) // 2, (size - nh) // 2
+    canvas.paste(img_resized, (pad_x, pad_y))
+
+    arr = np.array(canvas, dtype=np.float32) / 255.0
+    return arr, scale, (pad_x, pad_y)
+
+
+# ---------------------------------------------------------------------------
+# COCO 80 class names
+# ---------------------------------------------------------------------------
+
+COCO_CLASSES = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat",
+    "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat",
+    "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
+    "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball",
+    "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket",
+    "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
+    "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
+    "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
+    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush",
+]
+
+
+# ---------------------------------------------------------------------------
+# YOLO output parsing
+# ---------------------------------------------------------------------------
+
+def compute_iou(x1a: float, y1a: float, x2a: float, y2a: float,
+                x1b: float, y1b: float, x2b: float, y2b: float) -> float:
+    """Compute IoU between two bounding boxes."""
+    xi1 = max(x1a, x1b)
+    yi1 = max(y1a, y1b)
+    xi2 = min(x2a, x2b)
+    yi2 = min(y2a, y2b)
+    inter = max(0, xi2 - xi1) * max(0, yi2 - yi1)
+    area_a = (x2a - x1a) * (y2a - y1a)
+    area_b = (x2b - x1b) * (y2b - y1b)
+    union = area_a + area_b - inter
+    return inter / union if union > 0 else 0.0
+
+
+def parse_yolo_output(output: np.ndarray, conf_threshold: float,
+                      img_w: int, img_h: int, input_size: int,
+                      scale: float, pad: tuple[int, int]) -> list[dict]:
+    """Parse YOLO output tensor [1, 84, 8400] into normalised detections."""
+    if output.ndim == 3 and output.shape[0] == 1:
+        output = output[0]
+    if output.shape[0] == 84:
+        output = output.T  # [8400, 84]
+
+    cx, cy, w, h = output[:, 0], output[:, 1], output[:, 2], output[:, 3]
+    class_scores = output[:, 4:]
+
+    class_ids = np.argmax(class_scores, axis=1)
+    scores = class_scores[np.arange(len(class_ids)), class_ids]
+
+    mask = scores > conf_threshold
+    cx, cy, w, h = cx[mask], cy[mask], w[mask], h[mask]
+    scores, class_ids = scores[mask], class_ids[mask]
+
+    x1 = cx - w / 2
+    y1 = cy - h / 2
+    x2 = cx + w / 2
+    y2 = cy + h / 2
+
+    pad_x, pad_y = pad
+    x1 = (x1 - pad_x) / scale
+    y1 = (y1 - pad_y) / scale
+    x2 = (x2 - pad_x) / scale
+    y2 = (y2 - pad_y) / scale
+
+    x1 = np.clip(x1 / img_w, 0, 1)
+    y1 = np.clip(y1 / img_h, 0, 1)
+    x2 = np.clip(x2 / img_w, 0, 1)
+    y2 = np.clip(y2 / img_h, 0, 1)
+
+    detections: list[dict] = []
+    indices = np.argsort(-scores)
+    suppressed: set[int] = set()
+
+    for i in indices[:100]:
+        idx = int(i)
+        if idx in suppressed:
+            continue
+        detections.append({
+            "className": COCO_CLASSES[class_ids[idx]] if class_ids[idx] < len(COCO_CLASSES) else f"class_{class_ids[idx]}",
+            "score": round(float(scores[idx]), 4),
+            "bbox": [round(float(x1[idx]), 4), round(float(y1[idx]), 4),
+                     round(float(x2[idx]), 4), round(float(y2[idx]), 4)],
+        })
+        for j in indices:
+            jdx = int(j)
+            if jdx in suppressed or jdx == idx:
+                continue
+            iou = compute_iou(float(x1[idx]), float(y1[idx]), float(x2[idx]), float(y2[idx]),
+                              float(x1[jdx]), float(y1[jdx]), float(x2[jdx]), float(y2[jdx]))
+            if iou > 0.45:
+                suppressed.add(jdx)
+        suppressed.add(idx)
+
+    return detections
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="OpenVINO inference via binary IPC")
+    parser.add_argument("model_path", help="Path to .xml model (OpenVINO IR format)")
+    parser.add_argument("--device", default="AUTO", choices=["CPU", "GPU", "AUTO"],
+                        help="OpenVINO compute device")
+    parser.add_argument("--input-size", type=int, default=640,
+                        help="Model input size (square)")
+    parser.add_argument("--confidence", type=float, default=0.25,
+                        help="Confidence threshold")
+    args = parser.parse_args()
+
+    from openvino.runtime import Core
+
+    ie = Core()
+
+    sys.stderr.write(f"[openvino] Loading model: {args.model_path}\n")
+    sys.stderr.write(f"[openvino] Device: {args.device}\n")
+    sys.stderr.write(f"[openvino] Available devices: {ie.available_devices}\n")
+    sys.stderr.flush()
+
+    model = ie.read_model(args.model_path)
+    compiled = ie.compile_model(model, args.device)
+    infer_request = compiled.create_infer_request()
+
+    # Resolve input/output tensor info
+    input_layer = compiled.input(0)
+    output_layer = compiled.output(0)
+
+    sys.stderr.write(f"[openvino] Input shape: {input_layer.shape}, dtype: {input_layer.element_type}\n")
+    sys.stderr.write(f"[openvino] Output shape: {output_layer.shape}\n")
+    sys.stderr.write("[openvino] Model loaded, ready for inference\n")
+    sys.stderr.flush()
+
+    stdin_binary = sys.stdin.buffer
+    stdout_binary = sys.stdout.buffer
+
+    while True:
+        jpeg = read_frame(stdin_binary)
+        if not jpeg:
+            break
+
+        try:
+            start = time.perf_counter()
+
+            img = Image.open(io.BytesIO(jpeg)).convert("RGB")
+            orig_w, orig_h = img.size
+
+            arr, scale, pad = letterbox(img, args.input_size)
+
+            # OpenVINO expects [1, 3, H, W] float32
+            input_arr = arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32)
+
+            infer_request.infer({0: input_arr})
+            output = infer_request.get_output_tensor(0).data.copy()
+
+            detections = parse_yolo_output(
+                output, args.confidence, orig_w, orig_h,
+                args.input_size, scale, pad,
+            )
+
+            elapsed = (time.perf_counter() - start) * 1000
+            result = {"detections": detections, "inferenceMs": round(elapsed, 2)}
+        except Exception as exc:
+            sys.stderr.write(f"[openvino] Inference error: {exc}\n")
+            sys.stderr.flush()
+            result = {"detections": [], "error": str(exc), "inferenceMs": 0}
+
+        write_result(stdout_binary, result)
+
+
+if __name__ == "__main__":
+    main()

package/python/pytorch_inference.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""ONNX Runtime (Python) inference for YOLO object detection.
+
+Uses onnxruntime-python which can leverage CoreMLExecutionProvider on macOS,
+CUDAExecutionProvider on Linux/Windows with NVIDIA GPUs, or CPUExecutionProvider
+everywhere. This is simpler and more portable than requiring PyTorch directly.
+
+Binary IPC protocol over stdin/stdout:
+  Input:  [4 bytes LE uint32 length][JPEG bytes]
+  Output: [4 bytes LE uint32 length][JSON bytes]
+
+JSON output format:
+{
+  "detections": [
+    {"className": "person", "score": 0.92, "bbox": [x1, y1, x2, y2]},
+    ...
+  ],
+  "inferenceMs": 12.5
+}
+
+Bounding boxes are NORMALIZED (0-1 range).
+
+Usage:
+  python pytorch_inference.py <model_path> [--device cpu|cuda|mps] [--input-size 640] [--confidence 0.25]
+"""
+import sys
+import struct
+import json
+import argparse
+import time
+import io
+import numpy as np
+from PIL import Image
+
+
+# ---------------------------------------------------------------------------
+# Binary IPC helpers
+# ---------------------------------------------------------------------------
+
+def read_frame(stream) -> bytes:
+    """Read one JPEG frame from binary IPC stream."""
+    header = stream.read(4)
+    if len(header) < 4:
+        return b""
+    length = struct.unpack("<I", header)[0]
+    return stream.read(length)
+
+
+def write_result(stream, result: dict) -> None:
+    """Write JSON result to binary IPC stream."""
+    payload = json.dumps(result).encode("utf-8")
+    stream.write(struct.pack("<I", len(payload)) + payload)
+    stream.flush()
+
+
+# ---------------------------------------------------------------------------
+# Image preprocessing
+# ---------------------------------------------------------------------------
+
+def letterbox(img: Image.Image, size: int) -> tuple[np.ndarray, float, tuple[int, int]]:
+    """Resize with letterbox padding (maintain aspect ratio).
+
+    Returns (CHW float32 array normalised 0-1, scale, (pad_x, pad_y)).
+    """
+    w, h = img.size
+    scale = min(size / w, size / h)
+    nw, nh = int(w * scale), int(h * scale)
+    img_resized = img.resize((nw, nh), Image.BILINEAR)
+
+    canvas = Image.new("RGB", (size, size), (114, 114, 114))
+    pad_x, pad_y = (size - nw) // 2, (size - nh) // 2
+    canvas.paste(img_resized, (pad_x, pad_y))
+
+    arr = np.array(canvas, dtype=np.float32) / 255.0
+    return arr, scale, (pad_x, pad_y)
+
+
+# ---------------------------------------------------------------------------
+# COCO 80 class names
+# ---------------------------------------------------------------------------
+
+COCO_CLASSES = [
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat",
+    "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat",
+    "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
+    "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball",
+    "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket",
+    "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
+    "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
+    "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
+    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush",
+]
+
+
+# ---------------------------------------------------------------------------
+# YOLO output parsing
+# ---------------------------------------------------------------------------
+
+def compute_iou(x1a: float, y1a: float, x2a: float, y2a: float,
+                x1b: float, y1b: float, x2b: float, y2b: float) -> float:
+    """Compute IoU between two bounding boxes."""
+    xi1 = max(x1a, x1b)
+    yi1 = max(y1a, y1b)
+    xi2 = min(x2a, x2b)
+    yi2 = min(y2a, y2b)
+    inter = max(0, xi2 - xi1) * max(0, yi2 - yi1)
+    area_a = (x2a - x1a) * (y2a - y1a)
+    area_b = (x2b - x1b) * (y2b - y1b)
+    union = area_a + area_b - inter
+    return inter / union if union > 0 else 0.0
+
+
+def parse_yolo_output(output: np.ndarray, conf_threshold: float,
+                      img_w: int, img_h: int, input_size: int,
+                      scale: float, pad: tuple[int, int]) -> list[dict]:
+    """Parse YOLO output tensor [1, 84, 8400] into normalised detections."""
+    if output.ndim == 3 and output.shape[0] == 1:
+        output = output[0]
+    if output.shape[0] == 84:
+        output = output.T  # [8400, 84]
+
+    cx, cy, w, h = output[:, 0], output[:, 1], output[:, 2], output[:, 3]
+    class_scores = output[:, 4:]
+
+    class_ids = np.argmax(class_scores, axis=1)
+    scores = class_scores[np.arange(len(class_ids)), class_ids]
+
+    mask = scores > conf_threshold
+    cx, cy, w, h = cx[mask], cy[mask], w[mask], h[mask]
+    scores, class_ids = scores[mask], class_ids[mask]
+
+    x1 = cx - w / 2
+    y1 = cy - h / 2
+    x2 = cx + w / 2
+    y2 = cy + h / 2
+
+    pad_x, pad_y = pad
+    x1 = (x1 - pad_x) / scale
+    y1 = (y1 - pad_y) / scale
+    x2 = (x2 - pad_x) / scale
+    y2 = (y2 - pad_y) / scale
+
+    x1 = np.clip(x1 / img_w, 0, 1)
+    y1 = np.clip(y1 / img_h, 0, 1)
+    x2 = np.clip(x2 / img_w, 0, 1)
+    y2 = np.clip(y2 / img_h, 0, 1)
+
+    detections: list[dict] = []
+    indices = np.argsort(-scores)
+    suppressed: set[int] = set()
+
+    for i in indices[:100]:
+        idx = int(i)
+        if idx in suppressed:
+            continue
+        detections.append({
+            "className": COCO_CLASSES[class_ids[idx]] if class_ids[idx] < len(COCO_CLASSES) else f"class_{class_ids[idx]}",
+            "score": round(float(scores[idx]), 4),
+            "bbox": [round(float(x1[idx]), 4), round(float(y1[idx]), 4),
+                     round(float(x2[idx]), 4), round(float(y2[idx]), 4)],
+        })
+        for j in indices:
+            jdx = int(j)
+            if jdx in suppressed or jdx == idx:
+                continue
+            iou = compute_iou(float(x1[idx]), float(y1[idx]), float(x2[idx]), float(y2[idx]),
+                              float(x1[jdx]), float(y1[jdx]), float(x2[jdx]), float(y2[jdx]))
+            if iou > 0.45:
+                suppressed.add(jdx)
+        suppressed.add(idx)
+
+    return detections
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="ONNX Runtime (Python) inference via binary IPC")
+    parser.add_argument("model_path", help="Path to .onnx model")
+    parser.add_argument("--device", default="cpu", choices=["cpu", "cuda", "mps"],
+                        help="Compute device (selects execution provider)")
+    parser.add_argument("--input-size", type=int, default=640,
+                        help="Model input size (square)")
+    parser.add_argument("--confidence", type=float, default=0.25,
+                        help="Confidence threshold")
+    args = parser.parse_args()
+
+    import onnxruntime as ort
+
+    # Select execution providers based on requested device
+    if args.device == "cuda":
+        providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+    elif args.device == "mps" and sys.platform == "darwin":
+        providers = ["CoreMLExecutionProvider", "CPUExecutionProvider"]
+    elif sys.platform == "darwin":
+        # On macOS, prefer CoreML even for "cpu" — it may dispatch to ANE
+        providers = ["CoreMLExecutionProvider", "CPUExecutionProvider"]
+    else:
+        providers = ["CPUExecutionProvider"]
+
+    sys.stderr.write(f"[pytorch/ort] Loading model: {args.model_path}\n")
+    sys.stderr.write(f"[pytorch/ort] Requested device: {args.device}\n")
+    sys.stderr.write(f"[pytorch/ort] Providers: {providers}\n")
+    sys.stderr.flush()
+
+    session = ort.InferenceSession(args.model_path, providers=providers)
+    input_name = session.get_inputs()[0].name
+    active_providers = session.get_providers()
+
+    sys.stderr.write(f"[pytorch/ort] Active providers: {active_providers}\n")
+    sys.stderr.write("[pytorch/ort] Model loaded, ready for inference\n")
+    sys.stderr.flush()
+
+    stdin_binary = sys.stdin.buffer
+    stdout_binary = sys.stdout.buffer
+
+    while True:
+        jpeg = read_frame(stdin_binary)
+        if not jpeg:
+            break
+
+        try:
+            start = time.perf_counter()
+
+            img = Image.open(io.BytesIO(jpeg)).convert("RGB")
+            orig_w, orig_h = img.size
+
+            arr, scale, pad = letterbox(img, args.input_size)
+
+            # ONNX expects [1, 3, H, W] float32
+            input_arr = arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32)
+
+            outputs = session.run(None, {input_name: input_arr})
+            output = outputs[0]
+
+            detections = parse_yolo_output(
+                output, args.confidence, orig_w, orig_h,
+                args.input_size, scale, pad,
+            )
+
+            elapsed = (time.perf_counter() - start) * 1000
+            result = {"detections": detections, "inferenceMs": round(elapsed, 2)}
+        except Exception as exc:
+            sys.stderr.write(f"[pytorch/ort] Inference error: {exc}\n")
+            sys.stderr.flush()
+            result = {"detections": [], "error": str(exc), "inferenceMs": 0}
+
+        write_result(stdout_binary, result)
+
+
+if __name__ == "__main__":
+    main()