@camstack/addon-vision 0.1.7 → 0.1.9

package/dist/chunk-R5J3WAUI.mjs

@@ -0,0 +1,645 @@
+import {
+  nms,
+  yoloPostprocess
+} from "./chunk-KUO2BVFY.mjs";
+import {
+  MLPACKAGE_FILES,
+  OBJECT_DETECTION_MODELS
+} from "./chunk-BP7H4NFS.mjs";
+import {
+  letterbox
+} from "./chunk-22BHCDT5.mjs";
+import {
+  resolveEngine
+} from "./chunk-2IOKI4ES.mjs";
+
+// src/catalogs/segmentation-models.ts
+import { hfModelUrl, COCO_80_LABELS } from "@camstack/types";
+var HF_REPO = "camstack/camstack-models";
+var SEGMENTATION_MODELS = [
+  // ── YOLO11-seg ───────────────────────────────────────────────
+  {
+    id: "yolo11n-seg",
+    name: "YOLO11 Nano Segmentation",
+    description: "YOLO11 Nano \u2014 fastest, smallest YOLO11 instance segmentation model",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11n-seg.onnx"),
+        sizeMB: 11
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11n-seg.mlpackage"),
+        sizeMB: 6,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11n-seg.xml"),
+        sizeMB: 6,
+        runtimes: ["python"]
+      }
+    }
+  },
+  {
+    id: "yolo11s-seg",
+    name: "YOLO11 Small Segmentation",
+    description: "YOLO11 Small \u2014 balanced speed and accuracy for instance segmentation",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11s-seg.onnx"),
+        sizeMB: 39
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11s-seg.mlpackage"),
+        sizeMB: 20,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11s-seg.xml"),
+        sizeMB: 20,
+        runtimes: ["python"]
+      }
+    }
+  },
+  {
+    id: "yolo11m-seg",
+    name: "YOLO11 Medium Segmentation",
+    description: "YOLO11 Medium \u2014 higher accuracy instance segmentation",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11m-seg.onnx"),
+        sizeMB: 86
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11m-seg.mlpackage"),
+        sizeMB: 43,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11m-seg.xml"),
+        sizeMB: 43,
+        runtimes: ["python"]
+      }
+    }
+  },
+  // ── YOLOv8-seg — CoreML supported ─────────────────────────────
+  {
+    id: "yolov8n-seg",
+    name: "YOLOv8 Nano Segmentation",
+    description: "YOLOv8 Nano \u2014 fastest, smallest YOLOv8 instance segmentation model",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8n-seg.onnx"),
+        sizeMB: 13
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8n-seg.mlpackage"),
+        sizeMB: 7,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8n-seg.xml"),
+        sizeMB: 7,
+        runtimes: ["python"]
+      }
+    }
+  },
+  {
+    id: "yolov8s-seg",
+    name: "YOLOv8 Small Segmentation",
+    description: "YOLOv8 Small \u2014 balanced speed and accuracy for instance segmentation",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8s-seg.onnx"),
+        sizeMB: 45
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8s-seg.mlpackage"),
+        sizeMB: 23,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8s-seg.xml"),
+        sizeMB: 23,
+        runtimes: ["python"]
+      }
+    }
+  },
+  {
+    id: "yolov8m-seg",
+    name: "YOLOv8 Medium Segmentation",
+    description: "YOLOv8 Medium \u2014 higher accuracy instance segmentation",
+    inputSize: { width: 640, height: 640 },
+    labels: COCO_80_LABELS,
+    formats: {
+      onnx: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8m-seg.onnx"),
+        sizeMB: 104
+      },
+      coreml: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8m-seg.mlpackage"),
+        sizeMB: 52,
+        isDirectory: true,
+        files: MLPACKAGE_FILES,
+        runtimes: ["python"]
+      },
+      openvino: {
+        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8m-seg.xml"),
+        sizeMB: 53,
+        runtimes: ["python"]
+      }
+    }
+  }
+];
+
+// src/addons/object-detection/index.ts
+import { COCO_TO_MACRO, MACRO_LABELS, COCO_80_LABELS as COCO_80_LABELS2 } from "@camstack/types";
+
+// src/shared/postprocess/yolo-seg.ts
+function sigmoid(x) {
+  return 1 / (1 + Math.exp(-x));
+}
+function computeRawMask(coeffs, protos, numMaskCoeffs, maskH, maskW) {
+  const maskSize = maskH * maskW;
+  const rawMask = new Float32Array(maskSize);
+  for (let px = 0; px < maskSize; px++) {
+    let val = 0;
+    for (let k = 0; k < numMaskCoeffs; k++) {
+      val += (coeffs[k] ?? 0) * (protos[k * maskSize + px] ?? 0);
+    }
+    rawMask[px] = sigmoid(val);
+  }
+  return rawMask;
+}
+function cropAndThresholdMask(rawMask, maskH, maskW, bbox, maskThreshold, maskScale) {
+  const cropX1 = Math.max(0, Math.floor(bbox.x * maskScale));
+  const cropY1 = Math.max(0, Math.floor(bbox.y * maskScale));
+  const cropX2 = Math.min(maskW, Math.ceil((bbox.x + bbox.w) * maskScale));
+  const cropY2 = Math.min(maskH, Math.ceil((bbox.y + bbox.h) * maskScale));
+  const cropW = Math.max(1, cropX2 - cropX1);
+  const cropH = Math.max(1, cropY2 - cropY1);
+  const data = new Uint8Array(cropW * cropH);
+  for (let row = 0; row < cropH; row++) {
+    const srcRow = cropY1 + row;
+    for (let col = 0; col < cropW; col++) {
+      const srcCol = cropX1 + col;
+      const srcIdx = srcRow * maskW + srcCol;
+      data[row * cropW + col] = (rawMask[srcIdx] ?? 0) > maskThreshold ? 255 : 0;
+    }
+  }
+  return { data, width: cropW, height: cropH };
+}
+function yoloSegPostprocess(segOutput, options) {
+  const {
+    detectionOutput,
+    protoOutput,
+    numClasses,
+    numBoxes,
+    numMaskCoeffs,
+    maskHeight,
+    maskWidth
+  } = segOutput;
+  const {
+    confidence,
+    iouThreshold,
+    labels,
+    scale,
+    padX,
+    padY,
+    originalWidth,
+    originalHeight,
+    maskThreshold = 0.5
+  } = options;
+  const yoloInputSize = 640;
+  const maskScale = maskHeight / yoloInputSize;
+  const candidates = [];
+  for (let i = 0; i < numBoxes; i++) {
+    const cx = detectionOutput[0 * numBoxes + i] ?? 0;
+    const cy = detectionOutput[1 * numBoxes + i] ?? 0;
+    const w = detectionOutput[2 * numBoxes + i] ?? 0;
+    const h = detectionOutput[3 * numBoxes + i] ?? 0;
+    let bestScore = -Infinity;
+    let bestClass = 0;
+    for (let j = 0; j < numClasses; j++) {
+      const score = detectionOutput[(4 + j) * numBoxes + i] ?? 0;
+      if (score > bestScore) {
+        bestScore = score;
+        bestClass = j;
+      }
+    }
+    if (bestScore < confidence) continue;
+    const bbox = {
+      x: cx - w / 2,
+      y: cy - h / 2,
+      w,
+      h
+    };
+    const coeffs = new Float32Array(numMaskCoeffs);
+    for (let k = 0; k < numMaskCoeffs; k++) {
+      coeffs[k] = detectionOutput[(4 + numClasses + k) * numBoxes + i] ?? 0;
+    }
+    candidates.push({ bbox, score: bestScore, classIdx: bestClass, coeffs });
+  }
+  if (candidates.length === 0) return [];
+  const keptIndices = nms(candidates, iouThreshold);
+  return keptIndices.map((idx) => {
+    const { bbox, score, classIdx, coeffs } = candidates[idx];
+    const label = labels[classIdx] ?? String(classIdx);
+    const x = Math.max(0, Math.min(originalWidth, (bbox.x - padX) / scale));
+    const y = Math.max(0, Math.min(originalHeight, (bbox.y - padY) / scale));
+    const x2 = Math.max(0, Math.min(originalWidth, (bbox.x + bbox.w - padX) / scale));
+    const y2 = Math.max(0, Math.min(originalHeight, (bbox.y + bbox.h - padY) / scale));
+    const finalBbox = { x, y, w: x2 - x, h: y2 - y };
+    const rawMask = computeRawMask(coeffs, protoOutput, numMaskCoeffs, maskHeight, maskWidth);
+    const { data: maskData, width: mW, height: mH } = cropAndThresholdMask(
+      rawMask,
+      maskHeight,
+      maskWidth,
+      bbox,
+      maskThreshold,
+      maskScale
+    );
+    return {
+      class: label,
+      originalClass: label,
+      score,
+      bbox: finalBbox,
+      mask: maskData,
+      maskWidth: mW,
+      maskHeight: mH
+    };
+  });
+}
+
+// src/addons/object-detection/index.ts
+function isSegModel(modelId) {
+  return modelId.includes("-seg");
+}
+var ALL_DETECTION_MODELS = [
+  ...OBJECT_DETECTION_MODELS,
+  ...SEGMENTATION_MODELS
+];
+function applyClassMap(detections, classMap) {
+  return detections.filter((d) => classMap.mapping[d.class] !== void 0).map((d) => ({
+    ...d,
+    originalClass: d.class,
+    class: classMap.mapping[d.class]
+  }));
+}
+var RAM_ESTIMATES = {
+  "yolov8n": 80,
+  "yolov8s": 150,
+  "yolov8s-relu": 150,
+  "yolov8m": 300,
+  "yolov8l": 500,
+  "yolov8x": 800,
+  "yolov9t": 60,
+  "yolov9s": 120,
+  "yolov9c": 300,
+  "yolo11n": 70,
+  "yolo11s": 130,
+  "yolo11m": 280,
+  "yolo11l": 450,
+  "yolo11x": 750,
+  "yolo11n-seg": 84,
+  "yolo11s-seg": 156,
+  "yolo11m-seg": 336,
+  "yolov8n-seg": 96,
+  "yolov8s-seg": 180,
+  "yolov8m-seg": 360
+};
+var ACCURACY_SCORES = {
+  "yolov8n": 55,
+  "yolov8s": 70,
+  "yolov8s-relu": 68,
+  "yolov8m": 82,
+  "yolov8l": 88,
+  "yolov8x": 92,
+  "yolov9t": 58,
+  "yolov9s": 73,
+  "yolov9c": 86,
+  "yolo11n": 62,
+  "yolo11s": 78,
+  "yolo11m": 88,
+  "yolo11l": 93,
+  "yolo11x": 97,
+  "yolo11n-seg": 62,
+  "yolo11s-seg": 78,
+  "yolo11m-seg": 88,
+  "yolov8n-seg": 55,
+  "yolov8s-seg": 70,
+  "yolov8m-seg": 82
+};
+var ObjectDetectionAddon = class {
+  id = "object-detection";
+  slot = "detector";
+  inputClasses = null;
+  outputClasses = ["person", "vehicle", "animal"];
+  slotPriority = 0;
+  manifest = {
+    id: "object-detection",
+    name: "Object Detection",
+    version: "0.1.0",
+    description: "YOLO-based object detection \u2014 detects persons, vehicles, and animals",
+    slot: "detector",
+    inputClasses: void 0,
+    outputClasses: ["person", "vehicle", "animal"],
+    supportsCustomModels: true,
+    mayRequirePython: false,
+    defaultConfig: {
+      modelId: "yolo11n",
+      runtime: "node",
+      backend: "cpu",
+      confidence: 0.5,
+      iouThreshold: 0.45,
+      classMapMode: "macro"
+    }
+  };
+  engine = null;
+  modelEntry;
+  confidence = 0.5;
+  iouThreshold = 0.45;
+  classMapMode = "macro";
+  resolvedConfig = null;
+  ctx = null;
+  getModelRequirements() {
+    return ALL_DETECTION_MODELS.map((m) => ({
+      modelId: m.id,
+      name: m.name,
+      minRAM_MB: RAM_ESTIMATES[m.id] ?? 100,
+      accuracyScore: ACCURACY_SCORES[m.id] ?? 60,
+      formats: Object.keys(m.formats)
+    }));
+  }
+  configure(config) {
+    this.resolvedConfig = config;
+  }
+  async initialize(ctx) {
+    this.ctx = ctx;
+    const cfg = ctx.addonConfig;
+    const modelId = cfg["modelId"] ?? this.resolvedConfig?.modelId ?? "yolo11n";
+    this.confidence = cfg["confidence"] ?? 0.5;
+    this.iouThreshold = cfg["iouThreshold"] ?? 0.45;
+    this.classMapMode = cfg["classMapMode"] ?? "macro";
+    const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId);
+    if (!entry) {
+      throw new Error(`ObjectDetectionAddon: unknown modelId "${modelId}"`);
+    }
+    this.modelEntry = entry;
+  }
+  async detect(frame) {
+    if (!this.engine) await this.ensureEngine();
+    const start = Date.now();
+    if ("runJpeg" in this.engine && typeof this.engine.runJpeg === "function") {
+      const result = await this.engine.runJpeg(frame.data);
+      const rawDets = result.detections ?? [];
+      const detections2 = rawDets.map((d) => ({
+        class: this.classMapMode === "all" ? d.className : COCO_TO_MACRO.mapping[d.className] ?? d.className,
+        originalClass: d.className,
+        score: d.score,
+        bbox: {
+          x: d.bbox[0] * frame.width,
+          y: d.bbox[1] * frame.height,
+          w: (d.bbox[2] - d.bbox[0]) * frame.width,
+          h: (d.bbox[3] - d.bbox[1]) * frame.height
+        }
+      })).filter((d) => this.classMapMode === "all" || COCO_TO_MACRO.mapping[d.originalClass] !== void 0);
+      return {
+        detections: detections2,
+        inferenceMs: result.inferenceMs ?? Date.now() - start,
+        modelId: this.modelEntry.id
+      };
+    }
+    const { width: inputW, height: inputH } = this.modelEntry.inputSize;
+    const targetSize = Math.max(inputW, inputH);
+    const lb = await letterbox(frame.data, targetSize);
+    const numClasses = this.modelEntry.labels.length;
+    const labels = this.modelEntry.labels.map((l) => l.id);
+    const postprocessOpts = {
+      confidence: this.confidence,
+      iouThreshold: this.iouThreshold,
+      labels,
+      scale: lb.scale,
+      padX: lb.padX,
+      padY: lb.padY,
+      originalWidth: lb.originalWidth,
+      originalHeight: lb.originalHeight
+    };
+    let rawDetections;
+    if (isSegModel(this.modelEntry.id)) {
+      const outputs = await this.engine.runMultiOutput(lb.data, [1, 3, targetSize, targetSize]);
+      const outputNames = Object.keys(outputs);
+      if (outputNames.length < 2) {
+        throw new Error(
+          `ObjectDetectionAddon: seg model "${this.modelEntry.id}" returned ${outputNames.length} output(s); expected 2`
+        );
+      }
+      const detectionOutput = outputs[outputNames[0]];
+      const protoOutput = outputs[outputNames[1]];
+      const numMaskCoeffs = 32;
+      const numBoxes = detectionOutput.length / (4 + numClasses + numMaskCoeffs);
+      const maskHeight = 160;
+      const maskWidth = 160;
+      rawDetections = yoloSegPostprocess(
+        {
+          detectionOutput,
+          protoOutput,
+          numClasses,
+          numBoxes,
+          numMaskCoeffs,
+          maskHeight,
+          maskWidth
+        },
+        postprocessOpts
+      );
+    } else {
+      const output = await this.engine.run(lb.data, [1, 3, targetSize, targetSize]);
+      const numBoxes = output.length / (4 + numClasses);
+      rawDetections = yoloPostprocess(output, numClasses, numBoxes, postprocessOpts);
+    }
+    const detections = this.classMapMode === "all" ? rawDetections : applyClassMap(rawDetections, COCO_TO_MACRO);
+    return {
+      detections,
+      inferenceMs: Date.now() - start,
+      modelId: this.modelEntry.id
+    };
+  }
+  async ensureEngine() {
+    const config = this.resolvedConfig;
+    const modelId = config?.modelId ?? this.modelEntry.id;
+    const runtime = config?.runtime === "python" ? "coreml" : config?.runtime === "node" ? "onnx" : "auto";
+    const backend = config?.backend ?? "cpu";
+    const format = config?.format ?? "onnx";
+    const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId) ?? this.modelEntry;
+    this.modelEntry = entry;
+    const modelsDir = this.ctx.models?.getModelsDir() ?? this.ctx.locationPaths.models;
+    if (this.ctx.models) {
+      await this.ctx.models.ensure(modelId, format);
+    }
+    let pythonPath;
+    if (config?.runtime === "python") {
+      for (const cmd of ["python3", "python"]) {
+        try {
+          const { execSync } = await import("child_process");
+          execSync(`${cmd} --version`, { timeout: 3e3, stdio: "ignore" });
+          pythonPath = cmd;
+          break;
+        } catch {
+        }
+      }
+    }
+    const resolved = await resolveEngine({
+      runtime,
+      backend,
+      modelEntry: entry,
+      modelsDir,
+      pythonPath,
+      models: this.ctx.models
+    });
+    this.engine = resolved.engine;
+  }
+  async shutdown() {
+    await this.engine?.dispose();
+  }
+  getConfigSchema() {
+    return {
+      sections: [
+        {
+          id: "model",
+          title: "Model",
+          columns: 1,
+          fields: [
+            {
+              key: "modelId",
+              label: "Model",
+              type: "model-selector",
+              catalog: [...ALL_DETECTION_MODELS],
+              allowCustom: true,
+              allowConversion: true,
+              acceptFormats: ["onnx", "coreml", "openvino", "tflite"],
+              requiredMetadata: ["inputSize", "labels", "outputFormat"],
+              outputFormatHint: "yolo"
+            }
+          ]
+        },
+        {
+          id: "runtime",
+          title: "Runtime",
+          columns: 2,
+          fields: [
+            {
+              key: "runtime",
+              label: "Runtime",
+              type: "select",
+              options: [
+                { value: "auto", label: "Auto" },
+                { value: "onnx", label: "ONNX Runtime" },
+                { value: "coreml", label: "CoreML (Apple)" },
+                { value: "openvino", label: "OpenVINO (Intel)" }
+              ]
+            },
+            {
+              key: "backend",
+              label: "Backend",
+              type: "select",
+              showWhen: { field: "runtime", equals: "onnx" },
+              options: [
+                { value: "auto", label: "Auto" },
+                { value: "cpu", label: "CPU" },
+                { value: "coreml", label: "CoreML" },
+                { value: "cuda", label: "CUDA (NVIDIA)" },
+                { value: "tensorrt", label: "TensorRT (NVIDIA)" }
+              ]
+            }
+          ]
+        },
+        {
+          id: "thresholds",
+          title: "Detection Thresholds",
+          columns: 2,
+          fields: [
+            {
+              key: "confidence",
+              label: "Confidence Threshold",
+              type: "slider",
+              min: 0.1,
+              max: 1,
+              step: 0.05,
+              default: 0.5
+            },
+            {
+              key: "iouThreshold",
+              label: "IoU Threshold (NMS)",
+              type: "slider",
+              min: 0.1,
+              max: 1,
+              step: 0.05,
+              default: 0.45
+            }
+          ]
+        },
+        {
+          id: "classmap",
+          title: "Class Mapping",
+          columns: 1,
+          fields: [
+            {
+              key: "classMapMode",
+              label: "Output classes",
+              type: "select",
+              options: [
+                { value: "macro", label: "Macro (person / vehicle / animal)" },
+                { value: "all", label: "All COCO classes (80)" }
+              ]
+            }
+          ]
+        }
+      ]
+    };
+  }
+  getClassMap() {
+    return COCO_TO_MACRO;
+  }
+  getModelCatalog() {
+    return [...ALL_DETECTION_MODELS];
+  }
+  getAvailableModels() {
+    return [];
+  }
+  getActiveLabels() {
+    return this.classMapMode === "all" ? COCO_80_LABELS2 : MACRO_LABELS;
+  }
+  async probe() {
+    return {
+      available: true,
+      runtime: this.engine?.runtime ?? "onnx",
+      device: this.engine?.device ?? "cpu",
+      capabilities: ["fp32"]
+    };
+  }
+};
+
+export {
+  SEGMENTATION_MODELS,
+  ObjectDetectionAddon
+};
+//# sourceMappingURL=chunk-R5J3WAUI.mjs.map