@camstack/addon-vision 0.1.2 → 0.1.4

package/dist/chunk-R5J3WAUI.mjs

@@ -1,645 +0,0 @@
-import {
-  nms,
-  yoloPostprocess
-} from "./chunk-KUO2BVFY.mjs";
-import {
-  MLPACKAGE_FILES,
-  OBJECT_DETECTION_MODELS
-} from "./chunk-BP7H4NFS.mjs";
-import {
-  letterbox
-} from "./chunk-22BHCDT5.mjs";
-import {
-  resolveEngine
-} from "./chunk-2IOKI4ES.mjs";
-
-// src/catalogs/segmentation-models.ts
-import { hfModelUrl, COCO_80_LABELS } from "@camstack/types";
-var HF_REPO = "camstack/camstack-models";
-var SEGMENTATION_MODELS = [
-  // ── YOLO11-seg ───────────────────────────────────────────────
-  {
-    id: "yolo11n-seg",
-    name: "YOLO11 Nano Segmentation",
-    description: "YOLO11 Nano \u2014 fastest, smallest YOLO11 instance segmentation model",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11n-seg.onnx"),
-        sizeMB: 11
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11n-seg.mlpackage"),
-        sizeMB: 6,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11n-seg.xml"),
-        sizeMB: 6,
-        runtimes: ["python"]
-      }
-    }
-  },
-  {
-    id: "yolo11s-seg",
-    name: "YOLO11 Small Segmentation",
-    description: "YOLO11 Small \u2014 balanced speed and accuracy for instance segmentation",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11s-seg.onnx"),
-        sizeMB: 39
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11s-seg.mlpackage"),
-        sizeMB: 20,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11s-seg.xml"),
-        sizeMB: 20,
-        runtimes: ["python"]
-      }
-    }
-  },
-  {
-    id: "yolo11m-seg",
-    name: "YOLO11 Medium Segmentation",
-    description: "YOLO11 Medium \u2014 higher accuracy instance segmentation",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/onnx/camstack-yolo11m-seg.onnx"),
-        sizeMB: 86
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/coreml/camstack-yolo11m-seg.mlpackage"),
-        sizeMB: 43,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolo11-seg/openvino/camstack-yolo11m-seg.xml"),
-        sizeMB: 43,
-        runtimes: ["python"]
-      }
-    }
-  },
-  // ── YOLOv8-seg — CoreML supported ─────────────────────────────
-  {
-    id: "yolov8n-seg",
-    name: "YOLOv8 Nano Segmentation",
-    description: "YOLOv8 Nano \u2014 fastest, smallest YOLOv8 instance segmentation model",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8n-seg.onnx"),
-        sizeMB: 13
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8n-seg.mlpackage"),
-        sizeMB: 7,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8n-seg.xml"),
-        sizeMB: 7,
-        runtimes: ["python"]
-      }
-    }
-  },
-  {
-    id: "yolov8s-seg",
-    name: "YOLOv8 Small Segmentation",
-    description: "YOLOv8 Small \u2014 balanced speed and accuracy for instance segmentation",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8s-seg.onnx"),
-        sizeMB: 45
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8s-seg.mlpackage"),
-        sizeMB: 23,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8s-seg.xml"),
-        sizeMB: 23,
-        runtimes: ["python"]
-      }
-    }
-  },
-  {
-    id: "yolov8m-seg",
-    name: "YOLOv8 Medium Segmentation",
-    description: "YOLOv8 Medium \u2014 higher accuracy instance segmentation",
-    inputSize: { width: 640, height: 640 },
-    labels: COCO_80_LABELS,
-    formats: {
-      onnx: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/onnx/camstack-yolov8m-seg.onnx"),
-        sizeMB: 104
-      },
-      coreml: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/coreml/camstack-yolov8m-seg.mlpackage"),
-        sizeMB: 52,
-        isDirectory: true,
-        files: MLPACKAGE_FILES,
-        runtimes: ["python"]
-      },
-      openvino: {
-        url: hfModelUrl(HF_REPO, "segmentation/yolov8-seg/openvino/camstack-yolov8m-seg.xml"),
-        sizeMB: 53,
-        runtimes: ["python"]
-      }
-    }
-  }
-];
-
-// src/addons/object-detection/index.ts
-import { COCO_TO_MACRO, MACRO_LABELS, COCO_80_LABELS as COCO_80_LABELS2 } from "@camstack/types";
-
-// src/shared/postprocess/yolo-seg.ts
-function sigmoid(x) {
-  return 1 / (1 + Math.exp(-x));
-}
-function computeRawMask(coeffs, protos, numMaskCoeffs, maskH, maskW) {
-  const maskSize = maskH * maskW;
-  const rawMask = new Float32Array(maskSize);
-  for (let px = 0; px < maskSize; px++) {
-    let val = 0;
-    for (let k = 0; k < numMaskCoeffs; k++) {
-      val += (coeffs[k] ?? 0) * (protos[k * maskSize + px] ?? 0);
-    }
-    rawMask[px] = sigmoid(val);
-  }
-  return rawMask;
-}
-function cropAndThresholdMask(rawMask, maskH, maskW, bbox, maskThreshold, maskScale) {
-  const cropX1 = Math.max(0, Math.floor(bbox.x * maskScale));
-  const cropY1 = Math.max(0, Math.floor(bbox.y * maskScale));
-  const cropX2 = Math.min(maskW, Math.ceil((bbox.x + bbox.w) * maskScale));
-  const cropY2 = Math.min(maskH, Math.ceil((bbox.y + bbox.h) * maskScale));
-  const cropW = Math.max(1, cropX2 - cropX1);
-  const cropH = Math.max(1, cropY2 - cropY1);
-  const data = new Uint8Array(cropW * cropH);
-  for (let row = 0; row < cropH; row++) {
-    const srcRow = cropY1 + row;
-    for (let col = 0; col < cropW; col++) {
-      const srcCol = cropX1 + col;
-      const srcIdx = srcRow * maskW + srcCol;
-      data[row * cropW + col] = (rawMask[srcIdx] ?? 0) > maskThreshold ? 255 : 0;
-    }
-  }
-  return { data, width: cropW, height: cropH };
-}
-function yoloSegPostprocess(segOutput, options) {
-  const {
-    detectionOutput,
-    protoOutput,
-    numClasses,
-    numBoxes,
-    numMaskCoeffs,
-    maskHeight,
-    maskWidth
-  } = segOutput;
-  const {
-    confidence,
-    iouThreshold,
-    labels,
-    scale,
-    padX,
-    padY,
-    originalWidth,
-    originalHeight,
-    maskThreshold = 0.5
-  } = options;
-  const yoloInputSize = 640;
-  const maskScale = maskHeight / yoloInputSize;
-  const candidates = [];
-  for (let i = 0; i < numBoxes; i++) {
-    const cx = detectionOutput[0 * numBoxes + i] ?? 0;
-    const cy = detectionOutput[1 * numBoxes + i] ?? 0;
-    const w = detectionOutput[2 * numBoxes + i] ?? 0;
-    const h = detectionOutput[3 * numBoxes + i] ?? 0;
-    let bestScore = -Infinity;
-    let bestClass = 0;
-    for (let j = 0; j < numClasses; j++) {
-      const score = detectionOutput[(4 + j) * numBoxes + i] ?? 0;
-      if (score > bestScore) {
-        bestScore = score;
-        bestClass = j;
-      }
-    }
-    if (bestScore < confidence) continue;
-    const bbox = {
-      x: cx - w / 2,
-      y: cy - h / 2,
-      w,
-      h
-    };
-    const coeffs = new Float32Array(numMaskCoeffs);
-    for (let k = 0; k < numMaskCoeffs; k++) {
-      coeffs[k] = detectionOutput[(4 + numClasses + k) * numBoxes + i] ?? 0;
-    }
-    candidates.push({ bbox, score: bestScore, classIdx: bestClass, coeffs });
-  }
-  if (candidates.length === 0) return [];
-  const keptIndices = nms(candidates, iouThreshold);
-  return keptIndices.map((idx) => {
-    const { bbox, score, classIdx, coeffs } = candidates[idx];
-    const label = labels[classIdx] ?? String(classIdx);
-    const x = Math.max(0, Math.min(originalWidth, (bbox.x - padX) / scale));
-    const y = Math.max(0, Math.min(originalHeight, (bbox.y - padY) / scale));
-    const x2 = Math.max(0, Math.min(originalWidth, (bbox.x + bbox.w - padX) / scale));
-    const y2 = Math.max(0, Math.min(originalHeight, (bbox.y + bbox.h - padY) / scale));
-    const finalBbox = { x, y, w: x2 - x, h: y2 - y };
-    const rawMask = computeRawMask(coeffs, protoOutput, numMaskCoeffs, maskHeight, maskWidth);
-    const { data: maskData, width: mW, height: mH } = cropAndThresholdMask(
-      rawMask,
-      maskHeight,
-      maskWidth,
-      bbox,
-      maskThreshold,
-      maskScale
-    );
-    return {
-      class: label,
-      originalClass: label,
-      score,
-      bbox: finalBbox,
-      mask: maskData,
-      maskWidth: mW,
-      maskHeight: mH
-    };
-  });
-}
-
-// src/addons/object-detection/index.ts
-function isSegModel(modelId) {
-  return modelId.includes("-seg");
-}
-var ALL_DETECTION_MODELS = [
-  ...OBJECT_DETECTION_MODELS,
-  ...SEGMENTATION_MODELS
-];
-function applyClassMap(detections, classMap) {
-  return detections.filter((d) => classMap.mapping[d.class] !== void 0).map((d) => ({
-    ...d,
-    originalClass: d.class,
-    class: classMap.mapping[d.class]
-  }));
-}
-var RAM_ESTIMATES = {
-  "yolov8n": 80,
-  "yolov8s": 150,
-  "yolov8s-relu": 150,
-  "yolov8m": 300,
-  "yolov8l": 500,
-  "yolov8x": 800,
-  "yolov9t": 60,
-  "yolov9s": 120,
-  "yolov9c": 300,
-  "yolo11n": 70,
-  "yolo11s": 130,
-  "yolo11m": 280,
-  "yolo11l": 450,
-  "yolo11x": 750,
-  "yolo11n-seg": 84,
-  "yolo11s-seg": 156,
-  "yolo11m-seg": 336,
-  "yolov8n-seg": 96,
-  "yolov8s-seg": 180,
-  "yolov8m-seg": 360
-};
-var ACCURACY_SCORES = {
-  "yolov8n": 55,
-  "yolov8s": 70,
-  "yolov8s-relu": 68,
-  "yolov8m": 82,
-  "yolov8l": 88,
-  "yolov8x": 92,
-  "yolov9t": 58,
-  "yolov9s": 73,
-  "yolov9c": 86,
-  "yolo11n": 62,
-  "yolo11s": 78,
-  "yolo11m": 88,
-  "yolo11l": 93,
-  "yolo11x": 97,
-  "yolo11n-seg": 62,
-  "yolo11s-seg": 78,
-  "yolo11m-seg": 88,
-  "yolov8n-seg": 55,
-  "yolov8s-seg": 70,
-  "yolov8m-seg": 82
-};
-var ObjectDetectionAddon = class {
-  id = "object-detection";
-  slot = "detector";
-  inputClasses = null;
-  outputClasses = ["person", "vehicle", "animal"];
-  slotPriority = 0;
-  manifest = {
-    id: "object-detection",
-    name: "Object Detection",
-    version: "0.1.0",
-    description: "YOLO-based object detection \u2014 detects persons, vehicles, and animals",
-    slot: "detector",
-    inputClasses: void 0,
-    outputClasses: ["person", "vehicle", "animal"],
-    supportsCustomModels: true,
-    mayRequirePython: false,
-    defaultConfig: {
-      modelId: "yolo11n",
-      runtime: "node",
-      backend: "cpu",
-      confidence: 0.5,
-      iouThreshold: 0.45,
-      classMapMode: "macro"
-    }
-  };
-  engine = null;
-  modelEntry;
-  confidence = 0.5;
-  iouThreshold = 0.45;
-  classMapMode = "macro";
-  resolvedConfig = null;
-  ctx = null;
-  getModelRequirements() {
-    return ALL_DETECTION_MODELS.map((m) => ({
-      modelId: m.id,
-      name: m.name,
-      minRAM_MB: RAM_ESTIMATES[m.id] ?? 100,
-      accuracyScore: ACCURACY_SCORES[m.id] ?? 60,
-      formats: Object.keys(m.formats)
-    }));
-  }
-  configure(config) {
-    this.resolvedConfig = config;
-  }
-  async initialize(ctx) {
-    this.ctx = ctx;
-    const cfg = ctx.addonConfig;
-    const modelId = cfg["modelId"] ?? this.resolvedConfig?.modelId ?? "yolo11n";
-    this.confidence = cfg["confidence"] ?? 0.5;
-    this.iouThreshold = cfg["iouThreshold"] ?? 0.45;
-    this.classMapMode = cfg["classMapMode"] ?? "macro";
-    const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId);
-    if (!entry) {
-      throw new Error(`ObjectDetectionAddon: unknown modelId "${modelId}"`);
-    }
-    this.modelEntry = entry;
-  }
-  async detect(frame) {
-    if (!this.engine) await this.ensureEngine();
-    const start = Date.now();
-    if ("runJpeg" in this.engine && typeof this.engine.runJpeg === "function") {
-      const result = await this.engine.runJpeg(frame.data);
-      const rawDets = result.detections ?? [];
-      const detections2 = rawDets.map((d) => ({
-        class: this.classMapMode === "all" ? d.className : COCO_TO_MACRO.mapping[d.className] ?? d.className,
-        originalClass: d.className,
-        score: d.score,
-        bbox: {
-          x: d.bbox[0] * frame.width,
-          y: d.bbox[1] * frame.height,
-          w: (d.bbox[2] - d.bbox[0]) * frame.width,
-          h: (d.bbox[3] - d.bbox[1]) * frame.height
-        }
-      })).filter((d) => this.classMapMode === "all" || COCO_TO_MACRO.mapping[d.originalClass] !== void 0);
-      return {
-        detections: detections2,
-        inferenceMs: result.inferenceMs ?? Date.now() - start,
-        modelId: this.modelEntry.id
-      };
-    }
-    const { width: inputW, height: inputH } = this.modelEntry.inputSize;
-    const targetSize = Math.max(inputW, inputH);
-    const lb = await letterbox(frame.data, targetSize);
-    const numClasses = this.modelEntry.labels.length;
-    const labels = this.modelEntry.labels.map((l) => l.id);
-    const postprocessOpts = {
-      confidence: this.confidence,
-      iouThreshold: this.iouThreshold,
-      labels,
-      scale: lb.scale,
-      padX: lb.padX,
-      padY: lb.padY,
-      originalWidth: lb.originalWidth,
-      originalHeight: lb.originalHeight
-    };
-    let rawDetections;
-    if (isSegModel(this.modelEntry.id)) {
-      const outputs = await this.engine.runMultiOutput(lb.data, [1, 3, targetSize, targetSize]);
-      const outputNames = Object.keys(outputs);
-      if (outputNames.length < 2) {
-        throw new Error(
-          `ObjectDetectionAddon: seg model "${this.modelEntry.id}" returned ${outputNames.length} output(s); expected 2`
-        );
-      }
-      const detectionOutput = outputs[outputNames[0]];
-      const protoOutput = outputs[outputNames[1]];
-      const numMaskCoeffs = 32;
-      const numBoxes = detectionOutput.length / (4 + numClasses + numMaskCoeffs);
-      const maskHeight = 160;
-      const maskWidth = 160;
-      rawDetections = yoloSegPostprocess(
-        {
-          detectionOutput,
-          protoOutput,
-          numClasses,
-          numBoxes,
-          numMaskCoeffs,
-          maskHeight,
-          maskWidth
-        },
-        postprocessOpts
-      );
-    } else {
-      const output = await this.engine.run(lb.data, [1, 3, targetSize, targetSize]);
-      const numBoxes = output.length / (4 + numClasses);
-      rawDetections = yoloPostprocess(output, numClasses, numBoxes, postprocessOpts);
-    }
-    const detections = this.classMapMode === "all" ? rawDetections : applyClassMap(rawDetections, COCO_TO_MACRO);
-    return {
-      detections,
-      inferenceMs: Date.now() - start,
-      modelId: this.modelEntry.id
-    };
-  }
-  async ensureEngine() {
-    const config = this.resolvedConfig;
-    const modelId = config?.modelId ?? this.modelEntry.id;
-    const runtime = config?.runtime === "python" ? "coreml" : config?.runtime === "node" ? "onnx" : "auto";
-    const backend = config?.backend ?? "cpu";
-    const format = config?.format ?? "onnx";
-    const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId) ?? this.modelEntry;
-    this.modelEntry = entry;
-    const modelsDir = this.ctx.models?.getModelsDir() ?? this.ctx.locationPaths.models;
-    if (this.ctx.models) {
-      await this.ctx.models.ensure(modelId, format);
-    }
-    let pythonPath;
-    if (config?.runtime === "python") {
-      for (const cmd of ["python3", "python"]) {
-        try {
-          const { execSync } = await import("child_process");
-          execSync(`${cmd} --version`, { timeout: 3e3, stdio: "ignore" });
-          pythonPath = cmd;
-          break;
-        } catch {
-        }
-      }
-    }
-    const resolved = await resolveEngine({
-      runtime,
-      backend,
-      modelEntry: entry,
-      modelsDir,
-      pythonPath,
-      models: this.ctx.models
-    });
-    this.engine = resolved.engine;
-  }
-  async shutdown() {
-    await this.engine?.dispose();
-  }
-  getConfigSchema() {
-    return {
-      sections: [
-        {
-          id: "model",
-          title: "Model",
-          columns: 1,
-          fields: [
-            {
-              key: "modelId",
-              label: "Model",
-              type: "model-selector",
-              catalog: [...ALL_DETECTION_MODELS],
-              allowCustom: true,
-              allowConversion: true,
-              acceptFormats: ["onnx", "coreml", "openvino", "tflite"],
-              requiredMetadata: ["inputSize", "labels", "outputFormat"],
-              outputFormatHint: "yolo"
-            }
-          ]
-        },
-        {
-          id: "runtime",
-          title: "Runtime",
-          columns: 2,
-          fields: [
-            {
-              key: "runtime",
-              label: "Runtime",
-              type: "select",
-              options: [
-                { value: "auto", label: "Auto" },
-                { value: "onnx", label: "ONNX Runtime" },
-                { value: "coreml", label: "CoreML (Apple)" },
-                { value: "openvino", label: "OpenVINO (Intel)" }
-              ]
-            },
-            {
-              key: "backend",
-              label: "Backend",
-              type: "select",
-              showWhen: { field: "runtime", equals: "onnx" },
-              options: [
-                { value: "auto", label: "Auto" },
-                { value: "cpu", label: "CPU" },
-                { value: "coreml", label: "CoreML" },
-                { value: "cuda", label: "CUDA (NVIDIA)" },
-                { value: "tensorrt", label: "TensorRT (NVIDIA)" }
-              ]
-            }
-          ]
-        },
-        {
-          id: "thresholds",
-          title: "Detection Thresholds",
-          columns: 2,
-          fields: [
-            {
-              key: "confidence",
-              label: "Confidence Threshold",
-              type: "slider",
-              min: 0.1,
-              max: 1,
-              step: 0.05,
-              default: 0.5
-            },
-            {
-              key: "iouThreshold",
-              label: "IoU Threshold (NMS)",
-              type: "slider",
-              min: 0.1,
-              max: 1,
-              step: 0.05,
-              default: 0.45
-            }
-          ]
-        },
-        {
-          id: "classmap",
-          title: "Class Mapping",
-          columns: 1,
-          fields: [
-            {
-              key: "classMapMode",
-              label: "Output classes",
-              type: "select",
-              options: [
-                { value: "macro", label: "Macro (person / vehicle / animal)" },
-                { value: "all", label: "All COCO classes (80)" }
-              ]
-            }
-          ]
-        }
-      ]
-    };
-  }
-  getClassMap() {
-    return COCO_TO_MACRO;
-  }
-  getModelCatalog() {
-    return [...ALL_DETECTION_MODELS];
-  }
-  getAvailableModels() {
-    return [];
-  }
-  getActiveLabels() {
-    return this.classMapMode === "all" ? COCO_80_LABELS2 : MACRO_LABELS;
-  }
-  async probe() {
-    return {
-      available: true,
-      runtime: this.engine?.runtime ?? "onnx",
-      device: this.engine?.device ?? "cpu",
-      capabilities: ["fp32"]
-    };
-  }
-};
-
-export {
-  SEGMENTATION_MODELS,
-  ObjectDetectionAddon
-};
-//# sourceMappingURL=chunk-R5J3WAUI.mjs.map