@camstack/addon-vision 0.1.0

package/dist/addons/face-recognition/index.js

@@ -0,0 +1,603 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/addons/face-recognition/index.ts
+var face_recognition_exports = {};
+__export(face_recognition_exports, {
+  default: () => FaceRecognitionAddon
+});
+module.exports = __toCommonJS(face_recognition_exports);
+
+// src/catalogs/face-recognition-models.ts
+var import_types = require("@camstack/types");
+var HF_REPO = "camstack/camstack-models";
+var FACE_EMBEDDING_LABELS = [
+  { id: "embedding", name: "Face Embedding" }
+];
+var FACE_RECOGNITION_MODELS = [
+  {
+    id: "arcface-r100",
+    name: "ArcFace R100",
+    description: "ArcFace ResNet-100 \u2014 high-accuracy face recognition embeddings",
+    inputSize: { width: 112, height: 112 },
+    inputLayout: "nhwc",
+    labels: FACE_EMBEDDING_LABELS,
+    formats: {
+      onnx: {
+        url: (0, import_types.hfModelUrl)(HF_REPO, "faceRecognition/arcface/onnx/camstack-arcface-arcface.onnx"),
+        sizeMB: 130
+      },
+      coreml: {
+        url: (0, import_types.hfModelUrl)(HF_REPO, "faceRecognition/arcface/coreml/camstack-arcface-r100.mlpackage"),
+        sizeMB: 65
+      },
+      openvino: {
+        url: (0, import_types.hfModelUrl)(HF_REPO, "faceRecognition/arcface/openvino/camstack-arcface-r100.xml"),
+        sizeMB: 65
+      }
+    }
+  }
+];
+
+// src/shared/image-utils.ts
+var import_sharp = __toESM(require("sharp"));
+async function cropRegion(jpeg, roi) {
+  return (0, import_sharp.default)(jpeg).extract({
+    left: Math.round(roi.x),
+    top: Math.round(roi.y),
+    width: Math.round(roi.w),
+    height: Math.round(roi.h)
+  }).jpeg().toBuffer();
+}
+async function resizeAndNormalize(jpeg, targetWidth, targetHeight, normalization, layout) {
+  const { data } = await (0, import_sharp.default)(jpeg).resize(targetWidth, targetHeight).removeAlpha().raw().toBuffer({ resolveWithObject: true });
+  const numPixels = targetWidth * targetHeight;
+  const float32 = new Float32Array(3 * numPixels);
+  const mean = [0.485, 0.456, 0.406];
+  const std = [0.229, 0.224, 0.225];
+  if (layout === "nchw") {
+    for (let i = 0; i < numPixels; i++) {
+      const srcBase = i * 3;
+      for (let c = 0; c < 3; c++) {
+        const raw = data[srcBase + c] / 255;
+        let val;
+        if (normalization === "zero-one") {
+          val = raw;
+        } else if (normalization === "imagenet") {
+          val = (raw - mean[c]) / std[c];
+        } else {
+          val = data[srcBase + c];
+        }
+        float32[c * numPixels + i] = val;
+      }
+    }
+  } else {
+    for (let i = 0; i < numPixels; i++) {
+      const srcBase = i * 3;
+      for (let c = 0; c < 3; c++) {
+        const raw = data[srcBase + c] / 255;
+        let val;
+        if (normalization === "zero-one") {
+          val = raw;
+        } else if (normalization === "imagenet") {
+          val = (raw - mean[c]) / std[c];
+        } else {
+          val = data[srcBase + c];
+        }
+        float32[i * 3 + c] = val;
+      }
+    }
+  }
+  return float32;
+}
+
+// src/shared/postprocess/arcface.ts
+function l2Normalize(vec) {
+  let sumSq = 0;
+  for (let i = 0; i < vec.length; i++) {
+    sumSq += vec[i] * vec[i];
+  }
+  const norm = Math.sqrt(sumSq);
+  if (norm === 0) return new Float32Array(vec.length);
+  const out = new Float32Array(vec.length);
+  for (let i = 0; i < vec.length; i++) {
+    out[i] = vec[i] / norm;
+  }
+  return out;
+}
+
+// src/shared/engine-resolver.ts
+var fs = __toESM(require("fs"));
+var path2 = __toESM(require("path"));
+
+// src/shared/node-engine.ts
+var path = __toESM(require("path"));
+var BACKEND_TO_PROVIDER = {
+  cpu: "cpu",
+  coreml: "coreml",
+  cuda: "cuda",
+  tensorrt: "tensorrt",
+  dml: "dml"
+};
+var BACKEND_TO_DEVICE = {
+  cpu: "cpu",
+  coreml: "gpu-mps",
+  cuda: "gpu-cuda",
+  tensorrt: "tensorrt"
+};
+var NodeInferenceEngine = class {
+  constructor(modelPath, backend) {
+    this.modelPath = modelPath;
+    this.backend = backend;
+    this.device = BACKEND_TO_DEVICE[backend] ?? "cpu";
+  }
+  runtime = "onnx";
+  device;
+  session = null;
+  async initialize() {
+    const ort = await import("onnxruntime-node");
+    const provider = BACKEND_TO_PROVIDER[this.backend] ?? "cpu";
+    const absModelPath = path.isAbsolute(this.modelPath) ? this.modelPath : path.resolve(process.cwd(), this.modelPath);
+    const sessionOptions = {
+      executionProviders: [provider]
+    };
+    this.session = await ort.InferenceSession.create(absModelPath, sessionOptions);
+  }
+  async run(input, inputShape) {
+    if (!this.session) {
+      throw new Error("NodeInferenceEngine: not initialized \u2014 call initialize() first");
+    }
+    const ort = await import("onnxruntime-node");
+    const sess = this.session;
+    const inputName = sess.inputNames[0];
+    const tensor = new ort.Tensor("float32", input, [...inputShape]);
+    const feeds = { [inputName]: tensor };
+    const results = await sess.run(feeds);
+    const outputName = sess.outputNames[0];
+    const outputTensor = results[outputName];
+    return outputTensor.data;
+  }
+  async runMultiOutput(input, inputShape) {
+    if (!this.session) {
+      throw new Error("NodeInferenceEngine: not initialized \u2014 call initialize() first");
+    }
+    const ort = await import("onnxruntime-node");
+    const sess = this.session;
+    const inputName = sess.inputNames[0];
+    const tensor = new ort.Tensor("float32", input, [...inputShape]);
+    const feeds = { [inputName]: tensor };
+    const results = await sess.run(feeds);
+    const out = {};
+    for (const name of sess.outputNames) {
+      out[name] = results[name].data;
+    }
+    return out;
+  }
+  async dispose() {
+    this.session = null;
+  }
+};
+
+// src/shared/python-engine.ts
+var import_node_child_process = require("child_process");
+var PythonInferenceEngine = class {
+  constructor(pythonPath, scriptPath, runtime, modelPath, extraArgs = []) {
+    this.pythonPath = pythonPath;
+    this.scriptPath = scriptPath;
+    this.modelPath = modelPath;
+    this.extraArgs = extraArgs;
+    this.runtime = runtime;
+    const runtimeDeviceMap = {
+      onnx: "cpu",
+      coreml: "gpu-mps",
+      pytorch: "cpu",
+      openvino: "cpu",
+      tflite: "cpu"
+    };
+    this.device = runtimeDeviceMap[runtime];
+  }
+  runtime;
+  device;
+  process = null;
+  receiveBuffer = Buffer.alloc(0);
+  pendingResolve = null;
+  pendingReject = null;
+  async initialize() {
+    const args = [this.scriptPath, this.modelPath, ...this.extraArgs];
+    this.process = (0, import_node_child_process.spawn)(this.pythonPath, args, {
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    if (!this.process.stdout || !this.process.stdin) {
+      throw new Error("PythonInferenceEngine: failed to create process pipes");
+    }
+    this.process.stderr?.on("data", (chunk) => {
+      process.stderr.write(`[python-engine] ${chunk.toString()}`);
+    });
+    this.process.on("error", (err) => {
+      this.pendingReject?.(err);
+      this.pendingReject = null;
+      this.pendingResolve = null;
+    });
+    this.process.on("exit", (code) => {
+      if (code !== 0) {
+        const err = new Error(`PythonInferenceEngine: process exited with code ${code}`);
+        this.pendingReject?.(err);
+        this.pendingReject = null;
+        this.pendingResolve = null;
+      }
+    });
+    this.process.stdout.on("data", (chunk) => {
+      this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
+      this._tryReceive();
+    });
+    await new Promise((resolve2, reject) => {
+      const timeout = setTimeout(() => resolve2(), 2e3);
+      this.process?.on("error", (err) => {
+        clearTimeout(timeout);
+        reject(err);
+      });
+      this.process?.on("exit", (code) => {
+        clearTimeout(timeout);
+        if (code !== 0) {
+          reject(new Error(`PythonInferenceEngine: process exited early with code ${code}`));
+        }
+      });
+    });
+  }
+  _tryReceive() {
+    if (this.receiveBuffer.length < 4) return;
+    const length = this.receiveBuffer.readUInt32LE(0);
+    if (this.receiveBuffer.length < 4 + length) return;
+    const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
+    this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
+    const resolve2 = this.pendingResolve;
+    const reject = this.pendingReject;
+    this.pendingResolve = null;
+    this.pendingReject = null;
+    if (!resolve2) return;
+    try {
+      const parsed = JSON.parse(jsonBytes.toString("utf8"));
+      resolve2(parsed);
+    } catch (err) {
+      reject?.(err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+  /** Send JPEG buffer, receive JSON detection results */
+  async runJpeg(jpeg) {
+    if (!this.process?.stdin) {
+      throw new Error("PythonInferenceEngine: process not initialized");
+    }
+    return new Promise((resolve2, reject) => {
+      this.pendingResolve = resolve2;
+      this.pendingReject = reject;
+      const lengthBuf = Buffer.allocUnsafe(4);
+      lengthBuf.writeUInt32LE(jpeg.length, 0);
+      this.process.stdin.write(Buffer.concat([lengthBuf, jpeg]));
+    });
+  }
+  /** IInferenceEngine.run — wraps runJpeg for compatibility */
+  async run(_input, _inputShape) {
+    throw new Error(
+      "PythonInferenceEngine: use runJpeg() directly \u2014 this engine operates on JPEG input"
+    );
+  }
+  /** IInferenceEngine.runMultiOutput — not supported by Python engine (operates on JPEG input) */
+  async runMultiOutput(_input, _inputShape) {
+    throw new Error(
+      "PythonInferenceEngine: runMultiOutput() is not supported \u2014 this engine operates on JPEG input"
+    );
+  }
+  async dispose() {
+    if (this.process) {
+      this.process.stdin?.end();
+      this.process.kill("SIGTERM");
+      this.process = null;
+    }
+  }
+};
+
+// src/shared/engine-resolver.ts
+var AUTO_BACKEND_PRIORITY = ["coreml", "cuda", "tensorrt", "cpu"];
+var BACKEND_TO_FORMAT = {
+  cpu: "onnx",
+  coreml: "coreml",
+  cuda: "onnx",
+  tensorrt: "onnx"
+};
+var RUNTIME_TO_FORMAT = {
+  onnx: "onnx",
+  coreml: "coreml",
+  openvino: "openvino",
+  tflite: "tflite",
+  pytorch: "pt"
+};
+function modelFilePath(modelsDir, modelEntry, format) {
+  const formatEntry = modelEntry.formats[format];
+  if (!formatEntry) {
+    throw new Error(`Model ${modelEntry.id} has no ${format} format`);
+  }
+  const urlParts = formatEntry.url.split("/");
+  const filename = urlParts[urlParts.length - 1] ?? `${modelEntry.id}.${format}`;
+  return path2.join(modelsDir, filename);
+}
+function modelExists(filePath) {
+  try {
+    return fs.existsSync(filePath);
+  } catch {
+    return false;
+  }
+}
+async function resolveEngine(options) {
+  const { runtime, backend, modelEntry, modelsDir, downloadModel } = options;
+  let selectedFormat;
+  let selectedBackend;
+  if (runtime === "auto") {
+    const available = await probeOnnxBackends();
+    let chosen = null;
+    for (const b of AUTO_BACKEND_PRIORITY) {
+      if (!available.includes(b)) continue;
+      const fmt = BACKEND_TO_FORMAT[b];
+      if (!fmt) continue;
+      if (!modelEntry.formats[fmt]) continue;
+      chosen = { backend: b, format: fmt };
+      break;
+    }
+    if (!chosen) {
+      throw new Error(
+        `resolveEngine: no compatible backend found for model ${modelEntry.id}. Available backends: ${available.join(", ")}`
+      );
+    }
+    selectedFormat = chosen.format;
+    selectedBackend = chosen.backend;
+  } else {
+    const fmt = RUNTIME_TO_FORMAT[runtime];
+    if (!fmt) {
+      throw new Error(`resolveEngine: unsupported runtime "${runtime}"`);
+    }
+    if (!modelEntry.formats[fmt]) {
+      throw new Error(
+        `resolveEngine: model ${modelEntry.id} has no ${fmt} format for runtime ${runtime}`
+      );
+    }
+    selectedFormat = fmt;
+    selectedBackend = runtime === "onnx" ? backend || "cpu" : runtime;
+  }
+  let modelPath = modelFilePath(modelsDir, modelEntry, selectedFormat);
+  if (!modelExists(modelPath)) {
+    if (downloadModel) {
+      const formatEntry = modelEntry.formats[selectedFormat];
+      modelPath = await downloadModel(formatEntry.url, modelsDir);
+    } else {
+      throw new Error(
+        `resolveEngine: model file not found at ${modelPath} and no downloadModel function provided`
+      );
+    }
+  }
+  if (selectedFormat === "onnx" || selectedFormat === "coreml") {
+    const engine = new NodeInferenceEngine(modelPath, selectedBackend);
+    await engine.initialize();
+    return { engine, format: selectedFormat, modelPath };
+  }
+  const { pythonPath } = options;
+  const PYTHON_SCRIPT_MAP = {
+    coreml: "coreml_inference.py",
+    pytorch: "pytorch_inference.py",
+    openvino: "openvino_inference.py"
+  };
+  const effectiveRuntime = runtime === "auto" ? selectedBackend : runtime;
+  const scriptName = PYTHON_SCRIPT_MAP[effectiveRuntime];
+  if (scriptName && pythonPath) {
+    const scriptPath = path2.join(__dirname, "../../python", scriptName);
+    const inputSize = Math.max(modelEntry.inputSize.width, modelEntry.inputSize.height);
+    const engine = new PythonInferenceEngine(pythonPath, scriptPath, effectiveRuntime, modelPath, [
+      `--input-size=${inputSize}`,
+      `--confidence=0.25`
+    ]);
+    await engine.initialize();
+    return { engine, format: selectedFormat, modelPath };
+  }
+  const fallbackPath = modelFilePath(modelsDir, modelEntry, "onnx");
+  if (modelEntry.formats["onnx"] && modelExists(fallbackPath)) {
+    const engine = new NodeInferenceEngine(fallbackPath, "cpu");
+    await engine.initialize();
+    return { engine, format: "onnx", modelPath: fallbackPath };
+  }
+  throw new Error(
+    `resolveEngine: format ${selectedFormat} is not yet supported by NodeInferenceEngine, no Python runtime is available, and no ONNX fallback exists`
+  );
+}
+async function probeOnnxBackends() {
+  const available = ["cpu"];
+  try {
+    const ort = await import("onnxruntime-node");
+    const providers = ort.env?.webgl?.disabled !== void 0 ? ort.InferenceSession?.getAvailableProviders?.() ?? [] : [];
+    for (const p of providers) {
+      const normalized = p.toLowerCase().replace("executionprovider", "");
+      if (normalized === "coreml") available.push("coreml");
+      else if (normalized === "cuda") available.push("cuda");
+      else if (normalized === "tensorrt") available.push("tensorrt");
+    }
+  } catch {
+  }
+  if (process.platform === "darwin" && !available.includes("coreml")) {
+    available.push("coreml");
+  }
+  return [...new Set(available)];
+}
+
+// src/addons/face-recognition/index.ts
+var IDENTITY_LABEL = { id: "identity", name: "Identity" };
+var IDENTITY_LABELS = [IDENTITY_LABEL];
+var FACE_REC_CLASS_MAP = { mapping: {}, preserveOriginal: true };
+var REQUIRED_STEPS = [
+  { slot: "cropper", outputClasses: ["face"], description: "Requires a face detector" }
+];
+var FaceRecognitionAddon = class {
+  id = "face-recognition";
+  slot = "classifier";
+  inputClasses = ["face"];
+  outputClasses = ["identity:*"];
+  slotPriority = 0;
+  requiredSteps = REQUIRED_STEPS;
+  manifest = {
+    id: "face-recognition",
+    name: "Face Recognition",
+    version: "0.1.0",
+    description: "ArcFace-based face recognition \u2014 produces 512-d identity embeddings",
+    packageName: "@camstack/addon-vision",
+    slot: "classifier",
+    inputClasses: ["face"],
+    outputClasses: ["identity:*"],
+    requiredSteps: REQUIRED_STEPS,
+    supportsCustomModels: false,
+    mayRequirePython: false,
+    defaultConfig: {
+      modelId: "arcface-r100",
+      runtime: "auto",
+      backend: "cpu"
+    }
+  };
+  engine;
+  modelEntry;
+  async initialize(ctx) {
+    const cfg = ctx.addonConfig;
+    const modelId = cfg["modelId"] ?? "arcface-r100";
+    const runtime = cfg["runtime"] ?? "auto";
+    const backend = cfg["backend"] ?? "cpu";
+    const entry = FACE_RECOGNITION_MODELS.find((m) => m.id === modelId);
+    if (!entry) {
+      throw new Error(`FaceRecognitionAddon: unknown modelId "${modelId}"`);
+    }
+    this.modelEntry = entry;
+    const resolved = await resolveEngine({
+      runtime,
+      backend,
+      modelEntry: entry,
+      modelsDir: ctx.locationPaths.models
+    });
+    this.engine = resolved.engine;
+  }
+  async classify(input) {
+    const start = Date.now();
+    const { width: inputW, height: inputH } = this.modelEntry.inputSize;
+    const faceCrop = await cropRegion(input.frame.data, input.roi);
+    const layout = this.modelEntry.inputLayout ?? "nhwc";
+    const normalization = this.modelEntry.inputNormalization ?? "zero-one";
+    const normalized = await resizeAndNormalize(faceCrop, inputW, inputH, normalization, layout);
+    const rawEmbedding = await this.engine.run(normalized, [1, inputH, inputW, 3]);
+    const embedding = l2Normalize(rawEmbedding);
+    return {
+      classifications: [
+        {
+          class: "identity",
+          score: 1,
+          embedding
+        }
+      ],
+      inferenceMs: Date.now() - start,
+      modelId: this.modelEntry.id
+    };
+  }
+  async shutdown() {
+    await this.engine?.dispose();
+  }
+  getConfigSchema() {
+    return {
+      sections: [
+        {
+          id: "model",
+          title: "Model",
+          columns: 1,
+          fields: [
+            {
+              key: "modelId",
+              label: "Model",
+              type: "model-selector",
+              catalog: [...FACE_RECOGNITION_MODELS],
+              allowCustom: false,
+              allowConversion: false,
+              acceptFormats: ["onnx", "coreml", "openvino"],
+              requiredMetadata: ["inputSize", "labels", "outputFormat"],
+              outputFormatHint: "embedding"
+            }
+          ]
+        },
+        {
+          id: "runtime",
+          title: "Runtime",
+          columns: 2,
+          fields: [
+            {
+              key: "runtime",
+              label: "Runtime",
+              type: "select",
+              options: [
+                { value: "auto", label: "Auto (recommended)" },
+                { value: "onnx", label: "ONNX Runtime" },
+                { value: "coreml", label: "CoreML (Apple)" }
+              ]
+            },
+            {
+              key: "backend",
+              label: "Backend",
+              type: "select",
+              dependsOn: { runtime: "onnx" },
+              options: [
+                { value: "cpu", label: "CPU" },
+                { value: "coreml", label: "CoreML" },
+                { value: "cuda", label: "CUDA (NVIDIA)" }
+              ]
+            }
+          ]
+        }
+      ]
+    };
+  }
+  getClassMap() {
+    return FACE_REC_CLASS_MAP;
+  }
+  getModelCatalog() {
+    return [...FACE_RECOGNITION_MODELS];
+  }
+  getAvailableModels() {
+    return [];
+  }
+  getActiveLabels() {
+    return IDENTITY_LABELS;
+  }
+  async probe() {
+    return {
+      available: true,
+      runtime: this.engine?.runtime ?? "onnx",
+      device: this.engine?.device ?? "cpu",
+      capabilities: ["fp32"]
+    };
+  }
+};
+//# sourceMappingURL=index.js.map