@camstack/addon-vision 0.1.7 → 0.1.10

package/dist/addons/animal-classifier/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/addons/animal-classifier/index.ts"],"sourcesContent":["import type {\n  IClassifierProvider,\n  IDetectionAddon,\n  AddonManifest,\n  AddonContext,\n  CropInput,\n  ClassifierOutput,\n  ConfigUISchema,\n  ClassMapDefinition,\n  ProbeResult,\n  ModelCatalogEntry,\n  DetectionModel,\n  LabelDefinition,\n  IInferenceEngine,\n  ModelRequirement,\n  ResolvedInferenceConfig,\n} from '@camstack/types'\nimport { ANIMAL_TYPE_MODELS } from '../../catalogs/animal-classification-models.js'\nimport { cropRegion, resizeAndNormalize } from '../../shared/image-utils.js'\nimport { resolveEngine } from '../../shared/engine-resolver.js'\n\nconst ANIMAL_TYPE_LABEL: LabelDefinition = { id: 'animal-type', name: 'Animal Type' }\nconst ANIMAL_TYPE_LABELS: readonly LabelDefinition[] = [ANIMAL_TYPE_LABEL]\nconst ANIMAL_CLASS_MAP: ClassMapDefinition = { mapping: {}, preserveOriginal: true }\n\nconst ANIMAL_10_CLASSES = [\n  'cat',\n  'cow',\n  'dog',\n  'dolphin',\n  'eagle',\n  'giant panda',\n  'horse',\n  'monkey',\n  'sheep',\n  'spider',\n] as const\n\nfunction softmax(logits: Float32Array): Float32Array {\n  const max = logits.reduce((a, b) => Math.max(a, b), -Infinity)\n  const exps = logits.map((v) => Math.exp(v - max))\n  const sum = exps.reduce((a, b) => a + b, 0)\n  return exps.map((v) => v / sum) as unknown as Float32Array\n}\n\nexport default class AnimalClassifierAddon implements IClassifierProvider, IDetectionAddon {\n  readonly id = 'animal-classifier'\n  readonly slot = 'classifier' as const\n  readonly inputClasses = ['animal'] as const\n  readonly outputClasses = ['animal-type:*'] as const\n  readonly slotPriority = 0\n  readonly requiredSteps = [] as const\n  readonly manifest: AddonManifest = {\n    id: 'animal-classifier',\n    name: 'Animal Classifier',\n    version: '0.1.0',\n\n    description: 'ViT-based animal type classifier — 10 common species',\n\n    slot: 'classifier',\n    labelOutputType: 'classification',\n    inputClasses: ['animal'],\n    outputClasses: ['animal-type:*'],\n    supportsCustomModels: false,\n    mayRequirePython: false,\n    defaultConfig: {\n      modelId: 'animals-10',\n      runtime: 'node',\n      backend: 'cpu',\n      minConfidence: 0.3,\n    },\n  }\n\n  private engine: IInferenceEngine | null = null\n  private modelEntry!: ModelCatalogEntry\n  private minConfidence = 0.3\n  private resolvedConfig: ResolvedInferenceConfig | null = null\n  private ctx: AddonContext | null = null\n\n  getModelRequirements(): ModelRequirement[] {\n    return ANIMAL_TYPE_MODELS.map((m) => ({\n      modelId: m.id,\n      name: m.name,\n      minRAM_MB: 800,\n      accuracyScore: 75,\n      formats: Object.keys(m.formats) as readonly string[],\n    }))\n  }\n\n  configure(config: ResolvedInferenceConfig): void {\n    this.resolvedConfig = config\n  }\n\n  async initialize(ctx: AddonContext): Promise<void> {\n    this.ctx = ctx\n    const cfg = ctx.addonConfig\n    const modelId = (cfg['modelId'] as string | undefined) ?? this.resolvedConfig?.modelId ?? 'animals-10'\n    this.minConfidence = (cfg['minConfidence'] as number | undefined) ?? 0.3\n\n    const entry = ANIMAL_TYPE_MODELS.find((m) => m.id === modelId)\n    if (!entry) {\n      throw new Error(`AnimalClassifierAddon: unknown modelId \"${modelId}\"`)\n    }\n    this.modelEntry = entry\n  }\n\n  async classify(input: CropInput): Promise<ClassifierOutput> {\n    if (!this.engine) await this.ensureEngine()\n    const start = Date.now()\n    const { width: inputW, height: inputH } = this.modelEntry.inputSize\n\n    // Crop the animal region\n    const animalCrop = await cropRegion(input.frame.data, input.roi)\n\n    // Resize to 224x224, ImageNet normalization, NCHW\n    const normalized = await resizeAndNormalize(animalCrop, inputW, inputH, 'imagenet', 'nchw')\n\n    // Run inference — output shape: [1, 10]\n    const rawOutput = await this.engine!.run(normalized, [1, 3, inputH, inputW])\n\n    // Softmax to get probabilities\n    const probs = softmax(rawOutput)\n\n    // Find argmax\n    let maxIdx = 0\n    let maxScore = probs[0] ?? 0\n    for (let i = 1; i < probs.length; i++) {\n      const score = probs[i] ?? 0\n      if (score > maxScore) {\n        maxScore = score\n        maxIdx = i\n      }\n    }\n\n    if (maxScore < this.minConfidence) {\n      return {\n        classifications: [],\n        inferenceMs: Date.now() - start,\n        modelId: this.modelEntry.id,\n      }\n    }\n\n    const label = ANIMAL_10_CLASSES[maxIdx] ?? `animal_${maxIdx}`\n\n    return {\n      classifications: [\n        {\n          class: label,\n          score: maxScore,\n        },\n      ],\n      inferenceMs: Date.now() - start,\n      modelId: this.modelEntry.id,\n    }\n  }\n\n  private async ensureEngine(): Promise<void> {\n    const config = this.resolvedConfig\n    const modelId = config?.modelId ?? this.modelEntry.id\n    const runtime = config?.runtime === 'python' ? 'coreml' : (config?.runtime === 'node' ? 'onnx' : 'auto')\n    const backend = config?.backend ?? 'cpu'\n    const format = config?.format ?? 'onnx'\n\n    const entry = ANIMAL_TYPE_MODELS.find((m) => m.id === modelId) ?? this.modelEntry\n    this.modelEntry = entry\n\n    const modelsDir = this.ctx!.models?.getModelsDir() ?? this.ctx!.locationPaths.models\n\n    if (this.ctx!.models) {\n      await this.ctx!.models.ensure(modelId, format as any)\n    }\n\n    const resolved = await resolveEngine({\n      runtime: runtime as 'auto',\n      backend,\n      modelEntry: entry,\n      modelsDir,\n      models: this.ctx!.models,\n    })\n    this.engine = resolved.engine\n  }\n\n  async shutdown(): Promise<void> {\n    await this.engine?.dispose()\n  }\n\n  getConfigSchema(): ConfigUISchema {\n    return {\n      sections: [\n        {\n          id: 'model',\n          title: 'Model',\n          columns: 1,\n          fields: [\n            {\n              key: 'modelId',\n              label: 'Model',\n              type: 'model-selector',\n              catalog: [...ANIMAL_TYPE_MODELS],\n              allowCustom: false,\n              allowConversion: false,\n              acceptFormats: ['onnx', 'coreml', 'openvino'],\n              requiredMetadata: ['inputSize', 'labels'],\n              outputFormatHint: 'classification',\n            },\n          ],\n        },\n        {\n          id: 'runtime',\n          title: 'Runtime',\n          columns: 2,\n          fields: [\n            {\n              key: 'runtime',\n              label: 'Runtime',\n              type: 'select',\n              options: [\n                { value: 'auto', label: 'Auto' },\n                { value: 'onnx', label: 'ONNX Runtime' },\n                { value: 'coreml', label: 'CoreML (Apple)' },\n                { value: 'openvino', label: 'OpenVINO (Intel)' },\n              ],\n            },\n            {\n              key: 'backend',\n              label: 'Backend',\n              type: 'select',\n              showWhen: { field: 'runtime', equals: 'onnx' },\n              options: [\n                { value: 'auto', label: 'Auto' },\n                { value: 'cpu', label: 'CPU' },\n                { value: 'coreml', label: 'CoreML' },\n                { value: 'cuda', label: 'CUDA (NVIDIA)' },\n              ],\n            },\n          ],\n        },\n        {\n          id: 'thresholds',\n          title: 'Classification Settings',\n          columns: 1,\n          fields: [\n            {\n              key: 'minConfidence',\n              label: 'Minimum Confidence',\n              type: 'slider',\n              min: 0.05,\n              max: 1.0,\n              step: 0.05,\n              default: 0.3,\n            },\n          ],\n        },\n      ],\n    }\n  }\n\n  getClassMap(): ClassMapDefinition {\n    return ANIMAL_CLASS_MAP\n  }\n\n  getModelCatalog(): ModelCatalogEntry[] {\n    return [...ANIMAL_TYPE_MODELS]\n  }\n\n  getAvailableModels(): DetectionModel[] {\n    return []\n  }\n\n  getActiveLabels(): readonly LabelDefinition[] {\n    return ANIMAL_TYPE_LABELS\n  }\n\n  async probe(): Promise<ProbeResult> {\n    return {\n      available: true,\n      runtime: this.engine?.runtime ?? 'onnx',\n      device: this.engine?.device ?? 'cpu',\n      capabilities: ['fp32'],\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiBA,0CAAmC;AACnC,yBAA+C;AAC/C,6BAA8B;AAE9B,IAAM,oBAAqC,EAAE,IAAI,eAAe,MAAM,cAAc;AACpF,IAAM,qBAAiD,CAAC,iBAAiB;AACzE,IAAM,mBAAuC,EAAE,SAAS,CAAC,GAAG,kBAAkB,KAAK;AAEnF,IAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,SAAS,QAAQ,QAAoC;AACnD,QAAM,MAAM,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,SAAS;AAC7D,QAAM,OAAO,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC;AAChD,QAAM,MAAM,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC1C,SAAO,KAAK,IAAI,CAAC,MAAM,IAAI,GAAG;AAChC;AAEA,IAAqB,wBAArB,MAA2F;AAAA,EAChF,KAAK;AAAA,EACL,OAAO;AAAA,EACP,eAAe,CAAC,QAAQ;AAAA,EACxB,gBAAgB,CAAC,eAAe;AAAA,EAChC,eAAe;AAAA,EACf,gBAAgB,CAAC;AAAA,EACjB,WAA0B;AAAA,IACjC,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IAET,aAAa;AAAA,IAEb,MAAM;AAAA,IACN,iBAAiB;AAAA,IACjB,cAAc,CAAC,QAAQ;AAAA,IACvB,eAAe,CAAC,eAAe;AAAA,IAC/B,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,eAAe;AAAA,MACb,SAAS;AAAA,MACT,SAAS;AAAA,MACT,SAAS;AAAA,MACT,eAAe;AAAA,IACjB;AAAA,EACF;AAAA,EAEQ,SAAkC;AAAA,EAClC;AAAA,EACA,gBAAgB;AAAA,EAChB,iBAAiD;AAAA,EACjD,MAA2B;AAAA,EAEnC,uBAA2C;AACzC,WAAO,uDAAmB,IAAI,CAAC,OAAO;AAAA,MACpC,SAAS,EAAE;AAAA,MACX,MAAM,EAAE;AAAA,MACR,WAAW;AAAA,MACX,eAAe;AAAA,MACf,SAAS,OAAO,KAAK,EAAE,OAAO;AAAA,IAChC,EAAE;AAAA,EACJ;AAAA,EAEA,UAAU,QAAuC;AAC/C,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAM,WAAW,KAAkC;AACjD,SAAK,MAAM;AACX,UAAM,MAAM,IAAI;AAChB,UAAM,UAAW,IAAI,SAAS,KAA4B,KAAK,gBAAgB,WAAW;AAC1F,SAAK,gBAAiB,IAAI,eAAe,KAA4B;AAErE,UAAM,QAAQ,uDAAmB,KAAK,CAAC,MAAM,EAAE,OAAO,OAAO;AAC7D,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,MAAM,2CAA2C,OAAO,GAAG;AAAA,IACvE;AACA,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAM,SAAS,OAA6C;AAC1D,QAAI,CAAC,KAAK,OAAQ,OAAM,KAAK,aAAa;AAC1C,UAAM,QAAQ,KAAK,IAAI;AACvB,UAAM,EAAE,OAAO,QAAQ,QAAQ,OAAO,IAAI,KAAK,WAAW;AAG1D,UAAM,aAAa,UAAM,+BAAW,MAAM,MAAM,MAAM,MAAM,GAAG;AAG/D,UAAM,aAAa,UAAM,uCAAmB,YAAY,QAAQ,QAAQ,YAAY,MAAM;AAG1F,UAAM,YAAY,MAAM,KAAK,OAAQ,IAAI,YAAY,CAAC,GAAG,GAAG,QAAQ,MAAM,CAAC;AAG3E,UAAM,QAAQ,QAAQ,SAAS;AAG/B,QAAI,SAAS;AACb,QAAI,WAAW,MAAM,CAAC,KAAK;AAC3B,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,YAAM,QAAQ,MAAM,CAAC,KAAK;AAC1B,UAAI,QAAQ,UAAU;AACpB,mBAAW;AACX,iBAAS;AAAA,MACX;AAAA,IACF;AAEA,QAAI,WAAW,KAAK,eAAe;AACjC,aAAO;AAAA,QACL,iBAAiB,CAAC;AAAA,QAClB,aAAa,KAAK,IAAI,IAAI;AAAA,QAC1B,SAAS,KAAK,WAAW;AAAA,MAC3B;AAAA,IACF;AAEA,UAAM,QAAQ,kBAAkB,MAAM,KAAK,UAAU,MAAM;AAE3D,WAAO;AAAA,MACL,iBAAiB;AAAA,QACf;AAAA,UACE,OAAO;AAAA,UACP,OAAO;AAAA,QACT;AAAA,MACF;AAAA,MACA,aAAa,KAAK,IAAI,IAAI;AAAA,MAC1B,SAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,MAAc,eAA8B;AAC1C,UAAM,SAAS,KAAK;AACpB,UAAM,UAAU,QAAQ,WAAW,KAAK,WAAW;AACnD,UAAM,UAAU,QAAQ,YAAY,WAAW,WAAY,QAAQ,YAAY,SAAS,SAAS;AACjG,UAAM,UAAU,QAAQ,WAAW;AACnC,UAAM,SAAS,QAAQ,UAAU;AAEjC,UAAM,QAAQ,uDAAmB,KAAK,CAAC,MAAM,EAAE,OAAO,OAAO,KAAK,KAAK;AACvE,SAAK,aAAa;AAElB,UAAM,YAAY,KAAK,IAAK,QAAQ,aAAa,KAAK,KAAK,IAAK,cAAc;AAE9E,QAAI,KAAK,IAAK,QAAQ;AACpB,YAAM,KAAK,IAAK,OAAO,OAAO,SAAS,MAAa;AAAA,IACtD;AAEA,UAAM,WAAW,UAAM,sCAAc;AAAA,MACnC;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ;AAAA,MACA,QAAQ,KAAK,IAAK;AAAA,IACpB,CAAC;AACD,SAAK,SAAS,SAAS;AAAA,EACzB;AAAA,EAEA,MAAM,WAA0B;AAC9B,UAAM,KAAK,QAAQ,QAAQ;AAAA,EAC7B;AAAA,EAEA,kBAAkC;AAChC,WAAO;AAAA,MACL,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,SAAS,CAAC,GAAG,sDAAkB;AAAA,cAC/B,aAAa;AAAA,cACb,iBAAiB;AAAA,cACjB,eAAe,CAAC,QAAQ,UAAU,UAAU;AAAA,cAC5C,kBAAkB,CAAC,aAAa,QAAQ;AAAA,cACxC,kBAAkB;AAAA,YACpB;AAAA,UACF;AAAA,QACF;AAAA,QACA;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,SAAS;AAAA,gBACP,EAAE,OAAO,QAAQ,OAAO,OAAO;AAAA,gBAC/B,EAAE,OAAO,QAAQ,OAAO,eAAe;AAAA,gBACvC,EAAE,OAAO,UAAU,OAAO,iBAAiB;AAAA,gBAC3C,EAAE,OAAO,YAAY,OAAO,mBAAmB;AAAA,cACjD;AAAA,YACF;AAAA,YACA;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,UAAU,EAAE,OAAO,WAAW,QAAQ,OAAO;AAAA,cAC7C,SAAS;AAAA,gBACP,EAAE,OAAO,QAAQ,OAAO,OAAO;AAAA,gBAC/B,EAAE,OAAO,OAAO,OAAO,MAAM;AAAA,gBAC7B,EAAE,OAAO,UAAU,OAAO,SAAS;AAAA,gBACnC,EAAE,OAAO,QAAQ,OAAO,gBAAgB;AAAA,cAC1C;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,QACA;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,KAAK;AAAA,cACL,KAAK;AAAA,cACL,MAAM;AAAA,cACN,SAAS;AAAA,YACX;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,cAAkC;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,kBAAuC;AACrC,WAAO,CAAC,GAAG,sDAAkB;AAAA,EAC/B;AAAA,EAEA,qBAAuC;AACrC,WAAO,CAAC;AAAA,EACV;AAAA,EAEA,kBAA8C;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAA8B;AAClC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS,KAAK,QAAQ,WAAW;AAAA,MACjC,QAAQ,KAAK,QAAQ,UAAU;AAAA,MAC/B,cAAc,CAAC,MAAM;AAAA,IACvB;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/addons/audio-classification/index.d.mts

@@ -0,0 +1,36 @@
+import { IClassifierProvider, IDetectionAddon, AddonManifest, ModelRequirement, ResolvedInferenceConfig, AddonContext, CropInput, ClassifierOutput, ConfigUISchema, ClassMapDefinition, ModelCatalogEntry, DetectionModel, LabelDefinition, ProbeResult } from '@camstack/types';
+
+declare class AudioClassificationAddon implements IClassifierProvider, IDetectionAddon {
+    readonly id = "audio-classification";
+    readonly slot: "classifier";
+    readonly inputClasses: readonly string[] | null;
+    readonly outputClasses: readonly ["audio-event:*"];
+    readonly slotPriority = 0;
+    readonly manifest: AddonManifest;
+    private engine;
+    private modelEntry;
+    private minScore;
+    private resolvedConfig;
+    private ctx;
+    getModelRequirements(): ModelRequirement[];
+    configure(config: ResolvedInferenceConfig): void;
+    initialize(ctx: AddonContext): Promise<void>;
+    /**
+     * classify() receives a CropInput but internally treats input.frame.data as raw audio context.
+     * For audio, the actual audio chunk data should be stored in frame.data as a Float32Array
+     * serialized into a Buffer (little-endian float32 samples at 16 kHz).
+     *
+     * The CropInput.roi is not used for audio — it is ignored.
+     */
+    classify(input: CropInput): Promise<ClassifierOutput>;
+    private ensureEngine;
+    shutdown(): Promise<void>;
+    getConfigSchema(): ConfigUISchema;
+    getClassMap(): ClassMapDefinition;
+    getModelCatalog(): ModelCatalogEntry[];
+    getAvailableModels(): DetectionModel[];
+    getActiveLabels(): readonly LabelDefinition[];
+    probe(): Promise<ProbeResult>;
+}
+
+export { AudioClassificationAddon as default };

package/dist/addons/audio-classification/index.d.ts

@@ -0,0 +1,36 @@
+import { IClassifierProvider, IDetectionAddon, AddonManifest, ModelRequirement, ResolvedInferenceConfig, AddonContext, CropInput, ClassifierOutput, ConfigUISchema, ClassMapDefinition, ModelCatalogEntry, DetectionModel, LabelDefinition, ProbeResult } from '@camstack/types';
+
+declare class AudioClassificationAddon implements IClassifierProvider, IDetectionAddon {
+    readonly id = "audio-classification";
+    readonly slot: "classifier";
+    readonly inputClasses: readonly string[] | null;
+    readonly outputClasses: readonly ["audio-event:*"];
+    readonly slotPriority = 0;
+    readonly manifest: AddonManifest;
+    private engine;
+    private modelEntry;
+    private minScore;
+    private resolvedConfig;
+    private ctx;
+    getModelRequirements(): ModelRequirement[];
+    configure(config: ResolvedInferenceConfig): void;
+    initialize(ctx: AddonContext): Promise<void>;
+    /**
+     * classify() receives a CropInput but internally treats input.frame.data as raw audio context.
+     * For audio, the actual audio chunk data should be stored in frame.data as a Float32Array
+     * serialized into a Buffer (little-endian float32 samples at 16 kHz).
+     *
+     * The CropInput.roi is not used for audio — it is ignored.
+     */
+    classify(input: CropInput): Promise<ClassifierOutput>;
+    private ensureEngine;
+    shutdown(): Promise<void>;
+    getConfigSchema(): ConfigUISchema;
+    getClassMap(): ClassMapDefinition;
+    getModelCatalog(): ModelCatalogEntry[];
+    getAvailableModels(): DetectionModel[];
+    getActiveLabels(): readonly LabelDefinition[];
+    probe(): Promise<ProbeResult>;
+}
+
+export { AudioClassificationAddon as default };