@camstack/addon-pipeline 1.0.0 → 1.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/addon-pipeline",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "CamStack Pipeline bundle — runner, detection, motion, decoders, audio + stream broker. Multi-entry npm package shipping 7 addons under a single bundle.",
   "keywords": [
     "camstack",
@@ -69,9 +69,7 @@
       "description": "Detection, motion, decoders, audio + stream broker."
     },
     "nativeDependencies": {
-      "node-av": "^5.2.4",
-      "onnxruntime-node": "^1.24.3",
-      "sharp": "^0.34.0"
+      "node-av": "^5.2.4"
     },
     "addons": [
       {
@@ -296,7 +294,6 @@
     "@camstack/types": "^0.1.0",
     "react": ">=18",
     "react-dom": ">=18",
-    "sharp": "^0.34.0",
     "werift": "^0.22.9"
   },
   "dependencies": {
@@ -304,8 +301,6 @@
     "lucide-react": "^0.511.0",
     "mp4box": "0.5.4",
     "node-av": "^5.2.4",
-    "onnxruntime-node": "^1.24.3",
-    "sharp": "^0.35.2",
     "zod": "^4.3.6"
   },
   "devDependencies": {

package/python/requirements-audio.txt

@@ -0,0 +1,5 @@
+# YAMNet audio classification (yamnet_audio.py) — runs in the embedded
+# portable Python. Installed lazily via ctx.deps.installPythonRequirements
+# the first time the cross-platform audio backend boots.
+numpy>=1.26,<3
+onnxruntime>=1.20,<2

package/python/yamnet_audio.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""YAMNet audio classification subprocess.
+
+Runs YAMNet ONNX inference via the embedded portable Python's onnxruntime,
+replacing the former Node `onnxruntime-node` path. Persistent process: reads
+length-prefixed float32 waveforms (16 kHz mono) from stdin, writes
+length-prefixed JSON results to stdout. Mirrors the Apple SoundAnalysis Swift
+CLI wire protocol so the Node side (YamnetPythonPipeline) reuses one receive
+loop.
+
+Wire protocol (both directions): [4B little-endian length][payload]
+  stdin payload  = raw float32 waveform bytes (16 kHz mono)
+  stdout payload = JSON {"classifications":[{"className","score"}], "inferenceMs"}
+Startup: emits {"status":"ready"} once the model is loaded. Diagnostics go to
+stderr (stdout is the binary framing channel only).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import struct
+import sys
+import time
+
+import numpy as np
+import onnxruntime as ort
+
+# Match the former Node YamnetOnnxPipeline thresholds.
+MIN_SCORE = 0.05
+TOP_K = 10
+
+
+def write_msg(obj: dict) -> None:
+    data = json.dumps(obj).encode("utf-8")
+    sys.stdout.buffer.write(struct.pack("<I", len(data)))
+    sys.stdout.buffer.write(data)
+    sys.stdout.buffer.flush()
+
+
+def read_exact(n: int) -> bytes | None:
+    buf = bytearray()
+    while len(buf) < n:
+        chunk = sys.stdin.buffer.read(n - len(buf))
+        if not chunk:
+            return None
+        buf.extend(chunk)
+    return bytes(buf)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--labels", default="")
+    args = ap.parse_args()
+
+    labels: list[str] = []
+    if args.labels:
+        try:
+            with open(args.labels, "r", encoding="utf-8") as f:
+                labels = json.load(f)
+        except Exception as exc:  # noqa: BLE001 — labels are best-effort
+            print(f"yamnet_audio: failed to read labels: {exc}", file=sys.stderr)
+            labels = []
+
+    session = ort.InferenceSession(args.model, providers=["CPUExecutionProvider"])
+    input_name = session.get_inputs()[0].name
+    output_name = session.get_outputs()[0].name
+
+    write_msg({"status": "ready", "labels": len(labels)})
+
+    while True:
+        header = read_exact(4)
+        if header is None:
+            break
+        (length,) = struct.unpack("<I", header)
+        payload = read_exact(length)
+        if payload is None:
+            break
+
+        start = time.time()
+        try:
+            waveform = np.frombuffer(payload, dtype=np.float32)
+            outputs = session.run([output_name], {input_name: waveform})
+            scores = np.asarray(outputs[0], dtype=np.float32)
+            if scores.ndim == 1:
+                scores = scores.reshape(1, -1)
+            # Average across frames → [num_classes]
+            avg = np.mean(scores, axis=0)
+            results = []
+            for c in range(avg.shape[0]):
+                s = float(avg[c])
+                if s >= MIN_SCORE:
+                    label = labels[c] if c < len(labels) else str(c)
+                    results.append({"className": label, "score": round(s, 3)})
+            results.sort(key=lambda x: x["score"], reverse=True)
+            write_msg(
+                {
+                    "classifications": results[:TOP_K],
+                    "inferenceMs": int((time.time() - start) * 1000),
+                }
+            )
+        except Exception as exc:  # noqa: BLE001 — never kill the loop on one chunk
+            write_msg(
+                {
+                    "classifications": [],
+                    "inferenceMs": int((time.time() - start) * 1000),
+                    "error": str(exc),
+                }
+            )
+
+
+if __name__ == "__main__":
+    main()