@camstack/addon-pipeline 0.1.19 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-analyzer/index.js +736 -716
- package/dist/audio-analyzer/index.mjs +726 -676
- package/dist/audio-codec-nodeav/index.js +304 -461
- package/dist/audio-codec-nodeav/index.mjs +300 -462
- package/dist/chunk-BdkLduGY.mjs +5 -0
- package/dist/chunk-D6vf50IK.js +28 -0
- package/dist/codec-runtime-BOk-13PN.js +202 -0
- package/dist/codec-runtime-BsqlEjPi.mjs +197 -0
- package/dist/constants-B_b0a-6h.mjs +3119 -0
- package/dist/{index-D_cl0Qqb.js → constants-D65v6yp6.js} +3107 -2935
- package/dist/decoder-nodeav/index.js +1374 -1444
- package/dist/decoder-nodeav/index.mjs +1369 -1425
- package/dist/detection-pipeline/index.js +6462 -5613
- package/dist/detection-pipeline/index.mjs +6451 -5574
- package/dist/dist-7ewQjTle.js +22454 -0
- package/dist/dist-C5jnNl0n.mjs +22089 -0
- package/dist/motion-wasm/index.js +469 -467
- package/dist/motion-wasm/index.mjs +464 -446
- package/dist/pipeline-runner/index.js +2035 -1836
- package/dist/pipeline-runner/index.mjs +2031 -1820
- package/dist/recorder/index.js +2097 -0
- package/dist/recorder/index.mjs +2095 -0
- package/dist/stream-broker/_stub.js +1818 -734
- package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-D4-DHanK.mjs +156 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Tf-HACFd.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-C9WX5HNw.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-BO7TIbJV.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.js-C9j-2lBe.mjs +26 -0
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-XO0-Pyu6.mjs +26 -0
- package/dist/stream-broker/dist-CYZr2fwk.mjs +2726 -0
- package/dist/stream-broker/hostInit-Di6vceAU.mjs +129 -0
- package/dist/stream-broker/index.js +17837 -12904
- package/dist/stream-broker/index.mjs +17826 -12896
- package/dist/stream-broker/remoteEntry.js +134 -2973
- package/dist/stream-broker/remoteEntry.ssr.js +33 -0
- package/dist/stream-broker/virtualExposes-dYNvIwoR.mjs +27 -0
- package/dist/stream-broker/virtual_mf-exposes-ssr___mfe_internal__addon_stream_broker_widgets__remoteEntry_js-Cmqfp4i_.mjs +10 -0
- package/embed-dist/assets/index-B8VlSD0-.js +150 -0
- package/embed-dist/assets/index-ZhDdp1Nd.css +2 -0
- package/embed-dist/index.html +13 -0
- package/package.json +75 -9
- package/wasm/assembly/index.ts +41 -16
- package/dist/audio-analyzer/index.js.map +0 -1
- package/dist/audio-analyzer/index.mjs.map +0 -1
- package/dist/audio-codec-nodeav/index.js.map +0 -1
- package/dist/audio-codec-nodeav/index.mjs.map +0 -1
- package/dist/decoder-nodeav/index.js.map +0 -1
- package/dist/decoder-nodeav/index.mjs.map +0 -1
- package/dist/detection-pipeline/index.js.map +0 -1
- package/dist/detection-pipeline/index.mjs.map +0 -1
- package/dist/index-BbPPvoCx.js +0 -14682
- package/dist/index-BbPPvoCx.js.map +0 -1
- package/dist/index-Bmlkm0Fd.mjs +0 -14683
- package/dist/index-Bmlkm0Fd.mjs.map +0 -1
- package/dist/index-D_cl0Qqb.js.map +0 -1
- package/dist/index-UbcdLS7a.mjs +0 -5790
- package/dist/index-UbcdLS7a.mjs.map +0 -1
- package/dist/motion-wasm/index.js.map +0 -1
- package/dist/motion-wasm/index.mjs.map +0 -1
- package/dist/pipeline-runner/index.js.map +0 -1
- package/dist/pipeline-runner/index.mjs.map +0 -1
- package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/StreamBrokerPanel.d.ts +0 -21
- package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/index.d.ts +0 -13
- package/dist/stream-broker/@mf-types/widgets.d.ts +0 -2
- package/dist/stream-broker/@mf-types.d.ts +0 -3
- package/dist/stream-broker/@mf-types.zip +0 -0
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_sdk__loadShare__.mjs-h5aXOPSA.mjs +0 -12
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.mjs-NjF4kxzW.mjs +0 -19
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.mjs-BAv_5ISf.mjs +0 -20
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.mjs-U1EUeEPs.mjs +0 -104
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_client__loadShare__.mjs-DeouEaSs.mjs +0 -85
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_react_mf_2_query__loadShare__.mjs-DHUwjbb9.mjs +0 -62
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs-BsB2G7oY.mjs +0 -88
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs_commonjs-proxy-xrRiPUpA.mjs +0 -29
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.mjs-gBEZsQrp.mjs +0 -36
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs-DYEKzzY-.mjs +0 -45
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs_commonjs-proxy-C0E2yCzO.mjs +0 -6
- package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom_mf_1_client__loadShare__.mjs-DICOtMTl.mjs +0 -34
- package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CupRlwqG.mjs +0 -156
- package/dist/stream-broker/client-NPZqorv9.mjs +0 -9836
- package/dist/stream-broker/getErrorShape-BPSzUA7W-TlK8ipWe.mjs +0 -211
- package/dist/stream-broker/hostInit-Bh4w7o5_.mjs +0 -168
- package/dist/stream-broker/index-2Qp8vT3w.mjs +0 -185
- package/dist/stream-broker/index-BBcZvb5t.mjs +0 -435
- package/dist/stream-broker/index-CIJue-4t.mjs +0 -37880
- package/dist/stream-broker/index-CWkKuNLr.mjs +0 -232
- package/dist/stream-broker/index-Cc6QBqMk.mjs +0 -1655
- package/dist/stream-broker/index-D_1p2K9B.mjs +0 -2603
- package/dist/stream-broker/index-Dy2V7VOm.mjs +0 -14379
- package/dist/stream-broker/index-mX3Kgiv1.mjs +0 -725
- package/dist/stream-broker/index-xncRG7-x.mjs +0 -2713
- package/dist/stream-broker/index.js.map +0 -1
- package/dist/stream-broker/index.mjs.map +0 -1
- package/dist/stream-broker/jsx-runtime-lb0mH5st.mjs +0 -55
- package/dist/stream-broker/schemas-ClCuS4qa.mjs +0 -3594
- package/dist/stream-broker/virtualExposes-pCd777Rp.mjs +0 -42
|
@@ -1,686 +1,736 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import * as path from "node:path";
|
|
1
|
+
import { L as mapAudioLabelToMacro, P as hydrateSchema, S as audioAnalyzerCapability, c as DEFAULT_AUDIO_ANALYZER_CONFIG, i as BaseAddon, j as errMsg, m as HF_BASE_URL, n as AUDIO_BACKEND_CHOICES, x as audioAnalysisCapability } from "../dist-C5jnNl0n.mjs";
|
|
3
2
|
import * as fs from "node:fs";
|
|
3
|
+
import * as path$1 from "node:path";
|
|
4
4
|
import { downloadFile } from "@camstack/core";
|
|
5
|
+
//#region src/audio-analyzer/audio-pipeline.ts
|
|
6
|
+
/**
|
|
7
|
+
* Create the appropriate audio pipeline.
|
|
8
|
+
*
|
|
9
|
+
* - 'yamnet-onnx': Cross-platform YAMNet ONNX (requires model download)
|
|
10
|
+
* - 'apple-soundanalysis': macOS 12+ Apple SoundAnalysis (zero model download, Neural Engine)
|
|
11
|
+
* - undefined: auto-detect (Apple SA on macOS, YAMNet on Linux)
|
|
12
|
+
*/
|
|
5
13
|
async function createAudioPipeline(modelsDir, logger, options) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
return new AppleSoundAnalysisPipeline(logger);
|
|
9
|
-
}
|
|
10
|
-
return new YamnetOnnxPipeline(modelsDir, logger);
|
|
11
|
-
}
|
|
12
|
-
const YAMNET_MODEL_URL = `${HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
|
|
13
|
-
const YAMNET_LABELS_URL = `${HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
|
|
14
|
-
class YamnetOnnxPipeline {
|
|
15
|
-
constructor(modelsDir, logger) {
|
|
16
|
-
this.modelsDir = modelsDir;
|
|
17
|
-
this.log = logger;
|
|
18
|
-
}
|
|
19
|
-
session = null;
|
|
20
|
-
inputName = "";
|
|
21
|
-
labels = [];
|
|
22
|
-
log;
|
|
23
|
-
async initialize() {
|
|
24
|
-
const ort = await import("onnxruntime-node");
|
|
25
|
-
const modelPath = path.join(this.modelsDir, "camstack-yamnet.onnx");
|
|
26
|
-
const labelsPath = path.join(this.modelsDir, "camstack-yamnet-labels.json");
|
|
27
|
-
if (!fs.existsSync(modelPath)) {
|
|
28
|
-
this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", {
|
|
29
|
-
meta: { url: YAMNET_MODEL_URL, dest: modelPath }
|
|
30
|
-
});
|
|
31
|
-
await downloadFile(YAMNET_MODEL_URL, modelPath);
|
|
32
|
-
this.log.info("YAMNet ONNX model downloaded", {
|
|
33
|
-
meta: { sizeBytes: fs.statSync(modelPath).size }
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
if (!fs.existsSync(labelsPath)) {
|
|
37
|
-
this.log.info("YAMNet labels not found locally — downloading from HuggingFace", {
|
|
38
|
-
meta: { url: YAMNET_LABELS_URL, dest: labelsPath }
|
|
39
|
-
});
|
|
40
|
-
await downloadFile(YAMNET_LABELS_URL, labelsPath);
|
|
41
|
-
}
|
|
42
|
-
this.session = await ort.InferenceSession.create(modelPath);
|
|
43
|
-
this.inputName = this.session.inputNames[0] ?? "waveform";
|
|
44
|
-
if (fs.existsSync(labelsPath)) {
|
|
45
|
-
this.labels = JSON.parse(fs.readFileSync(labelsPath, "utf8"));
|
|
46
|
-
} else {
|
|
47
|
-
this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
|
|
48
|
-
}
|
|
49
|
-
this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
|
|
50
|
-
}
|
|
51
|
-
async classify(chunk) {
|
|
52
|
-
if (!this.session) {
|
|
53
|
-
throw new Error("YAMNet pipeline not initialized");
|
|
54
|
-
}
|
|
55
|
-
const start = Date.now();
|
|
56
|
-
const ort = await import("onnxruntime-node");
|
|
57
|
-
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
58
|
-
const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
|
|
59
|
-
const feeds = { [this.inputName]: tensor };
|
|
60
|
-
const results = await this.session.run(feeds);
|
|
61
|
-
const scoresData = results[this.session.outputNames[0]];
|
|
62
|
-
if (!scoresData) {
|
|
63
|
-
throw new Error("YAMNet returned no output");
|
|
64
|
-
}
|
|
65
|
-
const scores = scoresData.data;
|
|
66
|
-
const numClasses = 521;
|
|
67
|
-
const numFrames = scores.length / numClasses;
|
|
68
|
-
const avgScores = new Float32Array(numClasses);
|
|
69
|
-
for (let f = 0; f < numFrames; f++) {
|
|
70
|
-
for (let c = 0; c < numClasses; c++) {
|
|
71
|
-
avgScores[c] += scores[f * numClasses + c];
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
for (let c = 0; c < numClasses; c++) {
|
|
75
|
-
avgScores[c] = avgScores[c] / numFrames;
|
|
76
|
-
}
|
|
77
|
-
const minScore = 0.05;
|
|
78
|
-
const classifications = [];
|
|
79
|
-
for (let c = 0; c < numClasses; c++) {
|
|
80
|
-
const score = avgScores[c];
|
|
81
|
-
if (score >= minScore) {
|
|
82
|
-
const label = c < this.labels.length ? this.labels[c] : String(c);
|
|
83
|
-
classifications.push({ className: label, score: Math.round(score * 1e3) / 1e3 });
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
classifications.sort((a, b) => b.score - a.score);
|
|
87
|
-
return {
|
|
88
|
-
classifications: classifications.slice(0, 10),
|
|
89
|
-
inferenceMs: Date.now() - start
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
async dispose() {
|
|
93
|
-
if (this.session) {
|
|
94
|
-
await this.session.release();
|
|
95
|
-
this.session = null;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
class AppleSoundAnalysisPipeline {
|
|
100
|
-
log;
|
|
101
|
-
process = null;
|
|
102
|
-
receiveBuffer = Buffer.alloc(0);
|
|
103
|
-
pendingResolve = null;
|
|
104
|
-
pendingReject = null;
|
|
105
|
-
binaryPath = null;
|
|
106
|
-
debugCount = 0;
|
|
107
|
-
constructor(logger) {
|
|
108
|
-
this.log = logger;
|
|
109
|
-
}
|
|
110
|
-
async initialize() {
|
|
111
|
-
this.binaryPath = await this.resolveSwiftBinary();
|
|
112
|
-
if (!this.binaryPath) {
|
|
113
|
-
throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
|
|
114
|
-
}
|
|
115
|
-
const { spawn } = await import("node:child_process");
|
|
116
|
-
this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], {
|
|
117
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
118
|
-
});
|
|
119
|
-
this.process.stderr?.on("data", (chunk) => {
|
|
120
|
-
const lines = chunk.toString().split("\n");
|
|
121
|
-
for (const line of lines) {
|
|
122
|
-
const trimmed = line.trim();
|
|
123
|
-
if (trimmed) this.log.warn(trimmed);
|
|
124
|
-
}
|
|
125
|
-
});
|
|
126
|
-
this.process.on("error", (err) => {
|
|
127
|
-
this.log.error("Swift process error", { meta: { error: err.message } });
|
|
128
|
-
this.pendingReject?.(err);
|
|
129
|
-
this.pendingReject = null;
|
|
130
|
-
this.pendingResolve = null;
|
|
131
|
-
});
|
|
132
|
-
this.process.on("exit", (code) => {
|
|
133
|
-
if (code !== 0 && code !== null) {
|
|
134
|
-
this.log.error("Swift process exited", { meta: { code } });
|
|
135
|
-
const err = new Error(`Apple SoundAnalysis: process exited with code ${code}`);
|
|
136
|
-
this.pendingReject?.(err);
|
|
137
|
-
this.pendingReject = null;
|
|
138
|
-
this.pendingResolve = null;
|
|
139
|
-
}
|
|
140
|
-
});
|
|
141
|
-
this.process.stdout.on("data", (chunk) => {
|
|
142
|
-
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
143
|
-
this.tryReceive();
|
|
144
|
-
});
|
|
145
|
-
const ready = await this.receiveMessage();
|
|
146
|
-
if (ready["status"] !== "ready") {
|
|
147
|
-
throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
148
|
-
}
|
|
149
|
-
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
150
|
-
}
|
|
151
|
-
async classify(chunk) {
|
|
152
|
-
if (!this.process?.stdin) {
|
|
153
|
-
throw new Error("Apple SoundAnalysis: process not initialized");
|
|
154
|
-
}
|
|
155
|
-
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
156
|
-
const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
|
|
157
|
-
const lengthBuf = Buffer.allocUnsafe(4);
|
|
158
|
-
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
159
|
-
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
160
|
-
const result = await this.receiveMessage();
|
|
161
|
-
const classifications = result["classifications"] ?? [];
|
|
162
|
-
const inferenceMs = result["inferenceMs"] ?? 0;
|
|
163
|
-
if (this.debugCount < 3) {
|
|
164
|
-
const keys = Object.keys(result);
|
|
165
|
-
this.log.info("classify debug sample", {
|
|
166
|
-
meta: {
|
|
167
|
-
phase: "apple-sa",
|
|
168
|
-
index: this.debugCount,
|
|
169
|
-
keys,
|
|
170
|
-
classifications: classifications.length,
|
|
171
|
-
inferenceMs,
|
|
172
|
-
audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
|
|
173
|
-
sampleRate: chunk.sampleRate,
|
|
174
|
-
channels: chunk.channels
|
|
175
|
-
}
|
|
176
|
-
});
|
|
177
|
-
if (result["error"]) {
|
|
178
|
-
this.log.error("Swift error", { meta: { phase: "apple-sa", error: result["error"] } });
|
|
179
|
-
}
|
|
180
|
-
this.debugCount++;
|
|
181
|
-
}
|
|
182
|
-
return { classifications, inferenceMs };
|
|
183
|
-
}
|
|
184
|
-
async dispose() {
|
|
185
|
-
const proc = this.process;
|
|
186
|
-
if (!proc) return;
|
|
187
|
-
this.process = null;
|
|
188
|
-
proc.stdin?.end();
|
|
189
|
-
proc.kill("SIGTERM");
|
|
190
|
-
const exited = await new Promise((resolve) => {
|
|
191
|
-
const timer = setTimeout(() => resolve(false), 5e3);
|
|
192
|
-
proc.once("exit", () => {
|
|
193
|
-
clearTimeout(timer);
|
|
194
|
-
resolve(true);
|
|
195
|
-
});
|
|
196
|
-
});
|
|
197
|
-
if (!exited) {
|
|
198
|
-
try {
|
|
199
|
-
proc.kill("SIGKILL");
|
|
200
|
-
} catch {
|
|
201
|
-
}
|
|
202
|
-
this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
receiveMessage() {
|
|
206
|
-
return new Promise((resolve, reject) => {
|
|
207
|
-
this.pendingResolve = resolve;
|
|
208
|
-
this.pendingReject = reject;
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
tryReceive() {
|
|
212
|
-
if (this.receiveBuffer.length < 4) return;
|
|
213
|
-
const length = this.receiveBuffer.readUInt32LE(0);
|
|
214
|
-
if (this.receiveBuffer.length < 4 + length) return;
|
|
215
|
-
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
216
|
-
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
217
|
-
const resolve = this.pendingResolve;
|
|
218
|
-
const reject = this.pendingReject;
|
|
219
|
-
this.pendingResolve = null;
|
|
220
|
-
this.pendingReject = null;
|
|
221
|
-
if (!resolve) return;
|
|
222
|
-
try {
|
|
223
|
-
const parsed = JSON.parse(jsonBytes.toString("utf8"));
|
|
224
|
-
resolve(parsed);
|
|
225
|
-
} catch (err) {
|
|
226
|
-
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
/** Find pre-compiled binary or compile from Swift source. */
|
|
230
|
-
async resolveSwiftBinary() {
|
|
231
|
-
const candidates = [
|
|
232
|
-
path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
|
|
233
|
-
// Fallback for in-tree dev (src/<id>/swift/) and pre-merge layouts.
|
|
234
|
-
path.join(__dirname, "../swift/apple-sound-classifier"),
|
|
235
|
-
path.join(__dirname, "../../swift/apple-sound-classifier"),
|
|
236
|
-
path.join(__dirname, "../../../swift/apple-sound-classifier")
|
|
237
|
-
];
|
|
238
|
-
for (const p of candidates) {
|
|
239
|
-
if (fs.existsSync(p)) {
|
|
240
|
-
this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
|
|
241
|
-
return p;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
const sourceCandidates = [
|
|
245
|
-
path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
|
|
246
|
-
path.join(__dirname, "../swift/apple-sound-classifier.swift"),
|
|
247
|
-
path.join(__dirname, "../../swift/apple-sound-classifier.swift"),
|
|
248
|
-
path.join(__dirname, "../../../swift/apple-sound-classifier.swift")
|
|
249
|
-
];
|
|
250
|
-
const sourcePath = sourceCandidates.find((p) => fs.existsSync(p));
|
|
251
|
-
if (!sourcePath) {
|
|
252
|
-
this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
|
|
253
|
-
return null;
|
|
254
|
-
}
|
|
255
|
-
const outputPath = sourcePath.replace(".swift", "");
|
|
256
|
-
this.log.info("Compiling Swift CLI...", { meta: { source: sourcePath, output: outputPath } });
|
|
257
|
-
const { execFileSync } = await import("node:child_process");
|
|
258
|
-
try {
|
|
259
|
-
execFileSync("swiftc", ["-O", "-o", outputPath, sourcePath], {
|
|
260
|
-
timeout: 6e4,
|
|
261
|
-
stdio: "pipe"
|
|
262
|
-
});
|
|
263
|
-
this.log.info("Swift CLI compiled successfully");
|
|
264
|
-
return outputPath;
|
|
265
|
-
} catch (err) {
|
|
266
|
-
this.log.error("Swift compilation failed — install Xcode Command Line Tools", {
|
|
267
|
-
meta: { error: errMsg(err) }
|
|
268
|
-
});
|
|
269
|
-
return null;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
14
|
+
if ((options?.backend ?? (process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx")) === "apple-soundanalysis") return new AppleSoundAnalysisPipeline(logger);
|
|
15
|
+
return new YamnetOnnxPipeline(modelsDir, logger);
|
|
272
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Canonical model URLs on the camstack HuggingFace mirror. Mirrors the
|
|
19
|
+
* convention every detection model follows (single point of truth =
|
|
20
|
+
* `HF_BASE_URL` from `@camstack/types`); the auto-download path uses
|
|
21
|
+
* `downloadFile` from `@camstack/core`, the SAME helper detection-
|
|
22
|
+
* pipeline uses to materialise its YOLO/face/plate models. Missing
|
|
23
|
+
* model on disk → fetch from HF; cached file → no-op.
|
|
24
|
+
*
|
|
25
|
+
* Repo layout follows the detection-pipeline pattern:
|
|
26
|
+
* {domain}/{family}/{format}/{filename}
|
|
27
|
+
* For YAMNet that's `audioClassification/yamnet/onnx/camstack-yamnet.onnx`,
|
|
28
|
+
* with the labels JSON sitting one level up (`audioClassification/yamnet/`)
|
|
29
|
+
* because they're format-agnostic (same 521 AudioSet class names whether
|
|
30
|
+
* the runtime is ONNX, OpenVINO, or TF).
|
|
31
|
+
*/
|
|
32
|
+
var YAMNET_MODEL_URL = `${HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
|
|
33
|
+
var YAMNET_LABELS_URL = `${HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
|
|
34
|
+
var YamnetOnnxPipeline = class {
|
|
35
|
+
modelsDir;
|
|
36
|
+
session = null;
|
|
37
|
+
inputName = "";
|
|
38
|
+
labels = [];
|
|
39
|
+
log;
|
|
40
|
+
constructor(modelsDir, logger) {
|
|
41
|
+
this.modelsDir = modelsDir;
|
|
42
|
+
this.log = logger;
|
|
43
|
+
}
|
|
44
|
+
async initialize() {
|
|
45
|
+
const ort = await import("onnxruntime-node");
|
|
46
|
+
const modelPath = path$1.join(this.modelsDir, "camstack-yamnet.onnx");
|
|
47
|
+
const labelsPath = path$1.join(this.modelsDir, "camstack-yamnet-labels.json");
|
|
48
|
+
if (!fs.existsSync(modelPath)) {
|
|
49
|
+
this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", { meta: {
|
|
50
|
+
url: YAMNET_MODEL_URL,
|
|
51
|
+
dest: modelPath
|
|
52
|
+
} });
|
|
53
|
+
await downloadFile(YAMNET_MODEL_URL, modelPath);
|
|
54
|
+
this.log.info("YAMNet ONNX model downloaded", { meta: { sizeBytes: fs.statSync(modelPath).size } });
|
|
55
|
+
}
|
|
56
|
+
if (!fs.existsSync(labelsPath)) {
|
|
57
|
+
this.log.info("YAMNet labels not found locally — downloading from HuggingFace", { meta: {
|
|
58
|
+
url: YAMNET_LABELS_URL,
|
|
59
|
+
dest: labelsPath
|
|
60
|
+
} });
|
|
61
|
+
await downloadFile(YAMNET_LABELS_URL, labelsPath);
|
|
62
|
+
}
|
|
63
|
+
this.session = await ort.InferenceSession.create(modelPath);
|
|
64
|
+
this.inputName = this.session.inputNames[0] ?? "waveform";
|
|
65
|
+
if (fs.existsSync(labelsPath)) this.labels = JSON.parse(fs.readFileSync(labelsPath, "utf8"));
|
|
66
|
+
else this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
|
|
67
|
+
this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
|
|
68
|
+
}
|
|
69
|
+
async classify(chunk) {
|
|
70
|
+
if (!this.session) throw new Error("YAMNet pipeline not initialized");
|
|
71
|
+
const start = Date.now();
|
|
72
|
+
const ort = await import("onnxruntime-node");
|
|
73
|
+
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
74
|
+
const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
|
|
75
|
+
const feeds = { [this.inputName]: tensor };
|
|
76
|
+
const scoresData = (await this.session.run(feeds))[this.session.outputNames[0]];
|
|
77
|
+
if (!scoresData) throw new Error("YAMNet returned no output");
|
|
78
|
+
const scores = scoresData.data;
|
|
79
|
+
const numClasses = 521;
|
|
80
|
+
const numFrames = scores.length / numClasses;
|
|
81
|
+
const avgScores = new Float32Array(numClasses);
|
|
82
|
+
for (let f = 0; f < numFrames; f++) for (let c = 0; c < numClasses; c++) avgScores[c] += scores[f * numClasses + c];
|
|
83
|
+
for (let c = 0; c < numClasses; c++) avgScores[c] = avgScores[c] / numFrames;
|
|
84
|
+
const minScore = .05;
|
|
85
|
+
const classifications = [];
|
|
86
|
+
for (let c = 0; c < numClasses; c++) {
|
|
87
|
+
const score = avgScores[c];
|
|
88
|
+
if (score >= minScore) {
|
|
89
|
+
const label = c < this.labels.length ? this.labels[c] : String(c);
|
|
90
|
+
classifications.push({
|
|
91
|
+
className: label,
|
|
92
|
+
score: Math.round(score * 1e3) / 1e3
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
classifications.sort((a, b) => b.score - a.score);
|
|
97
|
+
return {
|
|
98
|
+
classifications: classifications.slice(0, 10),
|
|
99
|
+
inferenceMs: Date.now() - start
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
async dispose() {
|
|
103
|
+
if (this.session) {
|
|
104
|
+
await this.session.release();
|
|
105
|
+
this.session = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
var AppleSoundAnalysisPipeline = class {
|
|
110
|
+
log;
|
|
111
|
+
process = null;
|
|
112
|
+
receiveBuffer = Buffer.alloc(0);
|
|
113
|
+
pendingResolve = null;
|
|
114
|
+
pendingReject = null;
|
|
115
|
+
binaryPath = null;
|
|
116
|
+
debugCount = 0;
|
|
117
|
+
constructor(logger) {
|
|
118
|
+
this.log = logger;
|
|
119
|
+
}
|
|
120
|
+
async initialize() {
|
|
121
|
+
this.binaryPath = await this.resolveSwiftBinary();
|
|
122
|
+
if (!this.binaryPath) throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
|
|
123
|
+
const { spawn } = await import("node:child_process");
|
|
124
|
+
this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], { stdio: [
|
|
125
|
+
"pipe",
|
|
126
|
+
"pipe",
|
|
127
|
+
"pipe"
|
|
128
|
+
] });
|
|
129
|
+
this.process.stderr?.on("data", (chunk) => {
|
|
130
|
+
const lines = chunk.toString().split("\n");
|
|
131
|
+
for (const line of lines) {
|
|
132
|
+
const trimmed = line.trim();
|
|
133
|
+
if (trimmed) this.log.warn(trimmed);
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
this.process.on("error", (err) => {
|
|
137
|
+
this.log.error("Swift process error", { meta: { error: err.message } });
|
|
138
|
+
this.pendingReject?.(err);
|
|
139
|
+
this.pendingReject = null;
|
|
140
|
+
this.pendingResolve = null;
|
|
141
|
+
});
|
|
142
|
+
this.process.on("exit", (code) => {
|
|
143
|
+
if (code !== 0 && code !== null) {
|
|
144
|
+
this.log.error("Swift process exited", { meta: { code } });
|
|
145
|
+
const err = /* @__PURE__ */ new Error(`Apple SoundAnalysis: process exited with code ${code}`);
|
|
146
|
+
this.pendingReject?.(err);
|
|
147
|
+
this.pendingReject = null;
|
|
148
|
+
this.pendingResolve = null;
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
this.process.stdout.on("data", (chunk) => {
|
|
152
|
+
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
153
|
+
this.tryReceive();
|
|
154
|
+
});
|
|
155
|
+
const ready = await this.receiveMessage();
|
|
156
|
+
if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
157
|
+
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
158
|
+
}
|
|
159
|
+
async classify(chunk) {
|
|
160
|
+
if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
|
|
161
|
+
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
162
|
+
const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
|
|
163
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
164
|
+
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
165
|
+
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
166
|
+
const result = await this.receiveMessage();
|
|
167
|
+
const classifications = result["classifications"] ?? [];
|
|
168
|
+
const inferenceMs = result["inferenceMs"] ?? 0;
|
|
169
|
+
if (this.debugCount < 3) {
|
|
170
|
+
const keys = Object.keys(result);
|
|
171
|
+
this.log.info("classify debug sample", { meta: {
|
|
172
|
+
phase: "apple-sa",
|
|
173
|
+
index: this.debugCount,
|
|
174
|
+
keys,
|
|
175
|
+
classifications: classifications.length,
|
|
176
|
+
inferenceMs,
|
|
177
|
+
audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
|
|
178
|
+
sampleRate: chunk.sampleRate,
|
|
179
|
+
channels: chunk.channels
|
|
180
|
+
} });
|
|
181
|
+
if (result["error"]) this.log.error("Swift error", { meta: {
|
|
182
|
+
phase: "apple-sa",
|
|
183
|
+
error: result["error"]
|
|
184
|
+
} });
|
|
185
|
+
this.debugCount++;
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
classifications,
|
|
189
|
+
inferenceMs
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
async dispose() {
|
|
193
|
+
const proc = this.process;
|
|
194
|
+
if (!proc) return;
|
|
195
|
+
this.process = null;
|
|
196
|
+
proc.stdin?.end();
|
|
197
|
+
proc.kill("SIGTERM");
|
|
198
|
+
if (!await new Promise((resolve) => {
|
|
199
|
+
const timer = setTimeout(() => resolve(false), 5e3);
|
|
200
|
+
proc.once("exit", () => {
|
|
201
|
+
clearTimeout(timer);
|
|
202
|
+
resolve(true);
|
|
203
|
+
});
|
|
204
|
+
})) {
|
|
205
|
+
try {
|
|
206
|
+
proc.kill("SIGKILL");
|
|
207
|
+
} catch {}
|
|
208
|
+
this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
receiveMessage() {
|
|
212
|
+
return new Promise((resolve, reject) => {
|
|
213
|
+
this.pendingResolve = resolve;
|
|
214
|
+
this.pendingReject = reject;
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
tryReceive() {
|
|
218
|
+
if (this.receiveBuffer.length < 4) return;
|
|
219
|
+
const length = this.receiveBuffer.readUInt32LE(0);
|
|
220
|
+
if (this.receiveBuffer.length < 4 + length) return;
|
|
221
|
+
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
222
|
+
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
223
|
+
const resolve = this.pendingResolve;
|
|
224
|
+
const reject = this.pendingReject;
|
|
225
|
+
this.pendingResolve = null;
|
|
226
|
+
this.pendingReject = null;
|
|
227
|
+
if (!resolve) return;
|
|
228
|
+
try {
|
|
229
|
+
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
230
|
+
} catch (err) {
|
|
231
|
+
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/** Find pre-compiled binary or compile from Swift source. */
|
|
235
|
+
async resolveSwiftBinary() {
|
|
236
|
+
const candidates = [
|
|
237
|
+
path$1.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
|
|
238
|
+
path$1.join(__dirname, "../swift/apple-sound-classifier"),
|
|
239
|
+
path$1.join(__dirname, "../../swift/apple-sound-classifier"),
|
|
240
|
+
path$1.join(__dirname, "../../../swift/apple-sound-classifier")
|
|
241
|
+
];
|
|
242
|
+
for (const p of candidates) if (fs.existsSync(p)) {
|
|
243
|
+
this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
|
|
244
|
+
return p;
|
|
245
|
+
}
|
|
246
|
+
const sourceCandidates = [
|
|
247
|
+
path$1.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
|
|
248
|
+
path$1.join(__dirname, "../swift/apple-sound-classifier.swift"),
|
|
249
|
+
path$1.join(__dirname, "../../swift/apple-sound-classifier.swift"),
|
|
250
|
+
path$1.join(__dirname, "../../../swift/apple-sound-classifier.swift")
|
|
251
|
+
];
|
|
252
|
+
const sourcePath = sourceCandidates.find((p) => fs.existsSync(p));
|
|
253
|
+
if (!sourcePath) {
|
|
254
|
+
this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
const outputPath = sourcePath.replace(".swift", "");
|
|
258
|
+
this.log.info("Compiling Swift CLI...", { meta: {
|
|
259
|
+
source: sourcePath,
|
|
260
|
+
output: outputPath
|
|
261
|
+
} });
|
|
262
|
+
const { execFileSync } = await import("node:child_process");
|
|
263
|
+
try {
|
|
264
|
+
execFileSync("swiftc", [
|
|
265
|
+
"-O",
|
|
266
|
+
"-o",
|
|
267
|
+
outputPath,
|
|
268
|
+
sourcePath
|
|
269
|
+
], {
|
|
270
|
+
timeout: 6e4,
|
|
271
|
+
stdio: "pipe"
|
|
272
|
+
});
|
|
273
|
+
this.log.info("Swift CLI compiled successfully");
|
|
274
|
+
return outputPath;
|
|
275
|
+
} catch (err) {
|
|
276
|
+
this.log.error("Swift compilation failed — install Xcode Command Line Tools", { meta: { error: errMsg(err) } });
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
/** Simple resample to 16kHz mono by linear interpolation. */
|
|
273
282
|
function resampleMono16k(chunk) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
}
|
|
294
|
-
return out;
|
|
283
|
+
const { data, sampleRate, channels } = chunk;
|
|
284
|
+
const numSamples = data.length / channels;
|
|
285
|
+
const mono = new Float32Array(numSamples);
|
|
286
|
+
for (let i = 0; i < numSamples; i++) {
|
|
287
|
+
let sum = 0;
|
|
288
|
+
for (let c = 0; c < channels; c++) sum += data[i * channels + c];
|
|
289
|
+
mono[i] = sum / channels;
|
|
290
|
+
}
|
|
291
|
+
const ratio = 16e3 / sampleRate;
|
|
292
|
+
const outLen = Math.floor(numSamples * ratio);
|
|
293
|
+
const out = new Float32Array(outLen);
|
|
294
|
+
for (let i = 0; i < outLen; i++) {
|
|
295
|
+
const srcIdx = i / ratio;
|
|
296
|
+
const lo = Math.floor(srcIdx);
|
|
297
|
+
const hi = Math.min(lo + 1, numSamples - 1);
|
|
298
|
+
const frac = srcIdx - lo;
|
|
299
|
+
out[i] = mono[lo] * (1 - frac) + mono[hi] * frac;
|
|
300
|
+
}
|
|
301
|
+
return out;
|
|
295
302
|
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
303
|
+
//#endregion
|
|
304
|
+
//#region src/audio-analyzer/addons/analyzer/index.ts
|
|
305
|
+
/**
|
|
306
|
+
* Choices presented in the Audio Model dropdown. YAMNet runs via ONNX
|
|
307
|
+
* when backend=yamnet-onnx; Apple SoundAnalysis is a built-in macOS
|
|
308
|
+
* model and has no swappable modelId — the backend IS the model.
|
|
309
|
+
*/
|
|
310
|
+
var AUDIO_MODEL_OPTIONS = [
|
|
311
|
+
{
|
|
312
|
+
value: "",
|
|
313
|
+
label: "Auto (matches backend)"
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
value: "yamnet-onnx",
|
|
317
|
+
label: "YAMNet (ONNX)"
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
value: "apple-soundanalysis",
|
|
321
|
+
label: "Apple SoundAnalysis (built-in)"
|
|
322
|
+
}
|
|
300
323
|
];
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
324
|
+
/**
|
|
325
|
+
* AudioAnalyzerProvider — implements IAudioAnalyzer.
|
|
326
|
+
*
|
|
327
|
+
* Computes dB/RMS on every chunk and classifies via the in-process
|
|
328
|
+
* IAudioPipeline (YAMNet ONNX / Apple SoundAnalysis). No tRPC roundtrip
|
|
329
|
+
* to a separate audio-classifier addon.
|
|
330
|
+
*/
|
|
331
|
+
var CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
|
|
332
|
+
var CLASSIFY_MIN_INTERVAL_MS = 500;
|
|
333
|
+
var GLOBAL_DEVICE_KEY = -1;
|
|
334
|
+
/**
|
|
335
|
+
* Reconstruct a Float32 view over f32le PCM bytes carried in a Uint8Array.
|
|
336
|
+
*
|
|
337
|
+
* `@msgpack/msgpack` decodes a binary blob into a Uint8Array that is a subview
|
|
338
|
+
* of its internal decode buffer, whose `byteOffset` is NOT guaranteed to be
|
|
339
|
+
* 4-byte aligned (observed e.g. 9). `new Float32Array(buf, offset, …)` then
|
|
340
|
+
* throws "start offset of Float32Array should be a multiple of 4". When the
|
|
341
|
+
* offset is misaligned we copy into a fresh 0-offset buffer; the aligned
|
|
342
|
+
* fast-path reuses the existing view with no copy.
|
|
343
|
+
*/
|
|
304
344
|
function float32FromBytes(raw) {
|
|
305
|
-
|
|
306
|
-
|
|
345
|
+
const bytes = raw.byteOffset % 4 === 0 ? raw : new Uint8Array(raw);
|
|
346
|
+
return new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
|
|
307
347
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
348
|
+
var AudioAnalyzerProvider = class {
|
|
349
|
+
pipeline;
|
|
350
|
+
deviceSettingsResolver;
|
|
351
|
+
deviceContributionResolver;
|
|
352
|
+
deviceSettingsPatcher;
|
|
353
|
+
reprobeImpl;
|
|
354
|
+
log;
|
|
355
|
+
classifyCallCount = 0;
|
|
356
|
+
lastClassifyErrorMs = 0;
|
|
357
|
+
suppressedClassifyErrors = 0;
|
|
358
|
+
classifyCount = 0;
|
|
359
|
+
backendName = "unknown";
|
|
360
|
+
/** When true, logs a raw-label sample every 100 classifications (opt-in
|
|
361
|
+
* debug aid). Off by default — the watchdog heartbeat covers liveness. */
|
|
362
|
+
debugClassifySamples = false;
|
|
363
|
+
/** Per-camera in-flight state. Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
|
|
364
|
+
cameraState = /* @__PURE__ */ new Map();
|
|
365
|
+
/** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
|
|
366
|
+
* run at a time. Without this, concurrent calls from different cameras overwrite the
|
|
367
|
+
* single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
|
|
368
|
+
pipelineBusy = false;
|
|
369
|
+
constructor(logger, pipeline, backendName, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, reprobeImpl) {
|
|
370
|
+
this.pipeline = pipeline;
|
|
371
|
+
this.deviceSettingsResolver = deviceSettingsResolver;
|
|
372
|
+
this.deviceContributionResolver = deviceContributionResolver;
|
|
373
|
+
this.deviceSettingsPatcher = deviceSettingsPatcher;
|
|
374
|
+
this.reprobeImpl = reprobeImpl;
|
|
375
|
+
this.log = logger;
|
|
376
|
+
this.backendName = backendName;
|
|
377
|
+
}
|
|
378
|
+
async getDeviceSettingsContribution(input) {
|
|
379
|
+
return this.deviceContributionResolver(input.deviceId);
|
|
380
|
+
}
|
|
381
|
+
async getDeviceLiveContribution(_input) {
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
async applyDeviceSettingsPatch(input) {
|
|
385
|
+
await this.deviceSettingsPatcher(input.deviceId, input.patch);
|
|
386
|
+
return { success: true };
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Return the effective per-device audio-analyzer settings, resolved via
|
|
390
|
+
* the kernel's 3-level settings resolver (schema default → global →
|
|
391
|
+
* device override). Orchestrator consumers call this method so they
|
|
392
|
+
* never need to know the audio-analyzer schema field names.
|
|
393
|
+
*/
|
|
394
|
+
async resolveDeviceSettings({ deviceId }) {
|
|
395
|
+
return this.deviceSettingsResolver(deviceId);
|
|
396
|
+
}
|
|
397
|
+
async analyseChunk({ chunk, settings }) {
|
|
398
|
+
const samples = float32FromBytes(chunk.data);
|
|
399
|
+
let sumSquares = 0;
|
|
400
|
+
for (let i = 0; i < samples.length; i++) sumSquares += samples[i] * samples[i];
|
|
401
|
+
const rms = Math.sqrt(sumSquares / samples.length);
|
|
402
|
+
const dbfs = rms > 0 ? 20 * Math.log10(rms) : -96;
|
|
403
|
+
const level = {
|
|
404
|
+
rms: Math.round(rms * 1e4) / 1e4,
|
|
405
|
+
dbfs: Math.round(dbfs * 10) / 10
|
|
406
|
+
};
|
|
407
|
+
let classification;
|
|
408
|
+
try {
|
|
409
|
+
const result = await this.classify(chunk);
|
|
410
|
+
if (this.classifyCallCount < 3) {
|
|
411
|
+
const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
|
|
412
|
+
this.log.info("classify debug sample", {
|
|
413
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
414
|
+
meta: {
|
|
415
|
+
index: this.classifyCallCount,
|
|
416
|
+
labelCount: result.labels.length,
|
|
417
|
+
top: topRaw,
|
|
418
|
+
inferenceMs: result.inferenceMs,
|
|
419
|
+
minConf: settings.minConfidence,
|
|
420
|
+
allowedClasses: settings.allowedClasses
|
|
421
|
+
}
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
this.classifyCallCount++;
|
|
425
|
+
if (result.inferenceMs > 0) {
|
|
426
|
+
const minConf = settings.minConfidence;
|
|
427
|
+
const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
|
|
428
|
+
let filtered = result.labels.filter((c) => c.score >= minConf);
|
|
429
|
+
if (allowedSet) filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
|
|
430
|
+
if (filtered.length > 0) classification = {
|
|
431
|
+
labels: filtered,
|
|
432
|
+
inferenceMs: result.inferenceMs
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
} catch (err) {
|
|
436
|
+
const now = Date.now();
|
|
437
|
+
if (now - this.lastClassifyErrorMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
|
|
438
|
+
const suppressed = this.suppressedClassifyErrors;
|
|
439
|
+
this.suppressedClassifyErrors = 0;
|
|
440
|
+
this.lastClassifyErrorMs = now;
|
|
441
|
+
const msg = errMsg(err);
|
|
442
|
+
const stack = err instanceof Error ? err.stack : void 0;
|
|
443
|
+
this.log.warn("Audio classification failed", {
|
|
444
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
445
|
+
meta: {
|
|
446
|
+
error: msg,
|
|
447
|
+
stack,
|
|
448
|
+
suppressedSince: suppressed > 0 ? suppressed : void 0
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
} else this.suppressedClassifyErrors++;
|
|
452
|
+
}
|
|
453
|
+
return {
|
|
454
|
+
level,
|
|
455
|
+
classification,
|
|
456
|
+
timestamp: chunk.timestamp
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
async classify(chunk) {
|
|
460
|
+
const camKey = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
|
|
461
|
+
const now = Date.now();
|
|
462
|
+
const state = this.cameraState.get(camKey);
|
|
463
|
+
if (state?.inProgress || state !== void 0 && now - state.lastEndMs < CLASSIFY_MIN_INTERVAL_MS) return {
|
|
464
|
+
labels: [],
|
|
465
|
+
rawLabels: [],
|
|
466
|
+
inferenceMs: 0
|
|
467
|
+
};
|
|
468
|
+
if (this.pipelineBusy) return {
|
|
469
|
+
labels: [],
|
|
470
|
+
rawLabels: [],
|
|
471
|
+
inferenceMs: 0
|
|
472
|
+
};
|
|
473
|
+
this.cameraState.set(camKey, {
|
|
474
|
+
inProgress: true,
|
|
475
|
+
lastEndMs: state?.lastEndMs ?? 0
|
|
476
|
+
});
|
|
477
|
+
this.pipelineBusy = true;
|
|
478
|
+
const f32Data = float32FromBytes(chunk.data);
|
|
479
|
+
const result = await this.pipeline.classify({
|
|
480
|
+
data: f32Data,
|
|
481
|
+
sampleRate: chunk.sampleRate,
|
|
482
|
+
channels: chunk.channels
|
|
483
|
+
}).finally(() => {
|
|
484
|
+
this.pipelineBusy = false;
|
|
485
|
+
this.cameraState.set(camKey, {
|
|
486
|
+
inProgress: false,
|
|
487
|
+
lastEndMs: Date.now()
|
|
488
|
+
});
|
|
489
|
+
});
|
|
490
|
+
if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
|
|
491
|
+
const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
|
|
492
|
+
this.log.info("classify debug sample", {
|
|
493
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
494
|
+
meta: {
|
|
495
|
+
index: this.classifyCount,
|
|
496
|
+
engine: this.backendName,
|
|
497
|
+
rawLabelCount: result.classifications.length,
|
|
498
|
+
top: rawTop,
|
|
499
|
+
inferenceMs: result.inferenceMs
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
this.classifyCount++;
|
|
504
|
+
const macroAccum = /* @__PURE__ */ new Map();
|
|
505
|
+
for (const c of result.classifications) {
|
|
506
|
+
const macro = mapAudioLabelToMacro(c.className);
|
|
507
|
+
if (!macro) continue;
|
|
508
|
+
const prev = macroAccum.get(macro);
|
|
509
|
+
if (!prev || c.score > prev.score) macroAccum.set(macro, {
|
|
510
|
+
score: c.score,
|
|
511
|
+
rawTop: c.className
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
return {
|
|
515
|
+
labels: [...macroAccum.entries()].toSorted((a, b) => b[1].score - a[1].score).map(([className, { score, rawTop }]) => ({
|
|
516
|
+
className,
|
|
517
|
+
originalClass: rawTop,
|
|
518
|
+
score
|
|
519
|
+
})),
|
|
520
|
+
rawLabels: [...result.classifications].toSorted((a, b) => b.score - a.score).map((c) => ({
|
|
521
|
+
className: c.className,
|
|
522
|
+
originalClass: c.className,
|
|
523
|
+
score: c.score
|
|
524
|
+
})),
|
|
525
|
+
inferenceMs: result.inferenceMs
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
isReady() {
|
|
529
|
+
return this.pipeline !== null;
|
|
530
|
+
}
|
|
531
|
+
async dispose() {
|
|
532
|
+
await this.pipeline.dispose();
|
|
533
|
+
}
|
|
534
|
+
async reprobeAudioEngine() {
|
|
535
|
+
return this.reprobeImpl();
|
|
536
|
+
}
|
|
537
|
+
};
|
|
538
|
+
/**
|
|
539
|
+
* Audio Analyzer addon — provides the `audio-analyzer` capability.
|
|
540
|
+
*
|
|
541
|
+
* Owns the IAudioPipeline directly — no tRPC roundtrip to a separate
|
|
542
|
+
* audio-classifier addon.
|
|
543
|
+
*/
|
|
544
|
+
var AudioAnalyzerAddon = class extends BaseAddon {
|
|
545
|
+
id = "audio-analyzer";
|
|
546
|
+
provider = null;
|
|
547
|
+
pipeline = null;
|
|
548
|
+
constructor() {
|
|
549
|
+
super(DEFAULT_AUDIO_ANALYZER_CONFIG);
|
|
550
|
+
}
|
|
551
|
+
globalSettingsSchema() {
|
|
552
|
+
return { sections: [{
|
|
553
|
+
id: "audio-engine",
|
|
554
|
+
title: "Audio",
|
|
555
|
+
tab: "engine",
|
|
556
|
+
order: 10,
|
|
557
|
+
description: "Audio classification backend (Apple SoundAnalysis or YAMNet ONNX). Independent from the vision-detection engine above. \"Auto\" picks Apple SoundAnalysis on macOS, YAMNet on Linux. Click the refresh icon next to \"Probed best\" to re-run the probe.",
|
|
558
|
+
fields: [
|
|
559
|
+
{
|
|
560
|
+
type: "text",
|
|
561
|
+
key: "probedBestAudioBackend",
|
|
562
|
+
label: "Probed best",
|
|
563
|
+
description: "Auto-detected best audio backend on this host. Click the refresh icon to re-run the probe.",
|
|
564
|
+
readonlyField: true,
|
|
565
|
+
default: "",
|
|
566
|
+
actions: [{
|
|
567
|
+
action: "reprobe-audio-engine",
|
|
568
|
+
icon: "refresh-cw",
|
|
569
|
+
tooltip: "Re-probe audio engine"
|
|
570
|
+
}]
|
|
571
|
+
},
|
|
572
|
+
{
|
|
573
|
+
type: "select",
|
|
574
|
+
key: "audioBackend",
|
|
575
|
+
label: "Audio backend",
|
|
576
|
+
options: AUDIO_BACKEND_CHOICES.map((o) => ({
|
|
577
|
+
value: o.value,
|
|
578
|
+
label: o.label
|
|
579
|
+
})),
|
|
580
|
+
default: DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend,
|
|
581
|
+
immediate: true,
|
|
582
|
+
requiresRestart: true
|
|
583
|
+
},
|
|
584
|
+
{
|
|
585
|
+
type: "select",
|
|
586
|
+
key: "selectedAudioModel",
|
|
587
|
+
label: "Classification model",
|
|
588
|
+
description: "Empty = auto (matches backend). Device-level settings can only inherit / enable / disable this step; model + class filters live here at the node level.",
|
|
589
|
+
options: AUDIO_MODEL_OPTIONS.map((o) => ({
|
|
590
|
+
value: o.value,
|
|
591
|
+
label: o.label
|
|
592
|
+
})),
|
|
593
|
+
default: DEFAULT_AUDIO_ANALYZER_CONFIG.selectedAudioModel,
|
|
594
|
+
immediate: true,
|
|
595
|
+
requiresRestart: true
|
|
596
|
+
}
|
|
597
|
+
]
|
|
598
|
+
}] };
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Cascade override — narrow the `selectedAudioModel` options to the
|
|
602
|
+
* subset compatible with the currently-selected `audioBackend`.
|
|
603
|
+
*
|
|
604
|
+
* Same pattern as detection-pipeline's `engineRuntime → engineBackend
|
|
605
|
+
* → engineDevice` cascade: the base schema ships every option
|
|
606
|
+
* (Auto + YAMNet + Apple SA); this override drops the rows that
|
|
607
|
+
* belong to a backend the operator didn't pick. With `immediate:
|
|
608
|
+
* true` on the `audioBackend` select, the UI refetches schema after
|
|
609
|
+
* every flip and the model dropdown updates instantly.
|
|
610
|
+
*
|
|
611
|
+
* `overlay` carries the operator's tentative choices for benchmark/
|
|
612
|
+
* preview mode (operator typed but didn't save yet) — same
|
|
613
|
+
* semantics detection-pipeline relies on.
|
|
614
|
+
*/
|
|
615
|
+
async getGlobalSettings(overlay) {
|
|
616
|
+
const ctx = this.ctxIfReady;
|
|
617
|
+
const stored = ctx?.settings ? await ctx.settings.readAddonStore() ?? {} : {};
|
|
618
|
+
const merged = overlay ? {
|
|
619
|
+
...stored,
|
|
620
|
+
...overlay
|
|
621
|
+
} : stored;
|
|
622
|
+
const operatorChoice = typeof merged.audioBackend === "string" ? merged.audioBackend : DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend;
|
|
623
|
+
const effectiveBackend = operatorChoice === "apple-soundanalysis" ? "apple-soundanalysis" : operatorChoice === "yamnet-onnx" ? "yamnet-onnx" : process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
|
|
624
|
+
const filteredModels = AUDIO_MODEL_OPTIONS.filter((o) => o.value === "" || o.value === effectiveBackend);
|
|
625
|
+
const storedModel = typeof merged.selectedAudioModel === "string" ? merged.selectedAudioModel : "";
|
|
626
|
+
const validModel = filteredModels.find((o) => o.value === storedModel)?.value ?? "";
|
|
627
|
+
const raw = {
|
|
628
|
+
...merged,
|
|
629
|
+
selectedAudioModel: validModel
|
|
630
|
+
};
|
|
631
|
+
const schema = this.globalSettingsSchema();
|
|
632
|
+
return hydrateSchema({
|
|
633
|
+
...schema,
|
|
634
|
+
sections: schema.sections.map((section) => ({
|
|
635
|
+
...section,
|
|
636
|
+
fields: section.fields.map((field) => {
|
|
637
|
+
if (field.type === "select" && field.key === "selectedAudioModel") return {
|
|
638
|
+
...field,
|
|
639
|
+
options: filteredModels.map((o) => ({
|
|
640
|
+
value: o.value,
|
|
641
|
+
label: o.label
|
|
642
|
+
}))
|
|
643
|
+
};
|
|
644
|
+
return field;
|
|
645
|
+
})
|
|
646
|
+
}))
|
|
647
|
+
}, raw);
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Re-run the platform probe and persist the detected backend into
|
|
651
|
+
* `probedBestAudioBackend`. Operator `audioBackend` setting is not
|
|
652
|
+
* touched — only the hint.
|
|
653
|
+
*/
|
|
654
|
+
async reprobeAudioEngine() {
|
|
655
|
+
const backend = process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
|
|
656
|
+
await this.ctx.settings?.writeAddonStore({ probedBestAudioBackend: backend });
|
|
657
|
+
this.ctx.logger.info("reprobeAudioEngine: wrote probedBestAudioBackend", { meta: { backend } });
|
|
658
|
+
return { backend };
|
|
659
|
+
}
|
|
660
|
+
/** Resolve the effective backend from the operator choice, falling back to the platform heuristic when 'auto'. */
|
|
661
|
+
resolveAudioBackend() {
|
|
662
|
+
const choice = this.config.audioBackend;
|
|
663
|
+
if (choice === "apple-soundanalysis") return "apple-soundanalysis";
|
|
664
|
+
if (choice === "yamnet-onnx") return "yamnet-onnx";
|
|
665
|
+
return process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
|
|
666
|
+
}
|
|
667
|
+
async onInitialize() {
|
|
668
|
+
const logger = this.ctx.logger;
|
|
669
|
+
const modelsDir = await this.ctx.api.storage.resolve.query({
|
|
670
|
+
location: "models",
|
|
671
|
+
relativePath: ""
|
|
672
|
+
}).catch(() => "camstack-data/models");
|
|
673
|
+
const backend = this.resolveAudioBackend();
|
|
674
|
+
logger.info("audio-analyzer: resolving pipeline", { meta: {
|
|
675
|
+
operatorChoice: this.config.audioBackend,
|
|
676
|
+
effectiveBackend: backend,
|
|
677
|
+
selectedModel: this.config.selectedAudioModel || null
|
|
678
|
+
} });
|
|
679
|
+
const p = await createAudioPipeline(modelsDir, logger, { backend });
|
|
680
|
+
await p.initialize();
|
|
681
|
+
this.pipeline = p;
|
|
682
|
+
if (!this.config.probedBestAudioBackend) this.reprobeAudioEngine().catch((err) => {
|
|
683
|
+
logger.warn("audio: auto-reprobe failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
684
|
+
});
|
|
685
|
+
const self = this;
|
|
686
|
+
const deviceSettingsResolver = async (deviceId) => {
|
|
687
|
+
try {
|
|
688
|
+
const stepSettings = (await self.ctx.api.pipelineOrchestrator.resolvePipeline.query({ deviceId })).audio?.settings ?? {};
|
|
689
|
+
return {
|
|
690
|
+
minConfidence: typeof stepSettings["minConfidence"] === "number" ? stepSettings["minConfidence"] : .3,
|
|
691
|
+
allowedClasses: Array.isArray(stepSettings["enabledAudioClasses"]) ? stepSettings["enabledAudioClasses"] : []
|
|
692
|
+
};
|
|
693
|
+
} catch (err) {
|
|
694
|
+
logger.warn("audio: resolveDeviceSettings via orchestrator failed", {
|
|
695
|
+
tags: { deviceId },
|
|
696
|
+
meta: { error: err instanceof Error ? err.message : String(err) }
|
|
697
|
+
});
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
};
|
|
701
|
+
const deviceContributionResolver = async (_deviceId) => {
|
|
702
|
+
return null;
|
|
703
|
+
};
|
|
704
|
+
const deviceSettingsPatcher = async (_deviceId, _patch) => {};
|
|
705
|
+
this.provider = new AudioAnalyzerProvider(logger, this.pipeline, backend, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, () => this.reprobeAudioEngine());
|
|
706
|
+
return [{
|
|
707
|
+
capability: audioAnalyzerCapability,
|
|
708
|
+
provider: this.provider
|
|
709
|
+
}, {
|
|
710
|
+
capability: audioAnalysisCapability,
|
|
711
|
+
provider: this.provider
|
|
712
|
+
}];
|
|
713
|
+
}
|
|
714
|
+
async onShutdown() {
|
|
715
|
+
if (this.provider) {
|
|
716
|
+
await this.provider.dispose();
|
|
717
|
+
this.provider = null;
|
|
718
|
+
}
|
|
719
|
+
this.pipeline = null;
|
|
720
|
+
}
|
|
721
|
+
buildDeviceSchema() {
|
|
722
|
+
return { sections: [] };
|
|
723
|
+
}
|
|
724
|
+
async getDeviceSettings(deviceId) {
|
|
725
|
+
const raw = await this.ctx?.settings?.readDeviceStore(deviceId) ?? {};
|
|
726
|
+
return hydrateSchema(this.buildDeviceSchema(), raw);
|
|
727
|
+
}
|
|
728
|
+
async updateDeviceSettings(deviceId, patch) {
|
|
729
|
+
await this.ctx?.settings?.writeDeviceStore(deviceId, patch);
|
|
730
|
+
}
|
|
685
731
|
};
|
|
686
|
-
//#
|
|
732
|
+
//#endregion
|
|
733
|
+
//#region src/audio-analyzer/index.ts
|
|
734
|
+
var audio_analyzer_default = AudioAnalyzerAddon;
|
|
735
|
+
//#endregion
|
|
736
|
+
export { AudioAnalyzerAddon, AudioAnalyzerProvider, createAudioPipeline, audio_analyzer_default as default };
|