@camstack/addon-pipeline 0.1.19 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/dist/audio-analyzer/index.js +736 -716
  2. package/dist/audio-analyzer/index.mjs +726 -676
  3. package/dist/audio-codec-nodeav/index.js +304 -461
  4. package/dist/audio-codec-nodeav/index.mjs +300 -462
  5. package/dist/chunk-BdkLduGY.mjs +5 -0
  6. package/dist/chunk-D6vf50IK.js +28 -0
  7. package/dist/codec-runtime-BOk-13PN.js +202 -0
  8. package/dist/codec-runtime-BsqlEjPi.mjs +197 -0
  9. package/dist/constants-B_b0a-6h.mjs +3119 -0
  10. package/dist/{index-D_cl0Qqb.js → constants-D65v6yp6.js} +3107 -2935
  11. package/dist/decoder-nodeav/index.js +1374 -1444
  12. package/dist/decoder-nodeav/index.mjs +1369 -1425
  13. package/dist/detection-pipeline/index.js +6462 -5613
  14. package/dist/detection-pipeline/index.mjs +6451 -5574
  15. package/dist/dist-7ewQjTle.js +22454 -0
  16. package/dist/dist-C5jnNl0n.mjs +22089 -0
  17. package/dist/motion-wasm/index.js +469 -467
  18. package/dist/motion-wasm/index.mjs +464 -446
  19. package/dist/pipeline-runner/index.js +2035 -1836
  20. package/dist/pipeline-runner/index.mjs +2031 -1820
  21. package/dist/recorder/index.js +2097 -0
  22. package/dist/recorder/index.mjs +2095 -0
  23. package/dist/stream-broker/_stub.js +1818 -734
  24. package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-D4-DHanK.mjs +156 -0
  25. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Tf-HACFd.mjs +26 -0
  26. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-C9WX5HNw.mjs +26 -0
  27. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-BO7TIbJV.mjs +26 -0
  28. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.js-C9j-2lBe.mjs +26 -0
  29. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-XO0-Pyu6.mjs +26 -0
  30. package/dist/stream-broker/dist-CYZr2fwk.mjs +2726 -0
  31. package/dist/stream-broker/hostInit-Di6vceAU.mjs +129 -0
  32. package/dist/stream-broker/index.js +17837 -12904
  33. package/dist/stream-broker/index.mjs +17826 -12896
  34. package/dist/stream-broker/remoteEntry.js +134 -2973
  35. package/dist/stream-broker/remoteEntry.ssr.js +33 -0
  36. package/dist/stream-broker/virtualExposes-dYNvIwoR.mjs +27 -0
  37. package/dist/stream-broker/virtual_mf-exposes-ssr___mfe_internal__addon_stream_broker_widgets__remoteEntry_js-Cmqfp4i_.mjs +10 -0
  38. package/embed-dist/assets/index-B8VlSD0-.js +150 -0
  39. package/embed-dist/assets/index-ZhDdp1Nd.css +2 -0
  40. package/embed-dist/index.html +13 -0
  41. package/package.json +75 -9
  42. package/wasm/assembly/index.ts +41 -16
  43. package/dist/audio-analyzer/index.js.map +0 -1
  44. package/dist/audio-analyzer/index.mjs.map +0 -1
  45. package/dist/audio-codec-nodeav/index.js.map +0 -1
  46. package/dist/audio-codec-nodeav/index.mjs.map +0 -1
  47. package/dist/decoder-nodeav/index.js.map +0 -1
  48. package/dist/decoder-nodeav/index.mjs.map +0 -1
  49. package/dist/detection-pipeline/index.js.map +0 -1
  50. package/dist/detection-pipeline/index.mjs.map +0 -1
  51. package/dist/index-BbPPvoCx.js +0 -14682
  52. package/dist/index-BbPPvoCx.js.map +0 -1
  53. package/dist/index-Bmlkm0Fd.mjs +0 -14683
  54. package/dist/index-Bmlkm0Fd.mjs.map +0 -1
  55. package/dist/index-D_cl0Qqb.js.map +0 -1
  56. package/dist/index-UbcdLS7a.mjs +0 -5790
  57. package/dist/index-UbcdLS7a.mjs.map +0 -1
  58. package/dist/motion-wasm/index.js.map +0 -1
  59. package/dist/motion-wasm/index.mjs.map +0 -1
  60. package/dist/pipeline-runner/index.js.map +0 -1
  61. package/dist/pipeline-runner/index.mjs.map +0 -1
  62. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/StreamBrokerPanel.d.ts +0 -21
  63. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/index.d.ts +0 -13
  64. package/dist/stream-broker/@mf-types/widgets.d.ts +0 -2
  65. package/dist/stream-broker/@mf-types.d.ts +0 -3
  66. package/dist/stream-broker/@mf-types.zip +0 -0
  67. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_sdk__loadShare__.mjs-h5aXOPSA.mjs +0 -12
  68. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.mjs-NjF4kxzW.mjs +0 -19
  69. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.mjs-BAv_5ISf.mjs +0 -20
  70. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.mjs-U1EUeEPs.mjs +0 -104
  71. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_client__loadShare__.mjs-DeouEaSs.mjs +0 -85
  72. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_react_mf_2_query__loadShare__.mjs-DHUwjbb9.mjs +0 -62
  73. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs-BsB2G7oY.mjs +0 -88
  74. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs_commonjs-proxy-xrRiPUpA.mjs +0 -29
  75. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.mjs-gBEZsQrp.mjs +0 -36
  76. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs-DYEKzzY-.mjs +0 -45
  77. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs_commonjs-proxy-C0E2yCzO.mjs +0 -6
  78. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom_mf_1_client__loadShare__.mjs-DICOtMTl.mjs +0 -34
  79. package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CupRlwqG.mjs +0 -156
  80. package/dist/stream-broker/client-NPZqorv9.mjs +0 -9836
  81. package/dist/stream-broker/getErrorShape-BPSzUA7W-TlK8ipWe.mjs +0 -211
  82. package/dist/stream-broker/hostInit-Bh4w7o5_.mjs +0 -168
  83. package/dist/stream-broker/index-2Qp8vT3w.mjs +0 -185
  84. package/dist/stream-broker/index-BBcZvb5t.mjs +0 -435
  85. package/dist/stream-broker/index-CIJue-4t.mjs +0 -37880
  86. package/dist/stream-broker/index-CWkKuNLr.mjs +0 -232
  87. package/dist/stream-broker/index-Cc6QBqMk.mjs +0 -1655
  88. package/dist/stream-broker/index-D_1p2K9B.mjs +0 -2603
  89. package/dist/stream-broker/index-Dy2V7VOm.mjs +0 -14379
  90. package/dist/stream-broker/index-mX3Kgiv1.mjs +0 -725
  91. package/dist/stream-broker/index-xncRG7-x.mjs +0 -2713
  92. package/dist/stream-broker/index.js.map +0 -1
  93. package/dist/stream-broker/index.mjs.map +0 -1
  94. package/dist/stream-broker/jsx-runtime-lb0mH5st.mjs +0 -55
  95. package/dist/stream-broker/schemas-ClCuS4qa.mjs +0 -3594
  96. package/dist/stream-broker/virtualExposes-pCd777Rp.mjs +0 -42
@@ -1,726 +1,746 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __copyProps = (to, from, except, desc) => {
9
- if (from && typeof from === "object" || typeof from === "function") {
10
- for (let key of __getOwnPropNames(from))
11
- if (!__hasOwnProp.call(to, key) && key !== except)
12
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
13
- }
14
- return to;
15
- };
16
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
17
- // If the importer is in node compatibility mode or this is not an ESM
18
- // file that has been converted to a CommonJS file using a Babel-
19
- // compatible transform (i.e. "__esModule" has not been set), then set
20
- // "default" to the CommonJS "module.exports" for node compatibility.
21
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
22
- mod
23
- ));
24
- Object.defineProperties(exports, { __esModule: { value: true }, [Symbol.toStringTag]: { value: "Module" } });
25
- const index = require("../index-BbPPvoCx.js");
26
- const path = require("node:path");
27
- const fs = require("node:fs");
28
- const core = require("@camstack/core");
29
- function _interopNamespaceDefault(e) {
30
- const n = Object.create(null, { [Symbol.toStringTag]: { value: "Module" } });
31
- if (e) {
32
- for (const k in e) {
33
- if (k !== "default") {
34
- const d = Object.getOwnPropertyDescriptor(e, k);
35
- Object.defineProperty(n, k, d.get ? d : {
36
- enumerable: true,
37
- get: () => e[k]
38
- });
39
- }
40
- }
41
- }
42
- n.default = e;
43
- return Object.freeze(n);
44
- }
45
- const path__namespace = /* @__PURE__ */ _interopNamespaceDefault(path);
46
- const fs__namespace = /* @__PURE__ */ _interopNamespaceDefault(fs);
1
+ Object.defineProperties(exports, {
2
+ __esModule: { value: true },
3
+ [Symbol.toStringTag]: { value: "Module" }
4
+ });
5
+ const require_chunk = require("../chunk-D6vf50IK.js");
6
+ const require_dist = require("../dist-7ewQjTle.js");
7
+ let node_fs = require("node:fs");
8
+ node_fs = require_chunk.__toESM(node_fs);
9
+ let node_path = require("node:path");
10
+ node_path = require_chunk.__toESM(node_path);
11
+ let _camstack_core = require("@camstack/core");
12
+ //#region src/audio-analyzer/audio-pipeline.ts
13
+ /**
14
+ * Create the appropriate audio pipeline.
15
+ *
16
+ * - 'yamnet-onnx': Cross-platform YAMNet ONNX (requires model download)
17
+ * - 'apple-soundanalysis': macOS 12+ Apple SoundAnalysis (zero model download, Neural Engine)
18
+ * - undefined: auto-detect (Apple SA on macOS, YAMNet on Linux)
19
+ */
47
20
  async function createAudioPipeline(modelsDir, logger, options) {
48
- const backend = options?.backend ?? (process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx");
49
- if (backend === "apple-soundanalysis") {
50
- return new AppleSoundAnalysisPipeline(logger);
51
- }
52
- return new YamnetOnnxPipeline(modelsDir, logger);
53
- }
54
- const YAMNET_MODEL_URL = `${index.HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
55
- const YAMNET_LABELS_URL = `${index.HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
56
- class YamnetOnnxPipeline {
57
- constructor(modelsDir, logger) {
58
- this.modelsDir = modelsDir;
59
- this.log = logger;
60
- }
61
- session = null;
62
- inputName = "";
63
- labels = [];
64
- log;
65
- async initialize() {
66
- const ort = await import("onnxruntime-node");
67
- const modelPath = path__namespace.join(this.modelsDir, "camstack-yamnet.onnx");
68
- const labelsPath = path__namespace.join(this.modelsDir, "camstack-yamnet-labels.json");
69
- if (!fs__namespace.existsSync(modelPath)) {
70
- this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", {
71
- meta: { url: YAMNET_MODEL_URL, dest: modelPath }
72
- });
73
- await core.downloadFile(YAMNET_MODEL_URL, modelPath);
74
- this.log.info("YAMNet ONNX model downloaded", {
75
- meta: { sizeBytes: fs__namespace.statSync(modelPath).size }
76
- });
77
- }
78
- if (!fs__namespace.existsSync(labelsPath)) {
79
- this.log.info("YAMNet labels not found locally — downloading from HuggingFace", {
80
- meta: { url: YAMNET_LABELS_URL, dest: labelsPath }
81
- });
82
- await core.downloadFile(YAMNET_LABELS_URL, labelsPath);
83
- }
84
- this.session = await ort.InferenceSession.create(modelPath);
85
- this.inputName = this.session.inputNames[0] ?? "waveform";
86
- if (fs__namespace.existsSync(labelsPath)) {
87
- this.labels = JSON.parse(fs__namespace.readFileSync(labelsPath, "utf8"));
88
- } else {
89
- this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
90
- }
91
- this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
92
- }
93
- async classify(chunk) {
94
- if (!this.session) {
95
- throw new Error("YAMNet pipeline not initialized");
96
- }
97
- const start = Date.now();
98
- const ort = await import("onnxruntime-node");
99
- const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
100
- const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
101
- const feeds = { [this.inputName]: tensor };
102
- const results = await this.session.run(feeds);
103
- const scoresData = results[this.session.outputNames[0]];
104
- if (!scoresData) {
105
- throw new Error("YAMNet returned no output");
106
- }
107
- const scores = scoresData.data;
108
- const numClasses = 521;
109
- const numFrames = scores.length / numClasses;
110
- const avgScores = new Float32Array(numClasses);
111
- for (let f = 0; f < numFrames; f++) {
112
- for (let c = 0; c < numClasses; c++) {
113
- avgScores[c] += scores[f * numClasses + c];
114
- }
115
- }
116
- for (let c = 0; c < numClasses; c++) {
117
- avgScores[c] = avgScores[c] / numFrames;
118
- }
119
- const minScore = 0.05;
120
- const classifications = [];
121
- for (let c = 0; c < numClasses; c++) {
122
- const score = avgScores[c];
123
- if (score >= minScore) {
124
- const label = c < this.labels.length ? this.labels[c] : String(c);
125
- classifications.push({ className: label, score: Math.round(score * 1e3) / 1e3 });
126
- }
127
- }
128
- classifications.sort((a, b) => b.score - a.score);
129
- return {
130
- classifications: classifications.slice(0, 10),
131
- inferenceMs: Date.now() - start
132
- };
133
- }
134
- async dispose() {
135
- if (this.session) {
136
- await this.session.release();
137
- this.session = null;
138
- }
139
- }
140
- }
141
- class AppleSoundAnalysisPipeline {
142
- log;
143
- process = null;
144
- receiveBuffer = Buffer.alloc(0);
145
- pendingResolve = null;
146
- pendingReject = null;
147
- binaryPath = null;
148
- debugCount = 0;
149
- constructor(logger) {
150
- this.log = logger;
151
- }
152
- async initialize() {
153
- this.binaryPath = await this.resolveSwiftBinary();
154
- if (!this.binaryPath) {
155
- throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
156
- }
157
- const { spawn } = await import("node:child_process");
158
- this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], {
159
- stdio: ["pipe", "pipe", "pipe"]
160
- });
161
- this.process.stderr?.on("data", (chunk) => {
162
- const lines = chunk.toString().split("\n");
163
- for (const line of lines) {
164
- const trimmed = line.trim();
165
- if (trimmed) this.log.warn(trimmed);
166
- }
167
- });
168
- this.process.on("error", (err) => {
169
- this.log.error("Swift process error", { meta: { error: err.message } });
170
- this.pendingReject?.(err);
171
- this.pendingReject = null;
172
- this.pendingResolve = null;
173
- });
174
- this.process.on("exit", (code) => {
175
- if (code !== 0 && code !== null) {
176
- this.log.error("Swift process exited", { meta: { code } });
177
- const err = new Error(`Apple SoundAnalysis: process exited with code ${code}`);
178
- this.pendingReject?.(err);
179
- this.pendingReject = null;
180
- this.pendingResolve = null;
181
- }
182
- });
183
- this.process.stdout.on("data", (chunk) => {
184
- this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
185
- this.tryReceive();
186
- });
187
- const ready = await this.receiveMessage();
188
- if (ready["status"] !== "ready") {
189
- throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
190
- }
191
- this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
192
- }
193
- async classify(chunk) {
194
- if (!this.process?.stdin) {
195
- throw new Error("Apple SoundAnalysis: process not initialized");
196
- }
197
- const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
198
- const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
199
- const lengthBuf = Buffer.allocUnsafe(4);
200
- lengthBuf.writeUInt32LE(audioBuffer.length, 0);
201
- this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
202
- const result = await this.receiveMessage();
203
- const classifications = result["classifications"] ?? [];
204
- const inferenceMs = result["inferenceMs"] ?? 0;
205
- if (this.debugCount < 3) {
206
- const keys = Object.keys(result);
207
- this.log.info("classify debug sample", {
208
- meta: {
209
- phase: "apple-sa",
210
- index: this.debugCount,
211
- keys,
212
- classifications: classifications.length,
213
- inferenceMs,
214
- audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
215
- sampleRate: chunk.sampleRate,
216
- channels: chunk.channels
217
- }
218
- });
219
- if (result["error"]) {
220
- this.log.error("Swift error", { meta: { phase: "apple-sa", error: result["error"] } });
221
- }
222
- this.debugCount++;
223
- }
224
- return { classifications, inferenceMs };
225
- }
226
- async dispose() {
227
- const proc = this.process;
228
- if (!proc) return;
229
- this.process = null;
230
- proc.stdin?.end();
231
- proc.kill("SIGTERM");
232
- const exited = await new Promise((resolve) => {
233
- const timer = setTimeout(() => resolve(false), 5e3);
234
- proc.once("exit", () => {
235
- clearTimeout(timer);
236
- resolve(true);
237
- });
238
- });
239
- if (!exited) {
240
- try {
241
- proc.kill("SIGKILL");
242
- } catch {
243
- }
244
- this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
245
- }
246
- }
247
- receiveMessage() {
248
- return new Promise((resolve, reject) => {
249
- this.pendingResolve = resolve;
250
- this.pendingReject = reject;
251
- });
252
- }
253
- tryReceive() {
254
- if (this.receiveBuffer.length < 4) return;
255
- const length = this.receiveBuffer.readUInt32LE(0);
256
- if (this.receiveBuffer.length < 4 + length) return;
257
- const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
258
- this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
259
- const resolve = this.pendingResolve;
260
- const reject = this.pendingReject;
261
- this.pendingResolve = null;
262
- this.pendingReject = null;
263
- if (!resolve) return;
264
- try {
265
- const parsed = JSON.parse(jsonBytes.toString("utf8"));
266
- resolve(parsed);
267
- } catch (err) {
268
- reject?.(err instanceof Error ? err : new Error(String(err)));
269
- }
270
- }
271
- /** Find pre-compiled binary or compile from Swift source. */
272
- async resolveSwiftBinary() {
273
- const candidates = [
274
- path__namespace.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
275
- // Fallback for in-tree dev (src/<id>/swift/) and pre-merge layouts.
276
- path__namespace.join(__dirname, "../swift/apple-sound-classifier"),
277
- path__namespace.join(__dirname, "../../swift/apple-sound-classifier"),
278
- path__namespace.join(__dirname, "../../../swift/apple-sound-classifier")
279
- ];
280
- for (const p of candidates) {
281
- if (fs__namespace.existsSync(p)) {
282
- this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
283
- return p;
284
- }
285
- }
286
- const sourceCandidates = [
287
- path__namespace.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
288
- path__namespace.join(__dirname, "../swift/apple-sound-classifier.swift"),
289
- path__namespace.join(__dirname, "../../swift/apple-sound-classifier.swift"),
290
- path__namespace.join(__dirname, "../../../swift/apple-sound-classifier.swift")
291
- ];
292
- const sourcePath = sourceCandidates.find((p) => fs__namespace.existsSync(p));
293
- if (!sourcePath) {
294
- this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
295
- return null;
296
- }
297
- const outputPath = sourcePath.replace(".swift", "");
298
- this.log.info("Compiling Swift CLI...", { meta: { source: sourcePath, output: outputPath } });
299
- const { execFileSync } = await import("node:child_process");
300
- try {
301
- execFileSync("swiftc", ["-O", "-o", outputPath, sourcePath], {
302
- timeout: 6e4,
303
- stdio: "pipe"
304
- });
305
- this.log.info("Swift CLI compiled successfully");
306
- return outputPath;
307
- } catch (err) {
308
- this.log.error("Swift compilation failed — install Xcode Command Line Tools", {
309
- meta: { error: index.errMsg(err) }
310
- });
311
- return null;
312
- }
313
- }
21
+ if ((options?.backend ?? (process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx")) === "apple-soundanalysis") return new AppleSoundAnalysisPipeline(logger);
22
+ return new YamnetOnnxPipeline(modelsDir, logger);
314
23
  }
24
+ /**
25
+ * Canonical model URLs on the camstack HuggingFace mirror. Mirrors the
26
+ * convention every detection model follows (single point of truth =
27
+ * `HF_BASE_URL` from `@camstack/types`); the auto-download path uses
28
+ * `downloadFile` from `@camstack/core`, the SAME helper detection-
29
+ * pipeline uses to materialise its YOLO/face/plate models. Missing
30
+ * model on disk → fetch from HF; cached file → no-op.
31
+ *
32
+ * Repo layout follows the detection-pipeline pattern:
33
+ * {domain}/{family}/{format}/{filename}
34
+ * For YAMNet that's `audioClassification/yamnet/onnx/camstack-yamnet.onnx`,
35
+ * with the labels JSON sitting one level up (`audioClassification/yamnet/`)
36
+ * because they're format-agnostic (same 521 AudioSet class names whether
37
+ * the runtime is ONNX, OpenVINO, or TF).
38
+ */
39
+ var YAMNET_MODEL_URL = `${require_dist.HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
40
+ var YAMNET_LABELS_URL = `${require_dist.HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
41
+ var YamnetOnnxPipeline = class {
42
+ modelsDir;
43
+ session = null;
44
+ inputName = "";
45
+ labels = [];
46
+ log;
47
+ constructor(modelsDir, logger) {
48
+ this.modelsDir = modelsDir;
49
+ this.log = logger;
50
+ }
51
+ async initialize() {
52
+ const ort = await import("onnxruntime-node");
53
+ const modelPath = node_path.join(this.modelsDir, "camstack-yamnet.onnx");
54
+ const labelsPath = node_path.join(this.modelsDir, "camstack-yamnet-labels.json");
55
+ if (!node_fs.existsSync(modelPath)) {
56
+ this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", { meta: {
57
+ url: YAMNET_MODEL_URL,
58
+ dest: modelPath
59
+ } });
60
+ await (0, _camstack_core.downloadFile)(YAMNET_MODEL_URL, modelPath);
61
+ this.log.info("YAMNet ONNX model downloaded", { meta: { sizeBytes: node_fs.statSync(modelPath).size } });
62
+ }
63
+ if (!node_fs.existsSync(labelsPath)) {
64
+ this.log.info("YAMNet labels not found locally — downloading from HuggingFace", { meta: {
65
+ url: YAMNET_LABELS_URL,
66
+ dest: labelsPath
67
+ } });
68
+ await (0, _camstack_core.downloadFile)(YAMNET_LABELS_URL, labelsPath);
69
+ }
70
+ this.session = await ort.InferenceSession.create(modelPath);
71
+ this.inputName = this.session.inputNames[0] ?? "waveform";
72
+ if (node_fs.existsSync(labelsPath)) this.labels = JSON.parse(node_fs.readFileSync(labelsPath, "utf8"));
73
+ else this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
74
+ this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
75
+ }
76
+ async classify(chunk) {
77
+ if (!this.session) throw new Error("YAMNet pipeline not initialized");
78
+ const start = Date.now();
79
+ const ort = await import("onnxruntime-node");
80
+ const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
81
+ const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
82
+ const feeds = { [this.inputName]: tensor };
83
+ const scoresData = (await this.session.run(feeds))[this.session.outputNames[0]];
84
+ if (!scoresData) throw new Error("YAMNet returned no output");
85
+ const scores = scoresData.data;
86
+ const numClasses = 521;
87
+ const numFrames = scores.length / numClasses;
88
+ const avgScores = new Float32Array(numClasses);
89
+ for (let f = 0; f < numFrames; f++) for (let c = 0; c < numClasses; c++) avgScores[c] += scores[f * numClasses + c];
90
+ for (let c = 0; c < numClasses; c++) avgScores[c] = avgScores[c] / numFrames;
91
+ const minScore = .05;
92
+ const classifications = [];
93
+ for (let c = 0; c < numClasses; c++) {
94
+ const score = avgScores[c];
95
+ if (score >= minScore) {
96
+ const label = c < this.labels.length ? this.labels[c] : String(c);
97
+ classifications.push({
98
+ className: label,
99
+ score: Math.round(score * 1e3) / 1e3
100
+ });
101
+ }
102
+ }
103
+ classifications.sort((a, b) => b.score - a.score);
104
+ return {
105
+ classifications: classifications.slice(0, 10),
106
+ inferenceMs: Date.now() - start
107
+ };
108
+ }
109
+ async dispose() {
110
+ if (this.session) {
111
+ await this.session.release();
112
+ this.session = null;
113
+ }
114
+ }
115
+ };
116
+ var AppleSoundAnalysisPipeline = class {
117
+ log;
118
+ process = null;
119
+ receiveBuffer = Buffer.alloc(0);
120
+ pendingResolve = null;
121
+ pendingReject = null;
122
+ binaryPath = null;
123
+ debugCount = 0;
124
+ constructor(logger) {
125
+ this.log = logger;
126
+ }
127
+ async initialize() {
128
+ this.binaryPath = await this.resolveSwiftBinary();
129
+ if (!this.binaryPath) throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
130
+ const { spawn } = await import("node:child_process");
131
+ this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], { stdio: [
132
+ "pipe",
133
+ "pipe",
134
+ "pipe"
135
+ ] });
136
+ this.process.stderr?.on("data", (chunk) => {
137
+ const lines = chunk.toString().split("\n");
138
+ for (const line of lines) {
139
+ const trimmed = line.trim();
140
+ if (trimmed) this.log.warn(trimmed);
141
+ }
142
+ });
143
+ this.process.on("error", (err) => {
144
+ this.log.error("Swift process error", { meta: { error: err.message } });
145
+ this.pendingReject?.(err);
146
+ this.pendingReject = null;
147
+ this.pendingResolve = null;
148
+ });
149
+ this.process.on("exit", (code) => {
150
+ if (code !== 0 && code !== null) {
151
+ this.log.error("Swift process exited", { meta: { code } });
152
+ const err = /* @__PURE__ */ new Error(`Apple SoundAnalysis: process exited with code ${code}`);
153
+ this.pendingReject?.(err);
154
+ this.pendingReject = null;
155
+ this.pendingResolve = null;
156
+ }
157
+ });
158
+ this.process.stdout.on("data", (chunk) => {
159
+ this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
160
+ this.tryReceive();
161
+ });
162
+ const ready = await this.receiveMessage();
163
+ if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
164
+ this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
165
+ }
166
+ async classify(chunk) {
167
+ if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
168
+ const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
169
+ const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
170
+ const lengthBuf = Buffer.allocUnsafe(4);
171
+ lengthBuf.writeUInt32LE(audioBuffer.length, 0);
172
+ this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
173
+ const result = await this.receiveMessage();
174
+ const classifications = result["classifications"] ?? [];
175
+ const inferenceMs = result["inferenceMs"] ?? 0;
176
+ if (this.debugCount < 3) {
177
+ const keys = Object.keys(result);
178
+ this.log.info("classify debug sample", { meta: {
179
+ phase: "apple-sa",
180
+ index: this.debugCount,
181
+ keys,
182
+ classifications: classifications.length,
183
+ inferenceMs,
184
+ audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
185
+ sampleRate: chunk.sampleRate,
186
+ channels: chunk.channels
187
+ } });
188
+ if (result["error"]) this.log.error("Swift error", { meta: {
189
+ phase: "apple-sa",
190
+ error: result["error"]
191
+ } });
192
+ this.debugCount++;
193
+ }
194
+ return {
195
+ classifications,
196
+ inferenceMs
197
+ };
198
+ }
199
+ async dispose() {
200
+ const proc = this.process;
201
+ if (!proc) return;
202
+ this.process = null;
203
+ proc.stdin?.end();
204
+ proc.kill("SIGTERM");
205
+ if (!await new Promise((resolve) => {
206
+ const timer = setTimeout(() => resolve(false), 5e3);
207
+ proc.once("exit", () => {
208
+ clearTimeout(timer);
209
+ resolve(true);
210
+ });
211
+ })) {
212
+ try {
213
+ proc.kill("SIGKILL");
214
+ } catch {}
215
+ this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
216
+ }
217
+ }
218
+ receiveMessage() {
219
+ return new Promise((resolve, reject) => {
220
+ this.pendingResolve = resolve;
221
+ this.pendingReject = reject;
222
+ });
223
+ }
224
+ tryReceive() {
225
+ if (this.receiveBuffer.length < 4) return;
226
+ const length = this.receiveBuffer.readUInt32LE(0);
227
+ if (this.receiveBuffer.length < 4 + length) return;
228
+ const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
229
+ this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
230
+ const resolve = this.pendingResolve;
231
+ const reject = this.pendingReject;
232
+ this.pendingResolve = null;
233
+ this.pendingReject = null;
234
+ if (!resolve) return;
235
+ try {
236
+ resolve(JSON.parse(jsonBytes.toString("utf8")));
237
+ } catch (err) {
238
+ reject?.(err instanceof Error ? err : new Error(String(err)));
239
+ }
240
+ }
241
+ /** Find pre-compiled binary or compile from Swift source. */
242
+ async resolveSwiftBinary() {
243
+ const candidates = [
244
+ node_path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
245
+ node_path.join(__dirname, "../swift/apple-sound-classifier"),
246
+ node_path.join(__dirname, "../../swift/apple-sound-classifier"),
247
+ node_path.join(__dirname, "../../../swift/apple-sound-classifier")
248
+ ];
249
+ for (const p of candidates) if (node_fs.existsSync(p)) {
250
+ this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
251
+ return p;
252
+ }
253
+ const sourceCandidates = [
254
+ node_path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
255
+ node_path.join(__dirname, "../swift/apple-sound-classifier.swift"),
256
+ node_path.join(__dirname, "../../swift/apple-sound-classifier.swift"),
257
+ node_path.join(__dirname, "../../../swift/apple-sound-classifier.swift")
258
+ ];
259
+ const sourcePath = sourceCandidates.find((p) => node_fs.existsSync(p));
260
+ if (!sourcePath) {
261
+ this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
262
+ return null;
263
+ }
264
+ const outputPath = sourcePath.replace(".swift", "");
265
+ this.log.info("Compiling Swift CLI...", { meta: {
266
+ source: sourcePath,
267
+ output: outputPath
268
+ } });
269
+ const { execFileSync } = await import("node:child_process");
270
+ try {
271
+ execFileSync("swiftc", [
272
+ "-O",
273
+ "-o",
274
+ outputPath,
275
+ sourcePath
276
+ ], {
277
+ timeout: 6e4,
278
+ stdio: "pipe"
279
+ });
280
+ this.log.info("Swift CLI compiled successfully");
281
+ return outputPath;
282
+ } catch (err) {
283
+ this.log.error("Swift compilation failed — install Xcode Command Line Tools", { meta: { error: require_dist.errMsg(err) } });
284
+ return null;
285
+ }
286
+ }
287
+ };
288
+ /** Simple resample to 16kHz mono by linear interpolation. */
315
289
  function resampleMono16k(chunk) {
316
- const { data, sampleRate, channels } = chunk;
317
- const numSamples = data.length / channels;
318
- const mono = new Float32Array(numSamples);
319
- for (let i = 0; i < numSamples; i++) {
320
- let sum = 0;
321
- for (let c = 0; c < channels; c++) {
322
- sum += data[i * channels + c];
323
- }
324
- mono[i] = sum / channels;
325
- }
326
- const ratio = 16e3 / sampleRate;
327
- const outLen = Math.floor(numSamples * ratio);
328
- const out = new Float32Array(outLen);
329
- for (let i = 0; i < outLen; i++) {
330
- const srcIdx = i / ratio;
331
- const lo = Math.floor(srcIdx);
332
- const hi = Math.min(lo + 1, numSamples - 1);
333
- const frac = srcIdx - lo;
334
- out[i] = mono[lo] * (1 - frac) + mono[hi] * frac;
335
- }
336
- return out;
290
+ const { data, sampleRate, channels } = chunk;
291
+ const numSamples = data.length / channels;
292
+ const mono = new Float32Array(numSamples);
293
+ for (let i = 0; i < numSamples; i++) {
294
+ let sum = 0;
295
+ for (let c = 0; c < channels; c++) sum += data[i * channels + c];
296
+ mono[i] = sum / channels;
297
+ }
298
+ const ratio = 16e3 / sampleRate;
299
+ const outLen = Math.floor(numSamples * ratio);
300
+ const out = new Float32Array(outLen);
301
+ for (let i = 0; i < outLen; i++) {
302
+ const srcIdx = i / ratio;
303
+ const lo = Math.floor(srcIdx);
304
+ const hi = Math.min(lo + 1, numSamples - 1);
305
+ const frac = srcIdx - lo;
306
+ out[i] = mono[lo] * (1 - frac) + mono[hi] * frac;
307
+ }
308
+ return out;
337
309
  }
338
- const AUDIO_MODEL_OPTIONS = [
339
- { value: "", label: "Auto (matches backend)" },
340
- { value: "yamnet-onnx", label: "YAMNet (ONNX)" },
341
- { value: "apple-soundanalysis", label: "Apple SoundAnalysis (built-in)" }
310
+ //#endregion
311
+ //#region src/audio-analyzer/addons/analyzer/index.ts
312
+ /**
313
+ * Choices presented in the Audio Model dropdown. YAMNet runs via ONNX
314
+ * when backend=yamnet-onnx; Apple SoundAnalysis is a built-in macOS
315
+ * model and has no swappable modelId — the backend IS the model.
316
+ */
317
+ var AUDIO_MODEL_OPTIONS = [
318
+ {
319
+ value: "",
320
+ label: "Auto (matches backend)"
321
+ },
322
+ {
323
+ value: "yamnet-onnx",
324
+ label: "YAMNet (ONNX)"
325
+ },
326
+ {
327
+ value: "apple-soundanalysis",
328
+ label: "Apple SoundAnalysis (built-in)"
329
+ }
342
330
  ];
343
- const CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
344
- const CLASSIFY_MIN_INTERVAL_MS = 500;
345
- const GLOBAL_DEVICE_KEY = -1;
331
+ /**
332
+ * AudioAnalyzerProvider implements IAudioAnalyzer.
333
+ *
334
+ * Computes dB/RMS on every chunk and classifies via the in-process
335
+ * IAudioPipeline (YAMNet ONNX / Apple SoundAnalysis). No tRPC roundtrip
336
+ * to a separate audio-classifier addon.
337
+ */
338
+ var CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
339
+ var CLASSIFY_MIN_INTERVAL_MS = 500;
340
+ var GLOBAL_DEVICE_KEY = -1;
341
+ /**
342
+ * Reconstruct a Float32 view over f32le PCM bytes carried in a Uint8Array.
343
+ *
344
+ * `@msgpack/msgpack` decodes a binary blob into a Uint8Array that is a subview
345
+ * of its internal decode buffer, whose `byteOffset` is NOT guaranteed to be
346
+ * 4-byte aligned (observed e.g. 9). `new Float32Array(buf, offset, …)` then
347
+ * throws "start offset of Float32Array should be a multiple of 4". When the
348
+ * offset is misaligned we copy into a fresh 0-offset buffer; the aligned
349
+ * fast-path reuses the existing view with no copy.
350
+ */
346
351
  function float32FromBytes(raw) {
347
- const bytes = raw.byteOffset % 4 === 0 ? raw : new Uint8Array(raw);
348
- return new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
349
- }
350
- class AudioAnalyzerProvider {
351
- constructor(logger, pipeline, backendName, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, reprobeImpl) {
352
- this.pipeline = pipeline;
353
- this.deviceSettingsResolver = deviceSettingsResolver;
354
- this.deviceContributionResolver = deviceContributionResolver;
355
- this.deviceSettingsPatcher = deviceSettingsPatcher;
356
- this.reprobeImpl = reprobeImpl;
357
- this.log = logger;
358
- this.backendName = backendName;
359
- }
360
- log;
361
- classifyCallCount = 0;
362
- lastClassifyErrorMs = 0;
363
- suppressedClassifyErrors = 0;
364
- classifyCount = 0;
365
- backendName = "unknown";
366
- /** Per-camera in-flight state. Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
367
- cameraState = /* @__PURE__ */ new Map();
368
- /** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
369
- * run at a time. Without this, concurrent calls from different cameras overwrite the
370
- * single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
371
- pipelineBusy = false;
372
- // ── Device-details aggregator contribution ──────────────────────────────
373
- async getDeviceSettingsContribution(input) {
374
- return this.deviceContributionResolver(input.deviceId);
375
- }
376
- async getDeviceLiveContribution(_input) {
377
- return null;
378
- }
379
- async applyDeviceSettingsPatch(input) {
380
- await this.deviceSettingsPatcher(input.deviceId, input.patch);
381
- return { success: true };
382
- }
383
- /**
384
- * Return the effective per-device audio-analyzer settings, resolved via
385
- * the kernel's 3-level settings resolver (schema default → global →
386
- * device override). Orchestrator consumers call this method so they
387
- * never need to know the audio-analyzer schema field names.
388
- */
389
- async resolveDeviceSettings({ deviceId }) {
390
- return this.deviceSettingsResolver(deviceId);
391
- }
392
- async analyseChunk({
393
- chunk,
394
- settings
395
- }) {
396
- const samples = float32FromBytes(chunk.data);
397
- let sumSquares = 0;
398
- for (let i = 0; i < samples.length; i++) {
399
- sumSquares += samples[i] * samples[i];
400
- }
401
- const rms = Math.sqrt(sumSquares / samples.length);
402
- const dbfs = rms > 0 ? 20 * Math.log10(rms) : -96;
403
- const level = {
404
- rms: Math.round(rms * 1e4) / 1e4,
405
- dbfs: Math.round(dbfs * 10) / 10
406
- };
407
- let classification;
408
- try {
409
- const result = await this.classify(chunk);
410
- if (this.classifyCallCount < 3) {
411
- const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
412
- this.log.info("classify debug sample", {
413
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
414
- meta: {
415
- index: this.classifyCallCount,
416
- labelCount: result.labels.length,
417
- top: topRaw,
418
- inferenceMs: result.inferenceMs,
419
- minConf: settings.minConfidence,
420
- allowedClasses: settings.allowedClasses
421
- }
422
- });
423
- }
424
- this.classifyCallCount++;
425
- if (result.inferenceMs > 0) {
426
- const minConf = settings.minConfidence;
427
- const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
428
- let filtered = result.labels.filter((c) => c.score >= minConf);
429
- if (allowedSet) {
430
- filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
431
- }
432
- if (filtered.length > 0) {
433
- classification = {
434
- labels: filtered,
435
- inferenceMs: result.inferenceMs
436
- };
437
- }
438
- }
439
- } catch (err) {
440
- const now = Date.now();
441
- const sinceLastMs = now - this.lastClassifyErrorMs;
442
- if (sinceLastMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
443
- const suppressed = this.suppressedClassifyErrors;
444
- this.suppressedClassifyErrors = 0;
445
- this.lastClassifyErrorMs = now;
446
- const msg = index.errMsg(err);
447
- const stack = err instanceof Error ? err.stack : void 0;
448
- this.log.warn("Audio classification failed", {
449
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
450
- meta: { error: msg, stack, suppressedSince: suppressed > 0 ? suppressed : void 0 }
451
- });
452
- } else {
453
- this.suppressedClassifyErrors++;
454
- }
455
- }
456
- return { level, classification, timestamp: chunk.timestamp };
457
- }
458
- async classify(chunk) {
459
- const camKey = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
460
- const now = Date.now();
461
- const state = this.cameraState.get(camKey);
462
- if (state?.inProgress || state !== void 0 && now - state.lastEndMs < CLASSIFY_MIN_INTERVAL_MS) {
463
- return { labels: [], rawLabels: [], inferenceMs: 0 };
464
- }
465
- if (this.pipelineBusy) {
466
- return { labels: [], rawLabels: [], inferenceMs: 0 };
467
- }
468
- this.cameraState.set(camKey, { inProgress: true, lastEndMs: state?.lastEndMs ?? 0 });
469
- this.pipelineBusy = true;
470
- const f32Data = float32FromBytes(chunk.data);
471
- const result = await this.pipeline.classify({
472
- data: f32Data,
473
- sampleRate: chunk.sampleRate,
474
- channels: chunk.channels
475
- }).finally(() => {
476
- this.pipelineBusy = false;
477
- this.cameraState.set(camKey, { inProgress: false, lastEndMs: Date.now() });
478
- });
479
- if (this.classifyCount < 3 || this.classifyCount % 100 === 0) {
480
- const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
481
- this.log.info("classify debug sample", {
482
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
483
- meta: {
484
- index: this.classifyCount,
485
- engine: this.backendName,
486
- rawLabelCount: result.classifications.length,
487
- top: rawTop,
488
- inferenceMs: result.inferenceMs
489
- }
490
- });
491
- }
492
- this.classifyCount++;
493
- const macroAccum = /* @__PURE__ */ new Map();
494
- for (const c of result.classifications) {
495
- const macro = index.mapAudioLabelToMacro(c.className);
496
- if (!macro) continue;
497
- const prev = macroAccum.get(macro);
498
- if (!prev || c.score > prev.score) {
499
- macroAccum.set(macro, { score: c.score, rawTop: c.className });
500
- }
501
- }
502
- const labels = [...macroAccum.entries()].sort((a, b) => b[1].score - a[1].score).map(([className, { score, rawTop }]) => ({ className, originalClass: rawTop, score }));
503
- const rawLabels = [...result.classifications].sort((a, b) => b.score - a.score).map((c) => ({ className: c.className, originalClass: c.className, score: c.score }));
504
- return { labels, rawLabels, inferenceMs: result.inferenceMs };
505
- }
506
- isReady() {
507
- return this.pipeline !== null;
508
- }
509
- async dispose() {
510
- await this.pipeline.dispose();
511
- }
512
- // Expose via the cap so the reprobe button in the UI reaches the
513
- // right worker. Delegates to the addon-owned reprobe (it touches
514
- // `ctx.settings.writeAddonStore` which only the addon has access to).
515
- async reprobeAudioEngine() {
516
- return this.reprobeImpl();
517
- }
518
- }
519
- class AudioAnalyzerAddon extends index.BaseAddon {
520
- id = "audio-analyzer";
521
- provider = null;
522
- pipeline = null;
523
- constructor() {
524
- super(index.DEFAULT_AUDIO_ANALYZER_CONFIG);
525
- }
526
- globalSettingsSchema() {
527
- return {
528
- sections: [
529
- {
530
- id: "audio-engine",
531
- title: "Audio",
532
- // Co-located with detection-pipeline's `engine` section under
533
- // a single "Engine" tab. Both sections handle inference-engine
534
- // selection but for different modalities — distinct purposes
535
- // are conveyed through section titles ("Detection engine" vs
536
- // "Audio inference engine") and descriptions, not separate
537
- // tabs.
538
- tab: "engine",
539
- // Renders after detection-pipeline's `engine` section
540
- // (`order: 0`) on the "Inference Engine" tab.
541
- order: 10,
542
- description: 'Audio classification backend (Apple SoundAnalysis or YAMNet ONNX). Independent from the vision-detection engine above. "Auto" picks Apple SoundAnalysis on macOS, YAMNet on Linux. Click the refresh icon next to "Probed best" to re-run the probe.',
543
- // Field order — `probedBest*` lives at the top so the operator
544
- // sees the auto-detected hint first and can compare it against
545
- // their override below at a glance. Same convention as the
546
- // detection-pipeline section.
547
- fields: [
548
- {
549
- type: "text",
550
- key: "probedBestAudioBackend",
551
- label: "Probed best",
552
- description: "Auto-detected best audio backend on this host. Click the refresh icon to re-run the probe.",
553
- readonlyField: true,
554
- default: "",
555
- actions: [
556
- { action: "reprobe-audio-engine", icon: "refresh-cw", tooltip: "Re-probe audio engine" }
557
- ]
558
- },
559
- {
560
- type: "select",
561
- key: "audioBackend",
562
- label: "Audio backend",
563
- options: index.AUDIO_BACKEND_CHOICES.map((o) => ({ value: o.value, label: o.label })),
564
- default: index.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend,
565
- immediate: true,
566
- requiresRestart: true
567
- },
568
- {
569
- type: "select",
570
- key: "selectedAudioModel",
571
- label: "Classification model",
572
- description: "Empty = auto (matches backend). Device-level settings can only inherit / enable / disable this step; model + class filters live here at the node level.",
573
- options: AUDIO_MODEL_OPTIONS.map((o) => ({ value: o.value, label: o.label })),
574
- default: index.DEFAULT_AUDIO_ANALYZER_CONFIG.selectedAudioModel,
575
- immediate: true,
576
- requiresRestart: true
577
- }
578
- ]
579
- }
580
- ]
581
- };
582
- }
583
- /**
584
- * Cascade override — narrow the `selectedAudioModel` options to the
585
- * subset compatible with the currently-selected `audioBackend`.
586
- *
587
- * Same pattern as detection-pipeline's `engineRuntime → engineBackend
588
- * → engineDevice` cascade: the base schema ships every option
589
- * (Auto + YAMNet + Apple SA); this override drops the rows that
590
- * belong to a backend the operator didn't pick. With `immediate:
591
- * true` on the `audioBackend` select, the UI refetches schema after
592
- * every flip and the model dropdown updates instantly.
593
- *
594
- * `overlay` carries the operator's tentative choices for benchmark/
595
- * preview mode (operator typed but didn't save yet) — same
596
- * semantics detection-pipeline relies on.
597
- */
598
- async getGlobalSettings(overlay) {
599
- const ctx = this.ctxIfReady;
600
- const stored = ctx?.settings ? await ctx.settings.readAddonStore() ?? {} : {};
601
- const merged = overlay ? { ...stored, ...overlay } : stored;
602
- const operatorChoice = typeof merged.audioBackend === "string" ? merged.audioBackend : index.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend;
603
- const effectiveBackend = operatorChoice === "apple-soundanalysis" ? "apple-soundanalysis" : operatorChoice === "yamnet-onnx" ? "yamnet-onnx" : process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
604
- const filteredModels = AUDIO_MODEL_OPTIONS.filter(
605
- (o) => o.value === "" || o.value === effectiveBackend
606
- );
607
- const storedModel = typeof merged.selectedAudioModel === "string" ? merged.selectedAudioModel : "";
608
- const validModel = filteredModels.find((o) => o.value === storedModel)?.value ?? "";
609
- const raw = { ...merged, selectedAudioModel: validModel };
610
- const schema = this.globalSettingsSchema();
611
- const patched = {
612
- ...schema,
613
- sections: schema.sections.map((section) => ({
614
- ...section,
615
- fields: section.fields.map((field) => {
616
- if (field.type === "select" && field.key === "selectedAudioModel") {
617
- return { ...field, options: filteredModels.map((o) => ({ value: o.value, label: o.label })) };
618
- }
619
- return field;
620
- })
621
- }))
622
- };
623
- return index.hydrateSchema(patched, raw);
624
- }
625
- /**
626
- * Re-run the platform probe and persist the detected backend into
627
- * `probedBestAudioBackend`. Operator `audioBackend` setting is not
628
- * touched — only the hint.
629
- */
630
- async reprobeAudioEngine() {
631
- const backend = process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
632
- await this.ctx.settings?.writeAddonStore({ probedBestAudioBackend: backend });
633
- this.ctx.logger.info("reprobeAudioEngine: wrote probedBestAudioBackend", { meta: { backend } });
634
- return { backend };
635
- }
636
- /** Resolve the effective backend from the operator choice, falling back to the platform heuristic when 'auto'. */
637
- resolveAudioBackend() {
638
- const choice = this.config.audioBackend;
639
- if (choice === "apple-soundanalysis") return "apple-soundanalysis";
640
- if (choice === "yamnet-onnx") return "yamnet-onnx";
641
- return process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
642
- }
643
- async onInitialize() {
644
- const logger = this.ctx.logger;
645
- const modelsDir = await this.ctx.api.storage.resolve.query({ location: "models", relativePath: "" }).catch(() => "camstack-data/models");
646
- const backend = this.resolveAudioBackend();
647
- logger.info("audio-analyzer: resolving pipeline", {
648
- meta: { operatorChoice: this.config.audioBackend, effectiveBackend: backend, selectedModel: this.config.selectedAudioModel || null }
649
- });
650
- const p = await createAudioPipeline(modelsDir, logger, { backend });
651
- await p.initialize();
652
- this.pipeline = p;
653
- if (!this.config.probedBestAudioBackend) {
654
- this.reprobeAudioEngine().catch((err) => {
655
- logger.warn("audio: auto-reprobe failed", {
656
- meta: { error: err instanceof Error ? err.message : String(err) }
657
- });
658
- });
659
- }
660
- const self = this;
661
- const deviceSettingsResolver = async (deviceId) => {
662
- try {
663
- const resolved = await self.ctx.api.pipelineOrchestrator.resolvePipeline.query({ deviceId });
664
- const stepSettings = resolved.audio?.settings ?? {};
665
- const minConfidence = typeof stepSettings["minConfidence"] === "number" ? stepSettings["minConfidence"] : 0.3;
666
- const allowedClasses = Array.isArray(stepSettings["enabledAudioClasses"]) ? stepSettings["enabledAudioClasses"] : [];
667
- return { minConfidence, allowedClasses };
668
- } catch (err) {
669
- logger.warn("audio: resolveDeviceSettings via orchestrator failed", {
670
- tags: { deviceId },
671
- meta: { error: err instanceof Error ? err.message : String(err) }
672
- });
673
- return null;
674
- }
675
- };
676
- const deviceContributionResolver = async (_deviceId) => {
677
- return null;
678
- };
679
- const deviceSettingsPatcher = async (_deviceId, _patch) => {
680
- };
681
- this.provider = new AudioAnalyzerProvider(
682
- logger,
683
- this.pipeline,
684
- backend,
685
- deviceSettingsResolver,
686
- deviceContributionResolver,
687
- deviceSettingsPatcher,
688
- () => this.reprobeAudioEngine()
689
- );
690
- return [
691
- { capability: index.audioAnalyzerCapability, provider: this.provider },
692
- {
693
- capability: index.audioAnalysisCapability,
694
- provider: this.provider
695
- }
696
- ];
697
- }
698
- async onShutdown() {
699
- if (this.provider) {
700
- await this.provider.dispose();
701
- this.provider = null;
702
- }
703
- this.pipeline = null;
704
- }
705
- // ── Standard ICamstackAddon — three-level settings API ─────
706
- //
707
- // Per-device audio settings (audio class filter + minConfidence) moved
708
- // to the audio-classifier pipeline step's `getConfigSchema()` and the
709
- // orchestrator owns the audio node assignment (`audioNodeId`). No
710
- // device-level settings remain on this addon.
711
- buildDeviceSchema() {
712
- return { sections: [] };
713
- }
714
- async getDeviceSettings(deviceId) {
715
- const raw = await this.ctx?.settings?.readDeviceStore(deviceId) ?? {};
716
- return index.hydrateSchema(this.buildDeviceSchema(), raw);
717
- }
718
- async updateDeviceSettings(deviceId, patch) {
719
- await this.ctx?.settings?.writeDeviceStore(deviceId, patch);
720
- }
352
+ const bytes = raw.byteOffset % 4 === 0 ? raw : new Uint8Array(raw);
353
+ return new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
721
354
  }
355
+ var AudioAnalyzerProvider = class {
356
+ pipeline;
357
+ deviceSettingsResolver;
358
+ deviceContributionResolver;
359
+ deviceSettingsPatcher;
360
+ reprobeImpl;
361
+ log;
362
+ classifyCallCount = 0;
363
+ lastClassifyErrorMs = 0;
364
+ suppressedClassifyErrors = 0;
365
+ classifyCount = 0;
366
+ backendName = "unknown";
367
+ /** When true, logs a raw-label sample every 100 classifications (opt-in
368
+ * debug aid). Off by default — the watchdog heartbeat covers liveness. */
369
+ debugClassifySamples = false;
370
+ /** Per-camera in-flight state. Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
371
+ cameraState = /* @__PURE__ */ new Map();
372
+ /** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
373
+ * run at a time. Without this, concurrent calls from different cameras overwrite the
374
+ * single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
375
+ pipelineBusy = false;
376
+ constructor(logger, pipeline, backendName, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, reprobeImpl) {
377
+ this.pipeline = pipeline;
378
+ this.deviceSettingsResolver = deviceSettingsResolver;
379
+ this.deviceContributionResolver = deviceContributionResolver;
380
+ this.deviceSettingsPatcher = deviceSettingsPatcher;
381
+ this.reprobeImpl = reprobeImpl;
382
+ this.log = logger;
383
+ this.backendName = backendName;
384
+ }
385
+ async getDeviceSettingsContribution(input) {
386
+ return this.deviceContributionResolver(input.deviceId);
387
+ }
388
+ async getDeviceLiveContribution(_input) {
389
+ return null;
390
+ }
391
+ async applyDeviceSettingsPatch(input) {
392
+ await this.deviceSettingsPatcher(input.deviceId, input.patch);
393
+ return { success: true };
394
+ }
395
+ /**
396
+ * Return the effective per-device audio-analyzer settings, resolved via
397
+ * the kernel's 3-level settings resolver (schema default → global →
398
+ * device override). Orchestrator consumers call this method so they
399
+ * never need to know the audio-analyzer schema field names.
400
+ */
401
+ async resolveDeviceSettings({ deviceId }) {
402
+ return this.deviceSettingsResolver(deviceId);
403
+ }
404
+ async analyseChunk({ chunk, settings }) {
405
+ const samples = float32FromBytes(chunk.data);
406
+ let sumSquares = 0;
407
+ for (let i = 0; i < samples.length; i++) sumSquares += samples[i] * samples[i];
408
+ const rms = Math.sqrt(sumSquares / samples.length);
409
+ const dbfs = rms > 0 ? 20 * Math.log10(rms) : -96;
410
+ const level = {
411
+ rms: Math.round(rms * 1e4) / 1e4,
412
+ dbfs: Math.round(dbfs * 10) / 10
413
+ };
414
+ let classification;
415
+ try {
416
+ const result = await this.classify(chunk);
417
+ if (this.classifyCallCount < 3) {
418
+ const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
419
+ this.log.info("classify debug sample", {
420
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
421
+ meta: {
422
+ index: this.classifyCallCount,
423
+ labelCount: result.labels.length,
424
+ top: topRaw,
425
+ inferenceMs: result.inferenceMs,
426
+ minConf: settings.minConfidence,
427
+ allowedClasses: settings.allowedClasses
428
+ }
429
+ });
430
+ }
431
+ this.classifyCallCount++;
432
+ if (result.inferenceMs > 0) {
433
+ const minConf = settings.minConfidence;
434
+ const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
435
+ let filtered = result.labels.filter((c) => c.score >= minConf);
436
+ if (allowedSet) filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
437
+ if (filtered.length > 0) classification = {
438
+ labels: filtered,
439
+ inferenceMs: result.inferenceMs
440
+ };
441
+ }
442
+ } catch (err) {
443
+ const now = Date.now();
444
+ if (now - this.lastClassifyErrorMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
445
+ const suppressed = this.suppressedClassifyErrors;
446
+ this.suppressedClassifyErrors = 0;
447
+ this.lastClassifyErrorMs = now;
448
+ const msg = require_dist.errMsg(err);
449
+ const stack = err instanceof Error ? err.stack : void 0;
450
+ this.log.warn("Audio classification failed", {
451
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
452
+ meta: {
453
+ error: msg,
454
+ stack,
455
+ suppressedSince: suppressed > 0 ? suppressed : void 0
456
+ }
457
+ });
458
+ } else this.suppressedClassifyErrors++;
459
+ }
460
+ return {
461
+ level,
462
+ classification,
463
+ timestamp: chunk.timestamp
464
+ };
465
+ }
466
+ async classify(chunk) {
467
+ const camKey = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
468
+ const now = Date.now();
469
+ const state = this.cameraState.get(camKey);
470
+ if (state?.inProgress || state !== void 0 && now - state.lastEndMs < CLASSIFY_MIN_INTERVAL_MS) return {
471
+ labels: [],
472
+ rawLabels: [],
473
+ inferenceMs: 0
474
+ };
475
+ if (this.pipelineBusy) return {
476
+ labels: [],
477
+ rawLabels: [],
478
+ inferenceMs: 0
479
+ };
480
+ this.cameraState.set(camKey, {
481
+ inProgress: true,
482
+ lastEndMs: state?.lastEndMs ?? 0
483
+ });
484
+ this.pipelineBusy = true;
485
+ const f32Data = float32FromBytes(chunk.data);
486
+ const result = await this.pipeline.classify({
487
+ data: f32Data,
488
+ sampleRate: chunk.sampleRate,
489
+ channels: chunk.channels
490
+ }).finally(() => {
491
+ this.pipelineBusy = false;
492
+ this.cameraState.set(camKey, {
493
+ inProgress: false,
494
+ lastEndMs: Date.now()
495
+ });
496
+ });
497
+ if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
498
+ const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
499
+ this.log.info("classify debug sample", {
500
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
501
+ meta: {
502
+ index: this.classifyCount,
503
+ engine: this.backendName,
504
+ rawLabelCount: result.classifications.length,
505
+ top: rawTop,
506
+ inferenceMs: result.inferenceMs
507
+ }
508
+ });
509
+ }
510
+ this.classifyCount++;
511
+ const macroAccum = /* @__PURE__ */ new Map();
512
+ for (const c of result.classifications) {
513
+ const macro = require_dist.mapAudioLabelToMacro(c.className);
514
+ if (!macro) continue;
515
+ const prev = macroAccum.get(macro);
516
+ if (!prev || c.score > prev.score) macroAccum.set(macro, {
517
+ score: c.score,
518
+ rawTop: c.className
519
+ });
520
+ }
521
+ return {
522
+ labels: [...macroAccum.entries()].toSorted((a, b) => b[1].score - a[1].score).map(([className, { score, rawTop }]) => ({
523
+ className,
524
+ originalClass: rawTop,
525
+ score
526
+ })),
527
+ rawLabels: [...result.classifications].toSorted((a, b) => b.score - a.score).map((c) => ({
528
+ className: c.className,
529
+ originalClass: c.className,
530
+ score: c.score
531
+ })),
532
+ inferenceMs: result.inferenceMs
533
+ };
534
+ }
535
+ isReady() {
536
+ return this.pipeline !== null;
537
+ }
538
+ async dispose() {
539
+ await this.pipeline.dispose();
540
+ }
541
+ async reprobeAudioEngine() {
542
+ return this.reprobeImpl();
543
+ }
544
+ };
545
+ /**
546
+ * Audio Analyzer addon — provides the `audio-analyzer` capability.
547
+ *
548
+ * Owns the IAudioPipeline directly — no tRPC roundtrip to a separate
549
+ * audio-classifier addon.
550
+ */
551
+ var AudioAnalyzerAddon = class extends require_dist.BaseAddon {
552
+ id = "audio-analyzer";
553
+ provider = null;
554
+ pipeline = null;
555
+ constructor() {
556
+ super(require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG);
557
+ }
558
+ globalSettingsSchema() {
559
+ return { sections: [{
560
+ id: "audio-engine",
561
+ title: "Audio",
562
+ tab: "engine",
563
+ order: 10,
564
+ description: "Audio classification backend (Apple SoundAnalysis or YAMNet ONNX). Independent from the vision-detection engine above. \"Auto\" picks Apple SoundAnalysis on macOS, YAMNet on Linux. Click the refresh icon next to \"Probed best\" to re-run the probe.",
565
+ fields: [
566
+ {
567
+ type: "text",
568
+ key: "probedBestAudioBackend",
569
+ label: "Probed best",
570
+ description: "Auto-detected best audio backend on this host. Click the refresh icon to re-run the probe.",
571
+ readonlyField: true,
572
+ default: "",
573
+ actions: [{
574
+ action: "reprobe-audio-engine",
575
+ icon: "refresh-cw",
576
+ tooltip: "Re-probe audio engine"
577
+ }]
578
+ },
579
+ {
580
+ type: "select",
581
+ key: "audioBackend",
582
+ label: "Audio backend",
583
+ options: require_dist.AUDIO_BACKEND_CHOICES.map((o) => ({
584
+ value: o.value,
585
+ label: o.label
586
+ })),
587
+ default: require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend,
588
+ immediate: true,
589
+ requiresRestart: true
590
+ },
591
+ {
592
+ type: "select",
593
+ key: "selectedAudioModel",
594
+ label: "Classification model",
595
+ description: "Empty = auto (matches backend). Device-level settings can only inherit / enable / disable this step; model + class filters live here at the node level.",
596
+ options: AUDIO_MODEL_OPTIONS.map((o) => ({
597
+ value: o.value,
598
+ label: o.label
599
+ })),
600
+ default: require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.selectedAudioModel,
601
+ immediate: true,
602
+ requiresRestart: true
603
+ }
604
+ ]
605
+ }] };
606
+ }
607
+ /**
608
+ * Cascade override — narrow the `selectedAudioModel` options to the
609
+ * subset compatible with the currently-selected `audioBackend`.
610
+ *
611
+ * Same pattern as detection-pipeline's `engineRuntime → engineBackend
612
+ * → engineDevice` cascade: the base schema ships every option
613
+ * (Auto + YAMNet + Apple SA); this override drops the rows that
614
+ * belong to a backend the operator didn't pick. With `immediate:
615
+ * true` on the `audioBackend` select, the UI refetches schema after
616
+ * every flip and the model dropdown updates instantly.
617
+ *
618
+ * `overlay` carries the operator's tentative choices for benchmark/
619
+ * preview mode (operator typed but didn't save yet) — same
620
+ * semantics detection-pipeline relies on.
621
+ */
622
+ async getGlobalSettings(overlay) {
623
+ const ctx = this.ctxIfReady;
624
+ const stored = ctx?.settings ? await ctx.settings.readAddonStore() ?? {} : {};
625
+ const merged = overlay ? {
626
+ ...stored,
627
+ ...overlay
628
+ } : stored;
629
+ const operatorChoice = typeof merged.audioBackend === "string" ? merged.audioBackend : require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend;
630
+ const effectiveBackend = operatorChoice === "apple-soundanalysis" ? "apple-soundanalysis" : operatorChoice === "yamnet-onnx" ? "yamnet-onnx" : process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
631
+ const filteredModels = AUDIO_MODEL_OPTIONS.filter((o) => o.value === "" || o.value === effectiveBackend);
632
+ const storedModel = typeof merged.selectedAudioModel === "string" ? merged.selectedAudioModel : "";
633
+ const validModel = filteredModels.find((o) => o.value === storedModel)?.value ?? "";
634
+ const raw = {
635
+ ...merged,
636
+ selectedAudioModel: validModel
637
+ };
638
+ const schema = this.globalSettingsSchema();
639
+ return require_dist.hydrateSchema({
640
+ ...schema,
641
+ sections: schema.sections.map((section) => ({
642
+ ...section,
643
+ fields: section.fields.map((field) => {
644
+ if (field.type === "select" && field.key === "selectedAudioModel") return {
645
+ ...field,
646
+ options: filteredModels.map((o) => ({
647
+ value: o.value,
648
+ label: o.label
649
+ }))
650
+ };
651
+ return field;
652
+ })
653
+ }))
654
+ }, raw);
655
+ }
656
+ /**
657
+ * Re-run the platform probe and persist the detected backend into
658
+ * `probedBestAudioBackend`. Operator `audioBackend` setting is not
659
+ * touched — only the hint.
660
+ */
661
+ async reprobeAudioEngine() {
662
+ const backend = process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
663
+ await this.ctx.settings?.writeAddonStore({ probedBestAudioBackend: backend });
664
+ this.ctx.logger.info("reprobeAudioEngine: wrote probedBestAudioBackend", { meta: { backend } });
665
+ return { backend };
666
+ }
667
+ /** Resolve the effective backend from the operator choice, falling back to the platform heuristic when 'auto'. */
668
+ resolveAudioBackend() {
669
+ const choice = this.config.audioBackend;
670
+ if (choice === "apple-soundanalysis") return "apple-soundanalysis";
671
+ if (choice === "yamnet-onnx") return "yamnet-onnx";
672
+ return process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
673
+ }
674
+ async onInitialize() {
675
+ const logger = this.ctx.logger;
676
+ const modelsDir = await this.ctx.api.storage.resolve.query({
677
+ location: "models",
678
+ relativePath: ""
679
+ }).catch(() => "camstack-data/models");
680
+ const backend = this.resolveAudioBackend();
681
+ logger.info("audio-analyzer: resolving pipeline", { meta: {
682
+ operatorChoice: this.config.audioBackend,
683
+ effectiveBackend: backend,
684
+ selectedModel: this.config.selectedAudioModel || null
685
+ } });
686
+ const p = await createAudioPipeline(modelsDir, logger, { backend });
687
+ await p.initialize();
688
+ this.pipeline = p;
689
+ if (!this.config.probedBestAudioBackend) this.reprobeAudioEngine().catch((err) => {
690
+ logger.warn("audio: auto-reprobe failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
691
+ });
692
+ const self = this;
693
+ const deviceSettingsResolver = async (deviceId) => {
694
+ try {
695
+ const stepSettings = (await self.ctx.api.pipelineOrchestrator.resolvePipeline.query({ deviceId })).audio?.settings ?? {};
696
+ return {
697
+ minConfidence: typeof stepSettings["minConfidence"] === "number" ? stepSettings["minConfidence"] : .3,
698
+ allowedClasses: Array.isArray(stepSettings["enabledAudioClasses"]) ? stepSettings["enabledAudioClasses"] : []
699
+ };
700
+ } catch (err) {
701
+ logger.warn("audio: resolveDeviceSettings via orchestrator failed", {
702
+ tags: { deviceId },
703
+ meta: { error: err instanceof Error ? err.message : String(err) }
704
+ });
705
+ return null;
706
+ }
707
+ };
708
+ const deviceContributionResolver = async (_deviceId) => {
709
+ return null;
710
+ };
711
+ const deviceSettingsPatcher = async (_deviceId, _patch) => {};
712
+ this.provider = new AudioAnalyzerProvider(logger, this.pipeline, backend, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, () => this.reprobeAudioEngine());
713
+ return [{
714
+ capability: require_dist.audioAnalyzerCapability,
715
+ provider: this.provider
716
+ }, {
717
+ capability: require_dist.audioAnalysisCapability,
718
+ provider: this.provider
719
+ }];
720
+ }
721
+ async onShutdown() {
722
+ if (this.provider) {
723
+ await this.provider.dispose();
724
+ this.provider = null;
725
+ }
726
+ this.pipeline = null;
727
+ }
728
+ buildDeviceSchema() {
729
+ return { sections: [] };
730
+ }
731
+ async getDeviceSettings(deviceId) {
732
+ const raw = await this.ctx?.settings?.readDeviceStore(deviceId) ?? {};
733
+ return require_dist.hydrateSchema(this.buildDeviceSchema(), raw);
734
+ }
735
+ async updateDeviceSettings(deviceId, patch) {
736
+ await this.ctx?.settings?.writeDeviceStore(deviceId, patch);
737
+ }
738
+ };
739
+ //#endregion
740
+ //#region src/audio-analyzer/index.ts
741
+ var audio_analyzer_default = AudioAnalyzerAddon;
742
+ //#endregion
722
743
  exports.AudioAnalyzerAddon = AudioAnalyzerAddon;
723
744
  exports.AudioAnalyzerProvider = AudioAnalyzerProvider;
724
745
  exports.createAudioPipeline = createAudioPipeline;
725
- exports.default = AudioAnalyzerAddon;
726
- //# sourceMappingURL=index.js.map
746
+ exports.default = audio_analyzer_default;