@camstack/addon-pipeline 0.1.20 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dist/audio-analyzer/index.js +736 -719
  2. package/dist/audio-analyzer/index.mjs +726 -679
  3. package/dist/audio-codec-nodeav/index.js +304 -461
  4. package/dist/audio-codec-nodeav/index.mjs +300 -462
  5. package/dist/chunk-BdkLduGY.mjs +5 -0
  6. package/dist/chunk-D6vf50IK.js +28 -0
  7. package/dist/codec-runtime-BOk-13PN.js +202 -0
  8. package/dist/codec-runtime-BsqlEjPi.mjs +197 -0
  9. package/dist/constants-B_b0a-6h.mjs +3119 -0
  10. package/dist/{index-CMcx_k6Y.js → constants-D65v6yp6.js} +3107 -2935
  11. package/dist/decoder-nodeav/index.js +1374 -1444
  12. package/dist/decoder-nodeav/index.mjs +1369 -1425
  13. package/dist/detection-pipeline/index.js +6462 -5613
  14. package/dist/detection-pipeline/index.mjs +6451 -5574
  15. package/dist/dist-7ewQjTle.js +22454 -0
  16. package/dist/dist-C5jnNl0n.mjs +22089 -0
  17. package/dist/motion-wasm/index.js +469 -467
  18. package/dist/motion-wasm/index.mjs +464 -446
  19. package/dist/pipeline-runner/index.js +2029 -1827
  20. package/dist/pipeline-runner/index.mjs +2025 -1811
  21. package/dist/recorder/index.js +2045 -2157
  22. package/dist/recorder/index.mjs +2042 -2156
  23. package/dist/stream-broker/_stub.js +1806 -1352
  24. package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-D4-DHanK.mjs +156 -0
  25. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Tf-HACFd.mjs +26 -0
  26. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-C9WX5HNw.mjs +26 -0
  27. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-BO7TIbJV.mjs +26 -0
  28. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.js-C9j-2lBe.mjs +26 -0
  29. package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-XO0-Pyu6.mjs +26 -0
  30. package/dist/stream-broker/dist-CYZr2fwk.mjs +2726 -0
  31. package/dist/stream-broker/hostInit-Di6vceAU.mjs +129 -0
  32. package/dist/stream-broker/index.js +17778 -15470
  33. package/dist/stream-broker/index.mjs +17769 -15465
  34. package/dist/stream-broker/remoteEntry.js +134 -2973
  35. package/dist/stream-broker/remoteEntry.ssr.js +33 -0
  36. package/dist/stream-broker/virtualExposes-dYNvIwoR.mjs +27 -0
  37. package/dist/stream-broker/virtual_mf-exposes-ssr___mfe_internal__addon_stream_broker_widgets__remoteEntry_js-Cmqfp4i_.mjs +10 -0
  38. package/embed-dist/assets/index-B8VlSD0-.js +150 -0
  39. package/embed-dist/assets/index-ZhDdp1Nd.css +2 -0
  40. package/embed-dist/index.html +13 -0
  41. package/package.json +25 -7
  42. package/wasm/assembly/index.ts +41 -16
  43. package/dist/audio-analyzer/index.js.map +0 -1
  44. package/dist/audio-analyzer/index.mjs.map +0 -1
  45. package/dist/audio-codec-nodeav/index.js.map +0 -1
  46. package/dist/audio-codec-nodeav/index.mjs.map +0 -1
  47. package/dist/decoder-nodeav/index.js.map +0 -1
  48. package/dist/decoder-nodeav/index.mjs.map +0 -1
  49. package/dist/detection-pipeline/index.js.map +0 -1
  50. package/dist/detection-pipeline/index.mjs.map +0 -1
  51. package/dist/index-5aYef068.mjs +0 -17514
  52. package/dist/index-5aYef068.mjs.map +0 -1
  53. package/dist/index-B36NMAdu.js +0 -17513
  54. package/dist/index-B36NMAdu.js.map +0 -1
  55. package/dist/index-CMcx_k6Y.js.map +0 -1
  56. package/dist/index-CYb7cFrv.mjs +0 -5790
  57. package/dist/index-CYb7cFrv.mjs.map +0 -1
  58. package/dist/motion-wasm/index.js.map +0 -1
  59. package/dist/motion-wasm/index.mjs.map +0 -1
  60. package/dist/pipeline-runner/index.js.map +0 -1
  61. package/dist/pipeline-runner/index.mjs.map +0 -1
  62. package/dist/recorder/index.js.map +0 -1
  63. package/dist/recorder/index.mjs.map +0 -1
  64. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/FfmpegParamsField.d.ts +0 -41
  65. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/GeometryBuilder.d.ts +0 -54
  66. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/StreamBrokerPanel.d.ts +0 -21
  67. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/format-ua.d.ts +0 -13
  68. package/dist/stream-broker/@mf-types/compiled-types/stream-broker/widgets/index.d.ts +0 -15
  69. package/dist/stream-broker/@mf-types/widgets.d.ts +0 -2
  70. package/dist/stream-broker/@mf-types.d.ts +0 -3
  71. package/dist/stream-broker/@mf-types.zip +0 -0
  72. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_sdk__loadShare__.mjs-lantnv8e.mjs +0 -12
  73. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.mjs-DJ3UNg7O.mjs +0 -30
  74. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.mjs-CYXy_bhS.mjs +0 -21
  75. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.mjs-U1EUeEPs.mjs +0 -104
  76. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_client__loadShare__.mjs-DeouEaSs.mjs +0 -85
  77. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_trpc_mf_1_react_mf_2_query__loadShare__.mjs-DHUwjbb9.mjs +0 -62
  78. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs-CaDEYBIU.mjs +0 -89
  79. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react__loadShare__.mjs_commonjs-proxy-D6EROtlA.mjs +0 -29
  80. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.mjs-x6pP3Ghk.mjs +0 -36
  81. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs-DYEKzzY-.mjs +0 -45
  82. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom__loadShare__.mjs_commonjs-proxy-CcnN6sbA.mjs +0 -6
  83. package/dist/stream-broker/__mfe_internal__addon_stream_broker_widgets__loadShare__react_mf_2_dom_mf_1_client__loadShare__.mjs-DICOtMTl.mjs +0 -34
  84. package/dist/stream-broker/_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CL9DR49k.mjs +0 -156
  85. package/dist/stream-broker/client-BvTmMOQu.mjs +0 -9836
  86. package/dist/stream-broker/getErrorShape-BPSzUA7W-TlK8ipWe.mjs +0 -211
  87. package/dist/stream-broker/hostInit-ChmiMPS0.mjs +0 -168
  88. package/dist/stream-broker/index-BxsFuFmE.mjs +0 -2603
  89. package/dist/stream-broker/index-C-248uOU.mjs +0 -725
  90. package/dist/stream-broker/index-C05B6jqp.mjs +0 -185
  91. package/dist/stream-broker/index-CWkKuNLr.mjs +0 -232
  92. package/dist/stream-broker/index-DOJoSShD.mjs +0 -67784
  93. package/dist/stream-broker/index-DtOI1aTU.mjs +0 -18504
  94. package/dist/stream-broker/index-oMq6ilgR.mjs +0 -1641
  95. package/dist/stream-broker/index-vIWZQBIL.mjs +0 -435
  96. package/dist/stream-broker/index-xncRG7-x.mjs +0 -2713
  97. package/dist/stream-broker/index.js.map +0 -1
  98. package/dist/stream-broker/index.mjs.map +0 -1
  99. package/dist/stream-broker/jsx-runtime-BRT_HL0A.mjs +0 -55
  100. package/dist/stream-broker/schemas-B7L0qZtq.mjs +0 -3599
  101. package/dist/stream-broker/virtualExposes-pCd777Rp.mjs +0 -42
@@ -1,729 +1,746 @@
1
- "use strict";
2
- var __create = Object.create;
3
- var __defProp = Object.defineProperty;
4
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
- var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
- var __hasOwnProp = Object.prototype.hasOwnProperty;
8
- var __copyProps = (to, from, except, desc) => {
9
- if (from && typeof from === "object" || typeof from === "function") {
10
- for (let key of __getOwnPropNames(from))
11
- if (!__hasOwnProp.call(to, key) && key !== except)
12
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
13
- }
14
- return to;
15
- };
16
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
17
- // If the importer is in node compatibility mode or this is not an ESM
18
- // file that has been converted to a CommonJS file using a Babel-
19
- // compatible transform (i.e. "__esModule" has not been set), then set
20
- // "default" to the CommonJS "module.exports" for node compatibility.
21
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
22
- mod
23
- ));
24
- Object.defineProperties(exports, { __esModule: { value: true }, [Symbol.toStringTag]: { value: "Module" } });
25
- const index = require("../index-B36NMAdu.js");
26
- const path = require("node:path");
27
- const fs = require("node:fs");
28
- const core = require("@camstack/core");
29
- function _interopNamespaceDefault(e) {
30
- const n = Object.create(null, { [Symbol.toStringTag]: { value: "Module" } });
31
- if (e) {
32
- for (const k in e) {
33
- if (k !== "default") {
34
- const d = Object.getOwnPropertyDescriptor(e, k);
35
- Object.defineProperty(n, k, d.get ? d : {
36
- enumerable: true,
37
- get: () => e[k]
38
- });
39
- }
40
- }
41
- }
42
- n.default = e;
43
- return Object.freeze(n);
44
- }
45
- const path__namespace = /* @__PURE__ */ _interopNamespaceDefault(path);
46
- const fs__namespace = /* @__PURE__ */ _interopNamespaceDefault(fs);
1
+ Object.defineProperties(exports, {
2
+ __esModule: { value: true },
3
+ [Symbol.toStringTag]: { value: "Module" }
4
+ });
5
+ const require_chunk = require("../chunk-D6vf50IK.js");
6
+ const require_dist = require("../dist-7ewQjTle.js");
7
+ let node_fs = require("node:fs");
8
+ node_fs = require_chunk.__toESM(node_fs);
9
+ let node_path = require("node:path");
10
+ node_path = require_chunk.__toESM(node_path);
11
+ let _camstack_core = require("@camstack/core");
12
+ //#region src/audio-analyzer/audio-pipeline.ts
13
+ /**
14
+ * Create the appropriate audio pipeline.
15
+ *
16
+ * - 'yamnet-onnx': Cross-platform YAMNet ONNX (requires model download)
17
+ * - 'apple-soundanalysis': macOS 12+ Apple SoundAnalysis (zero model download, Neural Engine)
18
+ * - undefined: auto-detect (Apple SA on macOS, YAMNet on Linux)
19
+ */
47
20
  async function createAudioPipeline(modelsDir, logger, options) {
48
- const backend = options?.backend ?? (process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx");
49
- if (backend === "apple-soundanalysis") {
50
- return new AppleSoundAnalysisPipeline(logger);
51
- }
52
- return new YamnetOnnxPipeline(modelsDir, logger);
53
- }
54
- const YAMNET_MODEL_URL = `${index.HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
55
- const YAMNET_LABELS_URL = `${index.HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
56
- class YamnetOnnxPipeline {
57
- constructor(modelsDir, logger) {
58
- this.modelsDir = modelsDir;
59
- this.log = logger;
60
- }
61
- session = null;
62
- inputName = "";
63
- labels = [];
64
- log;
65
- async initialize() {
66
- const ort = await import("onnxruntime-node");
67
- const modelPath = path__namespace.join(this.modelsDir, "camstack-yamnet.onnx");
68
- const labelsPath = path__namespace.join(this.modelsDir, "camstack-yamnet-labels.json");
69
- if (!fs__namespace.existsSync(modelPath)) {
70
- this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", {
71
- meta: { url: YAMNET_MODEL_URL, dest: modelPath }
72
- });
73
- await core.downloadFile(YAMNET_MODEL_URL, modelPath);
74
- this.log.info("YAMNet ONNX model downloaded", {
75
- meta: { sizeBytes: fs__namespace.statSync(modelPath).size }
76
- });
77
- }
78
- if (!fs__namespace.existsSync(labelsPath)) {
79
- this.log.info("YAMNet labels not found locally — downloading from HuggingFace", {
80
- meta: { url: YAMNET_LABELS_URL, dest: labelsPath }
81
- });
82
- await core.downloadFile(YAMNET_LABELS_URL, labelsPath);
83
- }
84
- this.session = await ort.InferenceSession.create(modelPath);
85
- this.inputName = this.session.inputNames[0] ?? "waveform";
86
- if (fs__namespace.existsSync(labelsPath)) {
87
- this.labels = JSON.parse(fs__namespace.readFileSync(labelsPath, "utf8"));
88
- } else {
89
- this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
90
- }
91
- this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
92
- }
93
- async classify(chunk) {
94
- if (!this.session) {
95
- throw new Error("YAMNet pipeline not initialized");
96
- }
97
- const start = Date.now();
98
- const ort = await import("onnxruntime-node");
99
- const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
100
- const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
101
- const feeds = { [this.inputName]: tensor };
102
- const results = await this.session.run(feeds);
103
- const scoresData = results[this.session.outputNames[0]];
104
- if (!scoresData) {
105
- throw new Error("YAMNet returned no output");
106
- }
107
- const scores = scoresData.data;
108
- const numClasses = 521;
109
- const numFrames = scores.length / numClasses;
110
- const avgScores = new Float32Array(numClasses);
111
- for (let f = 0; f < numFrames; f++) {
112
- for (let c = 0; c < numClasses; c++) {
113
- avgScores[c] += scores[f * numClasses + c];
114
- }
115
- }
116
- for (let c = 0; c < numClasses; c++) {
117
- avgScores[c] = avgScores[c] / numFrames;
118
- }
119
- const minScore = 0.05;
120
- const classifications = [];
121
- for (let c = 0; c < numClasses; c++) {
122
- const score = avgScores[c];
123
- if (score >= minScore) {
124
- const label = c < this.labels.length ? this.labels[c] : String(c);
125
- classifications.push({ className: label, score: Math.round(score * 1e3) / 1e3 });
126
- }
127
- }
128
- classifications.sort((a, b) => b.score - a.score);
129
- return {
130
- classifications: classifications.slice(0, 10),
131
- inferenceMs: Date.now() - start
132
- };
133
- }
134
- async dispose() {
135
- if (this.session) {
136
- await this.session.release();
137
- this.session = null;
138
- }
139
- }
140
- }
141
- class AppleSoundAnalysisPipeline {
142
- log;
143
- process = null;
144
- receiveBuffer = Buffer.alloc(0);
145
- pendingResolve = null;
146
- pendingReject = null;
147
- binaryPath = null;
148
- debugCount = 0;
149
- constructor(logger) {
150
- this.log = logger;
151
- }
152
- async initialize() {
153
- this.binaryPath = await this.resolveSwiftBinary();
154
- if (!this.binaryPath) {
155
- throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
156
- }
157
- const { spawn } = await import("node:child_process");
158
- this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], {
159
- stdio: ["pipe", "pipe", "pipe"]
160
- });
161
- this.process.stderr?.on("data", (chunk) => {
162
- const lines = chunk.toString().split("\n");
163
- for (const line of lines) {
164
- const trimmed = line.trim();
165
- if (trimmed) this.log.warn(trimmed);
166
- }
167
- });
168
- this.process.on("error", (err) => {
169
- this.log.error("Swift process error", { meta: { error: err.message } });
170
- this.pendingReject?.(err);
171
- this.pendingReject = null;
172
- this.pendingResolve = null;
173
- });
174
- this.process.on("exit", (code) => {
175
- if (code !== 0 && code !== null) {
176
- this.log.error("Swift process exited", { meta: { code } });
177
- const err = new Error(`Apple SoundAnalysis: process exited with code ${code}`);
178
- this.pendingReject?.(err);
179
- this.pendingReject = null;
180
- this.pendingResolve = null;
181
- }
182
- });
183
- this.process.stdout.on("data", (chunk) => {
184
- this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
185
- this.tryReceive();
186
- });
187
- const ready = await this.receiveMessage();
188
- if (ready["status"] !== "ready") {
189
- throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
190
- }
191
- this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
192
- }
193
- async classify(chunk) {
194
- if (!this.process?.stdin) {
195
- throw new Error("Apple SoundAnalysis: process not initialized");
196
- }
197
- const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
198
- const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
199
- const lengthBuf = Buffer.allocUnsafe(4);
200
- lengthBuf.writeUInt32LE(audioBuffer.length, 0);
201
- this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
202
- const result = await this.receiveMessage();
203
- const classifications = result["classifications"] ?? [];
204
- const inferenceMs = result["inferenceMs"] ?? 0;
205
- if (this.debugCount < 3) {
206
- const keys = Object.keys(result);
207
- this.log.info("classify debug sample", {
208
- meta: {
209
- phase: "apple-sa",
210
- index: this.debugCount,
211
- keys,
212
- classifications: classifications.length,
213
- inferenceMs,
214
- audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
215
- sampleRate: chunk.sampleRate,
216
- channels: chunk.channels
217
- }
218
- });
219
- if (result["error"]) {
220
- this.log.error("Swift error", { meta: { phase: "apple-sa", error: result["error"] } });
221
- }
222
- this.debugCount++;
223
- }
224
- return { classifications, inferenceMs };
225
- }
226
- async dispose() {
227
- const proc = this.process;
228
- if (!proc) return;
229
- this.process = null;
230
- proc.stdin?.end();
231
- proc.kill("SIGTERM");
232
- const exited = await new Promise((resolve) => {
233
- const timer = setTimeout(() => resolve(false), 5e3);
234
- proc.once("exit", () => {
235
- clearTimeout(timer);
236
- resolve(true);
237
- });
238
- });
239
- if (!exited) {
240
- try {
241
- proc.kill("SIGKILL");
242
- } catch {
243
- }
244
- this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
245
- }
246
- }
247
- receiveMessage() {
248
- return new Promise((resolve, reject) => {
249
- this.pendingResolve = resolve;
250
- this.pendingReject = reject;
251
- });
252
- }
253
- tryReceive() {
254
- if (this.receiveBuffer.length < 4) return;
255
- const length = this.receiveBuffer.readUInt32LE(0);
256
- if (this.receiveBuffer.length < 4 + length) return;
257
- const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
258
- this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
259
- const resolve = this.pendingResolve;
260
- const reject = this.pendingReject;
261
- this.pendingResolve = null;
262
- this.pendingReject = null;
263
- if (!resolve) return;
264
- try {
265
- const parsed = JSON.parse(jsonBytes.toString("utf8"));
266
- resolve(parsed);
267
- } catch (err) {
268
- reject?.(err instanceof Error ? err : new Error(String(err)));
269
- }
270
- }
271
- /** Find pre-compiled binary or compile from Swift source. */
272
- async resolveSwiftBinary() {
273
- const candidates = [
274
- path__namespace.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
275
- // Fallback for in-tree dev (src/<id>/swift/) and pre-merge layouts.
276
- path__namespace.join(__dirname, "../swift/apple-sound-classifier"),
277
- path__namespace.join(__dirname, "../../swift/apple-sound-classifier"),
278
- path__namespace.join(__dirname, "../../../swift/apple-sound-classifier")
279
- ];
280
- for (const p of candidates) {
281
- if (fs__namespace.existsSync(p)) {
282
- this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
283
- return p;
284
- }
285
- }
286
- const sourceCandidates = [
287
- path__namespace.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
288
- path__namespace.join(__dirname, "../swift/apple-sound-classifier.swift"),
289
- path__namespace.join(__dirname, "../../swift/apple-sound-classifier.swift"),
290
- path__namespace.join(__dirname, "../../../swift/apple-sound-classifier.swift")
291
- ];
292
- const sourcePath = sourceCandidates.find((p) => fs__namespace.existsSync(p));
293
- if (!sourcePath) {
294
- this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
295
- return null;
296
- }
297
- const outputPath = sourcePath.replace(".swift", "");
298
- this.log.info("Compiling Swift CLI...", { meta: { source: sourcePath, output: outputPath } });
299
- const { execFileSync } = await import("node:child_process");
300
- try {
301
- execFileSync("swiftc", ["-O", "-o", outputPath, sourcePath], {
302
- timeout: 6e4,
303
- stdio: "pipe"
304
- });
305
- this.log.info("Swift CLI compiled successfully");
306
- return outputPath;
307
- } catch (err) {
308
- this.log.error("Swift compilation failed — install Xcode Command Line Tools", {
309
- meta: { error: index.errMsg(err) }
310
- });
311
- return null;
312
- }
313
- }
21
+ if ((options?.backend ?? (process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx")) === "apple-soundanalysis") return new AppleSoundAnalysisPipeline(logger);
22
+ return new YamnetOnnxPipeline(modelsDir, logger);
314
23
  }
24
+ /**
25
+ * Canonical model URLs on the camstack HuggingFace mirror. Mirrors the
26
+ * convention every detection model follows (single point of truth =
27
+ * `HF_BASE_URL` from `@camstack/types`); the auto-download path uses
28
+ * `downloadFile` from `@camstack/core`, the SAME helper detection-
29
+ * pipeline uses to materialise its YOLO/face/plate models. Missing
30
+ * model on disk → fetch from HF; cached file → no-op.
31
+ *
32
+ * Repo layout follows the detection-pipeline pattern:
33
+ * {domain}/{family}/{format}/{filename}
34
+ * For YAMNet that's `audioClassification/yamnet/onnx/camstack-yamnet.onnx`,
35
+ * with the labels JSON sitting one level up (`audioClassification/yamnet/`)
36
+ * because they're format-agnostic (same 521 AudioSet class names whether
37
+ * the runtime is ONNX, OpenVINO, or TF).
38
+ */
39
+ var YAMNET_MODEL_URL = `${require_dist.HF_BASE_URL}/audioClassification/yamnet/onnx/camstack-yamnet.onnx`;
40
+ var YAMNET_LABELS_URL = `${require_dist.HF_BASE_URL}/audioClassification/yamnet/camstack-yamnet-labels.json`;
41
+ var YamnetOnnxPipeline = class {
42
+ modelsDir;
43
+ session = null;
44
+ inputName = "";
45
+ labels = [];
46
+ log;
47
+ constructor(modelsDir, logger) {
48
+ this.modelsDir = modelsDir;
49
+ this.log = logger;
50
+ }
51
+ async initialize() {
52
+ const ort = await import("onnxruntime-node");
53
+ const modelPath = node_path.join(this.modelsDir, "camstack-yamnet.onnx");
54
+ const labelsPath = node_path.join(this.modelsDir, "camstack-yamnet-labels.json");
55
+ if (!node_fs.existsSync(modelPath)) {
56
+ this.log.info("YAMNet ONNX model not found locally — downloading from HuggingFace", { meta: {
57
+ url: YAMNET_MODEL_URL,
58
+ dest: modelPath
59
+ } });
60
+ await (0, _camstack_core.downloadFile)(YAMNET_MODEL_URL, modelPath);
61
+ this.log.info("YAMNet ONNX model downloaded", { meta: { sizeBytes: node_fs.statSync(modelPath).size } });
62
+ }
63
+ if (!node_fs.existsSync(labelsPath)) {
64
+ this.log.info("YAMNet labels not found locally — downloading from HuggingFace", { meta: {
65
+ url: YAMNET_LABELS_URL,
66
+ dest: labelsPath
67
+ } });
68
+ await (0, _camstack_core.downloadFile)(YAMNET_LABELS_URL, labelsPath);
69
+ }
70
+ this.session = await ort.InferenceSession.create(modelPath);
71
+ this.inputName = this.session.inputNames[0] ?? "waveform";
72
+ if (node_fs.existsSync(labelsPath)) this.labels = JSON.parse(node_fs.readFileSync(labelsPath, "utf8"));
73
+ else this.log.warn("YAMNet labels file not found — classifications will use numeric indices");
74
+ this.log.info(`YAMNet ONNX pipeline initialized (${this.labels.length} labels)`);
75
+ }
76
+ async classify(chunk) {
77
+ if (!this.session) throw new Error("YAMNet pipeline not initialized");
78
+ const start = Date.now();
79
+ const ort = await import("onnxruntime-node");
80
+ const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
81
+ const tensor = new ort.Tensor("float32", waveform, [waveform.length]);
82
+ const feeds = { [this.inputName]: tensor };
83
+ const scoresData = (await this.session.run(feeds))[this.session.outputNames[0]];
84
+ if (!scoresData) throw new Error("YAMNet returned no output");
85
+ const scores = scoresData.data;
86
+ const numClasses = 521;
87
+ const numFrames = scores.length / numClasses;
88
+ const avgScores = new Float32Array(numClasses);
89
+ for (let f = 0; f < numFrames; f++) for (let c = 0; c < numClasses; c++) avgScores[c] += scores[f * numClasses + c];
90
+ for (let c = 0; c < numClasses; c++) avgScores[c] = avgScores[c] / numFrames;
91
+ const minScore = .05;
92
+ const classifications = [];
93
+ for (let c = 0; c < numClasses; c++) {
94
+ const score = avgScores[c];
95
+ if (score >= minScore) {
96
+ const label = c < this.labels.length ? this.labels[c] : String(c);
97
+ classifications.push({
98
+ className: label,
99
+ score: Math.round(score * 1e3) / 1e3
100
+ });
101
+ }
102
+ }
103
+ classifications.sort((a, b) => b.score - a.score);
104
+ return {
105
+ classifications: classifications.slice(0, 10),
106
+ inferenceMs: Date.now() - start
107
+ };
108
+ }
109
+ async dispose() {
110
+ if (this.session) {
111
+ await this.session.release();
112
+ this.session = null;
113
+ }
114
+ }
115
+ };
116
+ var AppleSoundAnalysisPipeline = class {
117
+ log;
118
+ process = null;
119
+ receiveBuffer = Buffer.alloc(0);
120
+ pendingResolve = null;
121
+ pendingReject = null;
122
+ binaryPath = null;
123
+ debugCount = 0;
124
+ constructor(logger) {
125
+ this.log = logger;
126
+ }
127
+ async initialize() {
128
+ this.binaryPath = await this.resolveSwiftBinary();
129
+ if (!this.binaryPath) throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
130
+ const { spawn } = await import("node:child_process");
131
+ this.process = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], { stdio: [
132
+ "pipe",
133
+ "pipe",
134
+ "pipe"
135
+ ] });
136
+ this.process.stderr?.on("data", (chunk) => {
137
+ const lines = chunk.toString().split("\n");
138
+ for (const line of lines) {
139
+ const trimmed = line.trim();
140
+ if (trimmed) this.log.warn(trimmed);
141
+ }
142
+ });
143
+ this.process.on("error", (err) => {
144
+ this.log.error("Swift process error", { meta: { error: err.message } });
145
+ this.pendingReject?.(err);
146
+ this.pendingReject = null;
147
+ this.pendingResolve = null;
148
+ });
149
+ this.process.on("exit", (code) => {
150
+ if (code !== 0 && code !== null) {
151
+ this.log.error("Swift process exited", { meta: { code } });
152
+ const err = /* @__PURE__ */ new Error(`Apple SoundAnalysis: process exited with code ${code}`);
153
+ this.pendingReject?.(err);
154
+ this.pendingReject = null;
155
+ this.pendingResolve = null;
156
+ }
157
+ });
158
+ this.process.stdout.on("data", (chunk) => {
159
+ this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
160
+ this.tryReceive();
161
+ });
162
+ const ready = await this.receiveMessage();
163
+ if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
164
+ this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
165
+ }
166
+ async classify(chunk) {
167
+ if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
168
+ const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
169
+ const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
170
+ const lengthBuf = Buffer.allocUnsafe(4);
171
+ lengthBuf.writeUInt32LE(audioBuffer.length, 0);
172
+ this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
173
+ const result = await this.receiveMessage();
174
+ const classifications = result["classifications"] ?? [];
175
+ const inferenceMs = result["inferenceMs"] ?? 0;
176
+ if (this.debugCount < 3) {
177
+ const keys = Object.keys(result);
178
+ this.log.info("classify debug sample", { meta: {
179
+ phase: "apple-sa",
180
+ index: this.debugCount,
181
+ keys,
182
+ classifications: classifications.length,
183
+ inferenceMs,
184
+ audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
185
+ sampleRate: chunk.sampleRate,
186
+ channels: chunk.channels
187
+ } });
188
+ if (result["error"]) this.log.error("Swift error", { meta: {
189
+ phase: "apple-sa",
190
+ error: result["error"]
191
+ } });
192
+ this.debugCount++;
193
+ }
194
+ return {
195
+ classifications,
196
+ inferenceMs
197
+ };
198
+ }
199
+ async dispose() {
200
+ const proc = this.process;
201
+ if (!proc) return;
202
+ this.process = null;
203
+ proc.stdin?.end();
204
+ proc.kill("SIGTERM");
205
+ if (!await new Promise((resolve) => {
206
+ const timer = setTimeout(() => resolve(false), 5e3);
207
+ proc.once("exit", () => {
208
+ clearTimeout(timer);
209
+ resolve(true);
210
+ });
211
+ })) {
212
+ try {
213
+ proc.kill("SIGKILL");
214
+ } catch {}
215
+ this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
216
+ }
217
+ }
218
+ receiveMessage() {
219
+ return new Promise((resolve, reject) => {
220
+ this.pendingResolve = resolve;
221
+ this.pendingReject = reject;
222
+ });
223
+ }
224
+ tryReceive() {
225
+ if (this.receiveBuffer.length < 4) return;
226
+ const length = this.receiveBuffer.readUInt32LE(0);
227
+ if (this.receiveBuffer.length < 4 + length) return;
228
+ const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
229
+ this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
230
+ const resolve = this.pendingResolve;
231
+ const reject = this.pendingReject;
232
+ this.pendingResolve = null;
233
+ this.pendingReject = null;
234
+ if (!resolve) return;
235
+ try {
236
+ resolve(JSON.parse(jsonBytes.toString("utf8")));
237
+ } catch (err) {
238
+ reject?.(err instanceof Error ? err : new Error(String(err)));
239
+ }
240
+ }
241
+ /** Find pre-compiled binary or compile from Swift source. */
242
+ async resolveSwiftBinary() {
243
+ const candidates = [
244
+ node_path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier"),
245
+ node_path.join(__dirname, "../swift/apple-sound-classifier"),
246
+ node_path.join(__dirname, "../../swift/apple-sound-classifier"),
247
+ node_path.join(__dirname, "../../../swift/apple-sound-classifier")
248
+ ];
249
+ for (const p of candidates) if (node_fs.existsSync(p)) {
250
+ this.log.info("Found pre-compiled Swift CLI", { meta: { path: p } });
251
+ return p;
252
+ }
253
+ const sourceCandidates = [
254
+ node_path.join(__dirname, "../../swift/audio-analyzer/apple-sound-classifier.swift"),
255
+ node_path.join(__dirname, "../swift/apple-sound-classifier.swift"),
256
+ node_path.join(__dirname, "../../swift/apple-sound-classifier.swift"),
257
+ node_path.join(__dirname, "../../../swift/apple-sound-classifier.swift")
258
+ ];
259
+ const sourcePath = sourceCandidates.find((p) => node_fs.existsSync(p));
260
+ if (!sourcePath) {
261
+ this.log.error("Swift source not found", { meta: { searched: sourceCandidates } });
262
+ return null;
263
+ }
264
+ const outputPath = sourcePath.replace(".swift", "");
265
+ this.log.info("Compiling Swift CLI...", { meta: {
266
+ source: sourcePath,
267
+ output: outputPath
268
+ } });
269
+ const { execFileSync } = await import("node:child_process");
270
+ try {
271
+ execFileSync("swiftc", [
272
+ "-O",
273
+ "-o",
274
+ outputPath,
275
+ sourcePath
276
+ ], {
277
+ timeout: 6e4,
278
+ stdio: "pipe"
279
+ });
280
+ this.log.info("Swift CLI compiled successfully");
281
+ return outputPath;
282
+ } catch (err) {
283
+ this.log.error("Swift compilation failed — install Xcode Command Line Tools", { meta: { error: require_dist.errMsg(err) } });
284
+ return null;
285
+ }
286
+ }
287
+ };
288
+ /** Simple resample to 16kHz mono by linear interpolation. */
315
289
  function resampleMono16k(chunk) {
316
- const { data, sampleRate, channels } = chunk;
317
- const numSamples = data.length / channels;
318
- const mono = new Float32Array(numSamples);
319
- for (let i = 0; i < numSamples; i++) {
320
- let sum = 0;
321
- for (let c = 0; c < channels; c++) {
322
- sum += data[i * channels + c];
323
- }
324
- mono[i] = sum / channels;
325
- }
326
- const ratio = 16e3 / sampleRate;
327
- const outLen = Math.floor(numSamples * ratio);
328
- const out = new Float32Array(outLen);
329
- for (let i = 0; i < outLen; i++) {
330
- const srcIdx = i / ratio;
331
- const lo = Math.floor(srcIdx);
332
- const hi = Math.min(lo + 1, numSamples - 1);
333
- const frac = srcIdx - lo;
334
- out[i] = mono[lo] * (1 - frac) + mono[hi] * frac;
335
- }
336
- return out;
290
+ const { data, sampleRate, channels } = chunk;
291
+ const numSamples = data.length / channels;
292
+ const mono = new Float32Array(numSamples);
293
+ for (let i = 0; i < numSamples; i++) {
294
+ let sum = 0;
295
+ for (let c = 0; c < channels; c++) sum += data[i * channels + c];
296
+ mono[i] = sum / channels;
297
+ }
298
+ const ratio = 16e3 / sampleRate;
299
+ const outLen = Math.floor(numSamples * ratio);
300
+ const out = new Float32Array(outLen);
301
+ for (let i = 0; i < outLen; i++) {
302
+ const srcIdx = i / ratio;
303
+ const lo = Math.floor(srcIdx);
304
+ const hi = Math.min(lo + 1, numSamples - 1);
305
+ const frac = srcIdx - lo;
306
+ out[i] = mono[lo] * (1 - frac) + mono[hi] * frac;
307
+ }
308
+ return out;
337
309
  }
338
- const AUDIO_MODEL_OPTIONS = [
339
- { value: "", label: "Auto (matches backend)" },
340
- { value: "yamnet-onnx", label: "YAMNet (ONNX)" },
341
- { value: "apple-soundanalysis", label: "Apple SoundAnalysis (built-in)" }
310
+ //#endregion
311
+ //#region src/audio-analyzer/addons/analyzer/index.ts
312
+ /**
313
+ * Choices presented in the Audio Model dropdown. YAMNet runs via ONNX
314
+ * when backend=yamnet-onnx; Apple SoundAnalysis is a built-in macOS
315
+ * model and has no swappable modelId — the backend IS the model.
316
+ */
317
+ var AUDIO_MODEL_OPTIONS = [
318
+ {
319
+ value: "",
320
+ label: "Auto (matches backend)"
321
+ },
322
+ {
323
+ value: "yamnet-onnx",
324
+ label: "YAMNet (ONNX)"
325
+ },
326
+ {
327
+ value: "apple-soundanalysis",
328
+ label: "Apple SoundAnalysis (built-in)"
329
+ }
342
330
  ];
343
- const CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
344
- const CLASSIFY_MIN_INTERVAL_MS = 500;
345
- const GLOBAL_DEVICE_KEY = -1;
331
+ /**
332
+ * AudioAnalyzerProvider implements IAudioAnalyzer.
333
+ *
334
+ * Computes dB/RMS on every chunk and classifies via the in-process
335
+ * IAudioPipeline (YAMNet ONNX / Apple SoundAnalysis). No tRPC roundtrip
336
+ * to a separate audio-classifier addon.
337
+ */
338
+ var CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
339
+ var CLASSIFY_MIN_INTERVAL_MS = 500;
340
+ var GLOBAL_DEVICE_KEY = -1;
341
+ /**
342
+ * Reconstruct a Float32 view over f32le PCM bytes carried in a Uint8Array.
343
+ *
344
+ * `@msgpack/msgpack` decodes a binary blob into a Uint8Array that is a subview
345
+ * of its internal decode buffer, whose `byteOffset` is NOT guaranteed to be
346
+ * 4-byte aligned (observed e.g. 9). `new Float32Array(buf, offset, …)` then
347
+ * throws "start offset of Float32Array should be a multiple of 4". When the
348
+ * offset is misaligned we copy into a fresh 0-offset buffer; the aligned
349
+ * fast-path reuses the existing view with no copy.
350
+ */
346
351
  function float32FromBytes(raw) {
347
- const bytes = raw.byteOffset % 4 === 0 ? raw : new Uint8Array(raw);
348
- return new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
349
- }
350
- class AudioAnalyzerProvider {
351
- constructor(logger, pipeline, backendName, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, reprobeImpl) {
352
- this.pipeline = pipeline;
353
- this.deviceSettingsResolver = deviceSettingsResolver;
354
- this.deviceContributionResolver = deviceContributionResolver;
355
- this.deviceSettingsPatcher = deviceSettingsPatcher;
356
- this.reprobeImpl = reprobeImpl;
357
- this.log = logger;
358
- this.backendName = backendName;
359
- }
360
- log;
361
- classifyCallCount = 0;
362
- lastClassifyErrorMs = 0;
363
- suppressedClassifyErrors = 0;
364
- classifyCount = 0;
365
- backendName = "unknown";
366
- /** When true, logs a raw-label sample every 100 classifications (opt-in
367
- * debug aid). Off by default — the watchdog heartbeat covers liveness. */
368
- debugClassifySamples = false;
369
- /** Per-camera in-flight state. Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
370
- cameraState = /* @__PURE__ */ new Map();
371
- /** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
372
- * run at a time. Without this, concurrent calls from different cameras overwrite the
373
- * single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
374
- pipelineBusy = false;
375
- // ── Device-details aggregator contribution ──────────────────────────────
376
- async getDeviceSettingsContribution(input) {
377
- return this.deviceContributionResolver(input.deviceId);
378
- }
379
- async getDeviceLiveContribution(_input) {
380
- return null;
381
- }
382
- async applyDeviceSettingsPatch(input) {
383
- await this.deviceSettingsPatcher(input.deviceId, input.patch);
384
- return { success: true };
385
- }
386
- /**
387
- * Return the effective per-device audio-analyzer settings, resolved via
388
- * the kernel's 3-level settings resolver (schema default → global →
389
- * device override). Orchestrator consumers call this method so they
390
- * never need to know the audio-analyzer schema field names.
391
- */
392
- async resolveDeviceSettings({ deviceId }) {
393
- return this.deviceSettingsResolver(deviceId);
394
- }
395
- async analyseChunk({
396
- chunk,
397
- settings
398
- }) {
399
- const samples = float32FromBytes(chunk.data);
400
- let sumSquares = 0;
401
- for (let i = 0; i < samples.length; i++) {
402
- sumSquares += samples[i] * samples[i];
403
- }
404
- const rms = Math.sqrt(sumSquares / samples.length);
405
- const dbfs = rms > 0 ? 20 * Math.log10(rms) : -96;
406
- const level = {
407
- rms: Math.round(rms * 1e4) / 1e4,
408
- dbfs: Math.round(dbfs * 10) / 10
409
- };
410
- let classification;
411
- try {
412
- const result = await this.classify(chunk);
413
- if (this.classifyCallCount < 3) {
414
- const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
415
- this.log.info("classify debug sample", {
416
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
417
- meta: {
418
- index: this.classifyCallCount,
419
- labelCount: result.labels.length,
420
- top: topRaw,
421
- inferenceMs: result.inferenceMs,
422
- minConf: settings.minConfidence,
423
- allowedClasses: settings.allowedClasses
424
- }
425
- });
426
- }
427
- this.classifyCallCount++;
428
- if (result.inferenceMs > 0) {
429
- const minConf = settings.minConfidence;
430
- const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
431
- let filtered = result.labels.filter((c) => c.score >= minConf);
432
- if (allowedSet) {
433
- filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
434
- }
435
- if (filtered.length > 0) {
436
- classification = {
437
- labels: filtered,
438
- inferenceMs: result.inferenceMs
439
- };
440
- }
441
- }
442
- } catch (err) {
443
- const now = Date.now();
444
- const sinceLastMs = now - this.lastClassifyErrorMs;
445
- if (sinceLastMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
446
- const suppressed = this.suppressedClassifyErrors;
447
- this.suppressedClassifyErrors = 0;
448
- this.lastClassifyErrorMs = now;
449
- const msg = index.errMsg(err);
450
- const stack = err instanceof Error ? err.stack : void 0;
451
- this.log.warn("Audio classification failed", {
452
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
453
- meta: { error: msg, stack, suppressedSince: suppressed > 0 ? suppressed : void 0 }
454
- });
455
- } else {
456
- this.suppressedClassifyErrors++;
457
- }
458
- }
459
- return { level, classification, timestamp: chunk.timestamp };
460
- }
461
- async classify(chunk) {
462
- const camKey = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
463
- const now = Date.now();
464
- const state = this.cameraState.get(camKey);
465
- if (state?.inProgress || state !== void 0 && now - state.lastEndMs < CLASSIFY_MIN_INTERVAL_MS) {
466
- return { labels: [], rawLabels: [], inferenceMs: 0 };
467
- }
468
- if (this.pipelineBusy) {
469
- return { labels: [], rawLabels: [], inferenceMs: 0 };
470
- }
471
- this.cameraState.set(camKey, { inProgress: true, lastEndMs: state?.lastEndMs ?? 0 });
472
- this.pipelineBusy = true;
473
- const f32Data = float32FromBytes(chunk.data);
474
- const result = await this.pipeline.classify({
475
- data: f32Data,
476
- sampleRate: chunk.sampleRate,
477
- channels: chunk.channels
478
- }).finally(() => {
479
- this.pipelineBusy = false;
480
- this.cameraState.set(camKey, { inProgress: false, lastEndMs: Date.now() });
481
- });
482
- if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
483
- const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
484
- this.log.info("classify debug sample", {
485
- tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
486
- meta: {
487
- index: this.classifyCount,
488
- engine: this.backendName,
489
- rawLabelCount: result.classifications.length,
490
- top: rawTop,
491
- inferenceMs: result.inferenceMs
492
- }
493
- });
494
- }
495
- this.classifyCount++;
496
- const macroAccum = /* @__PURE__ */ new Map();
497
- for (const c of result.classifications) {
498
- const macro = index.mapAudioLabelToMacro(c.className);
499
- if (!macro) continue;
500
- const prev = macroAccum.get(macro);
501
- if (!prev || c.score > prev.score) {
502
- macroAccum.set(macro, { score: c.score, rawTop: c.className });
503
- }
504
- }
505
- const labels = [...macroAccum.entries()].sort((a, b) => b[1].score - a[1].score).map(([className, { score, rawTop }]) => ({ className, originalClass: rawTop, score }));
506
- const rawLabels = [...result.classifications].sort((a, b) => b.score - a.score).map((c) => ({ className: c.className, originalClass: c.className, score: c.score }));
507
- return { labels, rawLabels, inferenceMs: result.inferenceMs };
508
- }
509
- isReady() {
510
- return this.pipeline !== null;
511
- }
512
- async dispose() {
513
- await this.pipeline.dispose();
514
- }
515
- // Expose via the cap so the reprobe button in the UI reaches the
516
- // right worker. Delegates to the addon-owned reprobe (it touches
517
- // `ctx.settings.writeAddonStore` which only the addon has access to).
518
- async reprobeAudioEngine() {
519
- return this.reprobeImpl();
520
- }
521
- }
522
- class AudioAnalyzerAddon extends index.BaseAddon {
523
- id = "audio-analyzer";
524
- provider = null;
525
- pipeline = null;
526
- constructor() {
527
- super(index.DEFAULT_AUDIO_ANALYZER_CONFIG);
528
- }
529
- globalSettingsSchema() {
530
- return {
531
- sections: [
532
- {
533
- id: "audio-engine",
534
- title: "Audio",
535
- // Co-located with detection-pipeline's `engine` section under
536
- // a single "Engine" tab. Both sections handle inference-engine
537
- // selection but for different modalities — distinct purposes
538
- // are conveyed through section titles ("Detection engine" vs
539
- // "Audio inference engine") and descriptions, not separate
540
- // tabs.
541
- tab: "engine",
542
- // Renders after detection-pipeline's `engine` section
543
- // (`order: 0`) on the "Inference Engine" tab.
544
- order: 10,
545
- description: 'Audio classification backend (Apple SoundAnalysis or YAMNet ONNX). Independent from the vision-detection engine above. "Auto" picks Apple SoundAnalysis on macOS, YAMNet on Linux. Click the refresh icon next to "Probed best" to re-run the probe.',
546
- // Field order — `probedBest*` lives at the top so the operator
547
- // sees the auto-detected hint first and can compare it against
548
- // their override below at a glance. Same convention as the
549
- // detection-pipeline section.
550
- fields: [
551
- {
552
- type: "text",
553
- key: "probedBestAudioBackend",
554
- label: "Probed best",
555
- description: "Auto-detected best audio backend on this host. Click the refresh icon to re-run the probe.",
556
- readonlyField: true,
557
- default: "",
558
- actions: [
559
- { action: "reprobe-audio-engine", icon: "refresh-cw", tooltip: "Re-probe audio engine" }
560
- ]
561
- },
562
- {
563
- type: "select",
564
- key: "audioBackend",
565
- label: "Audio backend",
566
- options: index.AUDIO_BACKEND_CHOICES.map((o) => ({ value: o.value, label: o.label })),
567
- default: index.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend,
568
- immediate: true,
569
- requiresRestart: true
570
- },
571
- {
572
- type: "select",
573
- key: "selectedAudioModel",
574
- label: "Classification model",
575
- description: "Empty = auto (matches backend). Device-level settings can only inherit / enable / disable this step; model + class filters live here at the node level.",
576
- options: AUDIO_MODEL_OPTIONS.map((o) => ({ value: o.value, label: o.label })),
577
- default: index.DEFAULT_AUDIO_ANALYZER_CONFIG.selectedAudioModel,
578
- immediate: true,
579
- requiresRestart: true
580
- }
581
- ]
582
- }
583
- ]
584
- };
585
- }
586
- /**
587
- * Cascade override — narrow the `selectedAudioModel` options to the
588
- * subset compatible with the currently-selected `audioBackend`.
589
- *
590
- * Same pattern as detection-pipeline's `engineRuntime → engineBackend
591
- * → engineDevice` cascade: the base schema ships every option
592
- * (Auto + YAMNet + Apple SA); this override drops the rows that
593
- * belong to a backend the operator didn't pick. With `immediate:
594
- * true` on the `audioBackend` select, the UI refetches schema after
595
- * every flip and the model dropdown updates instantly.
596
- *
597
- * `overlay` carries the operator's tentative choices for benchmark/
598
- * preview mode (operator typed but didn't save yet) — same
599
- * semantics detection-pipeline relies on.
600
- */
601
- async getGlobalSettings(overlay) {
602
- const ctx = this.ctxIfReady;
603
- const stored = ctx?.settings ? await ctx.settings.readAddonStore() ?? {} : {};
604
- const merged = overlay ? { ...stored, ...overlay } : stored;
605
- const operatorChoice = typeof merged.audioBackend === "string" ? merged.audioBackend : index.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend;
606
- const effectiveBackend = operatorChoice === "apple-soundanalysis" ? "apple-soundanalysis" : operatorChoice === "yamnet-onnx" ? "yamnet-onnx" : process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
607
- const filteredModels = AUDIO_MODEL_OPTIONS.filter(
608
- (o) => o.value === "" || o.value === effectiveBackend
609
- );
610
- const storedModel = typeof merged.selectedAudioModel === "string" ? merged.selectedAudioModel : "";
611
- const validModel = filteredModels.find((o) => o.value === storedModel)?.value ?? "";
612
- const raw = { ...merged, selectedAudioModel: validModel };
613
- const schema = this.globalSettingsSchema();
614
- const patched = {
615
- ...schema,
616
- sections: schema.sections.map((section) => ({
617
- ...section,
618
- fields: section.fields.map((field) => {
619
- if (field.type === "select" && field.key === "selectedAudioModel") {
620
- return { ...field, options: filteredModels.map((o) => ({ value: o.value, label: o.label })) };
621
- }
622
- return field;
623
- })
624
- }))
625
- };
626
- return index.hydrateSchema(patched, raw);
627
- }
628
- /**
629
- * Re-run the platform probe and persist the detected backend into
630
- * `probedBestAudioBackend`. Operator `audioBackend` setting is not
631
- * touched — only the hint.
632
- */
633
- async reprobeAudioEngine() {
634
- const backend = process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
635
- await this.ctx.settings?.writeAddonStore({ probedBestAudioBackend: backend });
636
- this.ctx.logger.info("reprobeAudioEngine: wrote probedBestAudioBackend", { meta: { backend } });
637
- return { backend };
638
- }
639
- /** Resolve the effective backend from the operator choice, falling back to the platform heuristic when 'auto'. */
640
- resolveAudioBackend() {
641
- const choice = this.config.audioBackend;
642
- if (choice === "apple-soundanalysis") return "apple-soundanalysis";
643
- if (choice === "yamnet-onnx") return "yamnet-onnx";
644
- return process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
645
- }
646
- async onInitialize() {
647
- const logger = this.ctx.logger;
648
- const modelsDir = await this.ctx.api.storage.resolve.query({ location: "models", relativePath: "" }).catch(() => "camstack-data/models");
649
- const backend = this.resolveAudioBackend();
650
- logger.info("audio-analyzer: resolving pipeline", {
651
- meta: { operatorChoice: this.config.audioBackend, effectiveBackend: backend, selectedModel: this.config.selectedAudioModel || null }
652
- });
653
- const p = await createAudioPipeline(modelsDir, logger, { backend });
654
- await p.initialize();
655
- this.pipeline = p;
656
- if (!this.config.probedBestAudioBackend) {
657
- this.reprobeAudioEngine().catch((err) => {
658
- logger.warn("audio: auto-reprobe failed", {
659
- meta: { error: err instanceof Error ? err.message : String(err) }
660
- });
661
- });
662
- }
663
- const self = this;
664
- const deviceSettingsResolver = async (deviceId) => {
665
- try {
666
- const resolved = await self.ctx.api.pipelineOrchestrator.resolvePipeline.query({ deviceId });
667
- const stepSettings = resolved.audio?.settings ?? {};
668
- const minConfidence = typeof stepSettings["minConfidence"] === "number" ? stepSettings["minConfidence"] : 0.3;
669
- const allowedClasses = Array.isArray(stepSettings["enabledAudioClasses"]) ? stepSettings["enabledAudioClasses"] : [];
670
- return { minConfidence, allowedClasses };
671
- } catch (err) {
672
- logger.warn("audio: resolveDeviceSettings via orchestrator failed", {
673
- tags: { deviceId },
674
- meta: { error: err instanceof Error ? err.message : String(err) }
675
- });
676
- return null;
677
- }
678
- };
679
- const deviceContributionResolver = async (_deviceId) => {
680
- return null;
681
- };
682
- const deviceSettingsPatcher = async (_deviceId, _patch) => {
683
- };
684
- this.provider = new AudioAnalyzerProvider(
685
- logger,
686
- this.pipeline,
687
- backend,
688
- deviceSettingsResolver,
689
- deviceContributionResolver,
690
- deviceSettingsPatcher,
691
- () => this.reprobeAudioEngine()
692
- );
693
- return [
694
- { capability: index.audioAnalyzerCapability, provider: this.provider },
695
- {
696
- capability: index.audioAnalysisCapability,
697
- provider: this.provider
698
- }
699
- ];
700
- }
701
- async onShutdown() {
702
- if (this.provider) {
703
- await this.provider.dispose();
704
- this.provider = null;
705
- }
706
- this.pipeline = null;
707
- }
708
- // ── Standard ICamstackAddon — three-level settings API ─────
709
- //
710
- // Per-device audio settings (audio class filter + minConfidence) moved
711
- // to the audio-classifier pipeline step's `getConfigSchema()` and the
712
- // orchestrator owns the audio node assignment (`audioNodeId`). No
713
- // device-level settings remain on this addon.
714
- buildDeviceSchema() {
715
- return { sections: [] };
716
- }
717
- async getDeviceSettings(deviceId) {
718
- const raw = await this.ctx?.settings?.readDeviceStore(deviceId) ?? {};
719
- return index.hydrateSchema(this.buildDeviceSchema(), raw);
720
- }
721
- async updateDeviceSettings(deviceId, patch) {
722
- await this.ctx?.settings?.writeDeviceStore(deviceId, patch);
723
- }
352
+ const bytes = raw.byteOffset % 4 === 0 ? raw : new Uint8Array(raw);
353
+ return new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
724
354
  }
355
+ var AudioAnalyzerProvider = class {
356
+ pipeline;
357
+ deviceSettingsResolver;
358
+ deviceContributionResolver;
359
+ deviceSettingsPatcher;
360
+ reprobeImpl;
361
+ log;
362
+ classifyCallCount = 0;
363
+ lastClassifyErrorMs = 0;
364
+ suppressedClassifyErrors = 0;
365
+ classifyCount = 0;
366
+ backendName = "unknown";
367
+ /** When true, logs a raw-label sample every 100 classifications (opt-in
368
+ * debug aid). Off by default — the watchdog heartbeat covers liveness. */
369
+ debugClassifySamples = false;
370
+ /** Per-camera in-flight state. Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
371
+ cameraState = /* @__PURE__ */ new Map();
372
+ /** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
373
+ * run at a time. Without this, concurrent calls from different cameras overwrite the
374
+ * single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
375
+ pipelineBusy = false;
376
+ constructor(logger, pipeline, backendName, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, reprobeImpl) {
377
+ this.pipeline = pipeline;
378
+ this.deviceSettingsResolver = deviceSettingsResolver;
379
+ this.deviceContributionResolver = deviceContributionResolver;
380
+ this.deviceSettingsPatcher = deviceSettingsPatcher;
381
+ this.reprobeImpl = reprobeImpl;
382
+ this.log = logger;
383
+ this.backendName = backendName;
384
+ }
385
+ async getDeviceSettingsContribution(input) {
386
+ return this.deviceContributionResolver(input.deviceId);
387
+ }
388
+ async getDeviceLiveContribution(_input) {
389
+ return null;
390
+ }
391
+ async applyDeviceSettingsPatch(input) {
392
+ await this.deviceSettingsPatcher(input.deviceId, input.patch);
393
+ return { success: true };
394
+ }
395
+ /**
396
+ * Return the effective per-device audio-analyzer settings, resolved via
397
+ * the kernel's 3-level settings resolver (schema default → global →
398
+ * device override). Orchestrator consumers call this method so they
399
+ * never need to know the audio-analyzer schema field names.
400
+ */
401
+ async resolveDeviceSettings({ deviceId }) {
402
+ return this.deviceSettingsResolver(deviceId);
403
+ }
404
+ async analyseChunk({ chunk, settings }) {
405
+ const samples = float32FromBytes(chunk.data);
406
+ let sumSquares = 0;
407
+ for (let i = 0; i < samples.length; i++) sumSquares += samples[i] * samples[i];
408
+ const rms = Math.sqrt(sumSquares / samples.length);
409
+ const dbfs = rms > 0 ? 20 * Math.log10(rms) : -96;
410
+ const level = {
411
+ rms: Math.round(rms * 1e4) / 1e4,
412
+ dbfs: Math.round(dbfs * 10) / 10
413
+ };
414
+ let classification;
415
+ try {
416
+ const result = await this.classify(chunk);
417
+ if (this.classifyCallCount < 3) {
418
+ const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
419
+ this.log.info("classify debug sample", {
420
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
421
+ meta: {
422
+ index: this.classifyCallCount,
423
+ labelCount: result.labels.length,
424
+ top: topRaw,
425
+ inferenceMs: result.inferenceMs,
426
+ minConf: settings.minConfidence,
427
+ allowedClasses: settings.allowedClasses
428
+ }
429
+ });
430
+ }
431
+ this.classifyCallCount++;
432
+ if (result.inferenceMs > 0) {
433
+ const minConf = settings.minConfidence;
434
+ const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
435
+ let filtered = result.labels.filter((c) => c.score >= minConf);
436
+ if (allowedSet) filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
437
+ if (filtered.length > 0) classification = {
438
+ labels: filtered,
439
+ inferenceMs: result.inferenceMs
440
+ };
441
+ }
442
+ } catch (err) {
443
+ const now = Date.now();
444
+ if (now - this.lastClassifyErrorMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
445
+ const suppressed = this.suppressedClassifyErrors;
446
+ this.suppressedClassifyErrors = 0;
447
+ this.lastClassifyErrorMs = now;
448
+ const msg = require_dist.errMsg(err);
449
+ const stack = err instanceof Error ? err.stack : void 0;
450
+ this.log.warn("Audio classification failed", {
451
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
452
+ meta: {
453
+ error: msg,
454
+ stack,
455
+ suppressedSince: suppressed > 0 ? suppressed : void 0
456
+ }
457
+ });
458
+ } else this.suppressedClassifyErrors++;
459
+ }
460
+ return {
461
+ level,
462
+ classification,
463
+ timestamp: chunk.timestamp
464
+ };
465
+ }
466
+ async classify(chunk) {
467
+ const camKey = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
468
+ const now = Date.now();
469
+ const state = this.cameraState.get(camKey);
470
+ if (state?.inProgress || state !== void 0 && now - state.lastEndMs < CLASSIFY_MIN_INTERVAL_MS) return {
471
+ labels: [],
472
+ rawLabels: [],
473
+ inferenceMs: 0
474
+ };
475
+ if (this.pipelineBusy) return {
476
+ labels: [],
477
+ rawLabels: [],
478
+ inferenceMs: 0
479
+ };
480
+ this.cameraState.set(camKey, {
481
+ inProgress: true,
482
+ lastEndMs: state?.lastEndMs ?? 0
483
+ });
484
+ this.pipelineBusy = true;
485
+ const f32Data = float32FromBytes(chunk.data);
486
+ const result = await this.pipeline.classify({
487
+ data: f32Data,
488
+ sampleRate: chunk.sampleRate,
489
+ channels: chunk.channels
490
+ }).finally(() => {
491
+ this.pipelineBusy = false;
492
+ this.cameraState.set(camKey, {
493
+ inProgress: false,
494
+ lastEndMs: Date.now()
495
+ });
496
+ });
497
+ if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
498
+ const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
499
+ this.log.info("classify debug sample", {
500
+ tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
501
+ meta: {
502
+ index: this.classifyCount,
503
+ engine: this.backendName,
504
+ rawLabelCount: result.classifications.length,
505
+ top: rawTop,
506
+ inferenceMs: result.inferenceMs
507
+ }
508
+ });
509
+ }
510
+ this.classifyCount++;
511
+ const macroAccum = /* @__PURE__ */ new Map();
512
+ for (const c of result.classifications) {
513
+ const macro = require_dist.mapAudioLabelToMacro(c.className);
514
+ if (!macro) continue;
515
+ const prev = macroAccum.get(macro);
516
+ if (!prev || c.score > prev.score) macroAccum.set(macro, {
517
+ score: c.score,
518
+ rawTop: c.className
519
+ });
520
+ }
521
+ return {
522
+ labels: [...macroAccum.entries()].toSorted((a, b) => b[1].score - a[1].score).map(([className, { score, rawTop }]) => ({
523
+ className,
524
+ originalClass: rawTop,
525
+ score
526
+ })),
527
+ rawLabels: [...result.classifications].toSorted((a, b) => b.score - a.score).map((c) => ({
528
+ className: c.className,
529
+ originalClass: c.className,
530
+ score: c.score
531
+ })),
532
+ inferenceMs: result.inferenceMs
533
+ };
534
+ }
535
+ isReady() {
536
+ return this.pipeline !== null;
537
+ }
538
+ async dispose() {
539
+ await this.pipeline.dispose();
540
+ }
541
+ async reprobeAudioEngine() {
542
+ return this.reprobeImpl();
543
+ }
544
+ };
545
+ /**
546
+ * Audio Analyzer addon — provides the `audio-analyzer` capability.
547
+ *
548
+ * Owns the IAudioPipeline directly — no tRPC roundtrip to a separate
549
+ * audio-classifier addon.
550
+ */
551
+ var AudioAnalyzerAddon = class extends require_dist.BaseAddon {
552
+ id = "audio-analyzer";
553
+ provider = null;
554
+ pipeline = null;
555
+ constructor() {
556
+ super(require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG);
557
+ }
558
+ globalSettingsSchema() {
559
+ return { sections: [{
560
+ id: "audio-engine",
561
+ title: "Audio",
562
+ tab: "engine",
563
+ order: 10,
564
+ description: "Audio classification backend (Apple SoundAnalysis or YAMNet ONNX). Independent from the vision-detection engine above. \"Auto\" picks Apple SoundAnalysis on macOS, YAMNet on Linux. Click the refresh icon next to \"Probed best\" to re-run the probe.",
565
+ fields: [
566
+ {
567
+ type: "text",
568
+ key: "probedBestAudioBackend",
569
+ label: "Probed best",
570
+ description: "Auto-detected best audio backend on this host. Click the refresh icon to re-run the probe.",
571
+ readonlyField: true,
572
+ default: "",
573
+ actions: [{
574
+ action: "reprobe-audio-engine",
575
+ icon: "refresh-cw",
576
+ tooltip: "Re-probe audio engine"
577
+ }]
578
+ },
579
+ {
580
+ type: "select",
581
+ key: "audioBackend",
582
+ label: "Audio backend",
583
+ options: require_dist.AUDIO_BACKEND_CHOICES.map((o) => ({
584
+ value: o.value,
585
+ label: o.label
586
+ })),
587
+ default: require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend,
588
+ immediate: true,
589
+ requiresRestart: true
590
+ },
591
+ {
592
+ type: "select",
593
+ key: "selectedAudioModel",
594
+ label: "Classification model",
595
+ description: "Empty = auto (matches backend). Device-level settings can only inherit / enable / disable this step; model + class filters live here at the node level.",
596
+ options: AUDIO_MODEL_OPTIONS.map((o) => ({
597
+ value: o.value,
598
+ label: o.label
599
+ })),
600
+ default: require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.selectedAudioModel,
601
+ immediate: true,
602
+ requiresRestart: true
603
+ }
604
+ ]
605
+ }] };
606
+ }
607
+ /**
608
+ * Cascade override — narrow the `selectedAudioModel` options to the
609
+ * subset compatible with the currently-selected `audioBackend`.
610
+ *
611
+ * Same pattern as detection-pipeline's `engineRuntime → engineBackend
612
+ * → engineDevice` cascade: the base schema ships every option
613
+ * (Auto + YAMNet + Apple SA); this override drops the rows that
614
+ * belong to a backend the operator didn't pick. With `immediate:
615
+ * true` on the `audioBackend` select, the UI refetches schema after
616
+ * every flip and the model dropdown updates instantly.
617
+ *
618
+ * `overlay` carries the operator's tentative choices for benchmark/
619
+ * preview mode (operator typed but didn't save yet) — same
620
+ * semantics detection-pipeline relies on.
621
+ */
622
+ async getGlobalSettings(overlay) {
623
+ const ctx = this.ctxIfReady;
624
+ const stored = ctx?.settings ? await ctx.settings.readAddonStore() ?? {} : {};
625
+ const merged = overlay ? {
626
+ ...stored,
627
+ ...overlay
628
+ } : stored;
629
+ const operatorChoice = typeof merged.audioBackend === "string" ? merged.audioBackend : require_dist.DEFAULT_AUDIO_ANALYZER_CONFIG.audioBackend;
630
+ const effectiveBackend = operatorChoice === "apple-soundanalysis" ? "apple-soundanalysis" : operatorChoice === "yamnet-onnx" ? "yamnet-onnx" : process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
631
+ const filteredModels = AUDIO_MODEL_OPTIONS.filter((o) => o.value === "" || o.value === effectiveBackend);
632
+ const storedModel = typeof merged.selectedAudioModel === "string" ? merged.selectedAudioModel : "";
633
+ const validModel = filteredModels.find((o) => o.value === storedModel)?.value ?? "";
634
+ const raw = {
635
+ ...merged,
636
+ selectedAudioModel: validModel
637
+ };
638
+ const schema = this.globalSettingsSchema();
639
+ return require_dist.hydrateSchema({
640
+ ...schema,
641
+ sections: schema.sections.map((section) => ({
642
+ ...section,
643
+ fields: section.fields.map((field) => {
644
+ if (field.type === "select" && field.key === "selectedAudioModel") return {
645
+ ...field,
646
+ options: filteredModels.map((o) => ({
647
+ value: o.value,
648
+ label: o.label
649
+ }))
650
+ };
651
+ return field;
652
+ })
653
+ }))
654
+ }, raw);
655
+ }
656
+ /**
657
+ * Re-run the platform probe and persist the detected backend into
658
+ * `probedBestAudioBackend`. Operator `audioBackend` setting is not
659
+ * touched — only the hint.
660
+ */
661
+ async reprobeAudioEngine() {
662
+ const backend = process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
663
+ await this.ctx.settings?.writeAddonStore({ probedBestAudioBackend: backend });
664
+ this.ctx.logger.info("reprobeAudioEngine: wrote probedBestAudioBackend", { meta: { backend } });
665
+ return { backend };
666
+ }
667
+ /** Resolve the effective backend from the operator choice, falling back to the platform heuristic when 'auto'. */
668
+ resolveAudioBackend() {
669
+ const choice = this.config.audioBackend;
670
+ if (choice === "apple-soundanalysis") return "apple-soundanalysis";
671
+ if (choice === "yamnet-onnx") return "yamnet-onnx";
672
+ return process.platform === "darwin" ? "apple-soundanalysis" : "yamnet-onnx";
673
+ }
674
+ async onInitialize() {
675
+ const logger = this.ctx.logger;
676
+ const modelsDir = await this.ctx.api.storage.resolve.query({
677
+ location: "models",
678
+ relativePath: ""
679
+ }).catch(() => "camstack-data/models");
680
+ const backend = this.resolveAudioBackend();
681
+ logger.info("audio-analyzer: resolving pipeline", { meta: {
682
+ operatorChoice: this.config.audioBackend,
683
+ effectiveBackend: backend,
684
+ selectedModel: this.config.selectedAudioModel || null
685
+ } });
686
+ const p = await createAudioPipeline(modelsDir, logger, { backend });
687
+ await p.initialize();
688
+ this.pipeline = p;
689
+ if (!this.config.probedBestAudioBackend) this.reprobeAudioEngine().catch((err) => {
690
+ logger.warn("audio: auto-reprobe failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
691
+ });
692
+ const self = this;
693
+ const deviceSettingsResolver = async (deviceId) => {
694
+ try {
695
+ const stepSettings = (await self.ctx.api.pipelineOrchestrator.resolvePipeline.query({ deviceId })).audio?.settings ?? {};
696
+ return {
697
+ minConfidence: typeof stepSettings["minConfidence"] === "number" ? stepSettings["minConfidence"] : .3,
698
+ allowedClasses: Array.isArray(stepSettings["enabledAudioClasses"]) ? stepSettings["enabledAudioClasses"] : []
699
+ };
700
+ } catch (err) {
701
+ logger.warn("audio: resolveDeviceSettings via orchestrator failed", {
702
+ tags: { deviceId },
703
+ meta: { error: err instanceof Error ? err.message : String(err) }
704
+ });
705
+ return null;
706
+ }
707
+ };
708
+ const deviceContributionResolver = async (_deviceId) => {
709
+ return null;
710
+ };
711
+ const deviceSettingsPatcher = async (_deviceId, _patch) => {};
712
+ this.provider = new AudioAnalyzerProvider(logger, this.pipeline, backend, deviceSettingsResolver, deviceContributionResolver, deviceSettingsPatcher, () => this.reprobeAudioEngine());
713
+ return [{
714
+ capability: require_dist.audioAnalyzerCapability,
715
+ provider: this.provider
716
+ }, {
717
+ capability: require_dist.audioAnalysisCapability,
718
+ provider: this.provider
719
+ }];
720
+ }
721
+ async onShutdown() {
722
+ if (this.provider) {
723
+ await this.provider.dispose();
724
+ this.provider = null;
725
+ }
726
+ this.pipeline = null;
727
+ }
728
+ buildDeviceSchema() {
729
+ return { sections: [] };
730
+ }
731
+ async getDeviceSettings(deviceId) {
732
+ const raw = await this.ctx?.settings?.readDeviceStore(deviceId) ?? {};
733
+ return require_dist.hydrateSchema(this.buildDeviceSchema(), raw);
734
+ }
735
+ async updateDeviceSettings(deviceId, patch) {
736
+ await this.ctx?.settings?.writeDeviceStore(deviceId, patch);
737
+ }
738
+ };
739
+ //#endregion
740
+ //#region src/audio-analyzer/index.ts
741
+ var audio_analyzer_default = AudioAnalyzerAddon;
742
+ //#endregion
725
743
  exports.AudioAnalyzerAddon = AudioAnalyzerAddon;
726
744
  exports.AudioAnalyzerProvider = AudioAnalyzerProvider;
727
745
  exports.createAudioPipeline = createAudioPipeline;
728
- exports.default = AudioAnalyzerAddon;
729
- //# sourceMappingURL=index.js.map
746
+ exports.default = audio_analyzer_default;