@camstack/addon-pipeline 1.2.297 → 1.2.299
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_MODELS.md +126 -123
- package/dist/audio-analyzer/index.js +571 -305
- package/dist/audio-analyzer/index.mjs +571 -305
- package/dist/{default-detection-model-D6DypCkI.mjs → default-detection-model-DMcHBVmE.mjs} +391 -170
- package/dist/{default-detection-model-B2n_SFXt.js → default-detection-model-H_NzHI_P.js} +391 -170
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +4 -4
- package/dist/{dist-DUqr1zyq.js → dist-BzDPLvQ6.js} +73 -4
- package/dist/{dist-CxHIpsQs.mjs → dist-DNlqf8dr.mjs} +73 -4
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-Dx6DLQ1j.js → node-B-cB_Hdy.js} +1 -1
- package/dist/{node-DyeWu78a.mjs → node-g2ZRFDvc.mjs} +1 -1
- package/dist/pipeline-runner/index.js +3 -3
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/{process-memory-CMJ3NY-s.js → process-memory-Bo3-laZ_.js} +1 -1
- package/dist/{process-memory-BVR4592X.mjs → process-memory-kKYoJ9hl.mjs} +1 -1
- package/dist/recorder/index.js +3 -3
- package/dist/recorder/index.mjs +3 -3
- package/dist/{segment-demux-js-CmGIF_uK.js → segment-demux-js-DrDj8kP6.js} +1 -1
- package/dist/{segment-demux-js-Dzga-XgE.mjs → segment-demux-js-QK7qFMSj.mjs} +1 -1
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-P71ze8cu.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-Da_bbYQp.mjs} +1 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CxT9ry2j.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-CRlStbz4.mjs → hostInit-aFrOa5WT.mjs} +1 -1
- package/dist/stream-broker/index.js +3 -3
- package/dist/stream-broker/index.mjs +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/python/postprocessors/softmax.py +2 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CZkpFU-J.mjs +0 -26
|
@@ -3,13 +3,167 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../chunk-emK7D4bc.js");
|
|
6
|
-
const require_dist = require("../dist-
|
|
7
|
-
const require_process_memory = require("../process-memory-
|
|
6
|
+
const require_dist = require("../dist-BzDPLvQ6.js");
|
|
7
|
+
const require_process_memory = require("../process-memory-Bo3-laZ_.js");
|
|
8
8
|
let node_fs = require("node:fs");
|
|
9
9
|
node_fs = require_chunk.__toESM(node_fs);
|
|
10
10
|
let node_path = require("node:path");
|
|
11
11
|
node_path = require_chunk.__toESM(node_path);
|
|
12
12
|
let _camstack_system_addon_utils = require("@camstack/system/addon-utils");
|
|
13
|
+
//#region src/audio-analyzer/audio-subprocess-channel.ts
|
|
14
|
+
/** The backend process ended (or was disposed) while — or before — a request waited on it. */
|
|
15
|
+
var AudioBackendExitedError = class extends Error {
|
|
16
|
+
constructor(message) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.name = "AudioBackendExitedError";
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
/** How long `dispose()` waits for a graceful exit before SIGKILL. */
|
|
22
|
+
var DISPOSE_GRACE_MS = 5e3;
|
|
23
|
+
function isReplyObject(value) {
|
|
24
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
25
|
+
}
|
|
26
|
+
var AudioSubprocessChannel = class {
|
|
27
|
+
label;
|
|
28
|
+
log;
|
|
29
|
+
process = null;
|
|
30
|
+
receiveBuffer = Buffer.alloc(0);
|
|
31
|
+
pending = null;
|
|
32
|
+
constructor(label, log) {
|
|
33
|
+
this.label = label;
|
|
34
|
+
this.log = log;
|
|
35
|
+
}
|
|
36
|
+
/** Wire a freshly spawned process. Stderr stays the caller's (each backend formats it). */
|
|
37
|
+
attach(proc) {
|
|
38
|
+
this.process = proc;
|
|
39
|
+
proc.on("error", (err) => {
|
|
40
|
+
this.log.error(`${this.label} process error`, { meta: { error: err.message } });
|
|
41
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: process error: ${err.message}`));
|
|
42
|
+
});
|
|
43
|
+
proc.on("exit", (code, signal) => {
|
|
44
|
+
if (this.process === proc) this.process = null;
|
|
45
|
+
const clean = code === 0 && signal === null;
|
|
46
|
+
const meta = {
|
|
47
|
+
code,
|
|
48
|
+
signal
|
|
49
|
+
};
|
|
50
|
+
if (clean) this.log.info(`${this.label} process exited`, { meta });
|
|
51
|
+
else this.log.error(`${this.label} process exited`, { meta });
|
|
52
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: process exited (code ${String(code)}, signal ${String(signal)})`));
|
|
53
|
+
});
|
|
54
|
+
proc.stdout?.on("data", (chunk) => {
|
|
55
|
+
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
56
|
+
this.tryReceive();
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
get pid() {
|
|
60
|
+
return this.process?.pid ?? null;
|
|
61
|
+
}
|
|
62
|
+
get alive() {
|
|
63
|
+
return this.process !== null;
|
|
64
|
+
}
|
|
65
|
+
/** Wait for the next reply frame without sending (the init handshake). */
|
|
66
|
+
receive() {
|
|
67
|
+
if (this.process === null) return Promise.reject(new AudioBackendExitedError(`${this.label}: process not running`));
|
|
68
|
+
return new Promise((resolve, reject) => {
|
|
69
|
+
this.pending = {
|
|
70
|
+
resolve,
|
|
71
|
+
reject
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/** Send one payload frame and wait for its reply. */
|
|
76
|
+
request(payload) {
|
|
77
|
+
const stdin = this.process?.stdin;
|
|
78
|
+
if (!stdin) return Promise.reject(new AudioBackendExitedError(`${this.label}: process not initialized`));
|
|
79
|
+
const reply = this.receive();
|
|
80
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
81
|
+
lengthBuf.writeUInt32LE(payload.length, 0);
|
|
82
|
+
stdin.write(Buffer.concat([lengthBuf, payload]));
|
|
83
|
+
return reply;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Stop the process. The waiting request (if any) is rejected FIRST — the
|
|
87
|
+
* exit event would do it too, but a process that ignores SIGTERM for five
|
|
88
|
+
* seconds must not hold the caller for five seconds.
|
|
89
|
+
*/
|
|
90
|
+
async dispose() {
|
|
91
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: disposed`));
|
|
92
|
+
const proc = this.process;
|
|
93
|
+
if (!proc) return;
|
|
94
|
+
this.process = null;
|
|
95
|
+
proc.stdin?.end();
|
|
96
|
+
proc.kill("SIGTERM");
|
|
97
|
+
if (!await new Promise((resolve) => {
|
|
98
|
+
if (proc.exitCode !== null || proc.signalCode !== null) {
|
|
99
|
+
resolve(true);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const timer = setTimeout(() => resolve(false), DISPOSE_GRACE_MS);
|
|
103
|
+
proc.once("exit", () => {
|
|
104
|
+
clearTimeout(timer);
|
|
105
|
+
resolve(true);
|
|
106
|
+
});
|
|
107
|
+
})) {
|
|
108
|
+
try {
|
|
109
|
+
proc.kill("SIGKILL");
|
|
110
|
+
} catch {}
|
|
111
|
+
this.log.warn(`${this.label} process did not exit gracefully — sent SIGKILL`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
failPending(err) {
|
|
115
|
+
const pending = this.pending;
|
|
116
|
+
this.pending = null;
|
|
117
|
+
pending?.reject(err);
|
|
118
|
+
}
|
|
119
|
+
tryReceive() {
|
|
120
|
+
while (this.receiveBuffer.length >= 4) {
|
|
121
|
+
const length = this.receiveBuffer.readUInt32LE(0);
|
|
122
|
+
if (this.receiveBuffer.length < 4 + length) return;
|
|
123
|
+
const bytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
124
|
+
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
125
|
+
const pending = this.pending;
|
|
126
|
+
this.pending = null;
|
|
127
|
+
if (!pending) {
|
|
128
|
+
this.log.warn(`${this.label}: reply with no request waiting — discarded`, { meta: { bytes: length } });
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
let parsed;
|
|
132
|
+
try {
|
|
133
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
134
|
+
} catch (err) {
|
|
135
|
+
pending.reject(/* @__PURE__ */ new Error(`${this.label}: unparseable reply: ${require_dist.errMsg(err)}`));
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (isReplyObject(parsed)) pending.resolve(parsed);
|
|
139
|
+
else pending.reject(/* @__PURE__ */ new Error(`${this.label}: reply is not an object`));
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
function isBackendLabel(value) {
|
|
144
|
+
if (typeof value !== "object" || value === null) return false;
|
|
145
|
+
const className = Reflect.get(value, "className");
|
|
146
|
+
const score = Reflect.get(value, "score");
|
|
147
|
+
return typeof className === "string" && typeof score === "number";
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Read a classify reply. A reply carrying `error` is a FAILED inference and
|
|
151
|
+
* throws (D660): `yamnet_audio.py` answers an exception with an empty label
|
|
152
|
+
* list plus `error`, and reading that as "no labels" is the silence-vs-nobody-
|
|
153
|
+
* listened confusion D660 removed.
|
|
154
|
+
*/
|
|
155
|
+
function parseClassifyReply(label, reply) {
|
|
156
|
+
const error = reply["error"];
|
|
157
|
+
if (typeof error === "string" && error.length > 0) throw new Error(`${label}: inference failed: ${error}`);
|
|
158
|
+
const raw = reply["classifications"];
|
|
159
|
+
const classifications = Array.isArray(raw) ? raw.filter(isBackendLabel) : [];
|
|
160
|
+
const ms = reply["inferenceMs"];
|
|
161
|
+
return {
|
|
162
|
+
classifications,
|
|
163
|
+
inferenceMs: typeof ms === "number" ? ms : 0
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
//#endregion
|
|
13
167
|
//#region src/audio-analyzer/audio-pipeline.ts
|
|
14
168
|
/**
|
|
15
169
|
* Create the appropriate audio pipeline.
|
|
@@ -66,15 +220,14 @@ var YamnetPythonPipeline = class {
|
|
|
66
220
|
pythonPath;
|
|
67
221
|
installPythonRequirements;
|
|
68
222
|
log;
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
pendingResolve = null;
|
|
72
|
-
pendingReject = null;
|
|
223
|
+
/** The wire + reply slot; settles a waiting request on ANY exit (D660). */
|
|
224
|
+
channel;
|
|
73
225
|
constructor(modelsDir, logger, pythonPath, installPythonRequirements) {
|
|
74
226
|
this.modelsDir = modelsDir;
|
|
75
227
|
this.pythonPath = pythonPath;
|
|
76
228
|
this.installPythonRequirements = installPythonRequirements;
|
|
77
229
|
this.log = logger;
|
|
230
|
+
this.channel = new AudioSubprocessChannel("YAMNet Python", logger);
|
|
78
231
|
}
|
|
79
232
|
async initialize() {
|
|
80
233
|
const modelPath = node_path.join(this.modelsDir, "camstack-yamnet.onnx");
|
|
@@ -98,7 +251,7 @@ var YamnetPythonPipeline = class {
|
|
|
98
251
|
if (this.installPythonRequirements) await this.installPythonRequirements(node_path.join(pythonDir, "requirements-audio.txt"));
|
|
99
252
|
const scriptPath = node_path.join(pythonDir, "yamnet_audio.py");
|
|
100
253
|
const { spawn } = await import("node:child_process");
|
|
101
|
-
|
|
254
|
+
const proc = spawn(this.pythonPath, [
|
|
102
255
|
scriptPath,
|
|
103
256
|
"--model",
|
|
104
257
|
modelPath,
|
|
@@ -109,219 +262,79 @@ var YamnetPythonPipeline = class {
|
|
|
109
262
|
"pipe",
|
|
110
263
|
"pipe"
|
|
111
264
|
] });
|
|
112
|
-
|
|
265
|
+
proc.stderr?.on("data", (chunk) => {
|
|
113
266
|
const text = chunk.toString().trim();
|
|
114
267
|
if (text) this.log.warn(text);
|
|
115
268
|
});
|
|
116
|
-
this.
|
|
117
|
-
|
|
118
|
-
this.pendingReject?.(err);
|
|
119
|
-
this.pendingReject = null;
|
|
120
|
-
this.pendingResolve = null;
|
|
121
|
-
});
|
|
122
|
-
this.process.on("exit", (code) => {
|
|
123
|
-
if (code !== 0 && code !== null) {
|
|
124
|
-
this.log.error("YAMNet Python process exited", { meta: { code } });
|
|
125
|
-
const err = /* @__PURE__ */ new Error(`YAMNet Python: process exited with code ${code}`);
|
|
126
|
-
this.pendingReject?.(err);
|
|
127
|
-
this.pendingReject = null;
|
|
128
|
-
this.pendingResolve = null;
|
|
129
|
-
}
|
|
130
|
-
});
|
|
131
|
-
this.process.stdout.on("data", (chunk) => {
|
|
132
|
-
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
133
|
-
this.tryReceive();
|
|
134
|
-
});
|
|
135
|
-
const ready = await this.receiveMessage();
|
|
269
|
+
this.channel.attach(proc);
|
|
270
|
+
const ready = await this.channel.receive();
|
|
136
271
|
if (ready["status"] !== "ready") throw new Error(`YAMNet Python: unexpected init response: ${JSON.stringify(ready)}`);
|
|
137
272
|
this.log.info(`YAMNet Python pipeline initialized (${String(ready["labels"] ?? "?")} labels)`);
|
|
138
273
|
}
|
|
139
274
|
getPid() {
|
|
140
|
-
return this.
|
|
275
|
+
return this.channel.pid;
|
|
141
276
|
}
|
|
142
277
|
async classify(chunk) {
|
|
143
|
-
if (!this.process?.stdin) throw new Error("YAMNet Python: process not initialized");
|
|
144
278
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
145
|
-
|
|
146
|
-
const lengthBuf = Buffer.allocUnsafe(4);
|
|
147
|
-
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
148
|
-
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
149
|
-
const result = await this.receiveMessage();
|
|
150
|
-
return {
|
|
151
|
-
classifications: result["classifications"] ?? [],
|
|
152
|
-
inferenceMs: result["inferenceMs"] ?? 0
|
|
153
|
-
};
|
|
279
|
+
return parseClassifyReply("YAMNet Python", await this.channel.request(Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength)));
|
|
154
280
|
}
|
|
155
281
|
async dispose() {
|
|
156
|
-
|
|
157
|
-
if (!proc) return;
|
|
158
|
-
this.process = null;
|
|
159
|
-
proc.stdin?.end();
|
|
160
|
-
proc.kill("SIGTERM");
|
|
161
|
-
if (!await new Promise((resolve) => {
|
|
162
|
-
const timer = setTimeout(() => resolve(false), 5e3);
|
|
163
|
-
proc.once("exit", () => {
|
|
164
|
-
clearTimeout(timer);
|
|
165
|
-
resolve(true);
|
|
166
|
-
});
|
|
167
|
-
})) {
|
|
168
|
-
try {
|
|
169
|
-
proc.kill("SIGKILL");
|
|
170
|
-
} catch {}
|
|
171
|
-
this.log.warn("YAMNet Python process did not exit gracefully — sent SIGKILL");
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
receiveMessage() {
|
|
175
|
-
return new Promise((resolve, reject) => {
|
|
176
|
-
this.pendingResolve = resolve;
|
|
177
|
-
this.pendingReject = reject;
|
|
178
|
-
});
|
|
179
|
-
}
|
|
180
|
-
tryReceive() {
|
|
181
|
-
if (this.receiveBuffer.length < 4) return;
|
|
182
|
-
const length = this.receiveBuffer.readUInt32LE(0);
|
|
183
|
-
if (this.receiveBuffer.length < 4 + length) return;
|
|
184
|
-
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
185
|
-
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
186
|
-
const resolve = this.pendingResolve;
|
|
187
|
-
const reject = this.pendingReject;
|
|
188
|
-
this.pendingResolve = null;
|
|
189
|
-
this.pendingReject = null;
|
|
190
|
-
if (!resolve) return;
|
|
191
|
-
try {
|
|
192
|
-
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
193
|
-
} catch (err) {
|
|
194
|
-
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
195
|
-
}
|
|
282
|
+
await this.channel.dispose();
|
|
196
283
|
}
|
|
197
284
|
};
|
|
198
285
|
var AppleSoundAnalysisPipeline = class {
|
|
199
286
|
log;
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
pendingResolve = null;
|
|
203
|
-
pendingReject = null;
|
|
287
|
+
/** The wire + reply slot; settles a waiting request on ANY exit (D660). */
|
|
288
|
+
channel;
|
|
204
289
|
binaryPath = null;
|
|
205
290
|
debugCount = 0;
|
|
206
291
|
constructor(logger) {
|
|
207
292
|
this.log = logger;
|
|
293
|
+
this.channel = new AudioSubprocessChannel("Apple SoundAnalysis", logger);
|
|
208
294
|
}
|
|
209
295
|
async initialize() {
|
|
210
296
|
this.binaryPath = await this.resolveSwiftBinary();
|
|
211
297
|
if (!this.binaryPath) throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
|
|
212
298
|
const { spawn } = await import("node:child_process");
|
|
213
|
-
|
|
299
|
+
const proc = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], { stdio: [
|
|
214
300
|
"pipe",
|
|
215
301
|
"pipe",
|
|
216
302
|
"pipe"
|
|
217
303
|
] });
|
|
218
|
-
|
|
304
|
+
proc.stderr?.on("data", (chunk) => {
|
|
219
305
|
const lines = chunk.toString().split("\n");
|
|
220
306
|
for (const line of lines) {
|
|
221
307
|
const trimmed = line.trim();
|
|
222
308
|
if (trimmed) this.log.warn(trimmed);
|
|
223
309
|
}
|
|
224
310
|
});
|
|
225
|
-
this.
|
|
226
|
-
|
|
227
|
-
this.pendingReject?.(err);
|
|
228
|
-
this.pendingReject = null;
|
|
229
|
-
this.pendingResolve = null;
|
|
230
|
-
});
|
|
231
|
-
this.process.on("exit", (code) => {
|
|
232
|
-
if (code !== 0 && code !== null) {
|
|
233
|
-
this.log.error("Swift process exited", { meta: { code } });
|
|
234
|
-
const err = /* @__PURE__ */ new Error(`Apple SoundAnalysis: process exited with code ${code}`);
|
|
235
|
-
this.pendingReject?.(err);
|
|
236
|
-
this.pendingReject = null;
|
|
237
|
-
this.pendingResolve = null;
|
|
238
|
-
}
|
|
239
|
-
});
|
|
240
|
-
this.process.stdout.on("data", (chunk) => {
|
|
241
|
-
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
242
|
-
this.tryReceive();
|
|
243
|
-
});
|
|
244
|
-
const ready = await this.receiveMessage();
|
|
311
|
+
this.channel.attach(proc);
|
|
312
|
+
const ready = await this.channel.receive();
|
|
245
313
|
if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
246
314
|
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
247
315
|
}
|
|
248
316
|
getPid() {
|
|
249
|
-
return this.
|
|
317
|
+
return this.channel.pid;
|
|
250
318
|
}
|
|
251
319
|
async classify(chunk) {
|
|
252
|
-
if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
|
|
253
320
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
254
321
|
const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
|
|
255
|
-
const
|
|
256
|
-
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
257
|
-
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
258
|
-
const result = await this.receiveMessage();
|
|
259
|
-
const classifications = result["classifications"] ?? [];
|
|
260
|
-
const inferenceMs = result["inferenceMs"] ?? 0;
|
|
322
|
+
const reply = await this.channel.request(audioBuffer);
|
|
261
323
|
if (this.debugCount < 3) {
|
|
262
|
-
const keys = Object.keys(result);
|
|
263
324
|
this.log.info("classify debug sample", { meta: {
|
|
264
325
|
phase: "apple-sa",
|
|
265
326
|
index: this.debugCount,
|
|
266
|
-
keys,
|
|
267
|
-
|
|
268
|
-
inferenceMs,
|
|
269
|
-
audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
|
|
327
|
+
keys: Object.keys(reply),
|
|
328
|
+
audioBytes: audioBuffer.length,
|
|
270
329
|
sampleRate: chunk.sampleRate,
|
|
271
330
|
channels: chunk.channels
|
|
272
331
|
} });
|
|
273
|
-
if (result["error"]) this.log.error("Swift error", { meta: {
|
|
274
|
-
phase: "apple-sa",
|
|
275
|
-
error: result["error"]
|
|
276
|
-
} });
|
|
277
332
|
this.debugCount++;
|
|
278
333
|
}
|
|
279
|
-
return
|
|
280
|
-
classifications,
|
|
281
|
-
inferenceMs
|
|
282
|
-
};
|
|
334
|
+
return parseClassifyReply("Apple SoundAnalysis", reply);
|
|
283
335
|
}
|
|
284
336
|
async dispose() {
|
|
285
|
-
|
|
286
|
-
if (!proc) return;
|
|
287
|
-
this.process = null;
|
|
288
|
-
proc.stdin?.end();
|
|
289
|
-
proc.kill("SIGTERM");
|
|
290
|
-
if (!await new Promise((resolve) => {
|
|
291
|
-
const timer = setTimeout(() => resolve(false), 5e3);
|
|
292
|
-
proc.once("exit", () => {
|
|
293
|
-
clearTimeout(timer);
|
|
294
|
-
resolve(true);
|
|
295
|
-
});
|
|
296
|
-
})) {
|
|
297
|
-
try {
|
|
298
|
-
proc.kill("SIGKILL");
|
|
299
|
-
} catch {}
|
|
300
|
-
this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
receiveMessage() {
|
|
304
|
-
return new Promise((resolve, reject) => {
|
|
305
|
-
this.pendingResolve = resolve;
|
|
306
|
-
this.pendingReject = reject;
|
|
307
|
-
});
|
|
308
|
-
}
|
|
309
|
-
tryReceive() {
|
|
310
|
-
if (this.receiveBuffer.length < 4) return;
|
|
311
|
-
const length = this.receiveBuffer.readUInt32LE(0);
|
|
312
|
-
if (this.receiveBuffer.length < 4 + length) return;
|
|
313
|
-
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
314
|
-
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
315
|
-
const resolve = this.pendingResolve;
|
|
316
|
-
const reject = this.pendingReject;
|
|
317
|
-
this.pendingResolve = null;
|
|
318
|
-
this.pendingReject = null;
|
|
319
|
-
if (!resolve) return;
|
|
320
|
-
try {
|
|
321
|
-
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
322
|
-
} catch (err) {
|
|
323
|
-
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
324
|
-
}
|
|
337
|
+
await this.channel.dispose();
|
|
325
338
|
}
|
|
326
339
|
/** Find pre-compiled binary or compile from Swift source. */
|
|
327
340
|
async resolveSwiftBinary() {
|
|
@@ -784,6 +797,7 @@ function buildAudioResultFrame(deviceId, result) {
|
|
|
784
797
|
dbfs: result.level.dbfs
|
|
785
798
|
},
|
|
786
799
|
detections: audioDetections,
|
|
800
|
+
classifyOutcome: result.classifyOutcome,
|
|
787
801
|
debug: result.classification ? {
|
|
788
802
|
totalInferenceMs: result.classification.inferenceMs,
|
|
789
803
|
stepTimings: [{
|
|
@@ -794,49 +808,8 @@ function buildAudioResultFrame(deviceId, result) {
|
|
|
794
808
|
} : void 0
|
|
795
809
|
};
|
|
796
810
|
}
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
/**
|
|
800
|
-
* `AudioDeviceAttachments` — the analyzer's own audio subscriptions (D461).
|
|
801
|
-
*
|
|
802
|
-
* ## What this replaces
|
|
803
|
-
*
|
|
804
|
-
* Until 2026-09-11 the decoded audio-chunk plane ran through
|
|
805
|
-
* `pipeline-orchestrator`: it drained the broker's FIFO, accumulated ~1 s
|
|
806
|
-
* windows and pushed them back out to `audioAnalyzer.analyseChunk`. The
|
|
807
|
-
* orchestrator neither produced nor consumed the audio — it buffered it, and
|
|
808
|
-
* every byte crossed hub-main twice for the privilege. The poller's own
|
|
809
|
-
* docblock named the reason, and it was an accident: the code had been a
|
|
810
|
-
* closure INSIDE the broker's process, and when the `pipeline` co-location
|
|
811
|
-
* group was dissolved nobody moved it.
|
|
812
|
-
*
|
|
813
|
-
* D455 proved the relay was the PREMISE of the remaining cost rather than an
|
|
814
|
-
* alternative to it. It put the SOURCE bytes on the plane — one G.711 byte per
|
|
815
|
-
* sample instead of four f32le ones — but only for a consumer that declared it
|
|
816
|
-
* could read them, and the accumulator (the declaring consumer) sat in a third
|
|
817
|
-
* process that had to expand before forwarding. So legs one and two went coded
|
|
818
|
-
* and legs three and four stayed at ~49 MB/min each. Here the subscriber IS
|
|
819
|
-
* the decoder: there is no third interlocutor, the negotiation needs none, and
|
|
820
|
-
* legs three and four do not exist.
|
|
821
|
-
*
|
|
822
|
-
* ## What the orchestrator still owns
|
|
823
|
-
*
|
|
824
|
-
* Everything about WHETHER and WHERE, which is all policy: the `audioMode`
|
|
825
|
-
* gate, the on-motion window, the audio-track probe, the per-device node
|
|
826
|
-
* assignment and the settings read. It calls `attachDevice` / `detachDevice`
|
|
827
|
-
* and holds the teardown. What it no longer owns is the bytes.
|
|
828
|
-
*
|
|
829
|
-
* ## The failure this split ADDS, and where it is caught
|
|
830
|
-
*
|
|
831
|
-
* The broker subscription is now opened by a process the orchestrator does not
|
|
832
|
-
* supervise. A detach that never arrives — an analyzer runner that crashed, a
|
|
833
|
-
* node that went offline mid-teardown — leaves a FIFO nobody drains, and on
|
|
834
|
-
* this plane that is not merely a leak: `AudioChunkPlane.subscriberCount` is
|
|
835
|
-
* broker DEMAND, so an orphan pins the source dial and the decode session open
|
|
836
|
-
* for as long as the broker lives. It is caught in the broker
|
|
837
|
-
* (`AudioChunkPlane`'s idle lease), not here, because a process that has died
|
|
838
|
-
* cannot clean up after itself.
|
|
839
|
-
*/
|
|
811
|
+
/** One over-bound report per camera per this window; the rest are counted. */
|
|
812
|
+
var OVER_BOUND_LOG_INTERVAL_MS = 6e4;
|
|
840
813
|
var AudioDeviceAttachments = class {
|
|
841
814
|
deps;
|
|
842
815
|
attachments = /* @__PURE__ */ new Map();
|
|
@@ -905,6 +878,25 @@ var AudioDeviceAttachments = class {
|
|
|
905
878
|
meta
|
|
906
879
|
});
|
|
907
880
|
});
|
|
881
|
+
let emitChain = Promise.resolve();
|
|
882
|
+
let awaiting = 0;
|
|
883
|
+
let droppedOverBound = 0;
|
|
884
|
+
let lastOverBoundLogMs = null;
|
|
885
|
+
const reportOverBound = () => {
|
|
886
|
+
droppedOverBound += 1;
|
|
887
|
+
const now = Date.now();
|
|
888
|
+
if (lastOverBoundLogMs !== null && now - lastOverBoundLogMs < OVER_BOUND_LOG_INTERVAL_MS) return;
|
|
889
|
+
lastOverBoundLogMs = now;
|
|
890
|
+
logger.warn("audio windows dropped — too many already awaiting a result", {
|
|
891
|
+
tags: { deviceId },
|
|
892
|
+
meta: {
|
|
893
|
+
dropped: droppedOverBound,
|
|
894
|
+
bound: 16,
|
|
895
|
+
awaiting
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
droppedOverBound = 0;
|
|
899
|
+
};
|
|
908
900
|
const teardown = startAudioChunkPoller({
|
|
909
901
|
api,
|
|
910
902
|
brokerId,
|
|
@@ -913,19 +905,39 @@ var AudioDeviceAttachments = class {
|
|
|
913
905
|
accept: ["pcmu", "pcma"],
|
|
914
906
|
pollIntervalMs: isRemoteBroker ? 500 : 200,
|
|
915
907
|
logger,
|
|
916
|
-
onChunk:
|
|
908
|
+
onChunk: (chunk) => {
|
|
909
|
+
let window;
|
|
917
910
|
try {
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
911
|
+
window = accumulator.push(chunk);
|
|
912
|
+
} catch (err) {
|
|
913
|
+
logger.error("audio window accumulate failed — chunk dropped", {
|
|
914
|
+
tags: { deviceId },
|
|
915
|
+
meta: {
|
|
916
|
+
error: require_dist.errMsg(err),
|
|
917
|
+
timestamp: chunk.timestamp
|
|
918
|
+
}
|
|
919
|
+
});
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
if (!window) return;
|
|
923
|
+
if (awaiting >= 16) {
|
|
924
|
+
reportOverBound();
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
awaiting += 1;
|
|
928
|
+
const analysed = this.deps.analyse(window, settings);
|
|
929
|
+
analysed.catch(() => void 0);
|
|
930
|
+
emitChain = emitChain.then(() => analysed).then((result) => {
|
|
921
931
|
if (!result) return;
|
|
922
932
|
this.deps.emitResult(deviceId, buildAudioResultFrame(deviceId, result));
|
|
923
|
-
}
|
|
933
|
+
}).catch((err) => {
|
|
924
934
|
logger.error("Audio analysis failed", {
|
|
925
935
|
tags: { deviceId },
|
|
926
936
|
meta: { error: require_dist.errMsg(err) }
|
|
927
937
|
});
|
|
928
|
-
}
|
|
938
|
+
}).finally(() => {
|
|
939
|
+
awaiting -= 1;
|
|
940
|
+
});
|
|
929
941
|
}
|
|
930
942
|
});
|
|
931
943
|
this.attachments.set(deviceId, {
|
|
@@ -989,6 +1001,112 @@ var AudioDeviceAttachments = class {
|
|
|
989
1001
|
}
|
|
990
1002
|
};
|
|
991
1003
|
//#endregion
|
|
1004
|
+
//#region src/audio-analyzer/audio-classify-queue.ts
|
|
1005
|
+
function emptyCounts() {
|
|
1006
|
+
return {
|
|
1007
|
+
classified: 0,
|
|
1008
|
+
superseded: 0,
|
|
1009
|
+
errors: 0,
|
|
1010
|
+
disposed: 0
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
var AudioClassifyQueue = class {
|
|
1014
|
+
/** Pending job per camera, in arrival order. */
|
|
1015
|
+
pending = /* @__PURE__ */ new Map();
|
|
1016
|
+
counts = /* @__PURE__ */ new Map();
|
|
1017
|
+
running = false;
|
|
1018
|
+
closed = false;
|
|
1019
|
+
/** True while an inference is running or a window is waiting for one. */
|
|
1020
|
+
get busy() {
|
|
1021
|
+
return this.running || this.pending.size > 0;
|
|
1022
|
+
}
|
|
1023
|
+
/**
|
|
1024
|
+
* Queue `run` for camera `key`. Resolves when it ran (`ran: true`), or when
|
|
1025
|
+
* it was superseded / the queue closed first (`ran: false`). A job that
|
|
1026
|
+
* throws rejects this promise with its error, and is counted as an error.
|
|
1027
|
+
*/
|
|
1028
|
+
submit(key, run) {
|
|
1029
|
+
if (this.closed) {
|
|
1030
|
+
this.count(key, "disposed");
|
|
1031
|
+
return Promise.resolve({
|
|
1032
|
+
ran: false,
|
|
1033
|
+
reason: "disposed"
|
|
1034
|
+
});
|
|
1035
|
+
}
|
|
1036
|
+
return new Promise((resolve, reject) => {
|
|
1037
|
+
const previous = this.pending.get(key);
|
|
1038
|
+
if (previous) {
|
|
1039
|
+
this.count(key, "superseded");
|
|
1040
|
+
previous.settle({
|
|
1041
|
+
ran: false,
|
|
1042
|
+
reason: "superseded"
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
this.pending.set(key, {
|
|
1046
|
+
run,
|
|
1047
|
+
settle: resolve,
|
|
1048
|
+
fail: reject
|
|
1049
|
+
});
|
|
1050
|
+
this.drain();
|
|
1051
|
+
});
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Hand back, and reset, the per-camera counts for every camera that had any
|
|
1055
|
+
* activity since the last call.
|
|
1056
|
+
*/
|
|
1057
|
+
takeAccounting() {
|
|
1058
|
+
const out = [];
|
|
1059
|
+
for (const [key, c] of this.counts) out.push({
|
|
1060
|
+
key,
|
|
1061
|
+
...c
|
|
1062
|
+
});
|
|
1063
|
+
this.counts.clear();
|
|
1064
|
+
return out;
|
|
1065
|
+
}
|
|
1066
|
+
/** Answer every waiting window `disposed` and refuse new ones. */
|
|
1067
|
+
close() {
|
|
1068
|
+
this.closed = true;
|
|
1069
|
+
for (const [key, job] of this.pending) {
|
|
1070
|
+
this.count(key, "disposed");
|
|
1071
|
+
job.settle({
|
|
1072
|
+
ran: false,
|
|
1073
|
+
reason: "disposed"
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
this.pending.clear();
|
|
1077
|
+
}
|
|
1078
|
+
async drain() {
|
|
1079
|
+
if (this.running) return;
|
|
1080
|
+
this.running = true;
|
|
1081
|
+
try {
|
|
1082
|
+
for (;;) {
|
|
1083
|
+
const next = this.pending.entries().next();
|
|
1084
|
+
if (next.done === true) return;
|
|
1085
|
+
const [key, job] = next.value;
|
|
1086
|
+
this.pending.delete(key);
|
|
1087
|
+
try {
|
|
1088
|
+
const value = await job.run();
|
|
1089
|
+
this.count(key, "classified");
|
|
1090
|
+
job.settle({
|
|
1091
|
+
ran: true,
|
|
1092
|
+
value
|
|
1093
|
+
});
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
this.count(key, "errors");
|
|
1096
|
+
job.fail(err);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
} finally {
|
|
1100
|
+
this.running = false;
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
count(key, field) {
|
|
1104
|
+
const c = this.counts.get(key) ?? emptyCounts();
|
|
1105
|
+
c[field] += 1;
|
|
1106
|
+
this.counts.set(key, c);
|
|
1107
|
+
}
|
|
1108
|
+
};
|
|
1109
|
+
//#endregion
|
|
992
1110
|
//#region src/audio-analyzer/addons/analyzer/index.ts
|
|
993
1111
|
/**
|
|
994
1112
|
* AudioAnalyzerProvider — implements IAudioAnalyzer.
|
|
@@ -998,9 +1116,26 @@ var AudioDeviceAttachments = class {
|
|
|
998
1116
|
* to a separate audio-classifier addon.
|
|
999
1117
|
*/
|
|
1000
1118
|
var CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
|
|
1001
|
-
var
|
|
1119
|
+
var INFERENCE_TIMEOUT_MS = 5e3;
|
|
1120
|
+
/** An inference that did not answer within {@link INFERENCE_TIMEOUT_MS}. */
|
|
1121
|
+
var AudioInferenceTimeoutError = class extends Error {
|
|
1122
|
+
timeoutMs;
|
|
1123
|
+
constructor(timeoutMs) {
|
|
1124
|
+
super(`audio inference did not answer within ${timeoutMs} ms`);
|
|
1125
|
+
this.timeoutMs = timeoutMs;
|
|
1126
|
+
this.name = "AudioInferenceTimeoutError";
|
|
1127
|
+
}
|
|
1128
|
+
};
|
|
1129
|
+
var CLASSIFY_ACCOUNTING_PERIOD_MS = 6e4;
|
|
1130
|
+
var NEAR_MISS_WATCHED_MACROS = new Set(["crying", "scream"]);
|
|
1131
|
+
var NEAR_MISS_LOG_INTERVAL_MS = 6e4;
|
|
1002
1132
|
var PIPELINE_IDLE_DISPOSE_MS = 5 * 6e4;
|
|
1003
1133
|
var GLOBAL_DEVICE_KEY = -1;
|
|
1134
|
+
/** The labels a window cleared the camera's floor (and allow-list) with. */
|
|
1135
|
+
function labelsAboveFloor(result, settings) {
|
|
1136
|
+
const allowed = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
|
|
1137
|
+
return result.labels.filter((c) => c.score >= settings.minConfidence && (allowed === null || allowed.has(c.className.toLowerCase())));
|
|
1138
|
+
}
|
|
1004
1139
|
/** What the classifier consumes natively; anything else is resampled next to it. */
|
|
1005
1140
|
var CLASSIFIER_SAMPLE_RATE = 16e3;
|
|
1006
1141
|
var CLASSIFIER_CHANNELS = 1;
|
|
@@ -1034,14 +1169,15 @@ var AudioAnalyzerProvider = class {
|
|
|
1034
1169
|
/** When true, logs a raw-label sample every 100 classifications (opt-in
|
|
1035
1170
|
* debug aid). Off by default — the watchdog heartbeat covers liveness. */
|
|
1036
1171
|
debugClassifySamples = false;
|
|
1037
|
-
/**
|
|
1038
|
-
|
|
1172
|
+
/** The one line every camera's windows wait in for the single pipeline (D660).
|
|
1173
|
+
* Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
|
|
1174
|
+
classifyQueue = new AudioClassifyQueue();
|
|
1175
|
+
/** Periodic per-camera classify accounting (D660). */
|
|
1176
|
+
accountingTimer;
|
|
1177
|
+
/** Near-miss line rate limit, per camera (D660). */
|
|
1178
|
+
nearMissLog = /* @__PURE__ */ new Map();
|
|
1039
1179
|
/** Last window format seen per camera — the first and every change is logged (D450). */
|
|
1040
1180
|
windowFormats = /* @__PURE__ */ new Map();
|
|
1041
|
-
/** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
|
|
1042
|
-
* run at a time. Without this, concurrent calls from different cameras overwrite the
|
|
1043
|
-
* single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
|
|
1044
|
-
pipelineBusy = false;
|
|
1045
1181
|
/** Live inference pipeline. LAZY: null until the first classify() spawns it
|
|
1046
1182
|
* (see `ensurePipeline`); idle-disposed back to null after a long quiet
|
|
1047
1183
|
* window so a node hosting the addon with no consumer never keeps the
|
|
@@ -1075,6 +1211,29 @@ var AudioAnalyzerProvider = class {
|
|
|
1075
1211
|
restart: () => this.restartPipelineForMemory()
|
|
1076
1212
|
});
|
|
1077
1213
|
this.memoryGuard.start();
|
|
1214
|
+
this.accountingTimer = setInterval(() => this.reportClassifyAccounting(), CLASSIFY_ACCOUNTING_PERIOD_MS);
|
|
1215
|
+
if (typeof this.accountingTimer.unref === "function") this.accountingTimer.unref();
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* One line per camera that lost a window since the last report (D660):
|
|
1219
|
+
* how many were classified, superseded by the camera's own newer window,
|
|
1220
|
+
* failed, or cut off by shutdown. Never one line per window, and nothing
|
|
1221
|
+
* for a camera that lost none.
|
|
1222
|
+
*/
|
|
1223
|
+
reportClassifyAccounting() {
|
|
1224
|
+
for (const a of this.classifyQueue.takeAccounting()) {
|
|
1225
|
+
if (a.superseded + a.errors + a.disposed === 0) continue;
|
|
1226
|
+
this.log.info("audio classify accounting", {
|
|
1227
|
+
tags: a.key === GLOBAL_DEVICE_KEY ? void 0 : { deviceId: a.key },
|
|
1228
|
+
meta: {
|
|
1229
|
+
classified: a.classified,
|
|
1230
|
+
superseded: a.superseded,
|
|
1231
|
+
errors: a.errors,
|
|
1232
|
+
disposed: a.disposed,
|
|
1233
|
+
periodMs: CLASSIFY_ACCOUNTING_PERIOD_MS
|
|
1234
|
+
}
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
1078
1237
|
}
|
|
1079
1238
|
/** Memory watchdog for the inference subprocess (see constructor). */
|
|
1080
1239
|
memoryGuard;
|
|
@@ -1166,7 +1325,7 @@ var AudioAnalyzerProvider = class {
|
|
|
1166
1325
|
*/
|
|
1167
1326
|
async disposeIdlePipeline() {
|
|
1168
1327
|
if (this.disposed) return;
|
|
1169
|
-
if (this.
|
|
1328
|
+
if (this.classifyQueue.busy || this.pipelineInitPromise) {
|
|
1170
1329
|
this.scheduleIdleDispose();
|
|
1171
1330
|
return;
|
|
1172
1331
|
}
|
|
@@ -1227,67 +1386,133 @@ var AudioAnalyzerProvider = class {
|
|
|
1227
1386
|
rms: Math.round(rms * 1e4) / 1e4,
|
|
1228
1387
|
dbfs: Math.round(dbfs * 10) / 10
|
|
1229
1388
|
};
|
|
1230
|
-
|
|
1231
|
-
try {
|
|
1232
|
-
const result = await this.classify(chunk);
|
|
1233
|
-
if (this.classifyCallCount < 3) {
|
|
1234
|
-
const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
|
|
1235
|
-
this.log.info("classify debug sample", {
|
|
1236
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1237
|
-
meta: {
|
|
1238
|
-
index: this.classifyCallCount,
|
|
1239
|
-
labelCount: result.labels.length,
|
|
1240
|
-
top: topRaw,
|
|
1241
|
-
inferenceMs: result.inferenceMs,
|
|
1242
|
-
minConf: settings.minConfidence,
|
|
1243
|
-
allowedClasses: settings.allowedClasses
|
|
1244
|
-
}
|
|
1245
|
-
});
|
|
1246
|
-
}
|
|
1247
|
-
this.classifyCallCount++;
|
|
1248
|
-
const meaningful = result.labels.filter((l) => l.score >= .15 && l.className.toLowerCase() !== "silence");
|
|
1249
|
-
if (meaningful.length > 0) this.log.debug("audio classification", {
|
|
1250
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1251
|
-
meta: {
|
|
1252
|
-
top: meaningful.slice(0, 4).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1253
|
-
inferenceMs: result.inferenceMs
|
|
1254
|
-
}
|
|
1255
|
-
});
|
|
1256
|
-
if (result.inferenceMs > 0) {
|
|
1257
|
-
const minConf = settings.minConfidence;
|
|
1258
|
-
const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
|
|
1259
|
-
let filtered = result.labels.filter((c) => c.score >= minConf);
|
|
1260
|
-
if (allowedSet) filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
|
|
1261
|
-
if (filtered.length > 0) classification = {
|
|
1262
|
-
labels: filtered,
|
|
1263
|
-
inferenceMs: result.inferenceMs
|
|
1264
|
-
};
|
|
1265
|
-
}
|
|
1266
|
-
} catch (err) {
|
|
1267
|
-
const now = Date.now();
|
|
1268
|
-
if (now - this.lastClassifyErrorMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
|
|
1269
|
-
const suppressed = this.suppressedClassifyErrors;
|
|
1270
|
-
this.suppressedClassifyErrors = 0;
|
|
1271
|
-
this.lastClassifyErrorMs = now;
|
|
1272
|
-
const msg = require_dist.errMsg(err);
|
|
1273
|
-
const stack = err instanceof Error ? err.stack : void 0;
|
|
1274
|
-
this.log.warn("Audio classification failed", {
|
|
1275
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1276
|
-
meta: {
|
|
1277
|
-
error: msg,
|
|
1278
|
-
stack,
|
|
1279
|
-
suppressedSince: suppressed > 0 ? suppressed : void 0
|
|
1280
|
-
}
|
|
1281
|
-
});
|
|
1282
|
-
} else this.suppressedClassifyErrors++;
|
|
1283
|
-
}
|
|
1389
|
+
const { classification, classifyOutcome } = await this.classifyForAnalysis(chunk, settings);
|
|
1284
1390
|
return {
|
|
1285
1391
|
level,
|
|
1286
1392
|
classification,
|
|
1393
|
+
classifyOutcome,
|
|
1287
1394
|
timestamp: chunk.timestamp
|
|
1288
1395
|
};
|
|
1289
1396
|
}
|
|
1290
1397
|
/**
|
|
1398
|
+
* Run one window through the shared queue and apply the camera's floor.
|
|
1399
|
+
* Every exit names whether the model RAN (`classifyOutcome`, D660): a window
|
|
1400
|
+
* superseded by the camera's newer one, or whose inference failed, is
|
|
1401
|
+
* `not-classified` — never an empty classification.
|
|
1402
|
+
*/
|
|
1403
|
+
async classifyForAnalysis(chunk, settings) {
|
|
1404
|
+
let submission;
|
|
1405
|
+
try {
|
|
1406
|
+
submission = await this.classifyWindow(chunk);
|
|
1407
|
+
} catch (err) {
|
|
1408
|
+
const timedOut = err instanceof AudioInferenceTimeoutError;
|
|
1409
|
+
if (!timedOut) this.reportClassifyError(chunk, err);
|
|
1410
|
+
return {
|
|
1411
|
+
classification: void 0,
|
|
1412
|
+
classifyOutcome: {
|
|
1413
|
+
state: "not-classified",
|
|
1414
|
+
reason: timedOut ? "timeout" : "error"
|
|
1415
|
+
}
|
|
1416
|
+
};
|
|
1417
|
+
}
|
|
1418
|
+
if (!submission.ran) return {
|
|
1419
|
+
classification: void 0,
|
|
1420
|
+
classifyOutcome: {
|
|
1421
|
+
state: "not-classified",
|
|
1422
|
+
reason: submission.reason
|
|
1423
|
+
}
|
|
1424
|
+
};
|
|
1425
|
+
const result = submission.value;
|
|
1426
|
+
this.logClassifySample(chunk, result, settings);
|
|
1427
|
+
this.reportNearMiss(chunk, result, settings);
|
|
1428
|
+
const filtered = labelsAboveFloor(result, settings);
|
|
1429
|
+
return {
|
|
1430
|
+
classification: filtered.length > 0 ? {
|
|
1431
|
+
labels: filtered,
|
|
1432
|
+
inferenceMs: result.inferenceMs
|
|
1433
|
+
} : void 0,
|
|
1434
|
+
classifyOutcome: { state: "classified" }
|
|
1435
|
+
};
|
|
1436
|
+
}
|
|
1437
|
+
/** First-three debug samples + the per-window debug line (unchanged behaviour). */
|
|
1438
|
+
logClassifySample(chunk, result, settings) {
|
|
1439
|
+
const tags = chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0;
|
|
1440
|
+
if (this.classifyCallCount < 3) this.log.info("classify debug sample", {
|
|
1441
|
+
tags,
|
|
1442
|
+
meta: {
|
|
1443
|
+
index: this.classifyCallCount,
|
|
1444
|
+
labelCount: result.labels.length,
|
|
1445
|
+
top: result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1446
|
+
inferenceMs: result.inferenceMs,
|
|
1447
|
+
minConf: settings.minConfidence,
|
|
1448
|
+
allowedClasses: settings.allowedClasses
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
this.classifyCallCount++;
|
|
1452
|
+
const meaningful = result.labels.filter((l) => l.score >= .15 && l.className.toLowerCase() !== "silence");
|
|
1453
|
+
if (meaningful.length > 0) this.log.debug("audio classification", {
|
|
1454
|
+
tags,
|
|
1455
|
+
meta: {
|
|
1456
|
+
top: meaningful.slice(0, 4).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1457
|
+
inferenceMs: result.inferenceMs
|
|
1458
|
+
}
|
|
1459
|
+
});
|
|
1460
|
+
}
|
|
1461
|
+
/**
|
|
1462
|
+
* A watched macro (crying, scream) the model heard BELOW this camera's
|
|
1463
|
+
* floor — one line per camera per {@link NEAR_MISS_LOG_INTERVAL_MS}, the
|
|
1464
|
+
* rest counted into the next line (D660). This is the line that tells an
|
|
1465
|
+
* operator whose rule never fires whether the sound was heard at all.
|
|
1466
|
+
*/
|
|
1467
|
+
reportNearMiss(chunk, result, settings) {
|
|
1468
|
+
const miss = result.labels.find((l) => NEAR_MISS_WATCHED_MACROS.has(l.className) && l.score < settings.minConfidence);
|
|
1469
|
+
if (miss === void 0) return;
|
|
1470
|
+
const key = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
|
|
1471
|
+
const now = Date.now();
|
|
1472
|
+
const prev = this.nearMissLog.get(key) ?? null;
|
|
1473
|
+
if (prev !== null && now - prev.lastLoggedMs < NEAR_MISS_LOG_INTERVAL_MS) {
|
|
1474
|
+
this.nearMissLog.set(key, {
|
|
1475
|
+
...prev,
|
|
1476
|
+
suppressed: prev.suppressed + 1
|
|
1477
|
+
});
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
this.nearMissLog.set(key, {
|
|
1481
|
+
lastLoggedMs: now,
|
|
1482
|
+
suppressed: 0
|
|
1483
|
+
});
|
|
1484
|
+
this.log.info("audio near-miss below floor", {
|
|
1485
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1486
|
+
meta: {
|
|
1487
|
+
label: miss.className,
|
|
1488
|
+
originalClass: miss.originalClass,
|
|
1489
|
+
score: miss.score,
|
|
1490
|
+
floor: settings.minConfidence,
|
|
1491
|
+
suppressedSince: prev?.suppressed ?? 0,
|
|
1492
|
+
intervalMs: NEAR_MISS_LOG_INTERVAL_MS
|
|
1493
|
+
}
|
|
1494
|
+
});
|
|
1495
|
+
}
|
|
1496
|
+
/** Rate-limited classify-failure line (unchanged behaviour). */
|
|
1497
|
+
reportClassifyError(chunk, err) {
|
|
1498
|
+
const now = Date.now();
|
|
1499
|
+
if (now - this.lastClassifyErrorMs < CLASSIFY_ERROR_SUPPRESS_MS) {
|
|
1500
|
+
this.suppressedClassifyErrors++;
|
|
1501
|
+
return;
|
|
1502
|
+
}
|
|
1503
|
+
const suppressed = this.suppressedClassifyErrors;
|
|
1504
|
+
this.suppressedClassifyErrors = 0;
|
|
1505
|
+
this.lastClassifyErrorMs = now;
|
|
1506
|
+
this.log.warn("Audio classification failed", {
|
|
1507
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1508
|
+
meta: {
|
|
1509
|
+
error: require_dist.errMsg(err),
|
|
1510
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
1511
|
+
suppressedSince: suppressed > 0 ? suppressed : void 0
|
|
1512
|
+
}
|
|
1513
|
+
});
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1291
1516
|
* Say what this camera's windows arrive as — once, and again on every
|
|
1292
1517
|
* change. The plane carries the source rate (8 kHz for G.711 since D450,
|
|
1293
1518
|
* 16 kHz for AAC through the codec session), and the classifier resamples
|
|
@@ -1319,33 +1544,31 @@ var AudioAnalyzerProvider = class {
|
|
|
1319
1544
|
}
|
|
1320
1545
|
});
|
|
1321
1546
|
}
|
|
1547
|
+
/**
|
|
1548
|
+
* The cap's `classify`: one window through the same queue every camera
|
|
1549
|
+
* uses. A window that never reached the model is an ERROR here, not an
|
|
1550
|
+
* empty result — the caller asked for a classification and did not get one.
|
|
1551
|
+
*/
|
|
1322
1552
|
async classify(chunk) {
|
|
1323
|
-
const
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
lastEndMs: state?.lastEndMs ?? 0
|
|
1339
|
-
});
|
|
1340
|
-
this.pipelineBusy = true;
|
|
1553
|
+
const submission = await this.classifyWindow(chunk);
|
|
1554
|
+
if (!submission.ran) throw new Error(`audio classify not run: ${submission.reason}`);
|
|
1555
|
+
return submission.value;
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Queue one window for the single pipeline (D660). Per camera at most one
|
|
1559
|
+
* waits; a newer window from the same camera supersedes it and takes its
|
|
1560
|
+
* place in line. Cameras are served in arrival order. Rejects with the
|
|
1561
|
+
* inference's own error.
|
|
1562
|
+
*/
|
|
1563
|
+
classifyWindow(chunk) {
|
|
1564
|
+
return this.classifyQueue.submit(chunk.deviceId ?? GLOBAL_DEVICE_KEY, () => this.runInference(chunk));
|
|
1565
|
+
}
|
|
1566
|
+
/** One inference on the live pipeline. Only ever called by the queue — never two at once. */
|
|
1567
|
+
async runInference(chunk) {
|
|
1341
1568
|
try {
|
|
1342
1569
|
const pipeline = await this.ensurePipeline();
|
|
1343
1570
|
const f32Data = float32FromBytes(chunk.data);
|
|
1344
|
-
const result = await
|
|
1345
|
-
data: f32Data,
|
|
1346
|
-
sampleRate: chunk.sampleRate,
|
|
1347
|
-
channels: chunk.channels
|
|
1348
|
-
});
|
|
1571
|
+
const result = await this.boundedInference(pipeline, chunk, f32Data);
|
|
1349
1572
|
if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
|
|
1350
1573
|
const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
|
|
1351
1574
|
this.log.info("classify debug sample", {
|
|
@@ -1384,20 +1607,63 @@ var AudioAnalyzerProvider = class {
|
|
|
1384
1607
|
inferenceMs: result.inferenceMs
|
|
1385
1608
|
};
|
|
1386
1609
|
} finally {
|
|
1387
|
-
this.pipelineBusy = false;
|
|
1388
|
-
this.cameraState.set(camKey, {
|
|
1389
|
-
inProgress: false,
|
|
1390
|
-
lastEndMs: Date.now()
|
|
1391
|
-
});
|
|
1392
1610
|
this.scheduleIdleDispose();
|
|
1393
1611
|
}
|
|
1394
1612
|
}
|
|
1613
|
+
/**
|
|
1614
|
+
* One inference under {@link INFERENCE_TIMEOUT_MS}. A backend that timed
|
|
1615
|
+
* out, exited or was disposed under the call is DROPPED (killed, and the
|
|
1616
|
+
* next window lazily respawns it) — a dead pipeline kept as `this.pipeline`
|
|
1617
|
+
* would answer every later window with the same error forever.
|
|
1618
|
+
*/
|
|
1619
|
+
async boundedInference(pipeline, chunk, data) {
|
|
1620
|
+
const inference = pipeline.classify({
|
|
1621
|
+
data,
|
|
1622
|
+
sampleRate: chunk.sampleRate,
|
|
1623
|
+
channels: chunk.channels
|
|
1624
|
+
});
|
|
1625
|
+
inference.catch(() => void 0);
|
|
1626
|
+
let timer = null;
|
|
1627
|
+
const timeout = new Promise((_, reject) => {
|
|
1628
|
+
timer = setTimeout(() => reject(new AudioInferenceTimeoutError(INFERENCE_TIMEOUT_MS)), INFERENCE_TIMEOUT_MS);
|
|
1629
|
+
});
|
|
1630
|
+
try {
|
|
1631
|
+
return await Promise.race([inference, timeout]);
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
if (err instanceof AudioInferenceTimeoutError) {
|
|
1634
|
+
this.log.warn("audio inference timed out — killing the backend, next window respawns it", {
|
|
1635
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1636
|
+
meta: {
|
|
1637
|
+
timeoutMs: INFERENCE_TIMEOUT_MS,
|
|
1638
|
+
backend: this.backendName
|
|
1639
|
+
}
|
|
1640
|
+
});
|
|
1641
|
+
this.dropPipeline(pipeline);
|
|
1642
|
+
} else if (err instanceof AudioBackendExitedError) this.dropPipeline(pipeline);
|
|
1643
|
+
throw err;
|
|
1644
|
+
} finally {
|
|
1645
|
+
if (timer !== null) clearTimeout(timer);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
/** Forget (if still current) and dispose a pipeline that can no longer answer. */
|
|
1649
|
+
dropPipeline(pipeline) {
|
|
1650
|
+
if (this.pipeline === pipeline) this.pipeline = null;
|
|
1651
|
+
pipeline.dispose().catch((err) => {
|
|
1652
|
+
this.log.warn("audio pipeline dispose after failure threw", { meta: {
|
|
1653
|
+
error: require_dist.errMsg(err),
|
|
1654
|
+
backend: this.backendName
|
|
1655
|
+
} });
|
|
1656
|
+
});
|
|
1657
|
+
}
|
|
1395
1658
|
isReady() {
|
|
1396
1659
|
return !this.disposed;
|
|
1397
1660
|
}
|
|
1398
1661
|
async dispose() {
|
|
1399
1662
|
this.disposed = true;
|
|
1400
1663
|
this.memoryGuard.stop();
|
|
1664
|
+
clearInterval(this.accountingTimer);
|
|
1665
|
+
this.classifyQueue.close();
|
|
1666
|
+
this.reportClassifyAccounting();
|
|
1401
1667
|
if (this.idleTimer) {
|
|
1402
1668
|
clearTimeout(this.idleTimer);
|
|
1403
1669
|
this.idleTimer = null;
|