@camstack/addon-pipeline 1.2.297 → 1.2.299
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_MODELS.md +126 -123
- package/dist/audio-analyzer/index.js +571 -305
- package/dist/audio-analyzer/index.mjs +571 -305
- package/dist/{default-detection-model-D6DypCkI.mjs → default-detection-model-DMcHBVmE.mjs} +391 -170
- package/dist/{default-detection-model-B2n_SFXt.js → default-detection-model-H_NzHI_P.js} +391 -170
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +4 -4
- package/dist/{dist-DUqr1zyq.js → dist-BzDPLvQ6.js} +73 -4
- package/dist/{dist-CxHIpsQs.mjs → dist-DNlqf8dr.mjs} +73 -4
- package/dist/motion-wasm/index.js +1 -1
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/{node-Dx6DLQ1j.js → node-B-cB_Hdy.js} +1 -1
- package/dist/{node-DyeWu78a.mjs → node-g2ZRFDvc.mjs} +1 -1
- package/dist/pipeline-runner/index.js +3 -3
- package/dist/pipeline-runner/index.mjs +3 -3
- package/dist/{process-memory-CMJ3NY-s.js → process-memory-Bo3-laZ_.js} +1 -1
- package/dist/{process-memory-BVR4592X.mjs → process-memory-kKYoJ9hl.mjs} +1 -1
- package/dist/recorder/index.js +3 -3
- package/dist/recorder/index.mjs +3 -3
- package/dist/{segment-demux-js-CmGIF_uK.js → segment-demux-js-DrDj8kP6.js} +1 -1
- package/dist/{segment-demux-js-Dzga-XgE.mjs → segment-demux-js-QK7qFMSj.mjs} +1 -1
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-P71ze8cu.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-Da_bbYQp.mjs} +1 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CxT9ry2j.mjs +26 -0
- package/dist/stream-broker/demux-worker-child.js +1 -1
- package/dist/stream-broker/demux-worker-child.mjs +1 -1
- package/dist/stream-broker/{hostInit-CRlStbz4.mjs → hostInit-aFrOa5WT.mjs} +1 -1
- package/dist/stream-broker/index.js +3 -3
- package/dist/stream-broker/index.mjs +3 -3
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/python/postprocessors/softmax.py +2 -1
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-CZkpFU-J.mjs +0 -26
|
@@ -1,9 +1,163 @@
|
|
|
1
1
|
import { n as __require } from "../chunk-DnnnRqeS.mjs";
|
|
2
|
-
import { E as HF_BASE_URL, G as audioAnalyzerCapability, Rn as EventCategory, Ut as resolvePoolMemoryPolicy, W as audioAnalysisCapability, _n as hydrateSchema, bt as mapAudioLabelToMacro, en as errMsg, fn as audioChunkBytesPerSample, gn as expandAudioChunkToF32le, j as PoolMemoryWatchdog, n as AUDIO_BACKEND_CHOICES, p as DEFAULT_AUDIO_ANALYZER_CONFIG, sn as BaseAddon, xn as nodePin } from "../dist-
|
|
3
|
-
import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-
|
|
2
|
+
import { E as HF_BASE_URL, G as audioAnalyzerCapability, Rn as EventCategory, Ut as resolvePoolMemoryPolicy, W as audioAnalysisCapability, _n as hydrateSchema, bt as mapAudioLabelToMacro, en as errMsg, fn as audioChunkBytesPerSample, gn as expandAudioChunkToF32le, j as PoolMemoryWatchdog, n as AUDIO_BACKEND_CHOICES, p as DEFAULT_AUDIO_ANALYZER_CONFIG, sn as BaseAddon, xn as nodePin } from "../dist-DNlqf8dr.mjs";
|
|
3
|
+
import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-kKYoJ9hl.mjs";
|
|
4
4
|
import * as fs from "node:fs";
|
|
5
5
|
import * as path$1 from "node:path";
|
|
6
6
|
import { downloadFile } from "@camstack/system/addon-utils";
|
|
7
|
+
//#region src/audio-analyzer/audio-subprocess-channel.ts
|
|
8
|
+
/** The backend process ended (or was disposed) while — or before — a request waited on it. */
|
|
9
|
+
var AudioBackendExitedError = class extends Error {
|
|
10
|
+
constructor(message) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.name = "AudioBackendExitedError";
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
/** How long `dispose()` waits for a graceful exit before SIGKILL. */
|
|
16
|
+
var DISPOSE_GRACE_MS = 5e3;
|
|
17
|
+
function isReplyObject(value) {
|
|
18
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
19
|
+
}
|
|
20
|
+
var AudioSubprocessChannel = class {
|
|
21
|
+
label;
|
|
22
|
+
log;
|
|
23
|
+
process = null;
|
|
24
|
+
receiveBuffer = Buffer.alloc(0);
|
|
25
|
+
pending = null;
|
|
26
|
+
constructor(label, log) {
|
|
27
|
+
this.label = label;
|
|
28
|
+
this.log = log;
|
|
29
|
+
}
|
|
30
|
+
/** Wire a freshly spawned process. Stderr stays the caller's (each backend formats it). */
|
|
31
|
+
attach(proc) {
|
|
32
|
+
this.process = proc;
|
|
33
|
+
proc.on("error", (err) => {
|
|
34
|
+
this.log.error(`${this.label} process error`, { meta: { error: err.message } });
|
|
35
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: process error: ${err.message}`));
|
|
36
|
+
});
|
|
37
|
+
proc.on("exit", (code, signal) => {
|
|
38
|
+
if (this.process === proc) this.process = null;
|
|
39
|
+
const clean = code === 0 && signal === null;
|
|
40
|
+
const meta = {
|
|
41
|
+
code,
|
|
42
|
+
signal
|
|
43
|
+
};
|
|
44
|
+
if (clean) this.log.info(`${this.label} process exited`, { meta });
|
|
45
|
+
else this.log.error(`${this.label} process exited`, { meta });
|
|
46
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: process exited (code ${String(code)}, signal ${String(signal)})`));
|
|
47
|
+
});
|
|
48
|
+
proc.stdout?.on("data", (chunk) => {
|
|
49
|
+
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
50
|
+
this.tryReceive();
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
get pid() {
|
|
54
|
+
return this.process?.pid ?? null;
|
|
55
|
+
}
|
|
56
|
+
get alive() {
|
|
57
|
+
return this.process !== null;
|
|
58
|
+
}
|
|
59
|
+
/** Wait for the next reply frame without sending (the init handshake). */
|
|
60
|
+
receive() {
|
|
61
|
+
if (this.process === null) return Promise.reject(new AudioBackendExitedError(`${this.label}: process not running`));
|
|
62
|
+
return new Promise((resolve, reject) => {
|
|
63
|
+
this.pending = {
|
|
64
|
+
resolve,
|
|
65
|
+
reject
|
|
66
|
+
};
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/** Send one payload frame and wait for its reply. */
|
|
70
|
+
request(payload) {
|
|
71
|
+
const stdin = this.process?.stdin;
|
|
72
|
+
if (!stdin) return Promise.reject(new AudioBackendExitedError(`${this.label}: process not initialized`));
|
|
73
|
+
const reply = this.receive();
|
|
74
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
75
|
+
lengthBuf.writeUInt32LE(payload.length, 0);
|
|
76
|
+
stdin.write(Buffer.concat([lengthBuf, payload]));
|
|
77
|
+
return reply;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Stop the process. The waiting request (if any) is rejected FIRST — the
|
|
81
|
+
* exit event would do it too, but a process that ignores SIGTERM for five
|
|
82
|
+
* seconds must not hold the caller for five seconds.
|
|
83
|
+
*/
|
|
84
|
+
async dispose() {
|
|
85
|
+
this.failPending(new AudioBackendExitedError(`${this.label}: disposed`));
|
|
86
|
+
const proc = this.process;
|
|
87
|
+
if (!proc) return;
|
|
88
|
+
this.process = null;
|
|
89
|
+
proc.stdin?.end();
|
|
90
|
+
proc.kill("SIGTERM");
|
|
91
|
+
if (!await new Promise((resolve) => {
|
|
92
|
+
if (proc.exitCode !== null || proc.signalCode !== null) {
|
|
93
|
+
resolve(true);
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
const timer = setTimeout(() => resolve(false), DISPOSE_GRACE_MS);
|
|
97
|
+
proc.once("exit", () => {
|
|
98
|
+
clearTimeout(timer);
|
|
99
|
+
resolve(true);
|
|
100
|
+
});
|
|
101
|
+
})) {
|
|
102
|
+
try {
|
|
103
|
+
proc.kill("SIGKILL");
|
|
104
|
+
} catch {}
|
|
105
|
+
this.log.warn(`${this.label} process did not exit gracefully — sent SIGKILL`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
failPending(err) {
|
|
109
|
+
const pending = this.pending;
|
|
110
|
+
this.pending = null;
|
|
111
|
+
pending?.reject(err);
|
|
112
|
+
}
|
|
113
|
+
tryReceive() {
|
|
114
|
+
while (this.receiveBuffer.length >= 4) {
|
|
115
|
+
const length = this.receiveBuffer.readUInt32LE(0);
|
|
116
|
+
if (this.receiveBuffer.length < 4 + length) return;
|
|
117
|
+
const bytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
118
|
+
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
119
|
+
const pending = this.pending;
|
|
120
|
+
this.pending = null;
|
|
121
|
+
if (!pending) {
|
|
122
|
+
this.log.warn(`${this.label}: reply with no request waiting — discarded`, { meta: { bytes: length } });
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
let parsed;
|
|
126
|
+
try {
|
|
127
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
128
|
+
} catch (err) {
|
|
129
|
+
pending.reject(/* @__PURE__ */ new Error(`${this.label}: unparseable reply: ${errMsg(err)}`));
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (isReplyObject(parsed)) pending.resolve(parsed);
|
|
133
|
+
else pending.reject(/* @__PURE__ */ new Error(`${this.label}: reply is not an object`));
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
function isBackendLabel(value) {
|
|
138
|
+
if (typeof value !== "object" || value === null) return false;
|
|
139
|
+
const className = Reflect.get(value, "className");
|
|
140
|
+
const score = Reflect.get(value, "score");
|
|
141
|
+
return typeof className === "string" && typeof score === "number";
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Read a classify reply. A reply carrying `error` is a FAILED inference and
|
|
145
|
+
* throws (D660): `yamnet_audio.py` answers an exception with an empty label
|
|
146
|
+
* list plus `error`, and reading that as "no labels" is the silence-vs-nobody-
|
|
147
|
+
* listened confusion D660 removed.
|
|
148
|
+
*/
|
|
149
|
+
function parseClassifyReply(label, reply) {
|
|
150
|
+
const error = reply["error"];
|
|
151
|
+
if (typeof error === "string" && error.length > 0) throw new Error(`${label}: inference failed: ${error}`);
|
|
152
|
+
const raw = reply["classifications"];
|
|
153
|
+
const classifications = Array.isArray(raw) ? raw.filter(isBackendLabel) : [];
|
|
154
|
+
const ms = reply["inferenceMs"];
|
|
155
|
+
return {
|
|
156
|
+
classifications,
|
|
157
|
+
inferenceMs: typeof ms === "number" ? ms : 0
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
//#endregion
|
|
7
161
|
//#region src/audio-analyzer/audio-pipeline.ts
|
|
8
162
|
/**
|
|
9
163
|
* Create the appropriate audio pipeline.
|
|
@@ -60,15 +214,14 @@ var YamnetPythonPipeline = class {
|
|
|
60
214
|
pythonPath;
|
|
61
215
|
installPythonRequirements;
|
|
62
216
|
log;
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
pendingResolve = null;
|
|
66
|
-
pendingReject = null;
|
|
217
|
+
/** The wire + reply slot; settles a waiting request on ANY exit (D660). */
|
|
218
|
+
channel;
|
|
67
219
|
constructor(modelsDir, logger, pythonPath, installPythonRequirements) {
|
|
68
220
|
this.modelsDir = modelsDir;
|
|
69
221
|
this.pythonPath = pythonPath;
|
|
70
222
|
this.installPythonRequirements = installPythonRequirements;
|
|
71
223
|
this.log = logger;
|
|
224
|
+
this.channel = new AudioSubprocessChannel("YAMNet Python", logger);
|
|
72
225
|
}
|
|
73
226
|
async initialize() {
|
|
74
227
|
const modelPath = path$1.join(this.modelsDir, "camstack-yamnet.onnx");
|
|
@@ -92,7 +245,7 @@ var YamnetPythonPipeline = class {
|
|
|
92
245
|
if (this.installPythonRequirements) await this.installPythonRequirements(path$1.join(pythonDir, "requirements-audio.txt"));
|
|
93
246
|
const scriptPath = path$1.join(pythonDir, "yamnet_audio.py");
|
|
94
247
|
const { spawn } = await import("node:child_process");
|
|
95
|
-
|
|
248
|
+
const proc = spawn(this.pythonPath, [
|
|
96
249
|
scriptPath,
|
|
97
250
|
"--model",
|
|
98
251
|
modelPath,
|
|
@@ -103,219 +256,79 @@ var YamnetPythonPipeline = class {
|
|
|
103
256
|
"pipe",
|
|
104
257
|
"pipe"
|
|
105
258
|
] });
|
|
106
|
-
|
|
259
|
+
proc.stderr?.on("data", (chunk) => {
|
|
107
260
|
const text = chunk.toString().trim();
|
|
108
261
|
if (text) this.log.warn(text);
|
|
109
262
|
});
|
|
110
|
-
this.
|
|
111
|
-
|
|
112
|
-
this.pendingReject?.(err);
|
|
113
|
-
this.pendingReject = null;
|
|
114
|
-
this.pendingResolve = null;
|
|
115
|
-
});
|
|
116
|
-
this.process.on("exit", (code) => {
|
|
117
|
-
if (code !== 0 && code !== null) {
|
|
118
|
-
this.log.error("YAMNet Python process exited", { meta: { code } });
|
|
119
|
-
const err = /* @__PURE__ */ new Error(`YAMNet Python: process exited with code ${code}`);
|
|
120
|
-
this.pendingReject?.(err);
|
|
121
|
-
this.pendingReject = null;
|
|
122
|
-
this.pendingResolve = null;
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
this.process.stdout.on("data", (chunk) => {
|
|
126
|
-
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
127
|
-
this.tryReceive();
|
|
128
|
-
});
|
|
129
|
-
const ready = await this.receiveMessage();
|
|
263
|
+
this.channel.attach(proc);
|
|
264
|
+
const ready = await this.channel.receive();
|
|
130
265
|
if (ready["status"] !== "ready") throw new Error(`YAMNet Python: unexpected init response: ${JSON.stringify(ready)}`);
|
|
131
266
|
this.log.info(`YAMNet Python pipeline initialized (${String(ready["labels"] ?? "?")} labels)`);
|
|
132
267
|
}
|
|
133
268
|
getPid() {
|
|
134
|
-
return this.
|
|
269
|
+
return this.channel.pid;
|
|
135
270
|
}
|
|
136
271
|
async classify(chunk) {
|
|
137
|
-
if (!this.process?.stdin) throw new Error("YAMNet Python: process not initialized");
|
|
138
272
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
139
|
-
|
|
140
|
-
const lengthBuf = Buffer.allocUnsafe(4);
|
|
141
|
-
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
142
|
-
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
143
|
-
const result = await this.receiveMessage();
|
|
144
|
-
return {
|
|
145
|
-
classifications: result["classifications"] ?? [],
|
|
146
|
-
inferenceMs: result["inferenceMs"] ?? 0
|
|
147
|
-
};
|
|
273
|
+
return parseClassifyReply("YAMNet Python", await this.channel.request(Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength)));
|
|
148
274
|
}
|
|
149
275
|
async dispose() {
|
|
150
|
-
|
|
151
|
-
if (!proc) return;
|
|
152
|
-
this.process = null;
|
|
153
|
-
proc.stdin?.end();
|
|
154
|
-
proc.kill("SIGTERM");
|
|
155
|
-
if (!await new Promise((resolve) => {
|
|
156
|
-
const timer = setTimeout(() => resolve(false), 5e3);
|
|
157
|
-
proc.once("exit", () => {
|
|
158
|
-
clearTimeout(timer);
|
|
159
|
-
resolve(true);
|
|
160
|
-
});
|
|
161
|
-
})) {
|
|
162
|
-
try {
|
|
163
|
-
proc.kill("SIGKILL");
|
|
164
|
-
} catch {}
|
|
165
|
-
this.log.warn("YAMNet Python process did not exit gracefully — sent SIGKILL");
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
receiveMessage() {
|
|
169
|
-
return new Promise((resolve, reject) => {
|
|
170
|
-
this.pendingResolve = resolve;
|
|
171
|
-
this.pendingReject = reject;
|
|
172
|
-
});
|
|
173
|
-
}
|
|
174
|
-
tryReceive() {
|
|
175
|
-
if (this.receiveBuffer.length < 4) return;
|
|
176
|
-
const length = this.receiveBuffer.readUInt32LE(0);
|
|
177
|
-
if (this.receiveBuffer.length < 4 + length) return;
|
|
178
|
-
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
179
|
-
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
180
|
-
const resolve = this.pendingResolve;
|
|
181
|
-
const reject = this.pendingReject;
|
|
182
|
-
this.pendingResolve = null;
|
|
183
|
-
this.pendingReject = null;
|
|
184
|
-
if (!resolve) return;
|
|
185
|
-
try {
|
|
186
|
-
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
187
|
-
} catch (err) {
|
|
188
|
-
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
189
|
-
}
|
|
276
|
+
await this.channel.dispose();
|
|
190
277
|
}
|
|
191
278
|
};
|
|
192
279
|
var AppleSoundAnalysisPipeline = class {
|
|
193
280
|
log;
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
pendingResolve = null;
|
|
197
|
-
pendingReject = null;
|
|
281
|
+
/** The wire + reply slot; settles a waiting request on ANY exit (D660). */
|
|
282
|
+
channel;
|
|
198
283
|
binaryPath = null;
|
|
199
284
|
debugCount = 0;
|
|
200
285
|
constructor(logger) {
|
|
201
286
|
this.log = logger;
|
|
287
|
+
this.channel = new AudioSubprocessChannel("Apple SoundAnalysis", logger);
|
|
202
288
|
}
|
|
203
289
|
async initialize() {
|
|
204
290
|
this.binaryPath = await this.resolveSwiftBinary();
|
|
205
291
|
if (!this.binaryPath) throw new Error("Apple SoundAnalysis: Swift CLI not found and compilation failed. macOS with Xcode CLI tools required.");
|
|
206
292
|
const { spawn } = await import("node:child_process");
|
|
207
|
-
|
|
293
|
+
const proc = spawn(this.binaryPath, ["--sample-rate=16000", "--top-k=10"], { stdio: [
|
|
208
294
|
"pipe",
|
|
209
295
|
"pipe",
|
|
210
296
|
"pipe"
|
|
211
297
|
] });
|
|
212
|
-
|
|
298
|
+
proc.stderr?.on("data", (chunk) => {
|
|
213
299
|
const lines = chunk.toString().split("\n");
|
|
214
300
|
for (const line of lines) {
|
|
215
301
|
const trimmed = line.trim();
|
|
216
302
|
if (trimmed) this.log.warn(trimmed);
|
|
217
303
|
}
|
|
218
304
|
});
|
|
219
|
-
this.
|
|
220
|
-
|
|
221
|
-
this.pendingReject?.(err);
|
|
222
|
-
this.pendingReject = null;
|
|
223
|
-
this.pendingResolve = null;
|
|
224
|
-
});
|
|
225
|
-
this.process.on("exit", (code) => {
|
|
226
|
-
if (code !== 0 && code !== null) {
|
|
227
|
-
this.log.error("Swift process exited", { meta: { code } });
|
|
228
|
-
const err = /* @__PURE__ */ new Error(`Apple SoundAnalysis: process exited with code ${code}`);
|
|
229
|
-
this.pendingReject?.(err);
|
|
230
|
-
this.pendingReject = null;
|
|
231
|
-
this.pendingResolve = null;
|
|
232
|
-
}
|
|
233
|
-
});
|
|
234
|
-
this.process.stdout.on("data", (chunk) => {
|
|
235
|
-
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
236
|
-
this.tryReceive();
|
|
237
|
-
});
|
|
238
|
-
const ready = await this.receiveMessage();
|
|
305
|
+
this.channel.attach(proc);
|
|
306
|
+
const ready = await this.channel.receive();
|
|
239
307
|
if (ready["status"] !== "ready") throw new Error(`Apple SoundAnalysis: unexpected init response: ${JSON.stringify(ready)}`);
|
|
240
308
|
this.log.info("Apple SoundAnalysis pipeline initialized (macOS built-in, Swift CLI bridge)");
|
|
241
309
|
}
|
|
242
310
|
getPid() {
|
|
243
|
-
return this.
|
|
311
|
+
return this.channel.pid;
|
|
244
312
|
}
|
|
245
313
|
async classify(chunk) {
|
|
246
|
-
if (!this.process?.stdin) throw new Error("Apple SoundAnalysis: process not initialized");
|
|
247
314
|
const waveform = chunk.sampleRate === 16e3 && chunk.channels === 1 ? chunk.data : resampleMono16k(chunk);
|
|
248
315
|
const audioBuffer = Buffer.from(waveform.buffer, waveform.byteOffset, waveform.byteLength);
|
|
249
|
-
const
|
|
250
|
-
lengthBuf.writeUInt32LE(audioBuffer.length, 0);
|
|
251
|
-
this.process.stdin.write(Buffer.concat([lengthBuf, audioBuffer]));
|
|
252
|
-
const result = await this.receiveMessage();
|
|
253
|
-
const classifications = result["classifications"] ?? [];
|
|
254
|
-
const inferenceMs = result["inferenceMs"] ?? 0;
|
|
316
|
+
const reply = await this.channel.request(audioBuffer);
|
|
255
317
|
if (this.debugCount < 3) {
|
|
256
|
-
const keys = Object.keys(result);
|
|
257
318
|
this.log.info("classify debug sample", { meta: {
|
|
258
319
|
phase: "apple-sa",
|
|
259
320
|
index: this.debugCount,
|
|
260
|
-
keys,
|
|
261
|
-
|
|
262
|
-
inferenceMs,
|
|
263
|
-
audioBytes: Buffer.from(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength).length,
|
|
321
|
+
keys: Object.keys(reply),
|
|
322
|
+
audioBytes: audioBuffer.length,
|
|
264
323
|
sampleRate: chunk.sampleRate,
|
|
265
324
|
channels: chunk.channels
|
|
266
325
|
} });
|
|
267
|
-
if (result["error"]) this.log.error("Swift error", { meta: {
|
|
268
|
-
phase: "apple-sa",
|
|
269
|
-
error: result["error"]
|
|
270
|
-
} });
|
|
271
326
|
this.debugCount++;
|
|
272
327
|
}
|
|
273
|
-
return
|
|
274
|
-
classifications,
|
|
275
|
-
inferenceMs
|
|
276
|
-
};
|
|
328
|
+
return parseClassifyReply("Apple SoundAnalysis", reply);
|
|
277
329
|
}
|
|
278
330
|
async dispose() {
|
|
279
|
-
|
|
280
|
-
if (!proc) return;
|
|
281
|
-
this.process = null;
|
|
282
|
-
proc.stdin?.end();
|
|
283
|
-
proc.kill("SIGTERM");
|
|
284
|
-
if (!await new Promise((resolve) => {
|
|
285
|
-
const timer = setTimeout(() => resolve(false), 5e3);
|
|
286
|
-
proc.once("exit", () => {
|
|
287
|
-
clearTimeout(timer);
|
|
288
|
-
resolve(true);
|
|
289
|
-
});
|
|
290
|
-
})) {
|
|
291
|
-
try {
|
|
292
|
-
proc.kill("SIGKILL");
|
|
293
|
-
} catch {}
|
|
294
|
-
this.log.warn("Swift process did not exit gracefully — sent SIGKILL");
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
receiveMessage() {
|
|
298
|
-
return new Promise((resolve, reject) => {
|
|
299
|
-
this.pendingResolve = resolve;
|
|
300
|
-
this.pendingReject = reject;
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
tryReceive() {
|
|
304
|
-
if (this.receiveBuffer.length < 4) return;
|
|
305
|
-
const length = this.receiveBuffer.readUInt32LE(0);
|
|
306
|
-
if (this.receiveBuffer.length < 4 + length) return;
|
|
307
|
-
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
308
|
-
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
309
|
-
const resolve = this.pendingResolve;
|
|
310
|
-
const reject = this.pendingReject;
|
|
311
|
-
this.pendingResolve = null;
|
|
312
|
-
this.pendingReject = null;
|
|
313
|
-
if (!resolve) return;
|
|
314
|
-
try {
|
|
315
|
-
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
316
|
-
} catch (err) {
|
|
317
|
-
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
318
|
-
}
|
|
331
|
+
await this.channel.dispose();
|
|
319
332
|
}
|
|
320
333
|
/** Find pre-compiled binary or compile from Swift source. */
|
|
321
334
|
async resolveSwiftBinary() {
|
|
@@ -778,6 +791,7 @@ function buildAudioResultFrame(deviceId, result) {
|
|
|
778
791
|
dbfs: result.level.dbfs
|
|
779
792
|
},
|
|
780
793
|
detections: audioDetections,
|
|
794
|
+
classifyOutcome: result.classifyOutcome,
|
|
781
795
|
debug: result.classification ? {
|
|
782
796
|
totalInferenceMs: result.classification.inferenceMs,
|
|
783
797
|
stepTimings: [{
|
|
@@ -788,49 +802,8 @@ function buildAudioResultFrame(deviceId, result) {
|
|
|
788
802
|
} : void 0
|
|
789
803
|
};
|
|
790
804
|
}
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
/**
|
|
794
|
-
* `AudioDeviceAttachments` — the analyzer's own audio subscriptions (D461).
|
|
795
|
-
*
|
|
796
|
-
* ## What this replaces
|
|
797
|
-
*
|
|
798
|
-
* Until 2026-09-11 the decoded audio-chunk plane ran through
|
|
799
|
-
* `pipeline-orchestrator`: it drained the broker's FIFO, accumulated ~1 s
|
|
800
|
-
* windows and pushed them back out to `audioAnalyzer.analyseChunk`. The
|
|
801
|
-
* orchestrator neither produced nor consumed the audio — it buffered it, and
|
|
802
|
-
* every byte crossed hub-main twice for the privilege. The poller's own
|
|
803
|
-
* docblock named the reason, and it was an accident: the code had been a
|
|
804
|
-
* closure INSIDE the broker's process, and when the `pipeline` co-location
|
|
805
|
-
* group was dissolved nobody moved it.
|
|
806
|
-
*
|
|
807
|
-
* D455 proved the relay was the PREMISE of the remaining cost rather than an
|
|
808
|
-
* alternative to it. It put the SOURCE bytes on the plane — one G.711 byte per
|
|
809
|
-
* sample instead of four f32le ones — but only for a consumer that declared it
|
|
810
|
-
* could read them, and the accumulator (the declaring consumer) sat in a third
|
|
811
|
-
* process that had to expand before forwarding. So legs one and two went coded
|
|
812
|
-
* and legs three and four stayed at ~49 MB/min each. Here the subscriber IS
|
|
813
|
-
* the decoder: there is no third interlocutor, the negotiation needs none, and
|
|
814
|
-
* legs three and four do not exist.
|
|
815
|
-
*
|
|
816
|
-
* ## What the orchestrator still owns
|
|
817
|
-
*
|
|
818
|
-
* Everything about WHETHER and WHERE, which is all policy: the `audioMode`
|
|
819
|
-
* gate, the on-motion window, the audio-track probe, the per-device node
|
|
820
|
-
* assignment and the settings read. It calls `attachDevice` / `detachDevice`
|
|
821
|
-
* and holds the teardown. What it no longer owns is the bytes.
|
|
822
|
-
*
|
|
823
|
-
* ## The failure this split ADDS, and where it is caught
|
|
824
|
-
*
|
|
825
|
-
* The broker subscription is now opened by a process the orchestrator does not
|
|
826
|
-
* supervise. A detach that never arrives — an analyzer runner that crashed, a
|
|
827
|
-
* node that went offline mid-teardown — leaves a FIFO nobody drains, and on
|
|
828
|
-
* this plane that is not merely a leak: `AudioChunkPlane.subscriberCount` is
|
|
829
|
-
* broker DEMAND, so an orphan pins the source dial and the decode session open
|
|
830
|
-
* for as long as the broker lives. It is caught in the broker
|
|
831
|
-
* (`AudioChunkPlane`'s idle lease), not here, because a process that has died
|
|
832
|
-
* cannot clean up after itself.
|
|
833
|
-
*/
|
|
805
|
+
/** One over-bound report per camera per this window; the rest are counted. */
|
|
806
|
+
var OVER_BOUND_LOG_INTERVAL_MS = 6e4;
|
|
834
807
|
var AudioDeviceAttachments = class {
|
|
835
808
|
deps;
|
|
836
809
|
attachments = /* @__PURE__ */ new Map();
|
|
@@ -899,6 +872,25 @@ var AudioDeviceAttachments = class {
|
|
|
899
872
|
meta
|
|
900
873
|
});
|
|
901
874
|
});
|
|
875
|
+
let emitChain = Promise.resolve();
|
|
876
|
+
let awaiting = 0;
|
|
877
|
+
let droppedOverBound = 0;
|
|
878
|
+
let lastOverBoundLogMs = null;
|
|
879
|
+
const reportOverBound = () => {
|
|
880
|
+
droppedOverBound += 1;
|
|
881
|
+
const now = Date.now();
|
|
882
|
+
if (lastOverBoundLogMs !== null && now - lastOverBoundLogMs < OVER_BOUND_LOG_INTERVAL_MS) return;
|
|
883
|
+
lastOverBoundLogMs = now;
|
|
884
|
+
logger.warn("audio windows dropped — too many already awaiting a result", {
|
|
885
|
+
tags: { deviceId },
|
|
886
|
+
meta: {
|
|
887
|
+
dropped: droppedOverBound,
|
|
888
|
+
bound: 16,
|
|
889
|
+
awaiting
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
droppedOverBound = 0;
|
|
893
|
+
};
|
|
902
894
|
const teardown = startAudioChunkPoller({
|
|
903
895
|
api,
|
|
904
896
|
brokerId,
|
|
@@ -907,19 +899,39 @@ var AudioDeviceAttachments = class {
|
|
|
907
899
|
accept: ["pcmu", "pcma"],
|
|
908
900
|
pollIntervalMs: isRemoteBroker ? 500 : 200,
|
|
909
901
|
logger,
|
|
910
|
-
onChunk:
|
|
902
|
+
onChunk: (chunk) => {
|
|
903
|
+
let window;
|
|
911
904
|
try {
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
905
|
+
window = accumulator.push(chunk);
|
|
906
|
+
} catch (err) {
|
|
907
|
+
logger.error("audio window accumulate failed — chunk dropped", {
|
|
908
|
+
tags: { deviceId },
|
|
909
|
+
meta: {
|
|
910
|
+
error: errMsg(err),
|
|
911
|
+
timestamp: chunk.timestamp
|
|
912
|
+
}
|
|
913
|
+
});
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
if (!window) return;
|
|
917
|
+
if (awaiting >= 16) {
|
|
918
|
+
reportOverBound();
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
awaiting += 1;
|
|
922
|
+
const analysed = this.deps.analyse(window, settings);
|
|
923
|
+
analysed.catch(() => void 0);
|
|
924
|
+
emitChain = emitChain.then(() => analysed).then((result) => {
|
|
915
925
|
if (!result) return;
|
|
916
926
|
this.deps.emitResult(deviceId, buildAudioResultFrame(deviceId, result));
|
|
917
|
-
}
|
|
927
|
+
}).catch((err) => {
|
|
918
928
|
logger.error("Audio analysis failed", {
|
|
919
929
|
tags: { deviceId },
|
|
920
930
|
meta: { error: errMsg(err) }
|
|
921
931
|
});
|
|
922
|
-
}
|
|
932
|
+
}).finally(() => {
|
|
933
|
+
awaiting -= 1;
|
|
934
|
+
});
|
|
923
935
|
}
|
|
924
936
|
});
|
|
925
937
|
this.attachments.set(deviceId, {
|
|
@@ -983,6 +995,112 @@ var AudioDeviceAttachments = class {
|
|
|
983
995
|
}
|
|
984
996
|
};
|
|
985
997
|
//#endregion
|
|
998
|
+
//#region src/audio-analyzer/audio-classify-queue.ts
|
|
999
|
+
function emptyCounts() {
|
|
1000
|
+
return {
|
|
1001
|
+
classified: 0,
|
|
1002
|
+
superseded: 0,
|
|
1003
|
+
errors: 0,
|
|
1004
|
+
disposed: 0
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
var AudioClassifyQueue = class {
|
|
1008
|
+
/** Pending job per camera, in arrival order. */
|
|
1009
|
+
pending = /* @__PURE__ */ new Map();
|
|
1010
|
+
counts = /* @__PURE__ */ new Map();
|
|
1011
|
+
running = false;
|
|
1012
|
+
closed = false;
|
|
1013
|
+
/** True while an inference is running or a window is waiting for one. */
|
|
1014
|
+
get busy() {
|
|
1015
|
+
return this.running || this.pending.size > 0;
|
|
1016
|
+
}
|
|
1017
|
+
/**
|
|
1018
|
+
* Queue `run` for camera `key`. Resolves when it ran (`ran: true`), or when
|
|
1019
|
+
* it was superseded / the queue closed first (`ran: false`). A job that
|
|
1020
|
+
* throws rejects this promise with its error, and is counted as an error.
|
|
1021
|
+
*/
|
|
1022
|
+
submit(key, run) {
|
|
1023
|
+
if (this.closed) {
|
|
1024
|
+
this.count(key, "disposed");
|
|
1025
|
+
return Promise.resolve({
|
|
1026
|
+
ran: false,
|
|
1027
|
+
reason: "disposed"
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
return new Promise((resolve, reject) => {
|
|
1031
|
+
const previous = this.pending.get(key);
|
|
1032
|
+
if (previous) {
|
|
1033
|
+
this.count(key, "superseded");
|
|
1034
|
+
previous.settle({
|
|
1035
|
+
ran: false,
|
|
1036
|
+
reason: "superseded"
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1039
|
+
this.pending.set(key, {
|
|
1040
|
+
run,
|
|
1041
|
+
settle: resolve,
|
|
1042
|
+
fail: reject
|
|
1043
|
+
});
|
|
1044
|
+
this.drain();
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
/**
|
|
1048
|
+
* Hand back, and reset, the per-camera counts for every camera that had any
|
|
1049
|
+
* activity since the last call.
|
|
1050
|
+
*/
|
|
1051
|
+
takeAccounting() {
|
|
1052
|
+
const out = [];
|
|
1053
|
+
for (const [key, c] of this.counts) out.push({
|
|
1054
|
+
key,
|
|
1055
|
+
...c
|
|
1056
|
+
});
|
|
1057
|
+
this.counts.clear();
|
|
1058
|
+
return out;
|
|
1059
|
+
}
|
|
1060
|
+
/** Answer every waiting window `disposed` and refuse new ones. */
|
|
1061
|
+
close() {
|
|
1062
|
+
this.closed = true;
|
|
1063
|
+
for (const [key, job] of this.pending) {
|
|
1064
|
+
this.count(key, "disposed");
|
|
1065
|
+
job.settle({
|
|
1066
|
+
ran: false,
|
|
1067
|
+
reason: "disposed"
|
|
1068
|
+
});
|
|
1069
|
+
}
|
|
1070
|
+
this.pending.clear();
|
|
1071
|
+
}
|
|
1072
|
+
async drain() {
|
|
1073
|
+
if (this.running) return;
|
|
1074
|
+
this.running = true;
|
|
1075
|
+
try {
|
|
1076
|
+
for (;;) {
|
|
1077
|
+
const next = this.pending.entries().next();
|
|
1078
|
+
if (next.done === true) return;
|
|
1079
|
+
const [key, job] = next.value;
|
|
1080
|
+
this.pending.delete(key);
|
|
1081
|
+
try {
|
|
1082
|
+
const value = await job.run();
|
|
1083
|
+
this.count(key, "classified");
|
|
1084
|
+
job.settle({
|
|
1085
|
+
ran: true,
|
|
1086
|
+
value
|
|
1087
|
+
});
|
|
1088
|
+
} catch (err) {
|
|
1089
|
+
this.count(key, "errors");
|
|
1090
|
+
job.fail(err);
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
} finally {
|
|
1094
|
+
this.running = false;
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
count(key, field) {
|
|
1098
|
+
const c = this.counts.get(key) ?? emptyCounts();
|
|
1099
|
+
c[field] += 1;
|
|
1100
|
+
this.counts.set(key, c);
|
|
1101
|
+
}
|
|
1102
|
+
};
|
|
1103
|
+
//#endregion
|
|
986
1104
|
//#region src/audio-analyzer/addons/analyzer/index.ts
|
|
987
1105
|
/**
|
|
988
1106
|
* AudioAnalyzerProvider — implements IAudioAnalyzer.
|
|
@@ -992,9 +1110,26 @@ var AudioDeviceAttachments = class {
|
|
|
992
1110
|
* to a separate audio-classifier addon.
|
|
993
1111
|
*/
|
|
994
1112
|
var CLASSIFY_ERROR_SUPPRESS_MS = 3e4;
|
|
995
|
-
var
|
|
1113
|
+
var INFERENCE_TIMEOUT_MS = 5e3;
|
|
1114
|
+
/** An inference that did not answer within {@link INFERENCE_TIMEOUT_MS}. */
|
|
1115
|
+
var AudioInferenceTimeoutError = class extends Error {
|
|
1116
|
+
timeoutMs;
|
|
1117
|
+
constructor(timeoutMs) {
|
|
1118
|
+
super(`audio inference did not answer within ${timeoutMs} ms`);
|
|
1119
|
+
this.timeoutMs = timeoutMs;
|
|
1120
|
+
this.name = "AudioInferenceTimeoutError";
|
|
1121
|
+
}
|
|
1122
|
+
};
|
|
1123
|
+
var CLASSIFY_ACCOUNTING_PERIOD_MS = 6e4;
|
|
1124
|
+
var NEAR_MISS_WATCHED_MACROS = new Set(["crying", "scream"]);
|
|
1125
|
+
var NEAR_MISS_LOG_INTERVAL_MS = 6e4;
|
|
996
1126
|
var PIPELINE_IDLE_DISPOSE_MS = 5 * 6e4;
|
|
997
1127
|
var GLOBAL_DEVICE_KEY = -1;
|
|
1128
|
+
/** The labels a window cleared the camera's floor (and allow-list) with. */
|
|
1129
|
+
function labelsAboveFloor(result, settings) {
|
|
1130
|
+
const allowed = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
|
|
1131
|
+
return result.labels.filter((c) => c.score >= settings.minConfidence && (allowed === null || allowed.has(c.className.toLowerCase())));
|
|
1132
|
+
}
|
|
998
1133
|
/** What the classifier consumes natively; anything else is resampled next to it. */
|
|
999
1134
|
var CLASSIFIER_SAMPLE_RATE = 16e3;
|
|
1000
1135
|
var CLASSIFIER_CHANNELS = 1;
|
|
@@ -1028,14 +1163,15 @@ var AudioAnalyzerProvider = class {
|
|
|
1028
1163
|
/** When true, logs a raw-label sample every 100 classifications (opt-in
|
|
1029
1164
|
* debug aid). Off by default — the watchdog heartbeat covers liveness. */
|
|
1030
1165
|
debugClassifySamples = false;
|
|
1031
|
-
/**
|
|
1032
|
-
|
|
1166
|
+
/** The one line every camera's windows wait in for the single pipeline (D660).
|
|
1167
|
+
* Key = deviceId (or GLOBAL_DEVICE_KEY for legacy callers). */
|
|
1168
|
+
classifyQueue = new AudioClassifyQueue();
|
|
1169
|
+
/** Periodic per-camera classify accounting (D660). */
|
|
1170
|
+
accountingTimer;
|
|
1171
|
+
/** Near-miss line rate limit, per camera (D660). */
|
|
1172
|
+
nearMissLog = /* @__PURE__ */ new Map();
|
|
1033
1173
|
/** Last window format seen per camera — the first and every change is logged (D450). */
|
|
1034
1174
|
windowFormats = /* @__PURE__ */ new Map();
|
|
1035
|
-
/** Global pipeline lock — Apple SA and ONNX are single-channel: only one classify() can
|
|
1036
|
-
* run at a time. Without this, concurrent calls from different cameras overwrite the
|
|
1037
|
-
* single pendingResolve slot in AppleSoundAnalysisPipeline, causing 30s timeouts. */
|
|
1038
|
-
pipelineBusy = false;
|
|
1039
1175
|
/** Live inference pipeline. LAZY: null until the first classify() spawns it
|
|
1040
1176
|
* (see `ensurePipeline`); idle-disposed back to null after a long quiet
|
|
1041
1177
|
* window so a node hosting the addon with no consumer never keeps the
|
|
@@ -1069,6 +1205,29 @@ var AudioAnalyzerProvider = class {
|
|
|
1069
1205
|
restart: () => this.restartPipelineForMemory()
|
|
1070
1206
|
});
|
|
1071
1207
|
this.memoryGuard.start();
|
|
1208
|
+
this.accountingTimer = setInterval(() => this.reportClassifyAccounting(), CLASSIFY_ACCOUNTING_PERIOD_MS);
|
|
1209
|
+
if (typeof this.accountingTimer.unref === "function") this.accountingTimer.unref();
|
|
1210
|
+
}
|
|
1211
|
+
/**
|
|
1212
|
+
* One line per camera that lost a window since the last report (D660):
|
|
1213
|
+
* how many were classified, superseded by the camera's own newer window,
|
|
1214
|
+
* failed, or cut off by shutdown. Never one line per window, and nothing
|
|
1215
|
+
* for a camera that lost none.
|
|
1216
|
+
*/
|
|
1217
|
+
reportClassifyAccounting() {
|
|
1218
|
+
for (const a of this.classifyQueue.takeAccounting()) {
|
|
1219
|
+
if (a.superseded + a.errors + a.disposed === 0) continue;
|
|
1220
|
+
this.log.info("audio classify accounting", {
|
|
1221
|
+
tags: a.key === GLOBAL_DEVICE_KEY ? void 0 : { deviceId: a.key },
|
|
1222
|
+
meta: {
|
|
1223
|
+
classified: a.classified,
|
|
1224
|
+
superseded: a.superseded,
|
|
1225
|
+
errors: a.errors,
|
|
1226
|
+
disposed: a.disposed,
|
|
1227
|
+
periodMs: CLASSIFY_ACCOUNTING_PERIOD_MS
|
|
1228
|
+
}
|
|
1229
|
+
});
|
|
1230
|
+
}
|
|
1072
1231
|
}
|
|
1073
1232
|
/** Memory watchdog for the inference subprocess (see constructor). */
|
|
1074
1233
|
memoryGuard;
|
|
@@ -1160,7 +1319,7 @@ var AudioAnalyzerProvider = class {
|
|
|
1160
1319
|
*/
|
|
1161
1320
|
async disposeIdlePipeline() {
|
|
1162
1321
|
if (this.disposed) return;
|
|
1163
|
-
if (this.
|
|
1322
|
+
if (this.classifyQueue.busy || this.pipelineInitPromise) {
|
|
1164
1323
|
this.scheduleIdleDispose();
|
|
1165
1324
|
return;
|
|
1166
1325
|
}
|
|
@@ -1221,67 +1380,133 @@ var AudioAnalyzerProvider = class {
|
|
|
1221
1380
|
rms: Math.round(rms * 1e4) / 1e4,
|
|
1222
1381
|
dbfs: Math.round(dbfs * 10) / 10
|
|
1223
1382
|
};
|
|
1224
|
-
|
|
1225
|
-
try {
|
|
1226
|
-
const result = await this.classify(chunk);
|
|
1227
|
-
if (this.classifyCallCount < 3) {
|
|
1228
|
-
const topRaw = result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", ");
|
|
1229
|
-
this.log.info("classify debug sample", {
|
|
1230
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1231
|
-
meta: {
|
|
1232
|
-
index: this.classifyCallCount,
|
|
1233
|
-
labelCount: result.labels.length,
|
|
1234
|
-
top: topRaw,
|
|
1235
|
-
inferenceMs: result.inferenceMs,
|
|
1236
|
-
minConf: settings.minConfidence,
|
|
1237
|
-
allowedClasses: settings.allowedClasses
|
|
1238
|
-
}
|
|
1239
|
-
});
|
|
1240
|
-
}
|
|
1241
|
-
this.classifyCallCount++;
|
|
1242
|
-
const meaningful = result.labels.filter((l) => l.score >= .15 && l.className.toLowerCase() !== "silence");
|
|
1243
|
-
if (meaningful.length > 0) this.log.debug("audio classification", {
|
|
1244
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1245
|
-
meta: {
|
|
1246
|
-
top: meaningful.slice(0, 4).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1247
|
-
inferenceMs: result.inferenceMs
|
|
1248
|
-
}
|
|
1249
|
-
});
|
|
1250
|
-
if (result.inferenceMs > 0) {
|
|
1251
|
-
const minConf = settings.minConfidence;
|
|
1252
|
-
const allowedSet = settings.allowedClasses.length > 0 ? new Set(settings.allowedClasses.map((c) => c.toLowerCase())) : null;
|
|
1253
|
-
let filtered = result.labels.filter((c) => c.score >= minConf);
|
|
1254
|
-
if (allowedSet) filtered = filtered.filter((c) => allowedSet.has(c.className.toLowerCase()));
|
|
1255
|
-
if (filtered.length > 0) classification = {
|
|
1256
|
-
labels: filtered,
|
|
1257
|
-
inferenceMs: result.inferenceMs
|
|
1258
|
-
};
|
|
1259
|
-
}
|
|
1260
|
-
} catch (err) {
|
|
1261
|
-
const now = Date.now();
|
|
1262
|
-
if (now - this.lastClassifyErrorMs >= CLASSIFY_ERROR_SUPPRESS_MS) {
|
|
1263
|
-
const suppressed = this.suppressedClassifyErrors;
|
|
1264
|
-
this.suppressedClassifyErrors = 0;
|
|
1265
|
-
this.lastClassifyErrorMs = now;
|
|
1266
|
-
const msg = errMsg(err);
|
|
1267
|
-
const stack = err instanceof Error ? err.stack : void 0;
|
|
1268
|
-
this.log.warn("Audio classification failed", {
|
|
1269
|
-
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1270
|
-
meta: {
|
|
1271
|
-
error: msg,
|
|
1272
|
-
stack,
|
|
1273
|
-
suppressedSince: suppressed > 0 ? suppressed : void 0
|
|
1274
|
-
}
|
|
1275
|
-
});
|
|
1276
|
-
} else this.suppressedClassifyErrors++;
|
|
1277
|
-
}
|
|
1383
|
+
const { classification, classifyOutcome } = await this.classifyForAnalysis(chunk, settings);
|
|
1278
1384
|
return {
|
|
1279
1385
|
level,
|
|
1280
1386
|
classification,
|
|
1387
|
+
classifyOutcome,
|
|
1281
1388
|
timestamp: chunk.timestamp
|
|
1282
1389
|
};
|
|
1283
1390
|
}
|
|
1284
1391
|
/**
|
|
1392
|
+
* Run one window through the shared queue and apply the camera's floor.
|
|
1393
|
+
* Every exit names whether the model RAN (`classifyOutcome`, D660): a window
|
|
1394
|
+
* superseded by the camera's newer one, or whose inference failed, is
|
|
1395
|
+
* `not-classified` — never an empty classification.
|
|
1396
|
+
*/
|
|
1397
|
+
async classifyForAnalysis(chunk, settings) {
|
|
1398
|
+
let submission;
|
|
1399
|
+
try {
|
|
1400
|
+
submission = await this.classifyWindow(chunk);
|
|
1401
|
+
} catch (err) {
|
|
1402
|
+
const timedOut = err instanceof AudioInferenceTimeoutError;
|
|
1403
|
+
if (!timedOut) this.reportClassifyError(chunk, err);
|
|
1404
|
+
return {
|
|
1405
|
+
classification: void 0,
|
|
1406
|
+
classifyOutcome: {
|
|
1407
|
+
state: "not-classified",
|
|
1408
|
+
reason: timedOut ? "timeout" : "error"
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
1411
|
+
}
|
|
1412
|
+
if (!submission.ran) return {
|
|
1413
|
+
classification: void 0,
|
|
1414
|
+
classifyOutcome: {
|
|
1415
|
+
state: "not-classified",
|
|
1416
|
+
reason: submission.reason
|
|
1417
|
+
}
|
|
1418
|
+
};
|
|
1419
|
+
const result = submission.value;
|
|
1420
|
+
this.logClassifySample(chunk, result, settings);
|
|
1421
|
+
this.reportNearMiss(chunk, result, settings);
|
|
1422
|
+
const filtered = labelsAboveFloor(result, settings);
|
|
1423
|
+
return {
|
|
1424
|
+
classification: filtered.length > 0 ? {
|
|
1425
|
+
labels: filtered,
|
|
1426
|
+
inferenceMs: result.inferenceMs
|
|
1427
|
+
} : void 0,
|
|
1428
|
+
classifyOutcome: { state: "classified" }
|
|
1429
|
+
};
|
|
1430
|
+
}
|
|
1431
|
+
/** First-three debug samples + the per-window debug line (unchanged behaviour). */
|
|
1432
|
+
logClassifySample(chunk, result, settings) {
|
|
1433
|
+
const tags = chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0;
|
|
1434
|
+
if (this.classifyCallCount < 3) this.log.info("classify debug sample", {
|
|
1435
|
+
tags,
|
|
1436
|
+
meta: {
|
|
1437
|
+
index: this.classifyCallCount,
|
|
1438
|
+
labelCount: result.labels.length,
|
|
1439
|
+
top: result.labels.slice(0, 5).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1440
|
+
inferenceMs: result.inferenceMs,
|
|
1441
|
+
minConf: settings.minConfidence,
|
|
1442
|
+
allowedClasses: settings.allowedClasses
|
|
1443
|
+
}
|
|
1444
|
+
});
|
|
1445
|
+
this.classifyCallCount++;
|
|
1446
|
+
const meaningful = result.labels.filter((l) => l.score >= .15 && l.className.toLowerCase() !== "silence");
|
|
1447
|
+
if (meaningful.length > 0) this.log.debug("audio classification", {
|
|
1448
|
+
tags,
|
|
1449
|
+
meta: {
|
|
1450
|
+
top: meaningful.slice(0, 4).map((l) => `${l.className}(${(l.score * 100).toFixed(0)}%)`).join(", "),
|
|
1451
|
+
inferenceMs: result.inferenceMs
|
|
1452
|
+
}
|
|
1453
|
+
});
|
|
1454
|
+
}
|
|
1455
|
+
/**
|
|
1456
|
+
* A watched macro (crying, scream) the model heard BELOW this camera's
|
|
1457
|
+
* floor — one line per camera per {@link NEAR_MISS_LOG_INTERVAL_MS}, the
|
|
1458
|
+
* rest counted into the next line (D660). This is the line that tells an
|
|
1459
|
+
* operator whose rule never fires whether the sound was heard at all.
|
|
1460
|
+
*/
|
|
1461
|
+
reportNearMiss(chunk, result, settings) {
|
|
1462
|
+
const miss = result.labels.find((l) => NEAR_MISS_WATCHED_MACROS.has(l.className) && l.score < settings.minConfidence);
|
|
1463
|
+
if (miss === void 0) return;
|
|
1464
|
+
const key = chunk.deviceId ?? GLOBAL_DEVICE_KEY;
|
|
1465
|
+
const now = Date.now();
|
|
1466
|
+
const prev = this.nearMissLog.get(key) ?? null;
|
|
1467
|
+
if (prev !== null && now - prev.lastLoggedMs < NEAR_MISS_LOG_INTERVAL_MS) {
|
|
1468
|
+
this.nearMissLog.set(key, {
|
|
1469
|
+
...prev,
|
|
1470
|
+
suppressed: prev.suppressed + 1
|
|
1471
|
+
});
|
|
1472
|
+
return;
|
|
1473
|
+
}
|
|
1474
|
+
this.nearMissLog.set(key, {
|
|
1475
|
+
lastLoggedMs: now,
|
|
1476
|
+
suppressed: 0
|
|
1477
|
+
});
|
|
1478
|
+
this.log.info("audio near-miss below floor", {
|
|
1479
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1480
|
+
meta: {
|
|
1481
|
+
label: miss.className,
|
|
1482
|
+
originalClass: miss.originalClass,
|
|
1483
|
+
score: miss.score,
|
|
1484
|
+
floor: settings.minConfidence,
|
|
1485
|
+
suppressedSince: prev?.suppressed ?? 0,
|
|
1486
|
+
intervalMs: NEAR_MISS_LOG_INTERVAL_MS
|
|
1487
|
+
}
|
|
1488
|
+
});
|
|
1489
|
+
}
|
|
1490
|
+
/** Rate-limited classify-failure line (unchanged behaviour). */
|
|
1491
|
+
reportClassifyError(chunk, err) {
|
|
1492
|
+
const now = Date.now();
|
|
1493
|
+
if (now - this.lastClassifyErrorMs < CLASSIFY_ERROR_SUPPRESS_MS) {
|
|
1494
|
+
this.suppressedClassifyErrors++;
|
|
1495
|
+
return;
|
|
1496
|
+
}
|
|
1497
|
+
const suppressed = this.suppressedClassifyErrors;
|
|
1498
|
+
this.suppressedClassifyErrors = 0;
|
|
1499
|
+
this.lastClassifyErrorMs = now;
|
|
1500
|
+
this.log.warn("Audio classification failed", {
|
|
1501
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1502
|
+
meta: {
|
|
1503
|
+
error: errMsg(err),
|
|
1504
|
+
stack: err instanceof Error ? err.stack : void 0,
|
|
1505
|
+
suppressedSince: suppressed > 0 ? suppressed : void 0
|
|
1506
|
+
}
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1285
1510
|
* Say what this camera's windows arrive as — once, and again on every
|
|
1286
1511
|
* change. The plane carries the source rate (8 kHz for G.711 since D450,
|
|
1287
1512
|
* 16 kHz for AAC through the codec session), and the classifier resamples
|
|
@@ -1313,33 +1538,31 @@ var AudioAnalyzerProvider = class {
|
|
|
1313
1538
|
}
|
|
1314
1539
|
});
|
|
1315
1540
|
}
|
|
1541
|
+
/**
|
|
1542
|
+
* The cap's `classify`: one window through the same queue every camera
|
|
1543
|
+
* uses. A window that never reached the model is an ERROR here, not an
|
|
1544
|
+
* empty result — the caller asked for a classification and did not get one.
|
|
1545
|
+
*/
|
|
1316
1546
|
async classify(chunk) {
|
|
1317
|
-
const
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
lastEndMs: state?.lastEndMs ?? 0
|
|
1333
|
-
});
|
|
1334
|
-
this.pipelineBusy = true;
|
|
1547
|
+
const submission = await this.classifyWindow(chunk);
|
|
1548
|
+
if (!submission.ran) throw new Error(`audio classify not run: ${submission.reason}`);
|
|
1549
|
+
return submission.value;
|
|
1550
|
+
}
|
|
1551
|
+
/**
|
|
1552
|
+
* Queue one window for the single pipeline (D660). Per camera at most one
|
|
1553
|
+
* waits; a newer window from the same camera supersedes it and takes its
|
|
1554
|
+
* place in line. Cameras are served in arrival order. Rejects with the
|
|
1555
|
+
* inference's own error.
|
|
1556
|
+
*/
|
|
1557
|
+
classifyWindow(chunk) {
|
|
1558
|
+
return this.classifyQueue.submit(chunk.deviceId ?? GLOBAL_DEVICE_KEY, () => this.runInference(chunk));
|
|
1559
|
+
}
|
|
1560
|
+
/** One inference on the live pipeline. Only ever called by the queue — never two at once. */
|
|
1561
|
+
async runInference(chunk) {
|
|
1335
1562
|
try {
|
|
1336
1563
|
const pipeline = await this.ensurePipeline();
|
|
1337
1564
|
const f32Data = float32FromBytes(chunk.data);
|
|
1338
|
-
const result = await
|
|
1339
|
-
data: f32Data,
|
|
1340
|
-
sampleRate: chunk.sampleRate,
|
|
1341
|
-
channels: chunk.channels
|
|
1342
|
-
});
|
|
1565
|
+
const result = await this.boundedInference(pipeline, chunk, f32Data);
|
|
1343
1566
|
if (this.debugClassifySamples && (this.classifyCount < 3 || this.classifyCount % 100 === 0)) {
|
|
1344
1567
|
const rawTop = result.classifications.slice(0, 5).map((c) => `"${c.className}"(${(c.score * 100).toFixed(0)}%)`).join(", ");
|
|
1345
1568
|
this.log.info("classify debug sample", {
|
|
@@ -1378,20 +1601,63 @@ var AudioAnalyzerProvider = class {
|
|
|
1378
1601
|
inferenceMs: result.inferenceMs
|
|
1379
1602
|
};
|
|
1380
1603
|
} finally {
|
|
1381
|
-
this.pipelineBusy = false;
|
|
1382
|
-
this.cameraState.set(camKey, {
|
|
1383
|
-
inProgress: false,
|
|
1384
|
-
lastEndMs: Date.now()
|
|
1385
|
-
});
|
|
1386
1604
|
this.scheduleIdleDispose();
|
|
1387
1605
|
}
|
|
1388
1606
|
}
|
|
1607
|
+
/**
|
|
1608
|
+
* One inference under {@link INFERENCE_TIMEOUT_MS}. A backend that timed
|
|
1609
|
+
* out, exited or was disposed under the call is DROPPED (killed, and the
|
|
1610
|
+
* next window lazily respawns it) — a dead pipeline kept as `this.pipeline`
|
|
1611
|
+
* would answer every later window with the same error forever.
|
|
1612
|
+
*/
|
|
1613
|
+
async boundedInference(pipeline, chunk, data) {
|
|
1614
|
+
const inference = pipeline.classify({
|
|
1615
|
+
data,
|
|
1616
|
+
sampleRate: chunk.sampleRate,
|
|
1617
|
+
channels: chunk.channels
|
|
1618
|
+
});
|
|
1619
|
+
inference.catch(() => void 0);
|
|
1620
|
+
let timer = null;
|
|
1621
|
+
const timeout = new Promise((_, reject) => {
|
|
1622
|
+
timer = setTimeout(() => reject(new AudioInferenceTimeoutError(INFERENCE_TIMEOUT_MS)), INFERENCE_TIMEOUT_MS);
|
|
1623
|
+
});
|
|
1624
|
+
try {
|
|
1625
|
+
return await Promise.race([inference, timeout]);
|
|
1626
|
+
} catch (err) {
|
|
1627
|
+
if (err instanceof AudioInferenceTimeoutError) {
|
|
1628
|
+
this.log.warn("audio inference timed out — killing the backend, next window respawns it", {
|
|
1629
|
+
tags: chunk.deviceId !== void 0 ? { deviceId: chunk.deviceId } : void 0,
|
|
1630
|
+
meta: {
|
|
1631
|
+
timeoutMs: INFERENCE_TIMEOUT_MS,
|
|
1632
|
+
backend: this.backendName
|
|
1633
|
+
}
|
|
1634
|
+
});
|
|
1635
|
+
this.dropPipeline(pipeline);
|
|
1636
|
+
} else if (err instanceof AudioBackendExitedError) this.dropPipeline(pipeline);
|
|
1637
|
+
throw err;
|
|
1638
|
+
} finally {
|
|
1639
|
+
if (timer !== null) clearTimeout(timer);
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
/** Forget (if still current) and dispose a pipeline that can no longer answer. */
|
|
1643
|
+
dropPipeline(pipeline) {
|
|
1644
|
+
if (this.pipeline === pipeline) this.pipeline = null;
|
|
1645
|
+
pipeline.dispose().catch((err) => {
|
|
1646
|
+
this.log.warn("audio pipeline dispose after failure threw", { meta: {
|
|
1647
|
+
error: errMsg(err),
|
|
1648
|
+
backend: this.backendName
|
|
1649
|
+
} });
|
|
1650
|
+
});
|
|
1651
|
+
}
|
|
1389
1652
|
isReady() {
|
|
1390
1653
|
return !this.disposed;
|
|
1391
1654
|
}
|
|
1392
1655
|
async dispose() {
|
|
1393
1656
|
this.disposed = true;
|
|
1394
1657
|
this.memoryGuard.stop();
|
|
1658
|
+
clearInterval(this.accountingTimer);
|
|
1659
|
+
this.classifyQueue.close();
|
|
1660
|
+
this.reportClassifyAccounting();
|
|
1395
1661
|
if (this.idleTimer) {
|
|
1396
1662
|
clearTimeout(this.idleTimer);
|
|
1397
1663
|
this.idleTimer = null;
|