@homebridge-plugins/homebridge-eufy-security 5.0.0-beta.4 → 5.0.0-beta.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/CONTRIBUTING.md +9 -17
  2. package/config.schema.json +60 -0
  3. package/dist/account/ownership.js +16 -16
  4. package/dist/account/ownership.js.map +1 -1
  5. package/dist/account/persistence.js +24 -74
  6. package/dist/account/persistence.js.map +1 -1
  7. package/dist/account/temporary-authentication.js +5 -3
  8. package/dist/account/temporary-authentication.js.map +1 -1
  9. package/dist/configuration.js +50 -13
  10. package/dist/configuration.js.map +1 -1
  11. package/dist/device/member-evidence.js +11 -4
  12. package/dist/device/member-evidence.js.map +1 -1
  13. package/dist/device/member-trust.js +14 -32
  14. package/dist/device/member-trust.js.map +1 -1
  15. package/dist/device/snapshot.js +6 -18
  16. package/dist/device/snapshot.js.map +1 -1
  17. package/dist/diagnostics.js +951 -334
  18. package/dist/diagnostics.js.map +1 -1
  19. package/dist/homekit/adapter.js +4 -1
  20. package/dist/homekit/adapter.js.map +1 -1
  21. package/dist/homekit/adapters/battery.js +3 -3
  22. package/dist/homekit/adapters/battery.js.map +1 -1
  23. package/dist/homekit/adapters/camera-controls.js +15 -37
  24. package/dist/homekit/adapters/camera-controls.js.map +1 -1
  25. package/dist/homekit/adapters/camera-streaming.js +74 -48
  26. package/dist/homekit/adapters/camera-streaming.js.map +1 -1
  27. package/dist/homekit/adapters/contact.js +5 -5
  28. package/dist/homekit/adapters/contact.js.map +1 -1
  29. package/dist/homekit/adapters/doorbell.js +3 -15
  30. package/dist/homekit/adapters/doorbell.js.map +1 -1
  31. package/dist/homekit/adapters/lock.js +72 -221
  32. package/dist/homekit/adapters/lock.js.map +1 -1
  33. package/dist/homekit/adapters/motion.js +3 -3
  34. package/dist/homekit/adapters/motion.js.map +1 -1
  35. package/dist/homekit/adapters/package-sensor.js +63 -0
  36. package/dist/homekit/adapters/package-sensor.js.map +1 -0
  37. package/dist/homekit/adapters/registry.js +2 -0
  38. package/dist/homekit/adapters/registry.js.map +1 -1
  39. package/dist/homekit/adapters/security-system.js +106 -250
  40. package/dist/homekit/adapters/security-system.js.map +1 -1
  41. package/dist/homekit/adapters/siren.js +5 -33
  42. package/dist/homekit/adapters/siren.js.map +1 -1
  43. package/dist/homekit/adapters/smart-light.js +19 -217
  44. package/dist/homekit/adapters/smart-light.js.map +1 -1
  45. package/dist/homekit/coverage-matrix.js +8 -67
  46. package/dist/homekit/coverage-matrix.js.map +1 -1
  47. package/dist/homekit/device-control.js +249 -13
  48. package/dist/homekit/device-control.js.map +1 -1
  49. package/dist/homekit/reconciler.js +25 -9
  50. package/dist/homekit/reconciler.js.map +1 -1
  51. package/dist/media/last-successful-image.js +8 -23
  52. package/dist/media/last-successful-image.js.map +1 -1
  53. package/dist/media/live-preview.js +217 -0
  54. package/dist/media/live-preview.js.map +1 -0
  55. package/dist/media/live-stream.js +689 -160
  56. package/dist/media/live-stream.js.map +1 -1
  57. package/dist/media/recording.js +41 -118
  58. package/dist/media/recording.js.map +1 -1
  59. package/dist/media/snapshot.js +22 -17
  60. package/dist/media/snapshot.js.map +1 -1
  61. package/dist/media/station-live-sessions.js +40 -51
  62. package/dist/media/station-live-sessions.js.map +1 -1
  63. package/dist/platform.js +61 -10
  64. package/dist/platform.js.map +1 -1
  65. package/dist/runtime/channel.js +227 -40
  66. package/dist/runtime/channel.js.map +1 -1
  67. package/dist/runtime/device-settings.js +88 -0
  68. package/dist/runtime/device-settings.js.map +1 -0
  69. package/dist/runtime/owner.js +30 -5
  70. package/dist/runtime/owner.js.map +1 -1
  71. package/dist/runtime/sdk-client.js +21 -17
  72. package/dist/runtime/sdk-client.js.map +1 -1
  73. package/dist/runtime/tracker.js +8 -38
  74. package/dist/runtime/tracker.js.map +1 -1
  75. package/dist/storage.js +1 -1
  76. package/dist/storage.js.map +1 -1
  77. package/dist/ui/dashboard.js +36 -16
  78. package/dist/ui/dashboard.js.map +1 -1
  79. package/dist/ui/runtime-channel-client.js +155 -184
  80. package/dist/ui/runtime-channel-client.js.map +1 -1
  81. package/dist/ui/server.js +81 -38
  82. package/dist/ui/server.js.map +1 -1
  83. package/homebridge-ui/public/app.css +531 -239
  84. package/homebridge-ui/public/assets/icons/auto_fix_high.svg +1 -0
  85. package/homebridge-ui/public/assets/icons/battery_0.svg +1 -0
  86. package/homebridge-ui/public/assets/icons/battery_1.svg +1 -0
  87. package/homebridge-ui/public/assets/icons/battery_2.svg +1 -0
  88. package/homebridge-ui/public/assets/icons/battery_3.svg +1 -0
  89. package/homebridge-ui/public/assets/icons/battery_4.svg +1 -0
  90. package/homebridge-ui/public/assets/icons/battery_5.svg +1 -0
  91. package/homebridge-ui/public/assets/icons/battery_6.svg +1 -0
  92. package/homebridge-ui/public/assets/icons/battery_full.svg +1 -0
  93. package/homebridge-ui/public/assets/icons/contact_support.svg +1 -0
  94. package/homebridge-ui/public/assets/icons/delete.svg +1 -0
  95. package/homebridge-ui/public/assets/icons/download.svg +1 -0
  96. package/homebridge-ui/public/assets/icons/extension_off.svg +1 -0
  97. package/homebridge-ui/public/assets/icons/info.svg +1 -0
  98. package/homebridge-ui/public/assets/icons/key_off.svg +1 -0
  99. package/homebridge-ui/public/assets/icons/keyboard_arrow_left.svg +1 -0
  100. package/homebridge-ui/public/assets/icons/keyboard_arrow_right.svg +1 -0
  101. package/homebridge-ui/public/assets/icons/keyboard_double_arrow_left.svg +1 -0
  102. package/homebridge-ui/public/assets/icons/keyboard_double_arrow_right.svg +1 -0
  103. package/homebridge-ui/public/assets/icons/movie_off.svg +1 -0
  104. package/homebridge-ui/public/assets/icons/play_arrow.svg +1 -0
  105. package/homebridge-ui/public/assets/icons/refresh.svg +1 -0
  106. package/homebridge-ui/public/assets/icons/satellite_alt.svg +1 -0
  107. package/homebridge-ui/public/assets/icons/search_off.svg +1 -0
  108. package/homebridge-ui/public/assets/icons/sensors_off.svg +1 -0
  109. package/homebridge-ui/public/assets/icons/settings_photo_camera.svg +1 -0
  110. package/homebridge-ui/public/assets/icons/solar_power.svg +1 -0
  111. package/homebridge-ui/public/assets/icons/stop.svg +1 -0
  112. package/homebridge-ui/public/assets/icons/videocam_off.svg +1 -0
  113. package/homebridge-ui/public/i18n/en.json +160 -82
  114. package/homebridge-ui/public/i18n/fr.json +161 -83
  115. package/homebridge-ui/public/index.html +225 -96
  116. package/homebridge-ui/public/js/app.js +814 -435
  117. package/homebridge-ui/public/js/dashboard.js +148 -31
  118. package/i18n/runtime/en.json +10 -3
  119. package/package.json +10 -11
  120. package/homebridge-ui/public/js/legacy-settings.js +0 -37
  121. package/homebridge-ui/public/js/profile-wizard.js +0 -61
@@ -1,9 +1,18 @@
1
- import { LiveStreamStartError } from '@mega-yfue/eufy-sdk';
1
+ import { LiveStreamStartError, StationKeyUnavailableError, StationUnreachableError } from '@mega-yfue/eufy-sdk';
2
2
  import { createSocket } from 'node:dgram';
3
3
  import { execFile, spawn } from 'node:child_process';
4
+ import { promisify } from 'node:util';
4
5
  import { setTimeout as delay } from 'node:timers/promises';
5
6
  /** How long a deliberately stopped adaptation process is given to exit before it is killed. */
6
- export const PROCESS_STOP_GRACE_MS = 2_000;
7
+ const PROCESS_STOP_GRACE_MS = 2_000;
8
+ /** Closes an adaptation process's input and terminates it, killing it if it has not exited within the grace. */
9
+ export function stopProcess(process) {
10
+ process.stdin?.destroy();
11
+ process.kill('SIGTERM');
12
+ const killDeadline = setTimeout(() => process.kill('SIGKILL'), PROCESS_STOP_GRACE_MS);
13
+ killDeadline.unref?.();
14
+ process.on('exit', () => clearTimeout(killDeadline));
15
+ }
7
16
  /**
8
17
  * Whether an adaptation opened for `running` can code what the source announced as `announced`.
9
18
  *
@@ -34,7 +43,18 @@ function canCode(running, announced) {
34
43
  */
35
44
  const MAX_SOURCE_INPUT_REPLACEMENTS = 3;
36
45
  const INITIAL_RTCP_GRACE_MS = 15_000;
37
- const SOURCE_ACQUISITION_DEADLINE_MS = 10_000;
46
+ /**
47
+ * The longest a session waits for the SDK to hand over a source, after which it cancels the acquisition.
48
+ *
49
+ * Reaching a station is inside this wait: an unconnected station is reconnected within it, and a HomeBase
50
+ * resolves its session before it can start one. The bound is therefore above what reaching a station costs and
51
+ * below both {@link SOURCE_START_BACKSTOP_MS} and the window a started request is owed an outcome within, so
52
+ * the acquisition is what this expiry reports.
53
+ *
54
+ * Expiry cancels the acquisition. A station serves one media channel, and an uncancelled acquisition holds it
55
+ * until the SDK's own wait elapses, which refuses every camera behind that station meanwhile.
56
+ */
57
+ const SOURCE_ACQUISITION_DEADLINE_MS = 25_000;
38
58
  const RETURN_AUDIO_BIND_GRACE_MS = 250;
39
59
  /**
40
60
  * Backstop for a started media session that never produces adapted output. The SDK source owns the
@@ -44,6 +64,20 @@ const RETURN_AUDIO_BIND_GRACE_MS = 250;
44
64
  * reports nothing at all.
45
65
  */
46
66
  export const SOURCE_START_BACKSTOP_MS = 30_000;
67
+ /**
68
+ * How far the RTP sequence space is advanced for each adaptation a session sends on, in packets.
69
+ *
70
+ * One session keeps one SSRC and one SRTP key across a replaced adaptation, and the SRTP packet index is built
71
+ * from the sequence number — so a replacement that started where FFmpeg would have, at a random base, is read
72
+ * by the controller as replay half the time and its packets are discarded under a key that still
73
+ * authenticates. Nothing reports it: the controller goes on acknowledging what it received before, so RTCP
74
+ * liveness holds while the picture never advances again.
75
+ *
76
+ * Advancing instead of restarting keeps the index monotonic. The step is well under half the sequence space,
77
+ * which is what keeps the receiver's own index estimate correct across the wrap it eventually crosses; a gap
78
+ * this size reads as loss, which a controller already tolerates.
79
+ */
80
+ const OUTPUT_SEQUENCE_STEP = 4_096;
47
81
  const SOURCE_ACQUISITION_TIMEOUT = Symbol('source-acquisition-timeout');
48
82
  /**
49
83
  * The bounded reason one SDK source failure reports.
@@ -51,14 +85,30 @@ const SOURCE_ACQUISITION_TIMEOUT = Symbol('source-acquisition-timeout');
51
85
  * The SDK stages a start that produced no keyframe by what the source delivered, and a start that carried
52
86
  * audio and never a single video frame is the one signature worth naming apart: measured on a real camera
53
87
  * that had been switched off, the source accepted the start, delivered audio for the whole warm-up window,
54
- * and reported `audio-only`. Every other stage is a source error, because nothing else distinguishes it
55
- * from a transport that failed.
88
+ * and reported `audio-only`. Any other warm-up that ran out of time is `source-warm-timeout`: the camera answered
89
+ * and the transport worked, it just sent no usable picture. Everything else is a source error, because nothing
90
+ * else distinguishes it from a transport that failed.
91
+ */
92
+ function liveSourceFailure(error) {
93
+ if (!(error instanceof LiveStreamStartError)) {
94
+ return sourceFailure(error);
95
+ }
96
+ if (error.stage === 'audio-only') {
97
+ return 'source-audio-only';
98
+ }
99
+ return error.reason === 'warm-timeout' ? 'source-warm-timeout' : sourceFailure(error);
100
+ }
101
+ /**
102
+ * Why a source stopped producing, as this domain's bounded vocabulary.
103
+ *
104
+ * A station names its own failures apart from the camera's: every camera behind a base that did not connect
105
+ * fails together, and a sealed start the station gave no key for does not resolve by retrying.
56
106
  */
57
- function sourceFailure(error) {
58
- if (error instanceof Error && error.name === 'StationBusyError') {
59
- return 'station-busy';
107
+ export function sourceFailure(error) {
108
+ if (error instanceof StationUnreachableError) {
109
+ return 'station-unreachable';
60
110
  }
61
- return error instanceof LiveStreamStartError && error.stage === 'audio-only' ? 'source-audio-only' : 'source-error';
111
+ return error instanceof StationKeyUnavailableError ? 'station-key-unavailable' : 'source-error';
62
112
  }
63
113
  /** How many diagnostic lines one adaptation retains, which is what a failure is attributed from. */
64
114
  const ADAPTATION_STDERR_TAIL_LINES = 8;
@@ -136,12 +186,14 @@ const FFMPEG_VERSION_BANNER_LENGTH = 120;
136
186
  * to know whether this binary can be asked to encode: a missing file, a file that is not executable, one
137
187
  * that is not FFmpeg, and one that hangs all mean the same thing to the media that would have used it.
138
188
  */
139
- const probeFfmpegVersion = (path) => new Promise((resolve) => {
140
- execFile(path, ['-hide_banner', '-version'], { timeout: FFMPEG_VERSION_PROBE_TIMEOUT_MS, maxBuffer: 64 * 1024, windowsHide: true }, (error, stdout) => {
141
- const banner = error ? undefined : stdout.split(/\r?\n/, 1)[0]?.trim().slice(0, FFMPEG_VERSION_BANNER_LENGTH);
142
- resolve(banner === undefined || banner === '' ? undefined : banner);
143
- });
144
- });
189
+ const probeFfmpegVersion = async (path) => {
190
+ const { stdout } = await promisify(execFile)(path, ['-hide_banner', '-version'], {
191
+ timeout: FFMPEG_VERSION_PROBE_TIMEOUT_MS,
192
+ maxBuffer: 64 * 1024,
193
+ windowsHide: true,
194
+ }).catch(() => ({ stdout: '' }));
195
+ return stdout.split(/\r?\n/, 1)[0]?.trim().slice(0, FFMPEG_VERSION_BANNER_LENGTH) || undefined;
196
+ };
145
197
  /**
146
198
  * Resolves the complete identity of one adaptation binary, without opening any media.
147
199
  *
@@ -152,6 +204,281 @@ export async function resolveFfmpegIdentity(path, source, probe = probeFfmpegVer
152
204
  const version = await probe(path).catch(() => undefined);
153
205
  return { path, source, ...(version === undefined ? {} : { version }) };
154
206
  }
207
+ /**
208
+ * Reports what one adaptation process did to `diagnostics`, whether or not the session it belonged to fails for it.
209
+ *
210
+ * An audio adaptation is restarted rather than failed and a return-audio one fails only talkback, so without a
211
+ * report of their own those two exit with no account anywhere of why they stopped. An `output` report carries
212
+ * nothing but the tail, so it is made only when the process actually wrote something: a silent process that
213
+ * ended as intended has nothing to attribute.
214
+ */
215
+ export function adaptationReporter(diagnostics) {
216
+ return (role, event, stderr, code, signal, extras) => {
217
+ const tail = stderr.tail();
218
+ if (event === 'output' && tail.length === 0) {
219
+ return;
220
+ }
221
+ diagnostics?.report({
222
+ role,
223
+ event,
224
+ ...(typeof code === 'number' ? { code } : {}),
225
+ ...(typeof signal === 'string' ? { signal } : {}),
226
+ ...extras,
227
+ ...(tail.length ? { stderr: tail } : {}),
228
+ });
229
+ };
230
+ }
231
+ /** The `libx264` constant rate factor a live adaptation codes at when the negotiated bit rate is ignored. */
232
+ const CONSTANT_QUALITY_CRF = 23;
233
+ /** `libx264`'s profile, level, speed and pixel format, shared by its rate-capped and constant-quality forms. */
234
+ function libx264Arguments(selection) {
235
+ return [
236
+ '-preset',
237
+ 'superfast',
238
+ '-tune',
239
+ 'zerolatency',
240
+ '-profile:v',
241
+ selection.profile,
242
+ '-level:v',
243
+ selection.level,
244
+ '-pix_fmt',
245
+ 'yuv420p',
246
+ ];
247
+ }
248
+ /** The software encoder every selection no qualified hardware encoder serves is coded by. */
249
+ const LIBX264 = {
250
+ name: 'libx264',
251
+ args: (selection, kbps) => [...libx264Arguments(selection), '-b:v', kbps, '-maxrate', kbps, '-bufsize', kbps],
252
+ };
253
+ /** `libx264` at {@link CONSTANT_QUALITY_CRF} with no rate control, which no hardware encoder replaces. */
254
+ const CONSTANT_QUALITY = {
255
+ name: 'libx264',
256
+ args: (selection) => [...libx264Arguments(selection), '-crf', String(CONSTANT_QUALITY_CRF)],
257
+ };
258
+ /** The H.264 `profile_idc` of each negotiable profile. */
259
+ const PROFILE_IDC = { baseline: 66, main: 77, high: 100 };
260
+ /**
261
+ * The hardware encoders a selection may be served by, in order of preference, each with the hosts whose
262
+ * binary may carry it.
263
+ *
264
+ * Each states profile, level and rate control in its own encoder's spelling, with B-frames off and its
265
+ * lowest-latency mode, as docs/reference/hardware-encoder-viability.md records them. None is used for a
266
+ * selection until {@link FfmpegLiveMedia.qualify} has proven it against that selection.
267
+ */
268
+ const HARDWARE_ENCODERS = [
269
+ {
270
+ name: 'h264_videotoolbox',
271
+ triedOn: (host) => host.platform === 'darwin',
272
+ args: (selection, kbps) => [
273
+ '-profile:v',
274
+ selection.profile,
275
+ '-level:v',
276
+ selection.level,
277
+ '-realtime',
278
+ '1',
279
+ '-prio_speed',
280
+ '1',
281
+ '-bf',
282
+ '0',
283
+ '-pix_fmt',
284
+ 'yuv420p',
285
+ '-b:v',
286
+ kbps,
287
+ '-maxrate',
288
+ kbps,
289
+ ],
290
+ },
291
+ {
292
+ name: 'h264_qsv',
293
+ triedOn: (host) => host.platform === 'win32' || host.configured,
294
+ args: (selection, kbps) => [
295
+ '-profile:v',
296
+ selection.profile,
297
+ '-level:v',
298
+ selection.level.replace('.', ''),
299
+ '-pix_fmt',
300
+ 'nv12',
301
+ '-bf',
302
+ '0',
303
+ '-look_ahead',
304
+ '0',
305
+ '-async_depth',
306
+ '1',
307
+ '-b:v',
308
+ kbps,
309
+ '-maxrate',
310
+ kbps,
311
+ ],
312
+ },
313
+ {
314
+ name: 'h264_nvenc',
315
+ triedOn: (host) => host.configured,
316
+ args: (selection, kbps) => [
317
+ '-profile:v',
318
+ selection.profile,
319
+ '-level:v',
320
+ selection.level,
321
+ '-preset',
322
+ 'p1',
323
+ '-tune',
324
+ 'ull',
325
+ '-zerolatency',
326
+ '1',
327
+ '-bf',
328
+ '0',
329
+ '-pix_fmt',
330
+ 'yuv420p',
331
+ '-rc',
332
+ 'cbr',
333
+ '-b:v',
334
+ kbps,
335
+ '-maxrate',
336
+ kbps,
337
+ '-bufsize',
338
+ kbps,
339
+ ],
340
+ },
341
+ {
342
+ name: 'h264_amf',
343
+ triedOn: (host) => host.platform === 'linux' && host.arch === 'x64',
344
+ args: (selection, kbps) => [
345
+ '-profile:v',
346
+ selection.profile === 'baseline' ? 'constrained_baseline' : selection.profile,
347
+ '-level:v',
348
+ selection.level,
349
+ '-usage',
350
+ 'ultralowlatency',
351
+ '-bf',
352
+ '0',
353
+ '-pix_fmt',
354
+ 'yuv420p',
355
+ '-b:v',
356
+ kbps,
357
+ '-maxrate',
358
+ kbps,
359
+ '-bufsize',
360
+ kbps,
361
+ ],
362
+ },
363
+ {
364
+ name: 'h264_vaapi',
365
+ triedOn: (host) => host.platform === 'linux' && host.configured,
366
+ upload: ',format=nv12,hwupload',
367
+ args: (selection, kbps) => [
368
+ '-vaapi_device',
369
+ '/dev/dri/renderD128',
370
+ '-profile:v',
371
+ selection.profile === 'baseline' ? 'constrained_baseline' : selection.profile,
372
+ '-level:v',
373
+ selection.level,
374
+ '-bf',
375
+ '0',
376
+ '-async_depth',
377
+ '1',
378
+ '-b:v',
379
+ kbps,
380
+ '-maxrate',
381
+ kbps,
382
+ ],
383
+ },
384
+ {
385
+ name: 'h264_v4l2m2m',
386
+ triedOn: (host) => host.platform === 'linux' && (host.arch === 'arm' || host.arch === 'arm64'),
387
+ args: (selection, kbps) => [
388
+ '-profile:v',
389
+ String(PROFILE_IDC[selection.profile]),
390
+ '-pix_fmt',
391
+ 'yuv420p',
392
+ '-b:v',
393
+ kbps,
394
+ ],
395
+ },
396
+ ];
397
+ /** How many seconds of synthetic picture one qualification run codes. */
398
+ const QUALIFICATION_SECONDS = 2;
399
+ /** The longest one qualification process may run before its selection counts as unqualified. */
400
+ const QUALIFICATION_RUN_TIMEOUT_MS = 10_000;
401
+ /** The longest qualification as a whole may take; a selection it has not finished by then stays unqualified. */
402
+ const QUALIFICATION_BUDGET_MS = 60_000;
403
+ /**
404
+ * The selections every hardware encoder is qualified against: each advertised profile and level, at the
405
+ * largest advertised geometry that level's frame-size limit admits, at the advertised frame-rate ceiling, and
406
+ * at 299 kbps, the lowest bit rate a controller has been measured selecting.
407
+ */
408
+ const QUALIFICATION_SELECTIONS = ['baseline', 'main', 'high'].flatMap((profile) => [
409
+ ['3.1', 1280, 720],
410
+ ['3.2', 1280, 720],
411
+ ['4.0', 1920, 1080],
412
+ ].map(([level, width, height]) => ({
413
+ profile,
414
+ level,
415
+ width,
416
+ height,
417
+ fps: 30,
418
+ maxBitRate: 299,
419
+ mtu: 1378,
420
+ payloadType: 99,
421
+ ssrc: 1,
422
+ rtcpInterval: 0.5,
423
+ })));
424
+ const runQualification = (executable, args, input, timeoutMs) => new Promise((resolve) => {
425
+ const child = execFile(executable, args, { encoding: 'buffer', timeout: timeoutMs, maxBuffer: 16 * 1024 * 1024, windowsHide: true }, (error, stdout, stderr) => resolve(error ? undefined : { stdout, stderr: stderr.toString() }));
426
+ child.stdin?.on('error', () => undefined);
427
+ child.stdin?.end(input);
428
+ });
429
+ /**
430
+ * Qualifies `encoder` for `selection` before `deadline`.
431
+ *
432
+ * The encoder codes {@link QUALIFICATION_SECONDS} of moving synthetic picture through the session's own
433
+ * argument list, which proves it is named by the binary, accepts those arguments, and opens on this host's
434
+ * devices. A second FFmpeg then traces the coded parameter sets, which proves what was coded: a profile at or
435
+ * below the negotiated one, the negotiated level and macroblock geometry, every frame, and no more bytes than
436
+ * the encoder budget plus the one second of it the session's rate control may borrow.
437
+ */
438
+ async function qualification(executable, encoder, selection, run, deadline) {
439
+ const within = (args, input) => {
440
+ const left = deadline - Date.now();
441
+ return left > 0 ? run(executable, args, input, Math.min(left, QUALIFICATION_RUN_TIMEOUT_MS)) : 'budget';
442
+ };
443
+ const coded = await within([
444
+ '-hide_banner',
445
+ '-loglevel',
446
+ 'error',
447
+ '-f',
448
+ 'lavfi',
449
+ '-i',
450
+ `testsrc2=size=${selection.width}x${selection.height}:rate=${selection.fps}:duration=${QUALIFICATION_SECONDS}`,
451
+ ...encodeArguments(encoder, selection),
452
+ '-f',
453
+ 'h264',
454
+ 'pipe:1',
455
+ ]);
456
+ if (!coded || coded === 'budget') {
457
+ return coded ?? 'encode';
458
+ }
459
+ const traced = await within(['-hide_banner', '-f', 'h264', '-i', 'pipe:0', '-c', 'copy', '-bsf:v', 'trace_headers', '-f', 'null', '-'], coded.stdout);
460
+ if (!traced || traced === 'budget') {
461
+ return traced ?? 'trace';
462
+ }
463
+ const field = (name) => Number(new RegExp(`\\b${name}\\s.*= (\\d+)$`, 'm').exec(traced.stderr)?.[1]);
464
+ const profile = field('profile_idc');
465
+ const wanted = PROFILE_IDC[selection.profile];
466
+ const checks = [
467
+ ['profile', profile === wanted || (profile === 66 && field('constraint_set1_flag') === 1) || (profile === 77 && wanted === 100)],
468
+ ['level', field('level_idc') === Number(selection.level.replace('.', ''))],
469
+ [
470
+ 'geometry',
471
+ field('pic_width_in_mbs_minus1') + 1 === Math.ceil(selection.width / 16) &&
472
+ field('pic_height_in_map_units_minus1') + 1 === Math.ceil(selection.height / 16),
473
+ ],
474
+ ['frames', (traced.stderr.match(/\bPacket: /g)?.length ?? 0) === selection.fps * QUALIFICATION_SECONDS],
475
+ ];
476
+ const bound = videoBudgetInsideCeiling(selection) * 125 * (QUALIFICATION_SECONDS + 1);
477
+ return (checks.find(([, passed]) => !passed)?.[0] ??
478
+ (coded.stdout.length <= bound ? 'qualified' : ['bytes', coded.stdout.length, bound]));
479
+ }
480
+ /** The live video packet size an administrator can choose for networks that drop the negotiated one. */
481
+ export const SMALL_VIDEO_PACKET_SIZE = 1128;
155
482
  /** Adapts separate SDK elementary streams into independently failing HomeKit SRTP outputs. */
156
483
  export class FfmpegLiveMedia {
157
484
  executable;
@@ -159,12 +486,79 @@ export class FfmpegLiveMedia {
159
486
  createProcess;
160
487
  reservePort;
161
488
  createReturnAudioProcess;
162
- constructor(executable, adaptationDiagnostics, createProcess = spawnLiveMediaProcess, reservePort = reserveMediaPort, createReturnAudioProcess = spawnReturnAudioProcess) {
489
+ maxVideoPacketSize;
490
+ copyVideo;
491
+ constantQuality;
492
+ constructor(executable, adaptationDiagnostics, createProcess = spawnLiveMediaProcess, reservePort = reserveMediaPort, createReturnAudioProcess = spawnReturnAudioProcess, maxVideoPacketSize = Number.POSITIVE_INFINITY, copyVideo = false, constantQuality = false) {
163
493
  this.executable = executable;
164
494
  this.adaptationDiagnostics = adaptationDiagnostics;
165
495
  this.createProcess = createProcess;
166
496
  this.reservePort = reservePort;
167
497
  this.createReturnAudioProcess = createReturnAudioProcess;
498
+ this.maxVideoPacketSize = maxVideoPacketSize;
499
+ this.copyVideo = copyVideo;
500
+ this.constantQuality = constantQuality;
501
+ }
502
+ /**
503
+ * Each hardware encoder's qualified selections, as `profile@level`, for the rest of this run.
504
+ *
505
+ * A spawn reads it, so a selection is served by hardware from the moment it qualifies, and an encoder that
506
+ * exits before output in a session is removed from it and serves nothing again.
507
+ */
508
+ qualified = new Map();
509
+ /**
510
+ * Qualifies every hardware encoder `host`'s binary may carry against every selection it is qualified for,
511
+ * one run after another and within {@link QUALIFICATION_BUDGET_MS}, and returns each tried encoder's
512
+ * outcome per `profile@level`.
513
+ *
514
+ * Until a selection qualifies it is coded by `libx264`, and a selection no encoder qualifies for stays there.
515
+ */
516
+ async qualify(host, run = runQualification) {
517
+ const deadline = Date.now() + QUALIFICATION_BUDGET_MS;
518
+ const outcomes = {};
519
+ for (const encoder of HARDWARE_ENCODERS.filter((candidate) => candidate.triedOn(host))) {
520
+ const tried = (outcomes[encoder.name] = {});
521
+ for (const selection of QUALIFICATION_SELECTIONS) {
522
+ const key = `${selection.profile}@${selection.level}`;
523
+ tried[key] = await qualification(this.executable, encoder, selection, run, deadline);
524
+ if (tried[key] === 'qualified') {
525
+ this.qualified.set(encoder.name, (this.qualified.get(encoder.name) ?? new Set()).add(key));
526
+ }
527
+ }
528
+ }
529
+ return outcomes;
530
+ }
531
+ /**
532
+ * What a session codes `selection` from a `codec` source with: `copy` for a passed-through H.264 source,
533
+ * {@link CONSTANT_QUALITY} where the negotiated bit rate is ignored, else the first hardware encoder qualified
534
+ * for `selection`, or `libx264` where none is or `settings` turn hardware off. Without `settings`, this
535
+ * adapter's own.
536
+ */
537
+ encoderFor(codec, selection, settings) {
538
+ if ((settings?.liveVideoPassthrough ?? this.copyVideo) && codec === 'h264')
539
+ return 'copy';
540
+ if (settings?.ignoreHomeBitRate ?? this.constantQuality)
541
+ return CONSTANT_QUALITY;
542
+ if (settings?.hardwareVideoEncoder === false)
543
+ return LIBX264;
544
+ const key = `${selection.profile}@${selection.level}`;
545
+ return HARDWARE_ENCODERS.find((encoder) => this.qualified.get(encoder.name)?.has(key)) ?? LIBX264;
546
+ }
547
+ /** The largest video packet a session sends under `settings`, or under this adapter's own without them. */
548
+ packetSize(settings) {
549
+ if (!settings)
550
+ return this.maxVideoPacketSize;
551
+ return settings.smallVideoPackets ? SMALL_VIDEO_PACKET_SIZE : Number.POSITIVE_INFINITY;
552
+ }
553
+ /**
554
+ * The encoder and the argument list a session would code `selection` from a `codec` source with, its video
555
+ * written to `output` instead of RTP: every argument before `output` is the session's own, or that of a session
556
+ * of an adapter configured with `settings`.
557
+ */
558
+ videoAdaptation(codec, selection, output, settings) {
559
+ const video = { ...selection, mtu: Math.min(selection.mtu, this.packetSize(settings)) };
560
+ const encoder = this.encoderFor(codec, video, settings);
561
+ return { encoder: encoder === 'copy' ? encoder : encoder.name, args: videoArguments(codec, video, encoder, output) };
168
562
  }
169
563
  /**
170
564
  * Reserves the negotiated output ports so `SetupEndpoints` can be answered with them, and returns a
@@ -175,6 +569,7 @@ export class FfmpegLiveMedia {
175
569
  async prepare(transport) {
176
570
  const videoPort = await this.reservePort(transport.addressVersion);
177
571
  const targetAddress = transport.addressVersion === 'ipv6' ? `[${transport.targetAddress}]` : transport.targetAddress;
572
+ const maxVideoPacketSize = this.packetSize(transport.liveVideoSettings);
178
573
  let audioPort;
179
574
  try {
180
575
  audioPort = transport.audio ? await this.reservePort(transport.addressVersion) : undefined;
@@ -184,6 +579,13 @@ export class FfmpegLiveMedia {
184
579
  throw error;
185
580
  }
186
581
  let source;
582
+ /**
583
+ * Cancels the source acquisition in flight, and is held for exactly as long as one is.
584
+ *
585
+ * The station's one media channel is held for the whole of an uncancelled acquisition, so a session that
586
+ * ends during one cancels it rather than leaving it to the SDK's own wait.
587
+ */
588
+ let acquiring;
187
589
  let videoProcess;
188
590
  let audioProcess;
189
591
  let returnAudioProcess;
@@ -194,7 +596,6 @@ export class FfmpegLiveMedia {
194
596
  let negotiated;
195
597
  let stopped = false;
196
598
  let receivedVideoKeyframe = false;
197
- let reconfigurationPending = false;
198
599
  /**
199
600
  * The coded configuration the source last announced, and the one the running adaptation was opened for.
200
601
  *
@@ -208,11 +609,20 @@ export class FfmpegLiveMedia {
208
609
  /** How many adaptation replacements the source's own changes have asked this session for. */
209
610
  let sourceInputReplacements = 0;
210
611
  let audioInputCodec;
612
+ /** An audio adaptation that exited before any output cannot code this source, so none is started again. */
613
+ let audioFailed = false;
211
614
  let rtcpDeadline;
212
615
  let initialRtcpGrace;
213
616
  let videoStartBackstop;
214
617
  let videoFailed = false;
215
618
  let rtcpObserved = false;
619
+ /** How many receiver reports this session was acknowledged by, and when the last one arrived. */
620
+ /** Why the video adaptation now running was started, which the record reporting it carries. */
621
+ let videoCause = 'first';
622
+ /** Where the next adaptation this session sends on starts its RTP sequence, advanced for every one. */
623
+ let videoSequence = 0;
624
+ let rtcpReports = 0;
625
+ let lastRtcpAt;
216
626
  let streaming = false;
217
627
  const stoppingProcesses = new WeakSet();
218
628
  const congested = new Set();
@@ -253,38 +663,14 @@ export class FfmpegLiveMedia {
253
663
  }
254
664
  sink.once('drain', () => releaseAdaptation(sink));
255
665
  };
256
- /**
257
- * Reports what one adaptation process did, whether or not the session it belonged to fails for it.
258
- *
259
- * An audio adaptation is restarted rather than failed and a return-audio one fails only talkback, so
260
- * without a report of their own those two exit with no account anywhere of why they stopped. An `output`
261
- * report carries nothing but the tail, so it is made only when the process actually wrote something:
262
- * a silent process that ended as intended has nothing to attribute.
263
- */
264
- const reportAdaptation = (role, event, stderr, code, signal) => {
265
- const tail = stderr.tail();
266
- if (event === 'output' && tail.length === 0) {
267
- return;
268
- }
269
- adaptationDiagnostics?.report({
270
- role,
271
- event,
272
- ...(typeof code === 'number' ? { code } : {}),
273
- ...(typeof signal === 'string' ? { signal } : {}),
274
- ...(tail.length ? { stderr: tail } : {}),
275
- });
276
- };
277
- const stopProcess = (process) => {
666
+ const reportAdaptation = adaptationReporter(adaptationDiagnostics);
667
+ const stopAdaptation = (process) => {
278
668
  if (!process) {
279
669
  return;
280
670
  }
281
671
  stoppingProcesses.add(process);
282
- process.stdin.destroy();
672
+ stopProcess(process);
283
673
  releaseAdaptation(process.stdin);
284
- process.kill('SIGTERM');
285
- const killDeadline = setTimeout(() => process.kill('SIGKILL'), PROCESS_STOP_GRACE_MS);
286
- killDeadline.unref?.();
287
- process.on('exit', () => clearTimeout(killDeadline));
288
674
  };
289
675
  /** Stop a caller-owned SDK handle without allowing synchronous or asynchronous faults to abort cleanup. */
290
676
  const stopTalkbackHandle = (handle) => {
@@ -298,7 +684,9 @@ export class FfmpegLiveMedia {
298
684
  return;
299
685
  }
300
686
  talkbackEnded = true;
301
- stopProcess(returnAudioProcess);
687
+ stopAdaptation(returnAudioProcess);
688
+ returnAudioProcess?.stdout.destroy();
689
+ returnAudioProcess?.eld.destroy();
302
690
  returnAudioProcess = undefined;
303
691
  talkbackSink?.destroy();
304
692
  talkbackSink = undefined;
@@ -320,33 +708,43 @@ export class FfmpegLiveMedia {
320
708
  return;
321
709
  }
322
710
  stopped = true;
711
+ acquiring?.abort(new Error('live media session stopped'));
323
712
  clearTimeout(rtcpDeadline);
324
713
  clearTimeout(initialRtcpGrace);
325
714
  clearTimeout(videoStartBackstop);
326
715
  stopTalkback();
327
- stopProcess(videoProcess);
328
- stopProcess(audioProcess);
329
- if (source) {
330
- source.stop();
331
- transport.onSessionReleased?.();
716
+ stopAdaptation(videoProcess);
717
+ stopAdaptation(audioProcess);
718
+ source?.stop();
719
+ if (negotiated) {
720
+ transport.onSessionReleased?.(videoFailed ? 'failed' : 'requested', {
721
+ reports: rtcpReports,
722
+ ...(lastRtcpAt === undefined ? {} : { sinceLastReportMs: Date.now() - lastRtcpAt }),
723
+ }, ...(source === undefined ? ['sdk-source-acquisition'] : []));
332
724
  }
333
725
  videoPort.close();
334
726
  audioPort?.close();
335
727
  };
336
- const failVideo = (reason) => {
728
+ const failVideo = (reason, error) => {
337
729
  if (stopped || videoFailed) {
338
730
  return;
339
731
  }
340
732
  videoFailed = true;
341
733
  stop();
734
+ const warmUp = error instanceof LiveStreamStartError;
342
735
  const stage = reason === 'source-acquisition-timeout' || source === undefined
343
736
  ? 'sdk-source-acquisition'
344
737
  : reason === 'rtcp-timeout'
345
738
  ? 'controller-rtcp'
346
- : !receivedVideoKeyframe
739
+ : warmUp || !receivedVideoKeyframe
347
740
  ? 'first-source-keyframe'
348
741
  : 'first-adapted-output';
349
- transport.onSessionOutcome?.({ outcome: 'failed', reason, stage });
742
+ transport.onSessionOutcome?.({
743
+ outcome: 'failed',
744
+ reason,
745
+ stage,
746
+ ...(warmUp ? { sourceStage: error.stage, sourceReason: error.reason } : {}),
747
+ });
350
748
  transport.onVideoFailure?.();
351
749
  };
352
750
  /**
@@ -357,6 +755,8 @@ export class FfmpegLiveMedia {
357
755
  */
358
756
  const resetRtcpDeadline = () => {
359
757
  rtcpObserved = true;
758
+ rtcpReports += 1;
759
+ lastRtcpAt = Date.now();
360
760
  clearTimeout(initialRtcpGrace);
361
761
  clearTimeout(rtcpDeadline);
362
762
  if (!streaming) {
@@ -371,13 +771,9 @@ export class FfmpegLiveMedia {
371
771
  * Adaptation reached the negotiated output, so the session is bounded from here by RTCP liveness
372
772
  * rather than by the start backstop. The initial grace is armed from this point because media may
373
773
  * legitimately start well after the session does.
374
- *
375
- * A selection HomeKit has already replaced does not discharge the deadline for its replacement, however
376
- * much it keeps reporting: FFmpeg reports progress on a timer whether or not new media reaches it, so
377
- * only the adaptation that carries the current selection can say that selection is being served.
378
774
  */
379
- const observeAdaptationProgress = (reporter) => {
380
- if (stopped || (reconfigurationPending && reporter === videoProcess)) {
775
+ const observeAdaptationProgress = () => {
776
+ if (stopped) {
381
777
  return;
382
778
  }
383
779
  clearTimeout(videoStartBackstop);
@@ -406,13 +802,12 @@ export class FfmpegLiveMedia {
406
802
  * Writes one source access unit to the adaptation that is entitled to code it.
407
803
  *
408
804
  * A source configuration the running adaptation was not opened for cannot be coded by it at all, so those
409
- * frames wait for the keyframe that lets a replacement start. A reconfigured HomeKit selection changes
410
- * only the output, so the current process keeps coding and keeps the previous selection on the wire until
411
- * that same keyframe arrives; a controller reconfigures precisely when it is unhappy with what it
412
- * receives, and the source is then often the very thing not producing keyframes.
805
+ * frames wait for the keyframe that lets a replacement start.
413
806
  *
414
807
  * Source-driven replacement is bounded by {@link MAX_SOURCE_INPUT_REPLACEMENTS}, past which the session
415
- * fails rather than spawn again. A controller-driven one is not: the controller bounds how often it asks.
808
+ * fails rather than spawn again. It is the only replacement a session makes: a source that changes what it
809
+ * delivers cannot be coded by the running process at all, whereas a controller's renegotiated selection is
810
+ * a request this session declines in favour of the one it started on.
416
811
  */
417
812
  const writeVideo = (frame) => {
418
813
  if (stopped || !negotiated || !videoConfig) {
@@ -428,29 +823,39 @@ export class FfmpegLiveMedia {
428
823
  if (videoProcess && inputChanged && !frame.keyframe) {
429
824
  return;
430
825
  }
431
- if (videoProcess && frame.keyframe && (inputChanged || reconfigurationPending)) {
432
- if (inputChanged) {
433
- sourceInputReplacements += 1;
434
- }
826
+ if (videoProcess && frame.keyframe && inputChanged) {
827
+ sourceInputReplacements += 1;
435
828
  if (sourceInputReplacements > MAX_SOURCE_INPUT_REPLACEMENTS) {
436
829
  failVideo('source-input-unstable');
437
830
  return;
438
831
  }
439
- reconfigurationPending = false;
440
- stopProcess(videoProcess);
832
+ videoCause = 'source-configuration';
833
+ stopAdaptation(videoProcess);
441
834
  videoProcess = undefined;
442
835
  }
443
836
  if (!videoProcess) {
444
837
  adaptationConfig = videoConfig;
445
- const child = this.createProcess(this.executable, videoArguments(videoConfig, negotiated.video, targetAddress, transport.video));
838
+ videoSequence = (videoSequence + OUTPUT_SEQUENCE_STEP) & 0xffff;
839
+ const { video } = negotiated;
840
+ const encoder = this.encoderFor(videoConfig.codec, video, transport.liveVideoSettings);
841
+ const child = this.createProcess(this.executable, videoArguments(videoConfig.codec, video, encoder, outputArguments(targetAddress, transport.video, video.payloadType, video.ssrc, video.mtu, videoSequence)));
446
842
  videoProcess = child;
447
- adaptationDiagnostics?.report({ role: 'live-video', event: 'started' });
843
+ const encoderName = encoder === 'copy' ? encoder : encoder.name;
844
+ adaptationDiagnostics?.report({
845
+ role: 'live-video',
846
+ event: 'started',
847
+ cause: videoCause,
848
+ encoder: encoderName,
849
+ source: videoConfig.codec,
850
+ sourceWidth: videoConfig.width,
851
+ sourceHeight: videoConfig.height,
852
+ });
448
853
  const stderr = new AdaptationStderr();
449
854
  let producedOutput = false;
450
855
  child.stderr.on('data', (chunk) => {
451
856
  if (stderr.observe(chunk).some((line) => line.startsWith('progress='))) {
452
857
  producedOutput = true;
453
- observeAdaptationProgress(child);
858
+ observeAdaptationProgress();
454
859
  }
455
860
  });
456
861
  child.stdin.on('error', () => {
@@ -472,19 +877,25 @@ export class FfmpegLiveMedia {
472
877
  return;
473
878
  }
474
879
  reportAdaptation('live-video', producedOutput ? 'exited-while-streaming' : 'exited-before-output', stderr, code, signal);
880
+ if (!producedOutput && this.qualified.delete(encoderName)) {
881
+ adaptationDiagnostics?.report({ role: 'live-video', event: 'encoder-disqualified', encoder: encoderName });
882
+ }
475
883
  failVideo(producedOutput ? 'adaptation-exited-while-streaming' : 'adaptation-exited-before-output');
476
884
  });
477
885
  }
478
886
  writeToAdaptation(videoProcess.stdin, frame.data);
479
887
  };
480
888
  const writeAudio = (frame) => {
481
- if (stopped || !negotiated?.audio || !transport.audio || !audioPort) {
889
+ if (stopped || audioFailed || !negotiated?.audio || !transport.audio || !audioPort) {
482
890
  return;
483
891
  }
484
892
  if (!audioProcess) {
485
893
  audioInputCodec = frame.codec;
486
894
  const child = this.createProcess(this.executable, audioArguments(frame, negotiated.audio, targetAddress, transport.audio));
487
895
  audioProcess = child;
896
+ if (frame.config) {
897
+ writeToAdaptation(child.stdin, Buffer.concat([FLV_FILE_HEADER, flvAacTag(0, frame.config)]));
898
+ }
488
899
  adaptationDiagnostics?.report({ role: 'live-audio', event: 'started' });
489
900
  const stderr = new AdaptationStderr();
490
901
  let producedOutput = false;
@@ -509,44 +920,50 @@ export class FfmpegLiveMedia {
509
920
  if (!stoppingProcesses.has(child) && audioProcess === child) {
510
921
  reportAdaptation('live-audio', 'input-failed', stderr);
511
922
  }
512
- stopProcess(child);
923
+ stopAdaptation(child);
513
924
  clearChild();
514
925
  });
515
926
  child.on('error', () => {
516
927
  if (!stoppingProcesses.has(child)) {
517
928
  reportAdaptation('live-audio', producedOutput ? 'exited-while-streaming' : 'spawn-failed', stderr);
518
929
  }
519
- stopProcess(child);
930
+ stopAdaptation(child);
520
931
  clearChild();
521
932
  });
522
933
  child.on('exit', (code, signal) => {
523
- reportAdaptation('live-audio', stopped || stoppingProcesses.has(child)
934
+ const event = stopped || stoppingProcesses.has(child)
524
935
  ? 'output'
525
936
  : producedOutput
526
937
  ? 'exited-while-streaming'
527
- : 'exited-before-output', stderr, code, signal);
938
+ : 'exited-before-output';
939
+ audioFailed ||= event === 'exited-before-output';
940
+ reportAdaptation('live-audio', event, stderr, code, signal);
528
941
  clearChild();
529
942
  });
530
943
  }
531
944
  if (audioInputCodec !== frame.codec) {
532
- stopProcess(audioProcess);
945
+ stopAdaptation(audioProcess);
533
946
  audioProcess = undefined;
534
947
  audioInputCodec = undefined;
535
948
  writeAudio(frame);
536
949
  return;
537
950
  }
538
- writeToAdaptation(audioProcess.stdin, frame.data);
951
+ writeToAdaptation(audioProcess.stdin, frame.config ? flvAacTag(1, frame.data) : frame.data);
539
952
  };
540
953
  /**
541
954
  * Write one decoded return-audio chunk through the SDK writable that owns ADTS frame recovery,
542
955
  * validation, pacing, and transport backpressure. The first chunk opens exactly one SDK handle;
543
956
  * stdout stays paused while that handle is acquired so acquisition cannot create an unbounded queue.
957
+ * Once the handle's `codec` is `aac-eld`, ADTS output is discarded and fd 3 feeds the handle instead.
544
958
  */
545
959
  const writeReturnAudio = (camera, child, chunk) => {
546
960
  if (stopped || talkbackEnded) {
547
961
  return;
548
962
  }
549
963
  if (talkbackSink) {
964
+ if (talkbackHandle?.codec === 'aac-eld') {
965
+ return;
966
+ }
550
967
  try {
551
968
  if (!talkbackSink.write(chunk)) {
552
969
  child.stdout.pause();
@@ -596,6 +1013,14 @@ export class FfmpegLiveMedia {
596
1013
  }
597
1014
  });
598
1015
  sink.on('error', () => failTalkback('device-audio-failed'));
1016
+ if (handle.codec === 'aac-eld') {
1017
+ const reader = new FlvAacReader();
1018
+ child.eld.on('data', (eldChunk) => writeReturnFrames(child, reader, sink, eldChunk));
1019
+ child.stdout.resume();
1020
+ transport.onTalkbackOutcome?.({ outcome: 'talking' });
1021
+ return;
1022
+ }
1023
+ child.eld.resume();
599
1024
  const accepted = sink.write(chunk);
600
1025
  if (talkbackEnded) {
601
1026
  return;
@@ -617,10 +1042,33 @@ export class FfmpegLiveMedia {
617
1042
  }
618
1043
  }, () => failTalkback('source-unavailable'));
619
1044
  };
1045
+ /**
1046
+ * Write the access units completed by one ELD output chunk through the SDK's object-mode writable,
1047
+ * holding fd 3 from the first refusal until that writable drains. What stays buffered is the SDK's
1048
+ * bounded pacing queue and at most the rest of the one read that was in hand. A write that fails or
1049
+ * ends the talkback stops the loop.
1050
+ */
1051
+ const writeReturnFrames = (child, reader, sink, chunk) => {
1052
+ for (const data of reader.push(chunk)) {
1053
+ if (stopped || talkbackEnded) {
1054
+ return;
1055
+ }
1056
+ if (!sink.write(data) && !child.eld.isPaused()) {
1057
+ child.eld.pause();
1058
+ sink.once('drain', () => {
1059
+ if (!stopped && !talkbackEnded) {
1060
+ child.eld.resume();
1061
+ }
1062
+ });
1063
+ }
1064
+ }
1065
+ };
620
1066
  /**
621
1067
  * Hand the accessory audio endpoint to FFmpeg and give the spawned process a bounded bind grace before
622
1068
  * stream start is acknowledged. FFmpeg owns SRTP authentication, RTP AAC-ELD depacketization, decoding,
623
- * and AAC-LC ADTS encoding; the SDK handle remains unopened until the process produces decoded audio.
1069
+ * the AAC-LC ADTS encoding on stdout, and the copy of HomeKit's AAC-ELD access units on fd 3; the SDK
1070
+ * handle remains unopened until the process produces decoded audio, and the handle's `codec` then
1071
+ * selects the ADTS output (stdout) or the ELD output (fd 3).
624
1072
  */
625
1073
  const startReturnAudio = async (camera, selection) => {
626
1074
  if (!camera.talkback || !selection || !transport.audio || !audioPort) {
@@ -640,22 +1088,19 @@ export class FfmpegLiveMedia {
640
1088
  const stderr = new AdaptationStderr();
641
1089
  let producedOutput = false;
642
1090
  child.stderr.on('data', (chunk) => stderr.observe(chunk));
643
- child.stdin.on('error', () => {
1091
+ const failInput = () => {
644
1092
  if (!stopped && !talkbackEnded && !stoppingProcesses.has(child)) {
645
1093
  reportAdaptation('return-audio', 'input-failed', stderr);
646
1094
  failTalkback('adaptation-failed');
647
1095
  }
648
- });
1096
+ };
1097
+ child.stdin.on('error', failInput);
649
1098
  child.stdout.on('data', (chunk) => {
650
1099
  producedOutput = true;
651
1100
  writeReturnAudio(camera, child, chunk);
652
1101
  });
653
- child.stdout.on('error', () => {
654
- if (!stopped && !talkbackEnded && !stoppingProcesses.has(child)) {
655
- reportAdaptation('return-audio', 'input-failed', stderr);
656
- failTalkback('adaptation-failed');
657
- }
658
- });
1102
+ child.stdout.on('error', failInput);
1103
+ child.eld.on('error', failInput);
659
1104
  child.on('error', () => {
660
1105
  if (!stoppingProcesses.has(child)) {
661
1106
  reportAdaptation('return-audio', producedOutput ? 'exited-while-streaming' : 'spawn-failed', stderr);
@@ -668,7 +1113,11 @@ export class FfmpegLiveMedia {
668
1113
  return;
669
1114
  }
670
1115
  reportAdaptation('return-audio', producedOutput ? 'exited-while-streaming' : 'exited-before-output', stderr, code, signal);
671
- failTalkback('adaptation-failed');
1116
+ failTalkback(!stderr.tail().some((line) => line.includes('Error during demuxing: Operation timed out'))
1117
+ ? 'adaptation-failed'
1118
+ : receivedVideoKeyframe
1119
+ ? 'no-controller-audio'
1120
+ : 'source-unavailable');
672
1121
  });
673
1122
  child.stdin.end(returnAudioSdp(audioPort.port, selection, transport.audio, transport.addressVersion));
674
1123
  await delay(RETURN_AUDIO_BIND_GRACE_MS);
@@ -684,12 +1133,14 @@ export class FfmpegLiveMedia {
684
1133
  if (stopped) {
685
1134
  throw new Error('live media session is already stopped');
686
1135
  }
687
- negotiated = selection;
1136
+ negotiated = { ...selection, video: { ...selection.video, mtu: Math.min(selection.video.mtu, maxVideoPacketSize) } };
688
1137
  const returnAudioReady = startReturnAudio(camera, selection.audio);
689
1138
  let sourcePromise;
690
1139
  let acquisitionDeadline;
1140
+ const acquisition = new AbortController();
1141
+ acquiring = acquisition;
691
1142
  try {
692
- sourcePromise = camera.live();
1143
+ sourcePromise = camera.live({ signal: acquisition.signal });
693
1144
  void sourcePromise.then((lateSource) => {
694
1145
  if (stopped && source !== lateSource) {
695
1146
  lateSource.stop();
@@ -704,11 +1155,13 @@ export class FfmpegLiveMedia {
704
1155
  ]);
705
1156
  }
706
1157
  catch (error) {
707
- failVideo(error === SOURCE_ACQUISITION_TIMEOUT ? 'source-acquisition-timeout' : sourceFailure(error));
1158
+ acquisition.abort(new Error('live media source acquisition was cancelled'));
1159
+ failVideo(error === SOURCE_ACQUISITION_TIMEOUT ? 'source-acquisition-timeout' : liveSourceFailure(error), error);
708
1160
  throw error === SOURCE_ACQUISITION_TIMEOUT ? new Error('live media source acquisition timed out') : error;
709
1161
  }
710
1162
  finally {
711
1163
  clearTimeout(acquisitionDeadline);
1164
+ acquiring = undefined;
712
1165
  }
713
1166
  if (stopped) {
714
1167
  source.stop();
@@ -724,49 +1177,31 @@ export class FfmpegLiveMedia {
724
1177
  notice.extend();
725
1178
  }
726
1179
  });
727
- source.on('error', (error) => failVideo(sourceFailure(error)));
1180
+ source.on('error', (error) => failVideo(liveSourceFailure(error), error));
728
1181
  source.on('stop', () => failVideo('source-stopped'));
729
1182
  await returnAudioReady;
730
1183
  },
731
- /**
732
- * Acknowledges a reconfigured selection and applies it at the next source keyframe.
733
- *
734
- * Nothing is torn down here: the current adaptation keeps the previous selection on the wire until the
735
- * replacement has a keyframe to start from. The deferral is still bounded, because a session that never
736
- * applies the selection HomeKit asked for must end and be renegotiated rather than silently serve the
737
- * old one forever.
738
- */
739
- reconfigure(video) {
740
- if (!negotiated) {
741
- return;
742
- }
743
- negotiated = { ...negotiated, video };
744
- reconfigurationPending = videoProcess !== undefined;
745
- clearTimeout(videoStartBackstop);
746
- videoStartBackstop = setTimeout(() => failVideo('no-video-within-backstop'), SOURCE_START_BACKSTOP_MS);
747
- videoStartBackstop.unref?.();
748
- },
749
1184
  stop,
750
1185
  };
751
1186
  }
752
1187
  }
753
1188
  /**
754
- * Options every adapted elementary stream applies to its own input.
1189
+ * Options every adapted input applies to itself: a piped elementary stream, or audio framed as FLV.
755
1190
  *
756
- * A piped elementary stream carries no container, so FFmpeg's initial stream analysis is the only thing
757
- * standing between the first written access unit and the first coded frame. The caller already declares the
758
- * format, so that analysis has nothing left to discover and is bounded to its minimum; leaving it at the
1191
+ * FFmpeg's initial stream analysis is the only thing standing between the first written access unit and the
1192
+ * first coded frame. The caller already declares the format, and an FLV input states its decoder config in its
1193
+ * first tag, so that analysis has nothing left to discover and is bounded to its minimum; leaving it at the
759
1194
  * default delays first output by seconds and scales that delay with the source keyframe interval.
760
1195
  *
761
- * Every such input is read on one clock: when its media arrived. A pipe carries no timeline of its own, so
762
- * arrival is the only timeline there is, and two adapted streams of one session can be presented together
763
- * only while both are read on it.
1196
+ * Every such input is read on one clock: when its media arrived. An elementary stream carries no timeline of its
1197
+ * own and the FLV tags this plugin writes carry none either, so arrival is the only timeline there is, and two
1198
+ * adapted streams of one session can be presented together only while both are read on it.
764
1199
  *
765
1200
  * Nothing asks FFmpeg to discard or reinterpret what it read. Discarding the analysed packets throws away the
766
1201
  * leading keyframe the caller waited for, and a raw A-law demuxer stops emitting timestamps entirely, so both
767
1202
  * cost media that was already in hand.
768
1203
  */
769
- function commonArguments(inputFormat, inputOptions = []) {
1204
+ export function commonArguments(inputFormat, inputOptions = []) {
770
1205
  return [
771
1206
  '-hide_banner',
772
1207
  '-loglevel',
@@ -787,12 +1222,21 @@ function commonArguments(inputFormat, inputOptions = []) {
787
1222
  'pipe:0',
788
1223
  ];
789
1224
  }
790
- function outputArguments(targetAddress, target, payloadType, ssrc, packetSize) {
1225
+ /**
1226
+ * The RTP output one adaptation sends on: the controller's endpoint, the identity the session negotiated for
1227
+ * it, and where its sequence numbering starts.
1228
+ *
1229
+ * `sequence` is stated wherever a session may send on more than one adaptation, because the identity and the
1230
+ * SRTP key outlive the process: see {@link OUTPUT_SEQUENCE_STEP} for what a restarted numbering costs. An
1231
+ * output no replacement can follow omits it and keeps FFmpeg's own base.
1232
+ */
1233
+ function outputArguments(targetAddress, target, payloadType, ssrc, packetSize, sequence) {
791
1234
  return [
792
1235
  '-payload_type',
793
1236
  String(payloadType),
794
1237
  '-ssrc',
795
1238
  String(ssrc),
1239
+ ...(sequence === undefined ? [] : ['-seq', String(sequence)]),
796
1240
  '-f',
797
1241
  'rtp',
798
1242
  '-srtp_out_suite',
@@ -811,11 +1255,19 @@ function outputArguments(targetAddress, target, payloadType, ssrc, packetSize) {
811
1255
  * every frame it was given.
812
1256
  *
813
1257
  * A negotiated frame rate is a CEILING, not a cadence, so the rate is bounded rather than pinned. Pinning it
814
- * makes the encoder duplicate frames the source never sent, and each duplicate costs a full encode and a share
815
- * of the negotiated bit rate for a picture carrying nothing new. Measured on a 1600x1200 doorbell delivering
816
- * about 15 fps against a 30 fps selection: 264 of 267 emitted frames were duplicates and the adaptation ran at
817
- * 0.32x real time, so it fell further behind every second and the session died on its backstop with nothing
818
- * watchable reaching the controller.
1258
+ * with `-r` makes the encoder interpolate a cadence a bare Annex-B pipe never states, which collapses a session
1259
+ * onto one instant.
1260
+ *
1261
+ * The output is constant-rate, which fills a gap in arrival with a duplicate of the last picture rather than
1262
+ * carrying the gap. A duplicate costs a full encode and a share of the negotiated bit rate for a frame carrying
1263
+ * nothing new, and a source over this transport gaps as its normal case: measured on the bundled encoder against
1264
+ * a 15 fps arrival stalled 1.5 seconds after every second of media, a constant rate emitted 266 frames of which
1265
+ * 146 were duplicates, against 120 frames and none when the arrival timestamps were passed through, at the same
1266
+ * 0.90x real time either way. What the duplicates buy is a dense presentation timeline: a controller shown the
1267
+ * gaps instead displays a picture that visibly jumps at every one of them.
1268
+ *
1269
+ * `-fpsmax` states that ceiling for a constant-rate output only, and FFmpeg refuses it beside any other
1270
+ * frame-rate mode, so the two cannot be combined.
819
1271
  *
820
1272
  * A negotiated selection states no refresh cadence, so the keyframe interval is plugin policy: `-g` and
821
1273
  * `-keyint_min` are the negotiated rate doubled and scene-cut detection is off, which is a refresh every two
@@ -834,26 +1286,30 @@ function outputArguments(targetAddress, target, payloadType, ssrc, packetSize) {
834
1286
  * coded stream can carry a negotiated Main or High profile; `ultrafast` drops CABAC and codes Constrained
835
1287
  * Baseline whatever `-profile:v` asks for. `-tune zerolatency` pins the same `sliced_threads`, `bframes`
836
1288
  * and `rc_lookahead` at either preset, so this costs computation rather than frame delay.
1289
+ *
1290
+ * `copy` sends an H.264 source as the camera coded it, with no encoder in the process. Nothing above then holds:
1291
+ * the geometry, frame rate, bit rate, profile and keyframe interval are the camera's own, not the negotiated
1292
+ * selection. Only the RTP output is the negotiated one. An H.265 source is transcoded either way.
1293
+ *
1294
+ * {@link CONSTANT_QUALITY} codes at {@link CONSTANT_QUALITY_CRF} with no rate control, so the negotiated bit rate is not
1295
+ * honoured: the output spends what the picture needs. A controller asks 299 kbps for 720p and does not raise it
1296
+ * during a session, which the ceiling above codes as a blurred, then blocky, picture.
837
1297
  */
838
- function videoArguments(input, selection, targetAddress, target) {
839
- const budget = videoBudgetInsideCeiling(selection);
1298
+ function videoArguments(codec, selection, encoder, output) {
1299
+ if (encoder === 'copy') {
1300
+ return [...commonArguments('h264'), '-an', '-c:v', 'copy', ...output];
1301
+ }
1302
+ return [...commonArguments(codec === 'h265' ? 'hevc' : codec), ...encodeArguments(encoder, selection), ...output];
1303
+ }
1304
+ /** What one encoder is given to code `selection`, between its input and its output, in a session and in qualification alike. */
1305
+ function encodeArguments(encoder, selection) {
840
1306
  return [
841
- ...commonArguments(input.codec === 'h265' ? 'hevc' : input.codec),
842
1307
  '-an',
843
1308
  '-c:v',
844
- 'libx264',
845
- '-preset',
846
- 'superfast',
847
- '-tune',
848
- 'zerolatency',
849
- '-profile:v',
850
- selection.profile,
851
- '-level:v',
852
- selection.level,
853
- '-pix_fmt',
854
- 'yuv420p',
1309
+ encoder.name,
1310
+ ...encoder.args(selection, `${videoBudgetInsideCeiling(selection)}k`),
855
1311
  '-vf',
856
- `scale=${selection.width}:${selection.height}:force_original_aspect_ratio=decrease,pad=${selection.width}:${selection.height}:(ow-iw)/2:(oh-ih)/2`,
1312
+ `scale=${selection.width}:${selection.height}:force_original_aspect_ratio=decrease,pad=${selection.width}:${selection.height}:(ow-iw)/2:(oh-ih)/2${encoder.upload ?? ''}`,
857
1313
  '-fpsmax',
858
1314
  String(selection.fps),
859
1315
  '-g',
@@ -862,22 +1318,84 @@ function videoArguments(input, selection, targetAddress, target) {
862
1318
  String(selection.fps * 2),
863
1319
  '-sc_threshold',
864
1320
  '0',
865
- '-b:v',
866
- `${budget}k`,
867
- '-maxrate',
868
- `${budget}k`,
869
- '-bufsize',
870
- `${budget}k`,
871
- ...outputArguments(targetAddress, target, selection.payloadType, selection.ssrc, selection.mtu),
872
1321
  ];
873
1322
  }
1323
+ /** FLV signature, version 1, the audio-only flag, header size 9, and the zero size of the tag before the first. */
1324
+ const FLV_FILE_HEADER = Buffer.from([0x46, 0x4c, 0x56, 0x01, 0x04, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00]);
1325
+ /**
1326
+ * One FLV audio tag of an AAC elementary stream and its trailing size. The body is the AAC sound flags 0xaf, the AAC
1327
+ * packet type (0 for the AudioSpecificConfig, 1 for one access unit) and `payload` unchanged. The sound flags fix the
1328
+ * 44 kHz, 16-bit, stereo values the format requires; an AAC decoder takes the real parameters from the
1329
+ * AudioSpecificConfig. The timestamp, its extension and the stream id are zero: the stream carries no timeline of its
1330
+ * own, because its reader stamps every tag with the instant it arrived.
1331
+ */
1332
+ function flvAacTag(packetType, payload) {
1333
+ const header = Buffer.alloc(13);
1334
+ header[0] = 8;
1335
+ header.writeUIntBE(payload.length + 2, 1, 3);
1336
+ header[11] = 0xaf;
1337
+ header[12] = packetType;
1338
+ const previousTagSize = Buffer.alloc(4);
1339
+ previousTagSize.writeUInt32BE(13 + payload.length);
1340
+ return Buffer.concat([header, payload, previousTagSize]);
1341
+ }
874
1342
  /**
875
- * Adapts one source audio elementary stream to the negotiated AAC-ELD output.
1343
+ * Every access unit of one AAC elementary stream carried in FLV tags, in order. Bytes arrive in arbitrary pieces, so a
1344
+ * partial tag is held until the rest arrives; the sequence-header tag and non-AAC tags are skipped.
1345
+ */
1346
+ class FlvAacReader {
1347
+ held = Buffer.alloc(0);
1348
+ headerSkipped = false;
1349
+ /** Feed the next bytes of the stream and return the access units they complete. */
1350
+ push(chunk) {
1351
+ const bytes = this.held.length ? Buffer.concat([this.held, chunk]) : chunk;
1352
+ let at = 0;
1353
+ if (!this.headerSkipped) {
1354
+ if (bytes.length < FLV_FILE_HEADER.length) {
1355
+ this.held = Buffer.from(bytes);
1356
+ return [];
1357
+ }
1358
+ at = FLV_FILE_HEADER.length;
1359
+ this.headerSkipped = true;
1360
+ }
1361
+ const units = [];
1362
+ while (bytes.length - at >= 11) {
1363
+ const size = bytes.readUIntBE(at + 1, 3);
1364
+ const end = at + 11 + size + 4;
1365
+ if (bytes.length < end) {
1366
+ break;
1367
+ }
1368
+ if (bytes[at] === 8 && size > 2 && bytes[at + 11] === 0xaf && bytes[at + 12] === 1) {
1369
+ units.push(Buffer.from(bytes.subarray(at + 13, at + 11 + size)));
1370
+ }
1371
+ at = end;
1372
+ }
1373
+ this.held = Buffer.from(bytes.subarray(at));
1374
+ return units;
1375
+ }
1376
+ }
1377
+ /**
1378
+ * The FFmpeg input format and input options for one source audio stream.
876
1379
  *
877
- * The SDK reports no sample rate or channel count, because a station sends neither; 16 kHz mono is the
878
- * assumption every Eufy client applies. Raw A-law carries nothing at all and must be told that assumption,
879
- * while an ADTS input states its own rate in every frame header and rejects the option outright, which fails
880
- * the process before it reads a byte.
1380
+ * A stream whose decoder config travels apart from its access units is framed as FLV, which states that config
1381
+ * once ahead of them, and is decoded by `libfdk_aac`, which decodes AAC-ELD in the config the SDK declares. Outside
1382
+ * such a config the SDK reports no sample rate or channel count, because a station sends neither;
1383
+ * 16 kHz mono is the assumption every Eufy client applies. Raw A-law carries nothing at all and must be told that
1384
+ * assumption, while an ADTS input states its own rate in every frame header and rejects the option outright, which
1385
+ * fails the process before it reads a byte.
1386
+ */
1387
+ function audioInput(frame) {
1388
+ if (frame.config) {
1389
+ return ['flv', ['-c:a', 'libfdk_aac']];
1390
+ }
1391
+ if (frame.codec === 'g711a') {
1392
+ return ['alaw', ['-ar', '16k', '-ac', '1']];
1393
+ }
1394
+ return ['aac', []];
1395
+ }
1396
+ /**
1397
+ * Adapts one source audio elementary stream to the negotiated AAC-ELD output, reading it as {@link audioInput}
1398
+ * describes.
881
1399
  *
882
1400
  * Asynchronous resampling is what makes this output honour the arrival clock its input is read on. An encoder
883
1401
  * advances its output timeline by the samples in each coded frame it is handed, whatever timestamp arrived
@@ -890,13 +1408,12 @@ function videoArguments(input, selection, targetAddress, target) {
890
1408
  * accident of the filter: a live view owes its controller the newest audio, and audio which is already
891
1409
  * seconds stale by the time it arrives cannot be presented beside a picture timestamped on arrival.
892
1410
  *
893
- * `libfdk_aac` selects its transport from the requested output framing, and AAC-ELD cannot be carried in
1411
+ * The `libfdk_aac` encoder selects its transport from the requested output framing, and AAC-ELD cannot be carried in
894
1412
  * ADTS, so without an explicit global header the encoder refuses to initialise at all.
895
1413
  */
896
1414
  function audioArguments(frame, selection, targetAddress, target) {
897
- const rawAlaw = frame.codec === 'g711a';
898
1415
  return [
899
- ...commonArguments(rawAlaw ? 'alaw' : 'aac', rawAlaw ? ['-ar', '16k', '-ac', '1'] : []),
1416
+ ...commonArguments(...audioInput(frame)),
900
1417
  '-vn',
901
1418
  '-c:a',
902
1419
  'libfdk_aac',
@@ -916,9 +1433,10 @@ function audioArguments(frame, selection, targetAddress, target) {
916
1433
  ];
917
1434
  }
918
1435
  /**
919
- * Decode HomeKit's controller-to-accessory AAC-ELD SRTP and emit the SDK's exact talkback input: 16 kHz
920
- * mono AAC-LC in ADTS. The SDK owns complete-frame recovery, the 640-byte frame limit, and 64 ms pacing,
921
- * so FFmpeg must produce a byte stream rather than impose another clock.
1436
+ * Decode HomeKit's controller-to-accessory AAC-ELD SRTP and emit both talkback inputs the SDK accepts: 16 kHz mono
1437
+ * AAC-LC in ADTS on stdout, and HomeKit's own AAC-ELD access units, copied unchanged, in FLV on fd 3. The opened
1438
+ * handle's `codec` names the one that is fed; the other is drained. The SDK owns frame validation, the 640-byte
1439
+ * limit and pacing, so FFmpeg produces byte streams rather than imposing another clock.
922
1440
  */
923
1441
  function returnAudioArguments() {
924
1442
  return [
@@ -950,6 +1468,16 @@ function returnAudioArguments() {
950
1468
  '-f',
951
1469
  'adts',
952
1470
  'pipe:1',
1471
+ '-map',
1472
+ '0:a:0',
1473
+ '-vn',
1474
+ '-c:a',
1475
+ 'copy',
1476
+ '-flvflags',
1477
+ 'no_duration_filesize',
1478
+ '-f',
1479
+ 'flv',
1480
+ 'pipe:3',
953
1481
  ];
954
1482
  }
955
1483
  /**
@@ -980,7 +1508,8 @@ function spawnLiveMediaProcess(executable, args) {
980
1508
  return spawn(executable, args, { stdio: ['pipe', 'ignore', 'pipe'] });
981
1509
  }
982
1510
  function spawnReturnAudioProcess(executable, args) {
983
- return spawn(executable, args, { stdio: ['pipe', 'pipe', 'pipe'] });
1511
+ const child = spawn(executable, args, { stdio: ['pipe', 'pipe', 'pipe', 'pipe'] });
1512
+ return Object.assign(child, { eld: child.stdio[3] });
984
1513
  }
985
1514
  function reserveMediaPort(addressVersion) {
986
1515
  return new Promise((resolve, reject) => {