@camstack/addon-pipeline 1.2.93 → 1.2.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
- package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
- package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
- package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/process-memory-B9-UD7Rp.js +78 -0
- package/dist/process-memory-DXOB6eHX.mjs +66 -0
- package/dist/recorder/index.js +943 -109
- package/dist/recorder/index.mjs +942 -108
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
- package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
- package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
- package/package.json +13 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
package/dist/recorder/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const require_dist = require("../dist-
|
|
1
|
+
const require_dist = require("../dist-hyryByiw.js");
|
|
2
2
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
3
|
-
const require_addon_utils = require("../addon-utils-
|
|
3
|
+
const require_addon_utils = require("../addon-utils-UMWiMAxR.js");
|
|
4
4
|
let node_crypto = require("node:crypto");
|
|
5
5
|
let node_child_process = require("node:child_process");
|
|
6
6
|
let node_path = require("node:path");
|
|
@@ -457,6 +457,51 @@ var MfraTableStore = class {
|
|
|
457
457
|
}
|
|
458
458
|
}
|
|
459
459
|
};
|
|
460
|
+
var PendingGate = class {
|
|
461
|
+
deps;
|
|
462
|
+
inFlight = 0;
|
|
463
|
+
droppedSinceReport = 0;
|
|
464
|
+
lastReportAt = Number.NEGATIVE_INFINITY;
|
|
465
|
+
constructor(deps) {
|
|
466
|
+
this.deps = deps;
|
|
467
|
+
}
|
|
468
|
+
/** Tasks currently in flight. */
|
|
469
|
+
get pending() {
|
|
470
|
+
return this.inFlight;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Admit `task` unless the lane is saturated.
|
|
474
|
+
*
|
|
475
|
+
* Returns `false` when the task was REFUSED — the caller must log the drop
|
|
476
|
+
* (with its `deviceId` tag when the work is about a device); a branch that
|
|
477
|
+
* drops work silently reads as "never happened".
|
|
478
|
+
*/
|
|
479
|
+
run(task) {
|
|
480
|
+
if (this.inFlight >= this.deps.limit) {
|
|
481
|
+
this.droppedSinceReport += 1;
|
|
482
|
+
const now = this.deps.now?.() ?? Date.now();
|
|
483
|
+
const intervalMs = this.deps.reportIntervalMs ?? 3e4;
|
|
484
|
+
if (now - this.lastReportAt >= intervalMs) {
|
|
485
|
+
this.lastReportAt = now;
|
|
486
|
+
const dropped = this.droppedSinceReport;
|
|
487
|
+
this.droppedSinceReport = 0;
|
|
488
|
+
this.deps.onSaturated({
|
|
489
|
+
label: this.deps.label,
|
|
490
|
+
dropped,
|
|
491
|
+
pending: this.inFlight
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
this.inFlight += 1;
|
|
497
|
+
Promise.resolve().then(task).catch((error) => {
|
|
498
|
+
this.deps.onTaskError?.(error);
|
|
499
|
+
}).finally(() => {
|
|
500
|
+
this.inFlight -= 1;
|
|
501
|
+
});
|
|
502
|
+
return true;
|
|
503
|
+
}
|
|
504
|
+
};
|
|
460
505
|
//#endregion
|
|
461
506
|
//#region src/recorder/merge-ranges.ts
|
|
462
507
|
function mergeRanges(segments, gapToleranceMs) {
|
|
@@ -516,12 +561,33 @@ function* hourBuckets(fromMs, toMs) {
|
|
|
516
561
|
const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
|
|
517
562
|
for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
|
|
518
563
|
}
|
|
564
|
+
/**
|
|
565
|
+
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
566
|
+
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
567
|
+
* what a stable full sort of the backing map (insertion-ordered) produces, so
|
|
568
|
+
* an incrementally-maintained view is indistinguishable from a rebuilt one.
|
|
569
|
+
*/
|
|
570
|
+
function upperBoundByStart(rows, startMs) {
|
|
571
|
+
let lo = 0;
|
|
572
|
+
let hi = rows.length;
|
|
573
|
+
while (lo < hi) {
|
|
574
|
+
const mid = lo + hi >> 1;
|
|
575
|
+
if (rows[mid].startMs <= startMs) lo = mid + 1;
|
|
576
|
+
else hi = mid;
|
|
577
|
+
}
|
|
578
|
+
return lo;
|
|
579
|
+
}
|
|
519
580
|
var RecordingIndex = class {
|
|
520
581
|
byDevice = /* @__PURE__ */ new Map();
|
|
521
|
-
/** Start-sorted
|
|
522
|
-
*
|
|
523
|
-
*
|
|
582
|
+
/** Start-sorted views. Built lazily on first read (one copy+sort), then kept
|
|
583
|
+
* LIVE by `addSegment`'s ordered insert — rows arrive roughly ascending, so
|
|
584
|
+
* the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
|
|
585
|
+
* of an existing path) still invalidate; the steady-state finalize never
|
|
586
|
+
* does. Interactive locate/range calls reuse these instead of
|
|
587
|
+
* filtering+sorting the complete archive on every seek and scrub miss. */
|
|
524
588
|
sortedByDevice = /* @__PURE__ */ new Map();
|
|
589
|
+
fullRebuilds = 0;
|
|
590
|
+
incrementalInserts = 0;
|
|
525
591
|
/** Hour buckets this device has been walked for; `ALL_HOURS` after a full
|
|
526
592
|
* hydrate. Empty = nothing has been looked at, which is NOT the same as
|
|
527
593
|
* "there is nothing" — see {@link hydrationOf}. */
|
|
@@ -548,10 +614,34 @@ var RecordingIndex = class {
|
|
|
548
614
|
if (cached) return cached;
|
|
549
615
|
const m = this.byDevice.get(deviceId);
|
|
550
616
|
if (!m) return [];
|
|
617
|
+
this.fullRebuilds += 1;
|
|
551
618
|
const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
|
|
552
619
|
deviceCache.set(key, sorted);
|
|
553
620
|
return sorted;
|
|
554
621
|
}
|
|
622
|
+
/** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
|
|
623
|
+
sortedViewStats() {
|
|
624
|
+
return {
|
|
625
|
+
fullRebuilds: this.fullRebuilds,
|
|
626
|
+
incrementalInserts: this.incrementalInserts
|
|
627
|
+
};
|
|
628
|
+
}
|
|
629
|
+
/**
|
|
630
|
+
* Ordered insert of ONE NEW row into every live view it belongs to (the
|
|
631
|
+
* all-profiles view and its own profile's view). Views not yet materialised
|
|
632
|
+
* stay unmaterialised — their first read pays the one build. PRECONDITION:
|
|
633
|
+
* the row's path was NOT already in the device map; a replace must
|
|
634
|
+
* invalidate instead, because the views hold the old row object.
|
|
635
|
+
*/
|
|
636
|
+
insertIntoSortedViews(s) {
|
|
637
|
+
const deviceCache = this.sortedByDevice.get(s.deviceId);
|
|
638
|
+
if (!deviceCache) return;
|
|
639
|
+
for (const [key, rows] of deviceCache) {
|
|
640
|
+
if (key !== "" && key !== s.profile) continue;
|
|
641
|
+
rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
|
|
642
|
+
this.incrementalInserts += 1;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
555
645
|
/**
|
|
556
646
|
* Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
|
|
557
647
|
* location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
|
|
@@ -632,10 +722,26 @@ var RecordingIndex = class {
|
|
|
632
722
|
this.invalidateSorted(deviceId);
|
|
633
723
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
|
|
634
724
|
}
|
|
635
|
-
/**
|
|
725
|
+
/**
|
|
726
|
+
* Insert/replace one finalized segment (idempotent by path).
|
|
727
|
+
*
|
|
728
|
+
* THE hot mutation: every profile-writer lands here every `segmentSeconds`,
|
|
729
|
+
* ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
|
|
730
|
+
* live sorted views (rows arrive roughly ascending, so it is near-appending
|
|
731
|
+
* — O(log n) search + a short memmove). Only the rare RE-SET of an existing
|
|
732
|
+
* path (watcher replay after a playlist reset, or a relocate that changes
|
|
733
|
+
* `locationId`) invalidates: the views hold the old row object, and a
|
|
734
|
+
* one-off rebuild is the simple correct answer for a case that is not hot.
|
|
735
|
+
*/
|
|
636
736
|
addSegment(s) {
|
|
637
|
-
this.mapFor(s.deviceId)
|
|
638
|
-
|
|
737
|
+
const m = this.mapFor(s.deviceId);
|
|
738
|
+
const replacing = m.has(s.path);
|
|
739
|
+
m.set(s.path, s);
|
|
740
|
+
if (replacing) {
|
|
741
|
+
this.invalidateSorted(s.deviceId);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
this.insertIntoSortedViews(s);
|
|
639
745
|
}
|
|
640
746
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
|
|
641
747
|
removeSegments(paths) {
|
|
@@ -2901,6 +3007,67 @@ async function listExportsFor(state, deviceId) {
|
|
|
2901
3007
|
return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
|
|
2902
3008
|
}
|
|
2903
3009
|
//#endregion
|
|
3010
|
+
//#region src/recorder/addon/spawn-failure.ts
|
|
3011
|
+
/**
|
|
3012
|
+
* How the recorder reads a `child_process` `'error'` — ONE owner of the
|
|
3013
|
+
* transient-vs-permanent question.
|
|
3014
|
+
*
|
|
3015
|
+
* ── Why this file exists ───────────────────────────────────────────────────
|
|
3016
|
+
* In Node a child that cannot be spawned emits `'error'`, and an `'error'`
|
|
3017
|
+
* with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
|
|
3018
|
+
* cameras were unplugged, every writer for them re-dialled at once, the host
|
|
3019
|
+
* momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
|
|
3020
|
+
* runner died three times inside nine seconds. The D6 crash circuit-breaker
|
|
3021
|
+
* did its job and stopped respawning — leaving the whole fleet with no
|
|
3022
|
+
* recording for three hours and fifteen minutes while `/health` returned 200.
|
|
3023
|
+
*
|
|
3024
|
+
* Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
|
|
3025
|
+
* listener, and every one of them classifies the failure HERE rather than
|
|
3026
|
+
* inventing its own list.
|
|
3027
|
+
*/
|
|
3028
|
+
/**
|
|
3029
|
+
* Spawn failures that RETRYING CANNOT FIX.
|
|
3030
|
+
*
|
|
3031
|
+
* The EAGAIN-vs-ENOENT split, stated once:
|
|
3032
|
+
*
|
|
3033
|
+
* - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
|
|
3034
|
+
* resource comes back, often within one backoff. They take the ordinary
|
|
3035
|
+
* restart-with-backoff path, whose own comment already states the policy
|
|
3036
|
+
* for this shape: "only RAPID back-to-back failures count toward giving
|
|
3037
|
+
* up, so a brief blip never permanently stops a camera's recording".
|
|
3038
|
+
* - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
|
|
3039
|
+
* run". Nothing about the next ten attempts differs from this one, so the
|
|
3040
|
+
* caller stops immediately and surfaces it at ERROR level: burning a
|
|
3041
|
+
* ten-attempt budget on a missing binary only delays recovery and buries
|
|
3042
|
+
* the single actionable line under ten identical warnings.
|
|
3043
|
+
*
|
|
3044
|
+
* EPERM is deliberately absent: it is reported for transient sandbox/cgroup
|
|
3045
|
+
* refusals as well as for real permission faults, and treating an ambiguous
|
|
3046
|
+
* code as permanent stops a camera that would have recovered.
|
|
3047
|
+
*/
|
|
3048
|
+
var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
|
|
3049
|
+
/**
|
|
3050
|
+
* Read the `'error'` payload without a cast.
|
|
3051
|
+
*
|
|
3052
|
+
* An unknown code is treated as TRANSIENT: the cost of retrying something
|
|
3053
|
+
* permanent is a bounded restart budget, while the cost of giving up on
|
|
3054
|
+
* something transient is a camera that stops recording until an operator
|
|
3055
|
+
* notices.
|
|
3056
|
+
*/
|
|
3057
|
+
function readSpawnFailure(raw) {
|
|
3058
|
+
const code = errnoCode(raw);
|
|
3059
|
+
return {
|
|
3060
|
+
code,
|
|
3061
|
+
message: raw instanceof Error ? raw.message : String(raw),
|
|
3062
|
+
permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
|
|
3063
|
+
};
|
|
3064
|
+
}
|
|
3065
|
+
function errnoCode(raw) {
|
|
3066
|
+
if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
|
|
3067
|
+
const code = raw.code;
|
|
3068
|
+
return typeof code === "string" ? code : void 0;
|
|
3069
|
+
}
|
|
3070
|
+
//#endregion
|
|
2904
3071
|
//#region src/recorder/addon/export-engine.ts
|
|
2905
3072
|
/** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
|
|
2906
3073
|
* segment before the decisive "Error opening input" summary, so a short tail can
|
|
@@ -3070,7 +3237,7 @@ var ExportEngine = class {
|
|
|
3070
3237
|
outPath,
|
|
3071
3238
|
options
|
|
3072
3239
|
});
|
|
3073
|
-
const ran = await this.runFfmpeg(id, args, {
|
|
3240
|
+
const ran = await this.runFfmpeg(id, rec.deviceId, args, {
|
|
3074
3241
|
totalSec: expectedOutSeconds(rec),
|
|
3075
3242
|
floor: 0,
|
|
3076
3243
|
ceil: 99
|
|
@@ -3090,7 +3257,7 @@ var ExportEngine = class {
|
|
|
3090
3257
|
inputPlaylist: playlist,
|
|
3091
3258
|
outPath: concatPath
|
|
3092
3259
|
});
|
|
3093
|
-
const concatRan = await this.runFfmpeg(id, concatArgs, {
|
|
3260
|
+
const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
|
|
3094
3261
|
totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
|
|
3095
3262
|
floor: 0,
|
|
3096
3263
|
ceil: 49
|
|
@@ -3104,7 +3271,7 @@ var ExportEngine = class {
|
|
|
3104
3271
|
outPath,
|
|
3105
3272
|
options
|
|
3106
3273
|
});
|
|
3107
|
-
const selectRan = await this.runFfmpeg(id, selectArgs, {
|
|
3274
|
+
const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
|
|
3108
3275
|
totalSec: expectedOutSeconds(rec),
|
|
3109
3276
|
floor: 50,
|
|
3110
3277
|
ceil: 99
|
|
@@ -3116,7 +3283,7 @@ var ExportEngine = class {
|
|
|
3116
3283
|
} catch {}
|
|
3117
3284
|
}
|
|
3118
3285
|
}
|
|
3119
|
-
runFfmpeg(id, args, progress) {
|
|
3286
|
+
runFfmpeg(id, deviceId, args, progress) {
|
|
3120
3287
|
this.deps.logger.debug("export ffmpeg argv", { meta: {
|
|
3121
3288
|
exportId: id,
|
|
3122
3289
|
args
|
|
@@ -3153,6 +3320,24 @@ var ExportEngine = class {
|
|
|
3153
3320
|
stderrTail
|
|
3154
3321
|
});
|
|
3155
3322
|
});
|
|
3323
|
+
proc.on("error", (raw) => {
|
|
3324
|
+
this.activeProc = null;
|
|
3325
|
+
const failure = readSpawnFailure(raw);
|
|
3326
|
+
this.deps.logger.error("export ffmpeg could not be spawned", {
|
|
3327
|
+
tags: { deviceId },
|
|
3328
|
+
meta: {
|
|
3329
|
+
exportId: id,
|
|
3330
|
+
errorCode: failure.code,
|
|
3331
|
+
error: failure.message,
|
|
3332
|
+
permanent: failure.permanent
|
|
3333
|
+
}
|
|
3334
|
+
});
|
|
3335
|
+
stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
|
|
3336
|
+
resolve({
|
|
3337
|
+
code: null,
|
|
3338
|
+
stderrTail
|
|
3339
|
+
});
|
|
3340
|
+
});
|
|
3156
3341
|
});
|
|
3157
3342
|
}
|
|
3158
3343
|
async finish(id, code, outPath, stderrTail) {
|
|
@@ -5838,18 +6023,232 @@ function shouldRecordAt(config, triggers, at) {
|
|
|
5838
6023
|
return eventsBandDemanded(band, triggers, at.getTime());
|
|
5839
6024
|
}
|
|
5840
6025
|
/**
|
|
5841
|
-
*
|
|
5842
|
-
*
|
|
5843
|
-
*
|
|
5844
|
-
*
|
|
6026
|
+
* How late the FIRST segment of an attach may be NAMED, relative to the writer
|
|
6027
|
+
* spawn, and still count as "the segment this attach was spawned to produce".
|
|
6028
|
+
*
|
|
6029
|
+
* ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
|
|
6030
|
+
* first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
|
|
6031
|
+
* spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
|
|
6032
|
+
* 3.2 s. 5 s carries that with margin while staying far below the writer's own
|
|
6033
|
+
* "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
|
|
6034
|
+
* slow is not mistaken for one that is producing useful footage.
|
|
5845
6035
|
*/
|
|
5846
|
-
|
|
6036
|
+
var FIRST_SEGMENT_GRACE_MS = 5e3;
|
|
6037
|
+
/**
|
|
6038
|
+
* Latest segment START this attach may still produce and have kept, regardless
|
|
6039
|
+
* of where the operator's post-buffer ended: one full segment (nothing can be
|
|
6040
|
+
* finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
|
|
6041
|
+
* latency.
|
|
6042
|
+
*/
|
|
6043
|
+
function attachRetainDeadlineMs(attach) {
|
|
6044
|
+
return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
|
|
6045
|
+
}
|
|
6046
|
+
/**
|
|
6047
|
+
* Should a finalized segment `[segStartMs, segEndMs]` recorded under an
|
|
6048
|
+
* `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
|
|
6049
|
+
* trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
|
|
6050
|
+
* this is where `preBufferSec` retroactively retains pre-trigger footage — OR
|
|
6051
|
+
* it is the footage `attach` was spawned to produce (below).
|
|
6052
|
+
*
|
|
6053
|
+
* ## Why the attach matters here
|
|
6054
|
+
*
|
|
6055
|
+
* The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
|
|
6056
|
+
* gate is evaluated on segment timestamps. They shared one window, and a writer
|
|
6057
|
+
* physically cannot finalize anything before `spawn + segmentSeconds + dial`.
|
|
6058
|
+
* So the last ~`segmentSeconds` of every demand window was a zone where the
|
|
6059
|
+
* recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
|
|
6060
|
+
* result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
|
|
6061
|
+
* post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
|
|
6062
|
+
* finalized segments were unlinked.
|
|
6063
|
+
*
|
|
6064
|
+
* The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
|
|
6065
|
+
* `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
|
|
6066
|
+
* inside `trigger + postMs` and retains exactly what it retained before. It
|
|
6067
|
+
* only ever extends the tail of a LATE attach, by at most one segment plus the
|
|
6068
|
+
* grace, and only for a trigger whose live window was still open AT ATTACH — the
|
|
6069
|
+
* precise statement of "if the live gate said yes, the keep gate honours the
|
|
6070
|
+
* first segment that decision produces". A stale trigger extends nothing.
|
|
6071
|
+
*/
|
|
6072
|
+
function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
|
|
5847
6073
|
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
5848
|
-
const
|
|
6074
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
6075
|
+
const overlaps = (lastMs) => {
|
|
6076
|
+
if (lastMs == null) return false;
|
|
6077
|
+
if (segEndMs < lastMs - preMs) return false;
|
|
6078
|
+
if (segStartMs <= lastMs + postMs) return true;
|
|
6079
|
+
return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
|
|
6080
|
+
};
|
|
5849
6081
|
if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
|
|
5850
6082
|
if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
|
|
5851
6083
|
return false;
|
|
5852
6084
|
}
|
|
6085
|
+
/**
|
|
6086
|
+
* By how many ms a discarded segment missed being kept — its start past the
|
|
6087
|
+
* latest start that WOULD have been retained, mirroring
|
|
6088
|
+
* {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
|
|
6089
|
+
* the attach allowance for a trigger that was still live at attach). Null when
|
|
6090
|
+
* no qualifying trigger exists at all, which is a different fact and says so.
|
|
6091
|
+
*
|
|
6092
|
+
* Negative means the segment was dropped on the PRE-buffer side (it ended
|
|
6093
|
+
* before `trigger - preBufferSec`), not for being late.
|
|
6094
|
+
*
|
|
6095
|
+
* Reporting-only. The discard branch has to name a number: "outside the window"
|
|
6096
|
+
* with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
|
|
6097
|
+
*/
|
|
6098
|
+
function segmentMissedByMs(segStartMs, band, triggers, attach) {
|
|
6099
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
6100
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
6101
|
+
const latestKeptStart = (lastMs) => {
|
|
6102
|
+
if (lastMs == null) return null;
|
|
6103
|
+
const windowEndMs = lastMs + postMs;
|
|
6104
|
+
return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
|
|
6105
|
+
};
|
|
6106
|
+
const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
|
|
6107
|
+
if (bounds.length === 0) return null;
|
|
6108
|
+
return segStartMs - Math.max(...bounds);
|
|
6109
|
+
}
|
|
6110
|
+
/** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
|
|
6111
|
+
function localInstantMs(year, month, day, hhmm) {
|
|
6112
|
+
const [hours, minutes] = hhmm.split(":");
|
|
6113
|
+
return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
|
|
6114
|
+
}
|
|
6115
|
+
/**
|
|
6116
|
+
* Every instant inside the horizon at which `bandActiveAt` COULD change value:
|
|
6117
|
+
* each band's start and end on each local day, plus each local midnight (the
|
|
6118
|
+
* `days` membership boundary). A superset — cheap to compute (a few dozen
|
|
6119
|
+
* entries) and filtered against `activeBandAt` below.
|
|
6120
|
+
*/
|
|
6121
|
+
function candidateEdgesMs(bands, nowMs) {
|
|
6122
|
+
const base = new Date(nowMs);
|
|
6123
|
+
const year = base.getFullYear();
|
|
6124
|
+
const month = base.getMonth();
|
|
6125
|
+
const day = base.getDate();
|
|
6126
|
+
const out = [];
|
|
6127
|
+
for (let offset = 0; offset <= 8; offset++) {
|
|
6128
|
+
out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
|
|
6129
|
+
for (const band of bands) {
|
|
6130
|
+
out.push(localInstantMs(year, month, day + offset, band.start));
|
|
6131
|
+
out.push(localInstantMs(year, month, day + offset, band.end));
|
|
6132
|
+
}
|
|
6133
|
+
}
|
|
6134
|
+
return out;
|
|
6135
|
+
}
|
|
6136
|
+
/**
|
|
6137
|
+
* The next instant at which a DIFFERENT band (or no band) covers the clock, or
|
|
6138
|
+
* `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
|
|
6139
|
+
* band (`start === end`, empty `days`) being the common case that legitimately
|
|
6140
|
+
* has no edge at all.
|
|
6141
|
+
*
|
|
6142
|
+
* Identity is by band REFERENCE, which is what makes this exact: two distinct
|
|
6143
|
+
* band objects with the same fields yield an edge that changes nothing, which
|
|
6144
|
+
* costs one wasted wake-up and never costs footage.
|
|
6145
|
+
*/
|
|
6146
|
+
function nextBandEdgeMs(config, nowMs) {
|
|
6147
|
+
const bands = config.bands;
|
|
6148
|
+
if (!bands || bands.length === 0) return null;
|
|
6149
|
+
const current = activeBandAt({ bands }, new Date(nowMs));
|
|
6150
|
+
const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
|
|
6151
|
+
for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
|
|
6152
|
+
return null;
|
|
6153
|
+
}
|
|
6154
|
+
/**
|
|
6155
|
+
* The first instant an `events` band's demand window is CLOSED, given the
|
|
6156
|
+
* triggers it listens for, or `null` when no window is open at `nowMs`.
|
|
6157
|
+
*
|
|
6158
|
+
* Demand is an OR across the listened trigger kinds, so an open window ends
|
|
6159
|
+
* with the LATEST of them — a motion trigger followed by an audio trigger 5 s
|
|
6160
|
+
* later keeps the camera attached until the audio one expires, exactly as the
|
|
6161
|
+
* sliding window already behaved.
|
|
6162
|
+
*/
|
|
6163
|
+
function eventsWindowCloseMs(band, triggers, nowMs) {
|
|
6164
|
+
if (band.mode !== "events") return null;
|
|
6165
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
6166
|
+
const ends = [];
|
|
6167
|
+
if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
|
|
6168
|
+
if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
|
|
6169
|
+
const open = ends.filter((ms) => ms >= nowMs);
|
|
6170
|
+
if (open.length === 0) return null;
|
|
6171
|
+
return Math.max(...open) + 1;
|
|
6172
|
+
}
|
|
6173
|
+
/**
|
|
6174
|
+
* THE next instant the controller must re-evaluate this device, or `null` when
|
|
6175
|
+
* nothing can change without an event the controller already receives (a
|
|
6176
|
+
* trigger or a config write).
|
|
6177
|
+
*
|
|
6178
|
+
* Whichever of the two sources comes first wins, and the reason is carried so
|
|
6179
|
+
* the wake-up is attributable in the log without correlating timestamps.
|
|
6180
|
+
*/
|
|
6181
|
+
function nextDecisionChangeAt(input) {
|
|
6182
|
+
const { config, triggers, nowMs } = input;
|
|
6183
|
+
if (!config.enabled) return null;
|
|
6184
|
+
const bands = config.bands;
|
|
6185
|
+
if (!bands || bands.length === 0) return null;
|
|
6186
|
+
const band = activeBandAt({ bands }, new Date(nowMs));
|
|
6187
|
+
const edgeMs = nextBandEdgeMs(config, nowMs);
|
|
6188
|
+
const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
|
|
6189
|
+
if (closeMs === null) return edgeMs === null ? null : {
|
|
6190
|
+
atMs: edgeMs,
|
|
6191
|
+
reason: "band-edge"
|
|
6192
|
+
};
|
|
6193
|
+
if (edgeMs === null || closeMs <= edgeMs) return {
|
|
6194
|
+
atMs: closeMs,
|
|
6195
|
+
reason: "window-close"
|
|
6196
|
+
};
|
|
6197
|
+
return {
|
|
6198
|
+
atMs: edgeMs,
|
|
6199
|
+
reason: "band-edge"
|
|
6200
|
+
};
|
|
6201
|
+
}
|
|
6202
|
+
//#endregion
|
|
6203
|
+
//#region src/recorder/addon/device-wakeups.ts
|
|
6204
|
+
/** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
|
|
6205
|
+
var MAX_TIMER_DELAY_MS = 2147483647;
|
|
6206
|
+
var DeviceWakeups = class {
|
|
6207
|
+
deps;
|
|
6208
|
+
timers = /* @__PURE__ */ new Map();
|
|
6209
|
+
constructor(deps) {
|
|
6210
|
+
this.deps = deps;
|
|
6211
|
+
}
|
|
6212
|
+
/**
|
|
6213
|
+
* Wake this device at `atMs`, replacing any wake-up already armed for it.
|
|
6214
|
+
* `reason` is carried into the log line so a wake is attributable without
|
|
6215
|
+
* correlating it against the band model by hand.
|
|
6216
|
+
*/
|
|
6217
|
+
arm(deviceId, atMs, reason) {
|
|
6218
|
+
this.cancel(deviceId);
|
|
6219
|
+
const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
|
|
6220
|
+
const timer = setTimeout(() => {
|
|
6221
|
+
this.timers.delete(deviceId);
|
|
6222
|
+
this.deps.fire(deviceId);
|
|
6223
|
+
}, delayMs);
|
|
6224
|
+
timer.unref?.();
|
|
6225
|
+
this.timers.set(deviceId, timer);
|
|
6226
|
+
this.deps.logger.debug("recorder: device wake-up armed", {
|
|
6227
|
+
tags: { deviceId },
|
|
6228
|
+
meta: {
|
|
6229
|
+
atMs,
|
|
6230
|
+
delayMs,
|
|
6231
|
+
reason
|
|
6232
|
+
}
|
|
6233
|
+
});
|
|
6234
|
+
}
|
|
6235
|
+
/** Drop this device's pending wake-up, if any. Idempotent. */
|
|
6236
|
+
cancel(deviceId) {
|
|
6237
|
+
const timer = this.timers.get(deviceId);
|
|
6238
|
+
if (timer === void 0) return;
|
|
6239
|
+
clearTimeout(timer);
|
|
6240
|
+
this.timers.delete(deviceId);
|
|
6241
|
+
}
|
|
6242
|
+
/** Drop every pending wake-up — shutdown and maintenance pause. */
|
|
6243
|
+
cancelAll() {
|
|
6244
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
6245
|
+
this.timers.clear();
|
|
6246
|
+
}
|
|
6247
|
+
/** Devices with a wake-up pending — for tests and for shutdown assertions. */
|
|
6248
|
+
get pendingCount() {
|
|
6249
|
+
return this.timers.size;
|
|
6250
|
+
}
|
|
6251
|
+
};
|
|
5853
6252
|
//#endregion
|
|
5854
6253
|
//#region src/recorder/addon/periodic-pass.ts
|
|
5855
6254
|
function startPeriodicPass(deps) {
|
|
@@ -5916,7 +6315,9 @@ var ReadinessRestore = class {
|
|
|
5916
6315
|
this.deps = deps;
|
|
5917
6316
|
}
|
|
5918
6317
|
get concurrency() {
|
|
5919
|
-
|
|
6318
|
+
const declared = this.deps.concurrency;
|
|
6319
|
+
const value = typeof declared === "function" ? declared() : declared;
|
|
6320
|
+
return Math.max(1, value ?? 8);
|
|
5920
6321
|
}
|
|
5921
6322
|
/** Seed the pending set, subscribe to readiness, and drain once immediately
|
|
5922
6323
|
* (covers the already-ready case). Returns when the first drain settles. */
|
|
@@ -5975,6 +6376,70 @@ var ReadinessRestore = class {
|
|
|
5975
6376
|
}
|
|
5976
6377
|
}
|
|
5977
6378
|
};
|
|
6379
|
+
/** Adaptive bound for the recorder's attach parallelism. */
|
|
6380
|
+
var SpawnPressureGovernor = class {
|
|
6381
|
+
deps;
|
|
6382
|
+
limit;
|
|
6383
|
+
min;
|
|
6384
|
+
recoverAfterMs;
|
|
6385
|
+
/** When the limit last MOVED — the clock both directions are measured from. */
|
|
6386
|
+
lastChangeAt;
|
|
6387
|
+
underPressure = false;
|
|
6388
|
+
constructor(deps) {
|
|
6389
|
+
this.deps = deps;
|
|
6390
|
+
this.min = Math.max(1, deps.minConcurrency ?? 1);
|
|
6391
|
+
this.limit = Math.max(this.min, deps.maxConcurrency);
|
|
6392
|
+
this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
|
|
6393
|
+
this.lastChangeAt = this.now();
|
|
6394
|
+
}
|
|
6395
|
+
/**
|
|
6396
|
+
* How many attaches may overlap right now.
|
|
6397
|
+
*
|
|
6398
|
+
* Reading is what drives recovery: there is no timer to leak, no interval to
|
|
6399
|
+
* lose to a runner respawn, and a recorder that stops attaching entirely
|
|
6400
|
+
* simply keeps its reduced bound until it tries again — which is correct.
|
|
6401
|
+
*/
|
|
6402
|
+
get concurrency() {
|
|
6403
|
+
const max = Math.max(this.min, this.deps.maxConcurrency);
|
|
6404
|
+
if (this.limit >= max) return max;
|
|
6405
|
+
const elapsed = this.now() - this.lastChangeAt;
|
|
6406
|
+
if (elapsed < this.recoverAfterMs) return this.limit;
|
|
6407
|
+
const lanes = Math.floor(elapsed / this.recoverAfterMs);
|
|
6408
|
+
this.limit = Math.min(max, this.limit + lanes);
|
|
6409
|
+
this.lastChangeAt += lanes * this.recoverAfterMs;
|
|
6410
|
+
if (this.limit >= max && this.underPressure) {
|
|
6411
|
+
this.underPressure = false;
|
|
6412
|
+
this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
|
|
6413
|
+
} else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
|
|
6414
|
+
concurrency: this.limit,
|
|
6415
|
+
max
|
|
6416
|
+
} });
|
|
6417
|
+
return this.limit;
|
|
6418
|
+
}
|
|
6419
|
+
/**
|
|
6420
|
+
* A transient spawn failure happened on `deviceId`'s writer.
|
|
6421
|
+
*
|
|
6422
|
+
* Halves the CURRENT limit, not the max: two waves of pressure inside one
|
|
6423
|
+
* recovery window must compound, or a host under sustained load is asked for
|
|
6424
|
+
* the same too-large batch over and over.
|
|
6425
|
+
*/
|
|
6426
|
+
report(deviceId) {
|
|
6427
|
+
const before = this.limit;
|
|
6428
|
+
this.limit = Math.max(this.min, Math.floor(this.limit / 2));
|
|
6429
|
+
this.lastChangeAt = this.now();
|
|
6430
|
+
this.underPressure = true;
|
|
6431
|
+
this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
|
|
6432
|
+
tags: { deviceId },
|
|
6433
|
+
meta: {
|
|
6434
|
+
from: before,
|
|
6435
|
+
to: this.limit
|
|
6436
|
+
}
|
|
6437
|
+
});
|
|
6438
|
+
}
|
|
6439
|
+
now() {
|
|
6440
|
+
return this.deps.now?.() ?? Date.now();
|
|
6441
|
+
}
|
|
6442
|
+
};
|
|
5978
6443
|
//#endregion
|
|
5979
6444
|
//#region src/recorder/addon/segment-watcher.ts
|
|
5980
6445
|
/**
|
|
@@ -6360,36 +6825,87 @@ var SegmentWriter = class {
|
|
|
6360
6825
|
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
|
|
6361
6826
|
}
|
|
6362
6827
|
});
|
|
6828
|
+
let settled = false;
|
|
6829
|
+
const terminate = (termination) => {
|
|
6830
|
+
if (settled) return;
|
|
6831
|
+
settled = true;
|
|
6832
|
+
this.onTermination(termination, stderrTail);
|
|
6833
|
+
};
|
|
6363
6834
|
proc.on("exit", (code) => {
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
|
|
6835
|
+
terminate({
|
|
6836
|
+
reason: "exit",
|
|
6837
|
+
exitCode: typeof code === "number" ? code : null
|
|
6838
|
+
});
|
|
6839
|
+
});
|
|
6840
|
+
proc.on("error", (raw) => {
|
|
6841
|
+
const failure = readSpawnFailure(raw);
|
|
6842
|
+
terminate({
|
|
6843
|
+
reason: "spawn-error",
|
|
6844
|
+
errorCode: failure.code,
|
|
6845
|
+
message: failure.message,
|
|
6846
|
+
permanent: failure.permanent
|
|
6847
|
+
});
|
|
6848
|
+
});
|
|
6849
|
+
}
|
|
6850
|
+
/**
|
|
6851
|
+
* The ONE restart policy, reached from both `exit` and `error`.
|
|
6852
|
+
*
|
|
6853
|
+
* `deps.logger` is the device-scoped child the controller binds
|
|
6854
|
+
* (`logger.withTags({ deviceId })`), so every line below carries
|
|
6855
|
+
* `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
|
|
6856
|
+
* camera, and a writer that stops recording one camera must be greppable by
|
|
6857
|
+
* that camera.
|
|
6858
|
+
*/
|
|
6859
|
+
onTermination(termination, stderrTail) {
|
|
6860
|
+
this.resolveExit?.();
|
|
6861
|
+
this.resolveExit = null;
|
|
6862
|
+
this.proc = null;
|
|
6863
|
+
if (this.stopped) return;
|
|
6864
|
+
const ranMs = Date.now() - this.startedAt;
|
|
6865
|
+
if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
|
|
6866
|
+
if (termination.reason === "spawn-error") {
|
|
6867
|
+
this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
|
|
6868
|
+
outDir: this.cfg.outDir,
|
|
6869
|
+
errorCode: termination.errorCode,
|
|
6870
|
+
error: termination.message,
|
|
6871
|
+
permanent: termination.permanent
|
|
6872
|
+
} });
|
|
6873
|
+
if (termination.permanent) {
|
|
6874
|
+
this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
|
|
6372
6875
|
outDir: this.cfg.outDir,
|
|
6373
|
-
|
|
6374
|
-
|
|
6876
|
+
errorCode: termination.errorCode,
|
|
6877
|
+
error: termination.message
|
|
6375
6878
|
} });
|
|
6376
6879
|
this.stopped = true;
|
|
6377
6880
|
this.deps.onGaveUp?.();
|
|
6378
6881
|
return;
|
|
6379
6882
|
}
|
|
6380
|
-
this.
|
|
6381
|
-
|
|
6382
|
-
|
|
6883
|
+
this.deps.onResourcePressure?.();
|
|
6884
|
+
}
|
|
6885
|
+
if (this.restarts >= MAX_RESTARTS) {
|
|
6886
|
+
this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
|
|
6383
6887
|
outDir: this.cfg.outDir,
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
6888
|
+
reason: termination.reason,
|
|
6889
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
6890
|
+
stderrTail: stderrTail.join(" | ")
|
|
6387
6891
|
} });
|
|
6388
|
-
this.
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6892
|
+
this.stopped = true;
|
|
6893
|
+
this.deps.onGaveUp?.();
|
|
6894
|
+
return;
|
|
6895
|
+
}
|
|
6896
|
+
this.restarts++;
|
|
6897
|
+
const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
|
|
6898
|
+
this.deps.logger.info("SegmentWriter restarting", { meta: {
|
|
6899
|
+
outDir: this.cfg.outDir,
|
|
6900
|
+
attempt: this.restarts,
|
|
6901
|
+
delayMs,
|
|
6902
|
+
ranMs,
|
|
6903
|
+
reason: termination.reason
|
|
6904
|
+
} });
|
|
6905
|
+
this.restartTimer = setTimeout(() => {
|
|
6906
|
+
this.restartTimer = null;
|
|
6907
|
+
this.start();
|
|
6908
|
+
}, delayMs);
|
|
6393
6909
|
}
|
|
6394
6910
|
stop() {
|
|
6395
6911
|
this.stopAndWait();
|
|
@@ -6433,19 +6949,37 @@ var SegmentWriter = class {
|
|
|
6433
6949
|
* - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
|
|
6434
6950
|
* the v2 `SegmentStore.onFinalized` (stat + relocate + index).
|
|
6435
6951
|
*
|
|
6436
|
-
* Per device it decides whether
|
|
6437
|
-
* (`
|
|
6438
|
-
*
|
|
6439
|
-
* 1. `setDeviceConfig` (operator changed the bands / enable),
|
|
6440
|
-
* 2. a
|
|
6441
|
-
* 3. a
|
|
6442
|
-
*
|
|
6443
|
-
*
|
|
6444
|
-
*
|
|
6445
|
-
*
|
|
6446
|
-
*
|
|
6447
|
-
*
|
|
6448
|
-
*
|
|
6952
|
+
* Per device it decides whether the active band demands recording right now
|
|
6953
|
+
* (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
|
|
6954
|
+
* is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
|
|
6955
|
+
* 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
|
|
6956
|
+
* 2. a qualifying motion/audio trigger (`onTrigger`) — opens a window,
|
|
6957
|
+
* 3. a per-device WAKE-UP armed for the exact instant the decision could
|
|
6958
|
+
* change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
|
|
6959
|
+
* `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
|
|
6960
|
+
* 4. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`),
|
|
6961
|
+
* 5. the periodic RECONCILE (below), for everything the four above can lose.
|
|
6962
|
+
*
|
|
6963
|
+
* (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
|
|
6964
|
+
* demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
|
|
6965
|
+
* trigger, so nothing called `evaluateDevice` at the moment it closed and the
|
|
6966
|
+
* detach waited for the next pass. A 4K camera stayed dialled, decoded and
|
|
6967
|
+
* written for up to a whole tick past the operator's `postBufferSec`.
|
|
6968
|
+
*
|
|
6969
|
+
* There are THREE periodic passes, each on its own timer and each non-overlapping
|
|
6970
|
+
* (`periodic-pass.ts`):
|
|
6971
|
+
* - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
|
|
6972
|
+
* producing nothing?
|
|
6973
|
+
* - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
|
|
6974
|
+
* deliberately NOT on the attach beat, so a cold broker cannot delay it.
|
|
6975
|
+
* - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
|
|
6976
|
+
* broker, and the only backstop for a lost wake-up or a failed attach.
|
|
6977
|
+
*
|
|
6978
|
+
* They were ONE pass until 2026-08-13, when a convergence sweep wedged on
|
|
6979
|
+
* unbounded broker RPCs and took the liveness watchdog down with it for 9-27
|
|
6980
|
+
* minutes at a time; placement was split out of the second on 2026-08-18, after
|
|
6981
|
+
* a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
|
|
6982
|
+
* cold-dialling ~40 RTSP sources.
|
|
6449
6983
|
*
|
|
6450
6984
|
* EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
|
|
6451
6985
|
* trigger-gating. The enable intent is persisted in durable-state by the
|
|
@@ -6458,8 +6992,45 @@ var SegmentWriter = class {
|
|
|
6458
6992
|
* `ReadinessRegistry` is the single source of truth.
|
|
6459
6993
|
*/
|
|
6460
6994
|
var DEFAULT_WATCH_INTERVAL_MS = 2e3;
|
|
6461
|
-
/**
|
|
6462
|
-
|
|
6995
|
+
/**
|
|
6996
|
+
* Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
|
|
6997
|
+
* question is an in-memory scan over `active`, it costs nothing, and its bound
|
|
6998
|
+
* (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
|
|
6999
|
+
*/
|
|
7000
|
+
var LIVENESS_TICK_MS = 3e4;
|
|
7001
|
+
/**
|
|
7002
|
+
* Placement cadence — unchanged from the beat it used to share with
|
|
7003
|
+
* convergence, because nothing about placement wanted to be rarer. What changed
|
|
7004
|
+
* is that it no longer rides a pass that can block on broker RPCs: it is one
|
|
7005
|
+
* `statfs` per location and it re-plans only, so a plan reaches a camera at its
|
|
7006
|
+
* NEXT attach.
|
|
7007
|
+
*/
|
|
7008
|
+
var PLACEMENT_TICK_MS = 3e4;
|
|
7009
|
+
/**
|
|
7010
|
+
* Reconcile cadence — the SAFETY NET, not the mechanism.
|
|
7011
|
+
*
|
|
7012
|
+
* It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
|
|
7013
|
+
* window or crossed a band edge, so the loop had to keep asking. Both are now
|
|
7014
|
+
* armed for their exact instant (`decision-schedule.ts`), a config change is an
|
|
7015
|
+
* RPC, and a broker recovery is a readiness transition — so this pass exists
|
|
7016
|
+
* only for what those cannot cover: a wake-up lost because the event path threw
|
|
7017
|
+
* before it could arm one, an attach that failed with no later ready transition
|
|
7018
|
+
* to ride, a timer lost to a runner respawn, and clock drift.
|
|
7019
|
+
*
|
|
7020
|
+
* 2 minutes, and the number is chosen against a bound this recorder already
|
|
7021
|
+
* accepts. The liveness watchdog tolerates a pinned-but-silent writer for
|
|
7022
|
+
* `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
|
|
7023
|
+
* recording but is detached is the same class of fault — footage is not being
|
|
7024
|
+
* written and only a periodic check will notice — so its recovery is put on the
|
|
7025
|
+
* same order rather than an order slower. Four times rarer than the beat that
|
|
7026
|
+
* blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
|
|
7027
|
+
* ~2 minutes of footage for a camera whose event path failed entirely.
|
|
7028
|
+
*
|
|
7029
|
+
* Going rarer than this trades real footage for CPU that the pass no longer
|
|
7030
|
+
* spends anyway: with the broker-readiness pre-check below, a reconcile over a
|
|
7031
|
+
* healthy fleet is N config reads and no RPC at all.
|
|
7032
|
+
*/
|
|
7033
|
+
var RECONCILE_TICK_MS = 12e4;
|
|
6463
7034
|
/**
|
|
6464
7035
|
* Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
|
|
6465
7036
|
* teardown. The UDS request default is 60s, but the runner supervisor's
|
|
@@ -6498,11 +7069,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
|
|
|
6498
7069
|
*/
|
|
6499
7070
|
var ATTACH_RPC_TIMEOUT_MS = 2e4;
|
|
6500
7071
|
/**
|
|
6501
|
-
* A pass longer than this is reported with its duration.
|
|
6502
|
-
* cadence: a pass that cannot finish inside its own interval is
|
|
6503
|
-
* that used to go unlogged for half an hour.
|
|
7072
|
+
* A pass longer than this is reported with its duration. Each pass uses its OWN
|
|
7073
|
+
* cadence as the budget: a pass that cannot finish inside its own interval is
|
|
7074
|
+
* the condition that used to go unlogged for half an hour. Keeping it per-pass
|
|
7075
|
+
* matters now that the three cadences differ — a 40 s reconcile is healthy
|
|
7076
|
+
* against a 120 s beat and was a red flag against a 30 s one.
|
|
6504
7077
|
*/
|
|
6505
|
-
|
|
7078
|
+
function passSlowAfterMs(intervalMs) {
|
|
7079
|
+
return intervalMs;
|
|
7080
|
+
}
|
|
6506
7081
|
/**
|
|
6507
7082
|
* Race `promise` against a deadline. Handlers stay attached to the losing
|
|
6508
7083
|
* promise, so a post-deadline settlement can never surface as a process-level
|
|
@@ -6585,6 +7160,14 @@ var RecordingController = class {
|
|
|
6585
7160
|
*/
|
|
6586
7161
|
lastSegmentAt = /* @__PURE__ */ new Map();
|
|
6587
7162
|
/**
|
|
7163
|
+
* Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
|
|
7164
|
+
* at least one `events` segment. Read only to LEVEL the discard log: an
|
|
7165
|
+
* attach that kept nothing is a wasted pull (warn), a later discard from one
|
|
7166
|
+
* that kept something is the tail of the window (info). Bounded by
|
|
7167
|
+
* device×profile like {@link lastSegmentAt}, and cleared with it on detach.
|
|
7168
|
+
*/
|
|
7169
|
+
attachRetained = /* @__PURE__ */ new Map();
|
|
7170
|
+
/**
|
|
6588
7171
|
* Devices already named by {@link reportUnrecordable} — one warn per device
|
|
6589
7172
|
* per "enabled but can never record" episode, re-armed as soon as the device
|
|
6590
7173
|
* records again.
|
|
@@ -6598,24 +7181,91 @@ var RecordingController = class {
|
|
|
6598
7181
|
*/
|
|
6599
7182
|
skippedProfiles = /* @__PURE__ */ new Map();
|
|
6600
7183
|
restore = null;
|
|
7184
|
+
/**
|
|
7185
|
+
* One pending wake-up per device, armed for the exact instant its recording
|
|
7186
|
+
* decision could change on its own — an `events` window closing or a band
|
|
7187
|
+
* edge. THE reason a detach is punctual instead of up to a pass late.
|
|
7188
|
+
*/
|
|
7189
|
+
wakeups;
|
|
6601
7190
|
/** Cheap, never-delayed: is any pinned writer producing nothing? */
|
|
6602
7191
|
livenessPass = null;
|
|
6603
|
-
/**
|
|
6604
|
-
|
|
7192
|
+
/** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
|
|
7193
|
+
placementPass = null;
|
|
7194
|
+
/** The backstop: re-evaluate every tracked-or-configured device. */
|
|
7195
|
+
reconcilePass = null;
|
|
6605
7196
|
stopped = false;
|
|
6606
7197
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6607
7198
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6608
7199
|
paused = false;
|
|
7200
|
+
/**
|
|
7201
|
+
* The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
|
|
7202
|
+
* the host saying it cannot fork right now, and the answer is to attach
|
|
7203
|
+
* FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
|
|
7204
|
+
* for good reason. Every bounded-parallel attach in this file reads it.
|
|
7205
|
+
*/
|
|
7206
|
+
spawnPressure;
|
|
6609
7207
|
constructor(deps) {
|
|
6610
7208
|
this.deps = deps;
|
|
7209
|
+
this.spawnPressure = new SpawnPressureGovernor({
|
|
7210
|
+
maxConcurrency: 8,
|
|
7211
|
+
now: () => this.now(),
|
|
7212
|
+
logger: deps.logger
|
|
7213
|
+
});
|
|
7214
|
+
this.wakeups = new DeviceWakeups({
|
|
7215
|
+
logger: deps.logger,
|
|
7216
|
+
now: () => this.now(),
|
|
7217
|
+
fire: (deviceId) => this.onWakeup(deviceId)
|
|
7218
|
+
});
|
|
6611
7219
|
}
|
|
6612
7220
|
now() {
|
|
6613
7221
|
return this.deps.now?.() ?? Date.now();
|
|
6614
7222
|
}
|
|
6615
7223
|
/**
|
|
7224
|
+
* A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
|
|
7225
|
+
* cancels) the next one as part of deciding, so the chain is self-sustaining.
|
|
7226
|
+
*
|
|
7227
|
+
* A throw here breaks that chain — the device keeps whatever state it had and
|
|
7228
|
+
* has no timer behind it — so it is NAMED, and the reconcile is what picks it
|
|
7229
|
+
* up. Silence would read as "the window never closed".
|
|
7230
|
+
*/
|
|
7231
|
+
onWakeup(deviceId) {
|
|
7232
|
+
this.evaluateDevice(deviceId).catch((err) => {
|
|
7233
|
+
this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
|
|
7234
|
+
tags: { deviceId },
|
|
7235
|
+
meta: { error: require_dist.errMsg(err) }
|
|
7236
|
+
});
|
|
7237
|
+
});
|
|
7238
|
+
}
|
|
7239
|
+
/**
|
|
7240
|
+
* Arm (or cancel) this device's next wake-up from the config it was just
|
|
7241
|
+
* evaluated against.
|
|
7242
|
+
*
|
|
7243
|
+
* Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
|
|
7244
|
+
* attach chain can throw (broker not ready, stream unpublished) and the
|
|
7245
|
+
* schedule does not depend on whether it succeeded. Arming first means a
|
|
7246
|
+
* failed attach still gets its band edge, and only a failure to READ the
|
|
7247
|
+
* config can lose a wake-up.
|
|
7248
|
+
*/
|
|
7249
|
+
scheduleNextDecision(deviceId, config) {
|
|
7250
|
+
if (this.stopped || this.paused) {
|
|
7251
|
+
this.wakeups.cancel(deviceId);
|
|
7252
|
+
return;
|
|
7253
|
+
}
|
|
7254
|
+
const next = nextDecisionChangeAt({
|
|
7255
|
+
config,
|
|
7256
|
+
triggers: this.triggersFor(deviceId),
|
|
7257
|
+
nowMs: this.now()
|
|
7258
|
+
});
|
|
7259
|
+
if (next === null) {
|
|
7260
|
+
this.wakeups.cancel(deviceId);
|
|
7261
|
+
return;
|
|
7262
|
+
}
|
|
7263
|
+
this.wakeups.arm(deviceId, next.atMs, next.reason);
|
|
7264
|
+
}
|
|
7265
|
+
/**
|
|
6616
7266
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
6617
7267
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
6618
|
-
* periodic
|
|
7268
|
+
* periodic passes. Idempotent — safe to call once after the index is hydrated.
|
|
6619
7269
|
*/
|
|
6620
7270
|
async start() {
|
|
6621
7271
|
if (this.stopped) return;
|
|
@@ -6628,6 +7278,7 @@ var RecordingController = class {
|
|
|
6628
7278
|
}
|
|
6629
7279
|
this.restore = new ReadinessRestore({
|
|
6630
7280
|
subscribeReady: (handler) => this.deps.onBrokerReady(handler),
|
|
7281
|
+
concurrency: () => this.spawnPressure.concurrency,
|
|
6631
7282
|
attempt: async (deviceId) => {
|
|
6632
7283
|
await this.evaluateDevice(deviceId);
|
|
6633
7284
|
},
|
|
@@ -6636,45 +7287,83 @@ var RecordingController = class {
|
|
|
6636
7287
|
meta: { error: require_dist.errMsg(err) }
|
|
6637
7288
|
})
|
|
6638
7289
|
});
|
|
6639
|
-
if (this.
|
|
7290
|
+
if (this.reconcilePass === null) {
|
|
6640
7291
|
this.livenessPass = startPeriodicPass({
|
|
6641
7292
|
name: "liveness",
|
|
6642
|
-
intervalMs:
|
|
6643
|
-
slowAfterMs:
|
|
7293
|
+
intervalMs: LIVENESS_TICK_MS,
|
|
7294
|
+
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
6644
7295
|
logger: this.deps.logger,
|
|
6645
7296
|
now: () => this.now(),
|
|
6646
7297
|
run: () => this.checkIdleWriters()
|
|
6647
7298
|
});
|
|
6648
|
-
this.
|
|
6649
|
-
name: "
|
|
6650
|
-
intervalMs:
|
|
6651
|
-
slowAfterMs:
|
|
7299
|
+
this.placementPass = startPeriodicPass({
|
|
7300
|
+
name: "placement",
|
|
7301
|
+
intervalMs: PLACEMENT_TICK_MS,
|
|
7302
|
+
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
7303
|
+
logger: this.deps.logger,
|
|
7304
|
+
now: () => this.now(),
|
|
7305
|
+
run: () => this.replanPlacement()
|
|
7306
|
+
});
|
|
7307
|
+
this.reconcilePass = startPeriodicPass({
|
|
7308
|
+
name: "reconcile",
|
|
7309
|
+
intervalMs: RECONCILE_TICK_MS,
|
|
7310
|
+
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
6652
7311
|
logger: this.deps.logger,
|
|
6653
7312
|
now: () => this.now(),
|
|
6654
|
-
run: () => this.
|
|
7313
|
+
run: () => this.reconcile()
|
|
6655
7314
|
});
|
|
6656
7315
|
}
|
|
6657
7316
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
6658
7317
|
}
|
|
6659
|
-
/**
|
|
6660
|
-
|
|
6661
|
-
|
|
7318
|
+
/**
|
|
7319
|
+
* Every device the recorder is responsible for right now: those with writers
|
|
7320
|
+
* attached, plus every one with a persisted config. A failure to enumerate the
|
|
7321
|
+
* persisted set degrades to "the active ones" rather than aborting the pass.
|
|
7322
|
+
*/
|
|
7323
|
+
async trackedDeviceIds() {
|
|
6662
7324
|
const ids = new Set(this.active.keys());
|
|
6663
|
-
let persisted = [];
|
|
6664
7325
|
try {
|
|
6665
|
-
|
|
6666
|
-
} catch {
|
|
6667
|
-
|
|
7326
|
+
for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
|
|
7327
|
+
} catch (err) {
|
|
7328
|
+
this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
|
|
7329
|
+
error: require_dist.errMsg(err),
|
|
7330
|
+
activeDevices: this.active.size
|
|
7331
|
+
} });
|
|
7332
|
+
}
|
|
7333
|
+
return [...ids];
|
|
7334
|
+
}
|
|
7335
|
+
/**
|
|
7336
|
+
* Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
|
|
7337
|
+
* beyond a statfs per location), and it must not be starved by an attach that
|
|
7338
|
+
* is waiting on a 20 s broker timeout. It changes the PLAN only — a running
|
|
7339
|
+
* writer keeps its root until it next attaches, which is the boundary rule.
|
|
7340
|
+
*/
|
|
7341
|
+
async replanPlacement() {
|
|
7342
|
+
if (this.stopped || this.paused) return;
|
|
7343
|
+
if (this.deps.onPlacementTick === void 0) return;
|
|
7344
|
+
const ids = await this.trackedDeviceIds();
|
|
6668
7345
|
try {
|
|
6669
|
-
await this.deps.onPlacementTick
|
|
7346
|
+
await this.deps.onPlacementTick(ids);
|
|
6670
7347
|
} catch (err) {
|
|
6671
7348
|
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: require_dist.errMsg(err) } });
|
|
6672
7349
|
}
|
|
6673
|
-
|
|
7350
|
+
}
|
|
7351
|
+
/**
|
|
7352
|
+
* The backstop pass: re-evaluate every tracked-or-configured device.
|
|
7353
|
+
*
|
|
7354
|
+
* Nothing here is the normal path any more — a window close, a band edge, a
|
|
7355
|
+
* config change and a broker recovery all reach `evaluateDevice` on their own.
|
|
7356
|
+
* This exists for the four things that cannot: a wake-up the event path threw
|
|
7357
|
+
* before arming, an attach that failed with no later readiness transition to
|
|
7358
|
+
* ride, a timer lost to a runner respawn, and clock drift.
|
|
7359
|
+
*/
|
|
7360
|
+
async reconcile() {
|
|
7361
|
+
if (this.stopped) return;
|
|
7362
|
+
await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
|
|
6674
7363
|
try {
|
|
6675
7364
|
await this.evaluateDevice(id);
|
|
6676
7365
|
} catch (err) {
|
|
6677
|
-
this.deps.logger.warn("recorder controller:
|
|
7366
|
+
this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
|
|
6678
7367
|
tags: { deviceId: id },
|
|
6679
7368
|
meta: { error: require_dist.errMsg(err) }
|
|
6680
7369
|
});
|
|
@@ -6698,8 +7387,9 @@ var RecordingController = class {
|
|
|
6698
7387
|
* idle source can re-dial cleanly), then re-evaluate: if the band still demands
|
|
6699
7388
|
* recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
|
|
6700
7389
|
* the broker is not ready, the re-attach throws and the device stays on the
|
|
6701
|
-
* readiness queue (retried on the next broker ready) — and the periodic
|
|
6702
|
-
* the backstop. Bounded OUTER retry (
|
|
7390
|
+
* readiness queue (retried on the next broker ready) — and the periodic
|
|
7391
|
+
* reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
|
|
7392
|
+
* never a tight loop.
|
|
6703
7393
|
*/
|
|
6704
7394
|
async recoverWriterGaveUp(deviceId) {
|
|
6705
7395
|
if (this.stopped) return;
|
|
@@ -6709,7 +7399,7 @@ var RecordingController = class {
|
|
|
6709
7399
|
await this.evaluateDevice(deviceId);
|
|
6710
7400
|
} catch (err) {
|
|
6711
7401
|
this.restore?.add(deviceId, true);
|
|
6712
|
-
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/
|
|
7402
|
+
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
|
|
6713
7403
|
tags: { deviceId },
|
|
6714
7404
|
meta: { error: require_dist.errMsg(err) }
|
|
6715
7405
|
});
|
|
@@ -6760,10 +7450,16 @@ var RecordingController = class {
|
|
|
6760
7450
|
}
|
|
6761
7451
|
/**
|
|
6762
7452
|
* Record a qualifying trigger (motion/audio) for a device and converge: an
|
|
6763
|
-
* `events` band whose window this opens attaches now
|
|
6764
|
-
*
|
|
6765
|
-
*
|
|
6766
|
-
*
|
|
7453
|
+
* `events` band whose window this opens attaches now, and the same
|
|
7454
|
+
* `evaluateDevice` arms the wake-up that will CLOSE it at
|
|
7455
|
+
* `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
|
|
7456
|
+
* The qualification test (motion detected / audio over threshold) is done
|
|
7457
|
+
* upstream — by the time we're here, it qualified.
|
|
7458
|
+
*
|
|
7459
|
+
* A fresher trigger re-arms the wake-up later, which is exactly how the
|
|
7460
|
+
* sliding window already behaved: continuous motion stays ONE recording,
|
|
7461
|
+
* because `evaluateDevice` short-circuits at "already recording — converged"
|
|
7462
|
+
* and only the timer moves.
|
|
6767
7463
|
*/
|
|
6768
7464
|
async onTrigger(deviceId, source, atMs) {
|
|
6769
7465
|
if (this.stopped) return;
|
|
@@ -6775,7 +7471,7 @@ var RecordingController = class {
|
|
|
6775
7471
|
try {
|
|
6776
7472
|
await this.evaluateDevice(deviceId);
|
|
6777
7473
|
} catch (err) {
|
|
6778
|
-
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/
|
|
7474
|
+
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
|
|
6779
7475
|
tags: { deviceId },
|
|
6780
7476
|
meta: {
|
|
6781
7477
|
source,
|
|
@@ -6796,7 +7492,9 @@ var RecordingController = class {
|
|
|
6796
7492
|
if (this.stopped || this.paused) return;
|
|
6797
7493
|
const config = await this.deps.loadConfig(deviceId);
|
|
6798
7494
|
if (this.stopped || this.paused) return;
|
|
6799
|
-
|
|
7495
|
+
const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
|
|
7496
|
+
this.scheduleNextDecision(deviceId, config);
|
|
7497
|
+
if (!wantRecording) {
|
|
6800
7498
|
this.reportUnrecordable(deviceId, config);
|
|
6801
7499
|
await this.detachDevice(deviceId);
|
|
6802
7500
|
this.restore?.remove(deviceId);
|
|
@@ -6855,6 +7553,15 @@ var RecordingController = class {
|
|
|
6855
7553
|
});
|
|
6856
7554
|
}
|
|
6857
7555
|
/**
|
|
7556
|
+
* Throw before spending a single broker RPC when the registry already says the
|
|
7557
|
+
* `stream-broker` on the owner node is not ready. See
|
|
7558
|
+
* {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
|
|
7559
|
+
*/
|
|
7560
|
+
failFastIfBrokerCold(deviceId) {
|
|
7561
|
+
if (this.deps.brokerReady?.() !== false) return;
|
|
7562
|
+
throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
|
|
7563
|
+
}
|
|
7564
|
+
/**
|
|
6858
7565
|
* The attach body — extracted so the {@link attaching} in-flight guard in
|
|
6859
7566
|
* `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
|
|
6860
7567
|
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
@@ -6866,6 +7573,7 @@ var RecordingController = class {
|
|
|
6866
7573
|
*/
|
|
6867
7574
|
async performAttach(deviceId, config) {
|
|
6868
7575
|
this.restore?.add(deviceId, true);
|
|
7576
|
+
this.failFastIfBrokerCold(deviceId);
|
|
6869
7577
|
const profiles = await this.resolveProfiles(deviceId, config.profiles);
|
|
6870
7578
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
6871
7579
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
@@ -6923,6 +7631,16 @@ var RecordingController = class {
|
|
|
6923
7631
|
}
|
|
6924
7632
|
}
|
|
6925
7633
|
async runFillMissingProfiles(deviceId, config, current, skipped) {
|
|
7634
|
+
if (this.deps.brokerReady?.() === false) {
|
|
7635
|
+
this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
|
|
7636
|
+
tags: { deviceId },
|
|
7637
|
+
meta: {
|
|
7638
|
+
profiles: [...skipped],
|
|
7639
|
+
ownerNodeId: this.deps.ownerNodeId
|
|
7640
|
+
}
|
|
7641
|
+
});
|
|
7642
|
+
return;
|
|
7643
|
+
}
|
|
6926
7644
|
const have = new Set(current.map((r) => r.profile));
|
|
6927
7645
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
6928
7646
|
const added = [];
|
|
@@ -7000,7 +7718,8 @@ var RecordingController = class {
|
|
|
7000
7718
|
logger: deviceLog,
|
|
7001
7719
|
onGaveUp: () => {
|
|
7002
7720
|
this.recoverWriterGaveUp(deviceId);
|
|
7003
|
-
}
|
|
7721
|
+
},
|
|
7722
|
+
onResourcePressure: () => this.spawnPressure.report(deviceId)
|
|
7004
7723
|
});
|
|
7005
7724
|
const writerStartedMs = this.now();
|
|
7006
7725
|
writer.start();
|
|
@@ -7015,7 +7734,10 @@ var RecordingController = class {
|
|
|
7015
7734
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
7016
7735
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
7017
7736
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
7018
|
-
await this.handleFinalizedSegment(deviceId, profile, placement,
|
|
7737
|
+
await this.handleFinalizedSegment(deviceId, profile, placement, {
|
|
7738
|
+
attachedAtMs: writerStartedMs,
|
|
7739
|
+
segmentSeconds
|
|
7740
|
+
}, startMs, durMs, flatAbsPath);
|
|
7019
7741
|
},
|
|
7020
7742
|
logger: deviceLog
|
|
7021
7743
|
}),
|
|
@@ -7032,13 +7754,16 @@ var RecordingController = class {
|
|
|
7032
7754
|
* keep/discard gate touches `events`-mode segments only, so the continuous
|
|
7033
7755
|
* path is unchanged.
|
|
7034
7756
|
*/
|
|
7035
|
-
async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
|
|
7757
|
+
async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
|
|
7036
7758
|
try {
|
|
7037
7759
|
const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
|
|
7038
|
-
|
|
7760
|
+
const triggers = this.triggersFor(deviceId);
|
|
7761
|
+
if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
|
|
7762
|
+
this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
|
|
7039
7763
|
await node_fs.promises.unlink(flatAbsPath).catch(() => {});
|
|
7040
7764
|
return;
|
|
7041
7765
|
}
|
|
7766
|
+
if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
|
|
7042
7767
|
await this.deps.segmentStore.onFinalized({
|
|
7043
7768
|
deviceId,
|
|
7044
7769
|
profile,
|
|
@@ -7059,6 +7784,47 @@ var RecordingController = class {
|
|
|
7059
7784
|
}
|
|
7060
7785
|
}
|
|
7061
7786
|
/**
|
|
7787
|
+
* Name a discarded `events`-band segment. This branch deletes recorded
|
|
7788
|
+
* footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
|
|
7789
|
+
* then `unpinned` with an empty timeline in between and no line anywhere said
|
|
7790
|
+
* why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
|
|
7791
|
+
* every segment it produced.
|
|
7792
|
+
*
|
|
7793
|
+
* Level is chosen, not defaulted. A discard at the TAIL of a window is the
|
|
7794
|
+
* feature working (`info`) — the writer keeps running until the next converge
|
|
7795
|
+
* tick and those trailing segments were never asked for. A discard from an
|
|
7796
|
+
* attach that has retained NOTHING is a fault (`warn`): the whole pull was
|
|
7797
|
+
* wasted, which is exactly the shape of the 1439 loss and of any future
|
|
7798
|
+
* regression in the attach allowance.
|
|
7799
|
+
*/
|
|
7800
|
+
reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
|
|
7801
|
+
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
7802
|
+
const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
|
|
7803
|
+
const meta = {
|
|
7804
|
+
profile,
|
|
7805
|
+
startMs,
|
|
7806
|
+
durMs,
|
|
7807
|
+
lastMotionMs: triggers.lastMotionMs,
|
|
7808
|
+
lastAudioMs: triggers.lastAudioMs,
|
|
7809
|
+
postMs,
|
|
7810
|
+
preMs,
|
|
7811
|
+
attachedAtMs: attach.attachedAtMs,
|
|
7812
|
+
missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
|
|
7813
|
+
keptAnythingFromThisAttach: keptAnything
|
|
7814
|
+
};
|
|
7815
|
+
if (keptAnything) {
|
|
7816
|
+
this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
|
|
7817
|
+
tags: { deviceId },
|
|
7818
|
+
meta
|
|
7819
|
+
});
|
|
7820
|
+
return;
|
|
7821
|
+
}
|
|
7822
|
+
this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
|
|
7823
|
+
tags: { deviceId },
|
|
7824
|
+
meta
|
|
7825
|
+
});
|
|
7826
|
+
}
|
|
7827
|
+
/**
|
|
7062
7828
|
* Resolve the storage location a (device, profile) writes to, at an attach
|
|
7063
7829
|
* boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
|
|
7064
7830
|
* when the addon supplied one; otherwise to the pure `resolvePlacement` in
|
|
@@ -7100,7 +7866,10 @@ var RecordingController = class {
|
|
|
7100
7866
|
if (!recordings) return;
|
|
7101
7867
|
this.active.delete(deviceId);
|
|
7102
7868
|
this.skippedProfiles.delete(deviceId);
|
|
7103
|
-
for (const r of recordings)
|
|
7869
|
+
for (const r of recordings) {
|
|
7870
|
+
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
7871
|
+
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
7872
|
+
}
|
|
7104
7873
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
7105
7874
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
7106
7875
|
tags: { deviceId },
|
|
@@ -7111,6 +7880,7 @@ var RecordingController = class {
|
|
|
7111
7880
|
async pause() {
|
|
7112
7881
|
if (this.stopped || this.paused) return;
|
|
7113
7882
|
this.paused = true;
|
|
7883
|
+
this.wakeups.cancelAll();
|
|
7114
7884
|
while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
|
|
7115
7885
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
7116
7886
|
}
|
|
@@ -7126,13 +7896,16 @@ var RecordingController = class {
|
|
|
7126
7896
|
}
|
|
7127
7897
|
for (const deviceId of ids) await this.evaluateDevice(deviceId);
|
|
7128
7898
|
}
|
|
7129
|
-
/** Tear EVERYTHING down:
|
|
7899
|
+
/** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
|
|
7130
7900
|
async stop() {
|
|
7131
7901
|
this.stopped = true;
|
|
7902
|
+
this.wakeups.cancelAll();
|
|
7132
7903
|
this.livenessPass?.stop();
|
|
7133
7904
|
this.livenessPass = null;
|
|
7134
|
-
this.
|
|
7135
|
-
this.
|
|
7905
|
+
this.placementPass?.stop();
|
|
7906
|
+
this.placementPass = null;
|
|
7907
|
+
this.reconcilePass?.stop();
|
|
7908
|
+
this.reconcilePass = null;
|
|
7136
7909
|
this.restore?.stop();
|
|
7137
7910
|
this.restore = null;
|
|
7138
7911
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
@@ -8421,6 +9194,23 @@ var PLAYBACK_PREFIX = "playback";
|
|
|
8421
9194
|
/** Data-plane prefix for on-demand single stills →
|
|
8422
9195
|
* `/addon/recorder/still/<deviceId>/<epochSec>`. */
|
|
8423
9196
|
var STILL_PREFIX = "still";
|
|
9197
|
+
/**
|
|
9198
|
+
* Bound on pending hour-ledger writes (one per finalized segment, an RPC to
|
|
9199
|
+
* the hub store). Steady state completes in milliseconds and holds ~0–2; under
|
|
9200
|
+
* a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
|
|
9201
|
+
* grace before refusals begin. Each pending write holds one hour-row copy
|
|
9202
|
+
* (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
|
|
9203
|
+
* a dropped write is one silent re-seed from the archive walk (D148).
|
|
9204
|
+
*/
|
|
9205
|
+
var LEDGER_GATE_LIMIT = 256;
|
|
9206
|
+
/**
|
|
9207
|
+
* Bound on pending mfra tail reads (one per finalized RECENT segment; each is
|
|
9208
|
+
* an open + two reads on the same 4-thread libuv pool the live writers use,
|
|
9209
|
+
* holding read buffers while pending). 64 bounds both the held memory and the
|
|
9210
|
+
* pool contention; a dropped capture costs that segment's client one extra
|
|
9211
|
+
* round trip to the file's tail on the first seek.
|
|
9212
|
+
*/
|
|
9213
|
+
var MFRA_GATE_LIMIT = 64;
|
|
8424
9214
|
/** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
|
|
8425
9215
|
var EXPORTS_PREFIX = "exports";
|
|
8426
9216
|
/** How often the export janitor sweeps expired exports. */
|
|
@@ -8497,6 +9287,25 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8497
9287
|
/** Sync-sample tables captured at finalize, served by the directory so a
|
|
8498
9288
|
* playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
|
|
8499
9289
|
mfraTables = new MfraTableStore();
|
|
9290
|
+
/** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
|
|
9291
|
+
* disk/store stall must throttle this work, never balloon live heap — the
|
|
9292
|
+
* unbounded pending set is what fed the two `Ineffective mark-compacts`
|
|
9293
|
+
* fatals. Saturation is reported loudly but throttled; every individual
|
|
9294
|
+
* drop is logged with its deviceId at the call site. */
|
|
9295
|
+
ledgerWriteGate = new PendingGate({
|
|
9296
|
+
label: "hour-ledger",
|
|
9297
|
+
limit: LEDGER_GATE_LIMIT,
|
|
9298
|
+
onSaturated: (report) => {
|
|
9299
|
+
this.reportGateSaturated(report);
|
|
9300
|
+
}
|
|
9301
|
+
});
|
|
9302
|
+
mfraCaptureGate = new PendingGate({
|
|
9303
|
+
label: "mfra-capture",
|
|
9304
|
+
limit: MFRA_GATE_LIMIT,
|
|
9305
|
+
onSaturated: (report) => {
|
|
9306
|
+
this.reportGateSaturated(report);
|
|
9307
|
+
}
|
|
9308
|
+
});
|
|
8500
9309
|
/** Continuous + events band write-path controller (ffmpeg writers + watchers). */
|
|
8501
9310
|
controller = null;
|
|
8502
9311
|
/** Per-device motion/audio markers for playback (B3 events feed the controller). */
|
|
@@ -8534,6 +9343,20 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8534
9343
|
constructor() {
|
|
8535
9344
|
super({ ...DEFAULT_CONFIG });
|
|
8536
9345
|
}
|
|
9346
|
+
/**
|
|
9347
|
+
* One throttled line per saturated lane per report interval. Lane-level (no
|
|
9348
|
+
* deviceId — the lane aggregates every camera); the per-drop debug lines at
|
|
9349
|
+
* the call sites carry the tag. WARN because a saturated lane means the
|
|
9350
|
+
* store or the disk is stalling under live finalize load — the line that
|
|
9351
|
+
* explains the missing ledger hours and mfra misses an operator will see.
|
|
9352
|
+
*/
|
|
9353
|
+
reportGateSaturated(report) {
|
|
9354
|
+
this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
|
|
9355
|
+
lane: report.label,
|
|
9356
|
+
dropped: report.dropped,
|
|
9357
|
+
pending: report.pending
|
|
9358
|
+
} });
|
|
9359
|
+
}
|
|
8537
9360
|
async onInitialize() {
|
|
8538
9361
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
8539
9362
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
@@ -8597,18 +9420,28 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8597
9420
|
this.segmentHours?.dropSegments(rows);
|
|
8598
9421
|
},
|
|
8599
9422
|
onIndexed: (row, absPath) => {
|
|
8600
|
-
this.segmentHours
|
|
9423
|
+
const ledger = this.segmentHours;
|
|
9424
|
+
if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
|
|
9425
|
+
tags: { deviceId: row.deviceId },
|
|
9426
|
+
meta: { path: row.path }
|
|
9427
|
+
});
|
|
8601
9428
|
if (!isMfraTableServable(row.startMs, Date.now())) return;
|
|
8602
|
-
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
9429
|
+
if (!this.mfraCaptureGate.run(async () => {
|
|
9430
|
+
try {
|
|
9431
|
+
const table = await readMfraTable(absPath, row.bytes);
|
|
9432
|
+
this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
|
|
9433
|
+
} catch (err) {
|
|
9434
|
+
this.ctx.logger.debug("mfra capture failed", {
|
|
9435
|
+
tags: { deviceId: row.deviceId },
|
|
9436
|
+
meta: {
|
|
9437
|
+
path: row.path,
|
|
9438
|
+
error: require_dist.errMsg(err)
|
|
9439
|
+
}
|
|
9440
|
+
});
|
|
9441
|
+
}
|
|
9442
|
+
})) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
|
|
9443
|
+
tags: { deviceId: row.deviceId },
|
|
9444
|
+
meta: { path: row.path }
|
|
8612
9445
|
});
|
|
8613
9446
|
},
|
|
8614
9447
|
removeDirIfEmpty: async (locationId, relDir) => {
|
|
@@ -8666,7 +9499,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8666
9499
|
onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
|
|
8667
9500
|
if (state === "ready") handler();
|
|
8668
9501
|
}),
|
|
8669
|
-
brokerCall: makeBrokerCall(brokerHandle)
|
|
9502
|
+
brokerCall: makeBrokerCall(brokerHandle),
|
|
9503
|
+
brokerReady: () => brokerHandle.isReady
|
|
8670
9504
|
});
|
|
8671
9505
|
const relocateEngine = new RelocateEngine({
|
|
8672
9506
|
index: this.index,
|