@camstack/addon-pipeline 1.2.93 → 1.2.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-CTRpEiSd.js → addon-utils-UMWiMAxR.js} +1 -1
- package/dist/audio-analyzer/index.js +58 -4
- package/dist/audio-analyzer/index.mjs +56 -2
- package/dist/detection-pipeline/index.js +194 -6
- package/dist/detection-pipeline/index.mjs +191 -3
- package/dist/{dist-6POVIYmC.mjs → dist-C49o1k7u.mjs} +359 -21
- package/dist/{dist-D9To_r_d.js → dist-hyryByiw.js} +376 -20
- package/dist/{event-loop-stall-monitor-C-VvWOxS.mjs → event-loop-stall-monitor-CfxZBgNi.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-CGacbzXq.js → event-loop-stall-monitor-DPNTzGOT.js} +1 -1
- package/dist/{lazy-sharp-C3zOnmiU.js → lazy-sharp-Dxg6UeW-.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/process-memory-B9-UD7Rp.js +78 -0
- package/dist/process-memory-DXOB6eHX.mjs +66 -0
- package/dist/recorder/index.js +943 -109
- package/dist/recorder/index.mjs +942 -108
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +1 -1
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-C2woB6bY.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CF-c0cDZ.mjs} +3 -3
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-BjvmJK_d.mjs +26 -0
- package/dist/stream-broker/{hostInit-VCURcZLt.mjs → hostInit-Cpcf2hDB.mjs} +3 -3
- package/dist/stream-broker/index.js +25 -7
- package/dist/stream-broker/index.mjs +25 -7
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-Bq28qobd.js → worker-protocol-DR_a77D5.js} +1 -1
- package/dist/{worker-protocol-PzAKg8mC.mjs → worker-protocol-DqQ2GI4M.mjs} +1 -1
- package/package.json +13 -7
- package/python/inference_pool.py +178 -3
- package/python/test_inference_pool_backpressure.py +7 -1
- package/python/test_inference_pool_memstats.py +146 -0
- package/dist/node-topology-platform-BkR_k6WT.mjs +0 -15
- package/dist/node-topology-platform-CFZ7F4xW.js +0 -20
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Dut56EeQ.mjs +0 -26
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as recordingExportCapability, C as RecordingConfigSchema, Ct as nodePin, Et as selectAssignedProfileSlots, Ft as object, It as record, L as deriveRecordingMode, Lt as string, Pt as number, Q as recordingCapability, S as RECORDING_EXPORT_MAX_READ_BYTES, _t as DeviceType, ct as errMsg, f as EVENT_PAD_MS, it as storageEvictableCapability, kt as array, l as DEFAULT_EVENTS_BAND_BUFFER_SEC, m as ExportRecordSchema, mt as BaseAddon, nt as resolveRecordingProfiles, v as OpsLogEntrySchema, yt as hydrateSchema, zt as EventCategory } from "../dist-C49o1k7u.mjs";
|
|
2
2
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
3
3
|
import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-A2S9D7pu.mjs";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
@@ -456,6 +456,51 @@ var MfraTableStore = class {
|
|
|
456
456
|
}
|
|
457
457
|
}
|
|
458
458
|
};
|
|
459
|
+
var PendingGate = class {
|
|
460
|
+
deps;
|
|
461
|
+
inFlight = 0;
|
|
462
|
+
droppedSinceReport = 0;
|
|
463
|
+
lastReportAt = Number.NEGATIVE_INFINITY;
|
|
464
|
+
constructor(deps) {
|
|
465
|
+
this.deps = deps;
|
|
466
|
+
}
|
|
467
|
+
/** Tasks currently in flight. */
|
|
468
|
+
get pending() {
|
|
469
|
+
return this.inFlight;
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Admit `task` unless the lane is saturated.
|
|
473
|
+
*
|
|
474
|
+
* Returns `false` when the task was REFUSED — the caller must log the drop
|
|
475
|
+
* (with its `deviceId` tag when the work is about a device); a branch that
|
|
476
|
+
* drops work silently reads as "never happened".
|
|
477
|
+
*/
|
|
478
|
+
run(task) {
|
|
479
|
+
if (this.inFlight >= this.deps.limit) {
|
|
480
|
+
this.droppedSinceReport += 1;
|
|
481
|
+
const now = this.deps.now?.() ?? Date.now();
|
|
482
|
+
const intervalMs = this.deps.reportIntervalMs ?? 3e4;
|
|
483
|
+
if (now - this.lastReportAt >= intervalMs) {
|
|
484
|
+
this.lastReportAt = now;
|
|
485
|
+
const dropped = this.droppedSinceReport;
|
|
486
|
+
this.droppedSinceReport = 0;
|
|
487
|
+
this.deps.onSaturated({
|
|
488
|
+
label: this.deps.label,
|
|
489
|
+
dropped,
|
|
490
|
+
pending: this.inFlight
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
return false;
|
|
494
|
+
}
|
|
495
|
+
this.inFlight += 1;
|
|
496
|
+
Promise.resolve().then(task).catch((error) => {
|
|
497
|
+
this.deps.onTaskError?.(error);
|
|
498
|
+
}).finally(() => {
|
|
499
|
+
this.inFlight -= 1;
|
|
500
|
+
});
|
|
501
|
+
return true;
|
|
502
|
+
}
|
|
503
|
+
};
|
|
459
504
|
//#endregion
|
|
460
505
|
//#region src/recorder/merge-ranges.ts
|
|
461
506
|
function mergeRanges(segments, gapToleranceMs) {
|
|
@@ -515,12 +560,33 @@ function* hourBuckets(fromMs, toMs) {
|
|
|
515
560
|
const first = Math.floor(fromMs / HOUR_MS$5) * HOUR_MS$5;
|
|
516
561
|
for (let h = first; h < toMs; h += HOUR_MS$5) yield h;
|
|
517
562
|
}
|
|
563
|
+
/**
|
|
564
|
+
* Index of the first element whose `startMs` is GREATER than `startMs` (upper
|
|
565
|
+
* bound). Inserting there keeps equal-start rows in arrival order — exactly
|
|
566
|
+
* what a stable full sort of the backing map (insertion-ordered) produces, so
|
|
567
|
+
* an incrementally-maintained view is indistinguishable from a rebuilt one.
|
|
568
|
+
*/
|
|
569
|
+
function upperBoundByStart(rows, startMs) {
|
|
570
|
+
let lo = 0;
|
|
571
|
+
let hi = rows.length;
|
|
572
|
+
while (lo < hi) {
|
|
573
|
+
const mid = lo + hi >> 1;
|
|
574
|
+
if (rows[mid].startMs <= startMs) lo = mid + 1;
|
|
575
|
+
else hi = mid;
|
|
576
|
+
}
|
|
577
|
+
return lo;
|
|
578
|
+
}
|
|
518
579
|
var RecordingIndex = class {
|
|
519
580
|
byDevice = /* @__PURE__ */ new Map();
|
|
520
|
-
/** Start-sorted
|
|
521
|
-
*
|
|
522
|
-
*
|
|
581
|
+
/** Start-sorted views. Built lazily on first read (one copy+sort), then kept
|
|
582
|
+
* LIVE by `addSegment`'s ordered insert — rows arrive roughly ascending, so
|
|
583
|
+
* the insert is near-appending. Bulk mutations (hydrate, eviction, a re-set
|
|
584
|
+
* of an existing path) still invalidate; the steady-state finalize never
|
|
585
|
+
* does. Interactive locate/range calls reuse these instead of
|
|
586
|
+
* filtering+sorting the complete archive on every seek and scrub miss. */
|
|
523
587
|
sortedByDevice = /* @__PURE__ */ new Map();
|
|
588
|
+
fullRebuilds = 0;
|
|
589
|
+
incrementalInserts = 0;
|
|
524
590
|
/** Hour buckets this device has been walked for; `ALL_HOURS` after a full
|
|
525
591
|
* hydrate. Empty = nothing has been looked at, which is NOT the same as
|
|
526
592
|
* "there is nothing" — see {@link hydrationOf}. */
|
|
@@ -547,10 +613,34 @@ var RecordingIndex = class {
|
|
|
547
613
|
if (cached) return cached;
|
|
548
614
|
const m = this.byDevice.get(deviceId);
|
|
549
615
|
if (!m) return [];
|
|
616
|
+
this.fullRebuilds += 1;
|
|
550
617
|
const sorted = [...m.values()].filter((s) => profile == null || s.profile === profile).toSorted((a, b) => a.startMs - b.startMs);
|
|
551
618
|
deviceCache.set(key, sorted);
|
|
552
619
|
return sorted;
|
|
553
620
|
}
|
|
621
|
+
/** See {@link SortedViewStats}. Telemetry + the guard the churn test asserts on. */
|
|
622
|
+
sortedViewStats() {
|
|
623
|
+
return {
|
|
624
|
+
fullRebuilds: this.fullRebuilds,
|
|
625
|
+
incrementalInserts: this.incrementalInserts
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Ordered insert of ONE NEW row into every live view it belongs to (the
|
|
630
|
+
* all-profiles view and its own profile's view). Views not yet materialised
|
|
631
|
+
* stay unmaterialised — their first read pays the one build. PRECONDITION:
|
|
632
|
+
* the row's path was NOT already in the device map; a replace must
|
|
633
|
+
* invalidate instead, because the views hold the old row object.
|
|
634
|
+
*/
|
|
635
|
+
insertIntoSortedViews(s) {
|
|
636
|
+
const deviceCache = this.sortedByDevice.get(s.deviceId);
|
|
637
|
+
if (!deviceCache) return;
|
|
638
|
+
for (const [key, rows] of deviceCache) {
|
|
639
|
+
if (key !== "" && key !== s.profile) continue;
|
|
640
|
+
rows.splice(upperBoundByStart(rows, s.startMs), 0, s);
|
|
641
|
+
this.incrementalInserts += 1;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
554
644
|
/**
|
|
555
645
|
* Replace a device's segments FOR ONE LOCATION from a `storage.list` result (paths relative to that
|
|
556
646
|
* location root). Rows on other locations are preserved. Non-segment paths and other devices are ignored.
|
|
@@ -631,10 +721,26 @@ var RecordingIndex = class {
|
|
|
631
721
|
this.invalidateSorted(deviceId);
|
|
632
722
|
this.markHydrated(deviceId, hourStartMs, hourStartMs + HOUR_MS$5);
|
|
633
723
|
}
|
|
634
|
-
/**
|
|
724
|
+
/**
|
|
725
|
+
* Insert/replace one finalized segment (idempotent by path).
|
|
726
|
+
*
|
|
727
|
+
* THE hot mutation: every profile-writer lands here every `segmentSeconds`,
|
|
728
|
+
* ~3.4 times a second fleet-wide. A NEW path is an ordered insert into the
|
|
729
|
+
* live sorted views (rows arrive roughly ascending, so it is near-appending
|
|
730
|
+
* — O(log n) search + a short memmove). Only the rare RE-SET of an existing
|
|
731
|
+
* path (watcher replay after a playlist reset, or a relocate that changes
|
|
732
|
+
* `locationId`) invalidates: the views hold the old row object, and a
|
|
733
|
+
* one-off rebuild is the simple correct answer for a case that is not hot.
|
|
734
|
+
*/
|
|
635
735
|
addSegment(s) {
|
|
636
|
-
this.mapFor(s.deviceId)
|
|
637
|
-
|
|
736
|
+
const m = this.mapFor(s.deviceId);
|
|
737
|
+
const replacing = m.has(s.path);
|
|
738
|
+
m.set(s.path, s);
|
|
739
|
+
if (replacing) {
|
|
740
|
+
this.invalidateSorted(s.deviceId);
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
this.insertIntoSortedViews(s);
|
|
638
744
|
}
|
|
639
745
|
/** Remove segments by relative path, routing each path to the correct device map via its encoded deviceId. */
|
|
640
746
|
removeSegments(paths) {
|
|
@@ -2900,6 +3006,67 @@ async function listExportsFor(state, deviceId) {
|
|
|
2900
3006
|
return [...(await readExports(state)).values()].filter((r) => deviceId === void 0 || r.deviceId === deviceId).toSorted((a, b) => b.createdAt - a.createdAt);
|
|
2901
3007
|
}
|
|
2902
3008
|
//#endregion
|
|
3009
|
+
//#region src/recorder/addon/spawn-failure.ts
|
|
3010
|
+
/**
|
|
3011
|
+
* How the recorder reads a `child_process` `'error'` — ONE owner of the
|
|
3012
|
+
* transient-vs-permanent question.
|
|
3013
|
+
*
|
|
3014
|
+
* ── Why this file exists ───────────────────────────────────────────────────
|
|
3015
|
+
* In Node a child that cannot be spawned emits `'error'`, and an `'error'`
|
|
3016
|
+
* with no listener is re-thrown as an UNCAUGHT EXCEPTION. On 2026-08-18 three
|
|
3017
|
+
* cameras were unplugged, every writer for them re-dialled at once, the host
|
|
3018
|
+
* momentarily could not fork (`spawn ffmpeg EAGAIN`), and the `recorder`
|
|
3019
|
+
* runner died three times inside nine seconds. The D6 crash circuit-breaker
|
|
3020
|
+
* did its job and stopped respawning — leaving the whole fleet with no
|
|
3021
|
+
* recording for three hours and fifteen minutes while `/health` returned 200.
|
|
3022
|
+
*
|
|
3023
|
+
* Every ffmpeg spawn site in the recorder therefore attaches an `'error'`
|
|
3024
|
+
* listener, and every one of them classifies the failure HERE rather than
|
|
3025
|
+
* inventing its own list.
|
|
3026
|
+
*/
|
|
3027
|
+
/**
|
|
3028
|
+
* Spawn failures that RETRYING CANNOT FIX.
|
|
3029
|
+
*
|
|
3030
|
+
* The EAGAIN-vs-ENOENT split, stated once:
|
|
3031
|
+
*
|
|
3032
|
+
* - EAGAIN / EMFILE / ENOMEM are the host saying "not right now". The
|
|
3033
|
+
* resource comes back, often within one backoff. They take the ordinary
|
|
3034
|
+
* restart-with-backoff path, whose own comment already states the policy
|
|
3035
|
+
* for this shape: "only RAPID back-to-back failures count toward giving
|
|
3036
|
+
* up, so a brief blip never permanently stops a camera's recording".
|
|
3037
|
+
* - ENOENT / EACCES are the host saying "there is no ffmpeg here you may
|
|
3038
|
+
* run". Nothing about the next ten attempts differs from this one, so the
|
|
3039
|
+
* caller stops immediately and surfaces it at ERROR level: burning a
|
|
3040
|
+
* ten-attempt budget on a missing binary only delays recovery and buries
|
|
3041
|
+
* the single actionable line under ten identical warnings.
|
|
3042
|
+
*
|
|
3043
|
+
* EPERM is deliberately absent: it is reported for transient sandbox/cgroup
|
|
3044
|
+
* refusals as well as for real permission faults, and treating an ambiguous
|
|
3045
|
+
* code as permanent stops a camera that would have recovered.
|
|
3046
|
+
*/
|
|
3047
|
+
var PERMANENT_SPAWN_CODES = new Set(["ENOENT", "EACCES"]);
|
|
3048
|
+
/**
|
|
3049
|
+
* Read the `'error'` payload without a cast.
|
|
3050
|
+
*
|
|
3051
|
+
* An unknown code is treated as TRANSIENT: the cost of retrying something
|
|
3052
|
+
* permanent is a bounded restart budget, while the cost of giving up on
|
|
3053
|
+
* something transient is a camera that stops recording until an operator
|
|
3054
|
+
* notices.
|
|
3055
|
+
*/
|
|
3056
|
+
function readSpawnFailure(raw) {
|
|
3057
|
+
const code = errnoCode(raw);
|
|
3058
|
+
return {
|
|
3059
|
+
code,
|
|
3060
|
+
message: raw instanceof Error ? raw.message : String(raw),
|
|
3061
|
+
permanent: code !== void 0 && PERMANENT_SPAWN_CODES.has(code)
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
function errnoCode(raw) {
|
|
3065
|
+
if (typeof raw !== "object" || raw === null || !("code" in raw)) return void 0;
|
|
3066
|
+
const code = raw.code;
|
|
3067
|
+
return typeof code === "string" ? code : void 0;
|
|
3068
|
+
}
|
|
3069
|
+
//#endregion
|
|
2903
3070
|
//#region src/recorder/addon/export-engine.ts
|
|
2904
3071
|
/** Stderr lines retained for the failure LOG. ffmpeg prints one line per failed
|
|
2905
3072
|
* segment before the decisive "Error opening input" summary, so a short tail can
|
|
@@ -3069,7 +3236,7 @@ var ExportEngine = class {
|
|
|
3069
3236
|
outPath,
|
|
3070
3237
|
options
|
|
3071
3238
|
});
|
|
3072
|
-
const ran = await this.runFfmpeg(id, args, {
|
|
3239
|
+
const ran = await this.runFfmpeg(id, rec.deviceId, args, {
|
|
3073
3240
|
totalSec: expectedOutSeconds(rec),
|
|
3074
3241
|
floor: 0,
|
|
3075
3242
|
ceil: 99
|
|
@@ -3089,7 +3256,7 @@ var ExportEngine = class {
|
|
|
3089
3256
|
inputPlaylist: playlist,
|
|
3090
3257
|
outPath: concatPath
|
|
3091
3258
|
});
|
|
3092
|
-
const concatRan = await this.runFfmpeg(id, concatArgs, {
|
|
3259
|
+
const concatRan = await this.runFfmpeg(id, rec.deviceId, concatArgs, {
|
|
3093
3260
|
totalSec: Math.max(.001, (rec.toMs - rec.fromMs) / 1e3),
|
|
3094
3261
|
floor: 0,
|
|
3095
3262
|
ceil: 49
|
|
@@ -3103,7 +3270,7 @@ var ExportEngine = class {
|
|
|
3103
3270
|
outPath,
|
|
3104
3271
|
options
|
|
3105
3272
|
});
|
|
3106
|
-
const selectRan = await this.runFfmpeg(id, selectArgs, {
|
|
3273
|
+
const selectRan = await this.runFfmpeg(id, rec.deviceId, selectArgs, {
|
|
3107
3274
|
totalSec: expectedOutSeconds(rec),
|
|
3108
3275
|
floor: 50,
|
|
3109
3276
|
ceil: 99
|
|
@@ -3115,7 +3282,7 @@ var ExportEngine = class {
|
|
|
3115
3282
|
} catch {}
|
|
3116
3283
|
}
|
|
3117
3284
|
}
|
|
3118
|
-
runFfmpeg(id, args, progress) {
|
|
3285
|
+
runFfmpeg(id, deviceId, args, progress) {
|
|
3119
3286
|
this.deps.logger.debug("export ffmpeg argv", { meta: {
|
|
3120
3287
|
exportId: id,
|
|
3121
3288
|
args
|
|
@@ -3152,6 +3319,24 @@ var ExportEngine = class {
|
|
|
3152
3319
|
stderrTail
|
|
3153
3320
|
});
|
|
3154
3321
|
});
|
|
3322
|
+
proc.on("error", (raw) => {
|
|
3323
|
+
this.activeProc = null;
|
|
3324
|
+
const failure = readSpawnFailure(raw);
|
|
3325
|
+
this.deps.logger.error("export ffmpeg could not be spawned", {
|
|
3326
|
+
tags: { deviceId },
|
|
3327
|
+
meta: {
|
|
3328
|
+
exportId: id,
|
|
3329
|
+
errorCode: failure.code,
|
|
3330
|
+
error: failure.message,
|
|
3331
|
+
permanent: failure.permanent
|
|
3332
|
+
}
|
|
3333
|
+
});
|
|
3334
|
+
stderrTail.push(`ffmpeg spawn failed: ${failure.message}` + (failure.code === void 0 ? "" : ` (${failure.code})`));
|
|
3335
|
+
resolve({
|
|
3336
|
+
code: null,
|
|
3337
|
+
stderrTail
|
|
3338
|
+
});
|
|
3339
|
+
});
|
|
3155
3340
|
});
|
|
3156
3341
|
}
|
|
3157
3342
|
async finish(id, code, outPath, stderrTail) {
|
|
@@ -5837,18 +6022,232 @@ function shouldRecordAt(config, triggers, at) {
|
|
|
5837
6022
|
return eventsBandDemanded(band, triggers, at.getTime());
|
|
5838
6023
|
}
|
|
5839
6024
|
/**
|
|
5840
|
-
*
|
|
5841
|
-
*
|
|
5842
|
-
*
|
|
5843
|
-
*
|
|
6025
|
+
* How late the FIRST segment of an attach may be NAMED, relative to the writer
|
|
6026
|
+
* spawn, and still count as "the segment this attach was spawned to produce".
|
|
6027
|
+
*
|
|
6028
|
+
* ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
|
|
6029
|
+
* first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
|
|
6030
|
+
* spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
|
|
6031
|
+
* 3.2 s. 5 s carries that with margin while staying far below the writer's own
|
|
6032
|
+
* "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
|
|
6033
|
+
* slow is not mistaken for one that is producing useful footage.
|
|
5844
6034
|
*/
|
|
5845
|
-
|
|
6035
|
+
var FIRST_SEGMENT_GRACE_MS = 5e3;
|
|
6036
|
+
/**
|
|
6037
|
+
* Latest segment START this attach may still produce and have kept, regardless
|
|
6038
|
+
* of where the operator's post-buffer ended: one full segment (nothing can be
|
|
6039
|
+
* finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
|
|
6040
|
+
* latency.
|
|
6041
|
+
*/
|
|
6042
|
+
function attachRetainDeadlineMs(attach) {
|
|
6043
|
+
return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
|
|
6044
|
+
}
|
|
6045
|
+
/**
|
|
6046
|
+
* Should a finalized segment `[segStartMs, segEndMs]` recorded under an
|
|
6047
|
+
* `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
|
|
6048
|
+
* trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
|
|
6049
|
+
* this is where `preBufferSec` retroactively retains pre-trigger footage — OR
|
|
6050
|
+
* it is the footage `attach` was spawned to produce (below).
|
|
6051
|
+
*
|
|
6052
|
+
* ## Why the attach matters here
|
|
6053
|
+
*
|
|
6054
|
+
* The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
|
|
6055
|
+
* gate is evaluated on segment timestamps. They shared one window, and a writer
|
|
6056
|
+
* physically cannot finalize anything before `spawn + segmentSeconds + dial`.
|
|
6057
|
+
* So the last ~`segmentSeconds` of every demand window was a zone where the
|
|
6058
|
+
* recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
|
|
6059
|
+
* result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
|
|
6060
|
+
* post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
|
|
6061
|
+
* finalized segments were unlinked.
|
|
6062
|
+
*
|
|
6063
|
+
* The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
|
|
6064
|
+
* `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
|
|
6065
|
+
* inside `trigger + postMs` and retains exactly what it retained before. It
|
|
6066
|
+
* only ever extends the tail of a LATE attach, by at most one segment plus the
|
|
6067
|
+
* grace, and only for a trigger whose live window was still open AT ATTACH — the
|
|
6068
|
+
* precise statement of "if the live gate said yes, the keep gate honours the
|
|
6069
|
+
* first segment that decision produces". A stale trigger extends nothing.
|
|
6070
|
+
*/
|
|
6071
|
+
function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
|
|
5846
6072
|
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
5847
|
-
const
|
|
6073
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
6074
|
+
const overlaps = (lastMs) => {
|
|
6075
|
+
if (lastMs == null) return false;
|
|
6076
|
+
if (segEndMs < lastMs - preMs) return false;
|
|
6077
|
+
if (segStartMs <= lastMs + postMs) return true;
|
|
6078
|
+
return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
|
|
6079
|
+
};
|
|
5848
6080
|
if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
|
|
5849
6081
|
if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
|
|
5850
6082
|
return false;
|
|
5851
6083
|
}
|
|
6084
|
+
/**
|
|
6085
|
+
* By how many ms a discarded segment missed being kept — its start past the
|
|
6086
|
+
* latest start that WOULD have been retained, mirroring
|
|
6087
|
+
* {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
|
|
6088
|
+
* the attach allowance for a trigger that was still live at attach). Null when
|
|
6089
|
+
* no qualifying trigger exists at all, which is a different fact and says so.
|
|
6090
|
+
*
|
|
6091
|
+
* Negative means the segment was dropped on the PRE-buffer side (it ended
|
|
6092
|
+
* before `trigger - preBufferSec`), not for being late.
|
|
6093
|
+
*
|
|
6094
|
+
* Reporting-only. The discard branch has to name a number: "outside the window"
|
|
6095
|
+
* with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
|
|
6096
|
+
*/
|
|
6097
|
+
function segmentMissedByMs(segStartMs, band, triggers, attach) {
|
|
6098
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
6099
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
6100
|
+
const latestKeptStart = (lastMs) => {
|
|
6101
|
+
if (lastMs == null) return null;
|
|
6102
|
+
const windowEndMs = lastMs + postMs;
|
|
6103
|
+
return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
|
|
6104
|
+
};
|
|
6105
|
+
const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
|
|
6106
|
+
if (bounds.length === 0) return null;
|
|
6107
|
+
return segStartMs - Math.max(...bounds);
|
|
6108
|
+
}
|
|
6109
|
+
/** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
|
|
6110
|
+
function localInstantMs(year, month, day, hhmm) {
|
|
6111
|
+
const [hours, minutes] = hhmm.split(":");
|
|
6112
|
+
return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
|
|
6113
|
+
}
|
|
6114
|
+
/**
|
|
6115
|
+
* Every instant inside the horizon at which `bandActiveAt` COULD change value:
|
|
6116
|
+
* each band's start and end on each local day, plus each local midnight (the
|
|
6117
|
+
* `days` membership boundary). A superset — cheap to compute (a few dozen
|
|
6118
|
+
* entries) and filtered against `activeBandAt` below.
|
|
6119
|
+
*/
|
|
6120
|
+
function candidateEdgesMs(bands, nowMs) {
|
|
6121
|
+
const base = new Date(nowMs);
|
|
6122
|
+
const year = base.getFullYear();
|
|
6123
|
+
const month = base.getMonth();
|
|
6124
|
+
const day = base.getDate();
|
|
6125
|
+
const out = [];
|
|
6126
|
+
for (let offset = 0; offset <= 8; offset++) {
|
|
6127
|
+
out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
|
|
6128
|
+
for (const band of bands) {
|
|
6129
|
+
out.push(localInstantMs(year, month, day + offset, band.start));
|
|
6130
|
+
out.push(localInstantMs(year, month, day + offset, band.end));
|
|
6131
|
+
}
|
|
6132
|
+
}
|
|
6133
|
+
return out;
|
|
6134
|
+
}
|
|
6135
|
+
/**
|
|
6136
|
+
* The next instant at which a DIFFERENT band (or no band) covers the clock, or
|
|
6137
|
+
* `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
|
|
6138
|
+
* band (`start === end`, empty `days`) being the common case that legitimately
|
|
6139
|
+
* has no edge at all.
|
|
6140
|
+
*
|
|
6141
|
+
* Identity is by band REFERENCE, which is what makes this exact: two distinct
|
|
6142
|
+
* band objects with the same fields yield an edge that changes nothing, which
|
|
6143
|
+
* costs one wasted wake-up and never costs footage.
|
|
6144
|
+
*/
|
|
6145
|
+
function nextBandEdgeMs(config, nowMs) {
|
|
6146
|
+
const bands = config.bands;
|
|
6147
|
+
if (!bands || bands.length === 0) return null;
|
|
6148
|
+
const current = activeBandAt({ bands }, new Date(nowMs));
|
|
6149
|
+
const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
|
|
6150
|
+
for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
|
|
6151
|
+
return null;
|
|
6152
|
+
}
|
|
6153
|
+
/**
|
|
6154
|
+
* The first instant an `events` band's demand window is CLOSED, given the
|
|
6155
|
+
* triggers it listens for, or `null` when no window is open at `nowMs`.
|
|
6156
|
+
*
|
|
6157
|
+
* Demand is an OR across the listened trigger kinds, so an open window ends
|
|
6158
|
+
* with the LATEST of them — a motion trigger followed by an audio trigger 5 s
|
|
6159
|
+
* later keeps the camera attached until the audio one expires, exactly as the
|
|
6160
|
+
* sliding window already behaved.
|
|
6161
|
+
*/
|
|
6162
|
+
function eventsWindowCloseMs(band, triggers, nowMs) {
|
|
6163
|
+
if (band.mode !== "events") return null;
|
|
6164
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
6165
|
+
const ends = [];
|
|
6166
|
+
if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
|
|
6167
|
+
if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
|
|
6168
|
+
const open = ends.filter((ms) => ms >= nowMs);
|
|
6169
|
+
if (open.length === 0) return null;
|
|
6170
|
+
return Math.max(...open) + 1;
|
|
6171
|
+
}
|
|
6172
|
+
/**
|
|
6173
|
+
* THE next instant the controller must re-evaluate this device, or `null` when
|
|
6174
|
+
* nothing can change without an event the controller already receives (a
|
|
6175
|
+
* trigger or a config write).
|
|
6176
|
+
*
|
|
6177
|
+
* Whichever of the two sources comes first wins, and the reason is carried so
|
|
6178
|
+
* the wake-up is attributable in the log without correlating timestamps.
|
|
6179
|
+
*/
|
|
6180
|
+
function nextDecisionChangeAt(input) {
|
|
6181
|
+
const { config, triggers, nowMs } = input;
|
|
6182
|
+
if (!config.enabled) return null;
|
|
6183
|
+
const bands = config.bands;
|
|
6184
|
+
if (!bands || bands.length === 0) return null;
|
|
6185
|
+
const band = activeBandAt({ bands }, new Date(nowMs));
|
|
6186
|
+
const edgeMs = nextBandEdgeMs(config, nowMs);
|
|
6187
|
+
const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
|
|
6188
|
+
if (closeMs === null) return edgeMs === null ? null : {
|
|
6189
|
+
atMs: edgeMs,
|
|
6190
|
+
reason: "band-edge"
|
|
6191
|
+
};
|
|
6192
|
+
if (edgeMs === null || closeMs <= edgeMs) return {
|
|
6193
|
+
atMs: closeMs,
|
|
6194
|
+
reason: "window-close"
|
|
6195
|
+
};
|
|
6196
|
+
return {
|
|
6197
|
+
atMs: edgeMs,
|
|
6198
|
+
reason: "band-edge"
|
|
6199
|
+
};
|
|
6200
|
+
}
|
|
6201
|
+
//#endregion
|
|
6202
|
+
//#region src/recorder/addon/device-wakeups.ts
|
|
6203
|
+
/** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
|
|
6204
|
+
var MAX_TIMER_DELAY_MS = 2147483647;
|
|
6205
|
+
var DeviceWakeups = class {
|
|
6206
|
+
deps;
|
|
6207
|
+
timers = /* @__PURE__ */ new Map();
|
|
6208
|
+
constructor(deps) {
|
|
6209
|
+
this.deps = deps;
|
|
6210
|
+
}
|
|
6211
|
+
/**
|
|
6212
|
+
* Wake this device at `atMs`, replacing any wake-up already armed for it.
|
|
6213
|
+
* `reason` is carried into the log line so a wake is attributable without
|
|
6214
|
+
* correlating it against the band model by hand.
|
|
6215
|
+
*/
|
|
6216
|
+
arm(deviceId, atMs, reason) {
|
|
6217
|
+
this.cancel(deviceId);
|
|
6218
|
+
const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
|
|
6219
|
+
const timer = setTimeout(() => {
|
|
6220
|
+
this.timers.delete(deviceId);
|
|
6221
|
+
this.deps.fire(deviceId);
|
|
6222
|
+
}, delayMs);
|
|
6223
|
+
timer.unref?.();
|
|
6224
|
+
this.timers.set(deviceId, timer);
|
|
6225
|
+
this.deps.logger.debug("recorder: device wake-up armed", {
|
|
6226
|
+
tags: { deviceId },
|
|
6227
|
+
meta: {
|
|
6228
|
+
atMs,
|
|
6229
|
+
delayMs,
|
|
6230
|
+
reason
|
|
6231
|
+
}
|
|
6232
|
+
});
|
|
6233
|
+
}
|
|
6234
|
+
/** Drop this device's pending wake-up, if any. Idempotent. */
|
|
6235
|
+
cancel(deviceId) {
|
|
6236
|
+
const timer = this.timers.get(deviceId);
|
|
6237
|
+
if (timer === void 0) return;
|
|
6238
|
+
clearTimeout(timer);
|
|
6239
|
+
this.timers.delete(deviceId);
|
|
6240
|
+
}
|
|
6241
|
+
/** Drop every pending wake-up — shutdown and maintenance pause. */
|
|
6242
|
+
cancelAll() {
|
|
6243
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
6244
|
+
this.timers.clear();
|
|
6245
|
+
}
|
|
6246
|
+
/** Devices with a wake-up pending — for tests and for shutdown assertions. */
|
|
6247
|
+
get pendingCount() {
|
|
6248
|
+
return this.timers.size;
|
|
6249
|
+
}
|
|
6250
|
+
};
|
|
5852
6251
|
//#endregion
|
|
5853
6252
|
//#region src/recorder/addon/periodic-pass.ts
|
|
5854
6253
|
function startPeriodicPass(deps) {
|
|
@@ -5915,7 +6314,9 @@ var ReadinessRestore = class {
|
|
|
5915
6314
|
this.deps = deps;
|
|
5916
6315
|
}
|
|
5917
6316
|
get concurrency() {
|
|
5918
|
-
|
|
6317
|
+
const declared = this.deps.concurrency;
|
|
6318
|
+
const value = typeof declared === "function" ? declared() : declared;
|
|
6319
|
+
return Math.max(1, value ?? 8);
|
|
5919
6320
|
}
|
|
5920
6321
|
/** Seed the pending set, subscribe to readiness, and drain once immediately
|
|
5921
6322
|
* (covers the already-ready case). Returns when the first drain settles. */
|
|
@@ -5974,6 +6375,70 @@ var ReadinessRestore = class {
|
|
|
5974
6375
|
}
|
|
5975
6376
|
}
|
|
5976
6377
|
};
|
|
6378
|
+
/** Adaptive bound for the recorder's attach parallelism. */
|
|
6379
|
+
var SpawnPressureGovernor = class {
|
|
6380
|
+
deps;
|
|
6381
|
+
limit;
|
|
6382
|
+
min;
|
|
6383
|
+
recoverAfterMs;
|
|
6384
|
+
/** When the limit last MOVED — the clock both directions are measured from. */
|
|
6385
|
+
lastChangeAt;
|
|
6386
|
+
underPressure = false;
|
|
6387
|
+
constructor(deps) {
|
|
6388
|
+
this.deps = deps;
|
|
6389
|
+
this.min = Math.max(1, deps.minConcurrency ?? 1);
|
|
6390
|
+
this.limit = Math.max(this.min, deps.maxConcurrency);
|
|
6391
|
+
this.recoverAfterMs = deps.recoverAfterMs ?? 3e4;
|
|
6392
|
+
this.lastChangeAt = this.now();
|
|
6393
|
+
}
|
|
6394
|
+
/**
|
|
6395
|
+
* How many attaches may overlap right now.
|
|
6396
|
+
*
|
|
6397
|
+
* Reading is what drives recovery: there is no timer to leak, no interval to
|
|
6398
|
+
* lose to a runner respawn, and a recorder that stops attaching entirely
|
|
6399
|
+
* simply keeps its reduced bound until it tries again — which is correct.
|
|
6400
|
+
*/
|
|
6401
|
+
get concurrency() {
|
|
6402
|
+
const max = Math.max(this.min, this.deps.maxConcurrency);
|
|
6403
|
+
if (this.limit >= max) return max;
|
|
6404
|
+
const elapsed = this.now() - this.lastChangeAt;
|
|
6405
|
+
if (elapsed < this.recoverAfterMs) return this.limit;
|
|
6406
|
+
const lanes = Math.floor(elapsed / this.recoverAfterMs);
|
|
6407
|
+
this.limit = Math.min(max, this.limit + lanes);
|
|
6408
|
+
this.lastChangeAt += lanes * this.recoverAfterMs;
|
|
6409
|
+
if (this.limit >= max && this.underPressure) {
|
|
6410
|
+
this.underPressure = false;
|
|
6411
|
+
this.deps.logger?.info("recorder: spawn pressure cleared — attach concurrency restored", { meta: { concurrency: this.limit } });
|
|
6412
|
+
} else this.deps.logger?.info("recorder: recovering attach concurrency", { meta: {
|
|
6413
|
+
concurrency: this.limit,
|
|
6414
|
+
max
|
|
6415
|
+
} });
|
|
6416
|
+
return this.limit;
|
|
6417
|
+
}
|
|
6418
|
+
/**
|
|
6419
|
+
* A transient spawn failure happened on `deviceId`'s writer.
|
|
6420
|
+
*
|
|
6421
|
+
* Halves the CURRENT limit, not the max: two waves of pressure inside one
|
|
6422
|
+
* recovery window must compound, or a host under sustained load is asked for
|
|
6423
|
+
* the same too-large batch over and over.
|
|
6424
|
+
*/
|
|
6425
|
+
report(deviceId) {
|
|
6426
|
+
const before = this.limit;
|
|
6427
|
+
this.limit = Math.max(this.min, Math.floor(this.limit / 2));
|
|
6428
|
+
this.lastChangeAt = this.now();
|
|
6429
|
+
this.underPressure = true;
|
|
6430
|
+
this.deps.logger?.warn("recorder: spawn pressure — reducing attach concurrency", {
|
|
6431
|
+
tags: { deviceId },
|
|
6432
|
+
meta: {
|
|
6433
|
+
from: before,
|
|
6434
|
+
to: this.limit
|
|
6435
|
+
}
|
|
6436
|
+
});
|
|
6437
|
+
}
|
|
6438
|
+
now() {
|
|
6439
|
+
return this.deps.now?.() ?? Date.now();
|
|
6440
|
+
}
|
|
6441
|
+
};
|
|
5977
6442
|
//#endregion
|
|
5978
6443
|
//#region src/recorder/addon/segment-watcher.ts
|
|
5979
6444
|
/**
|
|
@@ -6359,36 +6824,87 @@ var SegmentWriter = class {
|
|
|
6359
6824
|
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
|
|
6360
6825
|
}
|
|
6361
6826
|
});
|
|
6827
|
+
let settled = false;
|
|
6828
|
+
const terminate = (termination) => {
|
|
6829
|
+
if (settled) return;
|
|
6830
|
+
settled = true;
|
|
6831
|
+
this.onTermination(termination, stderrTail);
|
|
6832
|
+
};
|
|
6362
6833
|
proc.on("exit", (code) => {
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6834
|
+
terminate({
|
|
6835
|
+
reason: "exit",
|
|
6836
|
+
exitCode: typeof code === "number" ? code : null
|
|
6837
|
+
});
|
|
6838
|
+
});
|
|
6839
|
+
proc.on("error", (raw) => {
|
|
6840
|
+
const failure = readSpawnFailure(raw);
|
|
6841
|
+
terminate({
|
|
6842
|
+
reason: "spawn-error",
|
|
6843
|
+
errorCode: failure.code,
|
|
6844
|
+
message: failure.message,
|
|
6845
|
+
permanent: failure.permanent
|
|
6846
|
+
});
|
|
6847
|
+
});
|
|
6848
|
+
}
|
|
6849
|
+
/**
|
|
6850
|
+
* The ONE restart policy, reached from both `exit` and `error`.
|
|
6851
|
+
*
|
|
6852
|
+
* `deps.logger` is the device-scoped child the controller binds
|
|
6853
|
+
* (`logger.withTags({ deviceId })`), so every line below carries
|
|
6854
|
+
* `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
|
|
6855
|
+
* camera, and a writer that stops recording one camera must be greppable by
|
|
6856
|
+
* that camera.
|
|
6857
|
+
*/
|
|
6858
|
+
onTermination(termination, stderrTail) {
|
|
6859
|
+
this.resolveExit?.();
|
|
6860
|
+
this.resolveExit = null;
|
|
6861
|
+
this.proc = null;
|
|
6862
|
+
if (this.stopped) return;
|
|
6863
|
+
const ranMs = Date.now() - this.startedAt;
|
|
6864
|
+
if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
|
|
6865
|
+
if (termination.reason === "spawn-error") {
|
|
6866
|
+
this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
|
|
6867
|
+
outDir: this.cfg.outDir,
|
|
6868
|
+
errorCode: termination.errorCode,
|
|
6869
|
+
error: termination.message,
|
|
6870
|
+
permanent: termination.permanent
|
|
6871
|
+
} });
|
|
6872
|
+
if (termination.permanent) {
|
|
6873
|
+
this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
|
|
6371
6874
|
outDir: this.cfg.outDir,
|
|
6372
|
-
|
|
6373
|
-
|
|
6875
|
+
errorCode: termination.errorCode,
|
|
6876
|
+
error: termination.message
|
|
6374
6877
|
} });
|
|
6375
6878
|
this.stopped = true;
|
|
6376
6879
|
this.deps.onGaveUp?.();
|
|
6377
6880
|
return;
|
|
6378
6881
|
}
|
|
6379
|
-
this.
|
|
6380
|
-
|
|
6381
|
-
|
|
6882
|
+
this.deps.onResourcePressure?.();
|
|
6883
|
+
}
|
|
6884
|
+
if (this.restarts >= MAX_RESTARTS) {
|
|
6885
|
+
this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
|
|
6382
6886
|
outDir: this.cfg.outDir,
|
|
6383
|
-
|
|
6384
|
-
|
|
6385
|
-
|
|
6887
|
+
reason: termination.reason,
|
|
6888
|
+
code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
|
|
6889
|
+
stderrTail: stderrTail.join(" | ")
|
|
6386
6890
|
} });
|
|
6387
|
-
this.
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6891
|
+
this.stopped = true;
|
|
6892
|
+
this.deps.onGaveUp?.();
|
|
6893
|
+
return;
|
|
6894
|
+
}
|
|
6895
|
+
this.restarts++;
|
|
6896
|
+
const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
|
|
6897
|
+
this.deps.logger.info("SegmentWriter restarting", { meta: {
|
|
6898
|
+
outDir: this.cfg.outDir,
|
|
6899
|
+
attempt: this.restarts,
|
|
6900
|
+
delayMs,
|
|
6901
|
+
ranMs,
|
|
6902
|
+
reason: termination.reason
|
|
6903
|
+
} });
|
|
6904
|
+
this.restartTimer = setTimeout(() => {
|
|
6905
|
+
this.restartTimer = null;
|
|
6906
|
+
this.start();
|
|
6907
|
+
}, delayMs);
|
|
6392
6908
|
}
|
|
6393
6909
|
stop() {
|
|
6394
6910
|
this.stopAndWait();
|
|
@@ -6432,19 +6948,37 @@ var SegmentWriter = class {
|
|
|
6432
6948
|
* - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
|
|
6433
6949
|
* the v2 `SegmentStore.onFinalized` (stat + relocate + index).
|
|
6434
6950
|
*
|
|
6435
|
-
* Per device it decides whether
|
|
6436
|
-
* (`
|
|
6437
|
-
*
|
|
6438
|
-
* 1. `setDeviceConfig` (operator changed the bands / enable),
|
|
6439
|
-
* 2. a
|
|
6440
|
-
* 3. a
|
|
6441
|
-
*
|
|
6442
|
-
*
|
|
6443
|
-
*
|
|
6444
|
-
*
|
|
6445
|
-
*
|
|
6446
|
-
*
|
|
6447
|
-
*
|
|
6951
|
+
* Per device it decides whether the active band demands recording right now
|
|
6952
|
+
* (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
|
|
6953
|
+
* is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
|
|
6954
|
+
* 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
|
|
6955
|
+
* 2. a qualifying motion/audio trigger (`onTrigger`) — opens a window,
|
|
6956
|
+
* 3. a per-device WAKE-UP armed for the exact instant the decision could
|
|
6957
|
+
* change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
|
|
6958
|
+
* `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
|
|
6959
|
+
* 4. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`),
|
|
6960
|
+
* 5. the periodic RECONCILE (below), for everything the four above can lose.
|
|
6961
|
+
*
|
|
6962
|
+
* (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
|
|
6963
|
+
* demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
|
|
6964
|
+
* trigger, so nothing called `evaluateDevice` at the moment it closed and the
|
|
6965
|
+
* detach waited for the next pass. A 4K camera stayed dialled, decoded and
|
|
6966
|
+
* written for up to a whole tick past the operator's `postBufferSec`.
|
|
6967
|
+
*
|
|
6968
|
+
* There are THREE periodic passes, each on its own timer and each non-overlapping
|
|
6969
|
+
* (`periodic-pass.ts`):
|
|
6970
|
+
* - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
|
|
6971
|
+
* producing nothing?
|
|
6972
|
+
* - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
|
|
6973
|
+
* deliberately NOT on the attach beat, so a cold broker cannot delay it.
|
|
6974
|
+
* - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
|
|
6975
|
+
* broker, and the only backstop for a lost wake-up or a failed attach.
|
|
6976
|
+
*
|
|
6977
|
+
* They were ONE pass until 2026-08-13, when a convergence sweep wedged on
|
|
6978
|
+
* unbounded broker RPCs and took the liveness watchdog down with it for 9-27
|
|
6979
|
+
* minutes at a time; placement was split out of the second on 2026-08-18, after
|
|
6980
|
+
* a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
|
|
6981
|
+
* cold-dialling ~40 RTSP sources.
|
|
6448
6982
|
*
|
|
6449
6983
|
* EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
|
|
6450
6984
|
* trigger-gating. The enable intent is persisted in durable-state by the
|
|
@@ -6457,8 +6991,45 @@ var SegmentWriter = class {
|
|
|
6457
6991
|
* `ReadinessRegistry` is the single source of truth.
|
|
6458
6992
|
*/
|
|
6459
6993
|
var DEFAULT_WATCH_INTERVAL_MS = 2e3;
|
|
6460
|
-
/**
|
|
6461
|
-
|
|
6994
|
+
/**
|
|
6995
|
+
* Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
|
|
6996
|
+
* question is an in-memory scan over `active`, it costs nothing, and its bound
|
|
6997
|
+
* (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
|
|
6998
|
+
*/
|
|
6999
|
+
var LIVENESS_TICK_MS = 3e4;
|
|
7000
|
+
/**
|
|
7001
|
+
* Placement cadence — unchanged from the beat it used to share with
|
|
7002
|
+
* convergence, because nothing about placement wanted to be rarer. What changed
|
|
7003
|
+
* is that it no longer rides a pass that can block on broker RPCs: it is one
|
|
7004
|
+
* `statfs` per location and it re-plans only, so a plan reaches a camera at its
|
|
7005
|
+
* NEXT attach.
|
|
7006
|
+
*/
|
|
7007
|
+
var PLACEMENT_TICK_MS = 3e4;
|
|
7008
|
+
/**
|
|
7009
|
+
* Reconcile cadence — the SAFETY NET, not the mechanism.
|
|
7010
|
+
*
|
|
7011
|
+
* It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
|
|
7012
|
+
* window or crossed a band edge, so the loop had to keep asking. Both are now
|
|
7013
|
+
* armed for their exact instant (`decision-schedule.ts`), a config change is an
|
|
7014
|
+
* RPC, and a broker recovery is a readiness transition — so this pass exists
|
|
7015
|
+
* only for what those cannot cover: a wake-up lost because the event path threw
|
|
7016
|
+
* before it could arm one, an attach that failed with no later ready transition
|
|
7017
|
+
* to ride, a timer lost to a runner respawn, and clock drift.
|
|
7018
|
+
*
|
|
7019
|
+
* 2 minutes, and the number is chosen against a bound this recorder already
|
|
7020
|
+
* accepts. The liveness watchdog tolerates a pinned-but-silent writer for
|
|
7021
|
+
* `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
|
|
7022
|
+
* recording but is detached is the same class of fault — footage is not being
|
|
7023
|
+
* written and only a periodic check will notice — so its recovery is put on the
|
|
7024
|
+
* same order rather than an order slower. Four times rarer than the beat that
|
|
7025
|
+
* blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
|
|
7026
|
+
* ~2 minutes of footage for a camera whose event path failed entirely.
|
|
7027
|
+
*
|
|
7028
|
+
* Going rarer than this trades real footage for CPU that the pass no longer
|
|
7029
|
+
* spends anyway: with the broker-readiness pre-check below, a reconcile over a
|
|
7030
|
+
* healthy fleet is N config reads and no RPC at all.
|
|
7031
|
+
*/
|
|
7032
|
+
var RECONCILE_TICK_MS = 12e4;
|
|
6462
7033
|
/**
|
|
6463
7034
|
* Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
|
|
6464
7035
|
* teardown. The UDS request default is 60s, but the runner supervisor's
|
|
@@ -6497,11 +7068,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
|
|
|
6497
7068
|
*/
|
|
6498
7069
|
var ATTACH_RPC_TIMEOUT_MS = 2e4;
|
|
6499
7070
|
/**
|
|
6500
|
-
* A pass longer than this is reported with its duration.
|
|
6501
|
-
* cadence: a pass that cannot finish inside its own interval is
|
|
6502
|
-
* that used to go unlogged for half an hour.
|
|
7071
|
+
* A pass longer than this is reported with its duration. Each pass uses its OWN
|
|
7072
|
+
* cadence as the budget: a pass that cannot finish inside its own interval is
|
|
7073
|
+
* the condition that used to go unlogged for half an hour. Keeping it per-pass
|
|
7074
|
+
* matters now that the three cadences differ — a 40 s reconcile is healthy
|
|
7075
|
+
* against a 120 s beat and was a red flag against a 30 s one.
|
|
6503
7076
|
*/
|
|
6504
|
-
|
|
7077
|
+
function passSlowAfterMs(intervalMs) {
|
|
7078
|
+
return intervalMs;
|
|
7079
|
+
}
|
|
6505
7080
|
/**
|
|
6506
7081
|
* Race `promise` against a deadline. Handlers stay attached to the losing
|
|
6507
7082
|
* promise, so a post-deadline settlement can never surface as a process-level
|
|
@@ -6584,6 +7159,14 @@ var RecordingController = class {
|
|
|
6584
7159
|
*/
|
|
6585
7160
|
lastSegmentAt = /* @__PURE__ */ new Map();
|
|
6586
7161
|
/**
|
|
7162
|
+
* Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
|
|
7163
|
+
* at least one `events` segment. Read only to LEVEL the discard log: an
|
|
7164
|
+
* attach that kept nothing is a wasted pull (warn), a later discard from one
|
|
7165
|
+
* that kept something is the tail of the window (info). Bounded by
|
|
7166
|
+
* device×profile like {@link lastSegmentAt}, and cleared with it on detach.
|
|
7167
|
+
*/
|
|
7168
|
+
attachRetained = /* @__PURE__ */ new Map();
|
|
7169
|
+
/**
|
|
6587
7170
|
* Devices already named by {@link reportUnrecordable} — one warn per device
|
|
6588
7171
|
* per "enabled but can never record" episode, re-armed as soon as the device
|
|
6589
7172
|
* records again.
|
|
@@ -6597,24 +7180,91 @@ var RecordingController = class {
|
|
|
6597
7180
|
*/
|
|
6598
7181
|
skippedProfiles = /* @__PURE__ */ new Map();
|
|
6599
7182
|
restore = null;
|
|
7183
|
+
/**
|
|
7184
|
+
* One pending wake-up per device, armed for the exact instant its recording
|
|
7185
|
+
* decision could change on its own — an `events` window closing or a band
|
|
7186
|
+
* edge. THE reason a detach is punctual instead of up to a pass late.
|
|
7187
|
+
*/
|
|
7188
|
+
wakeups;
|
|
6600
7189
|
/** Cheap, never-delayed: is any pinned writer producing nothing? */
|
|
6601
7190
|
livenessPass = null;
|
|
6602
|
-
/**
|
|
6603
|
-
|
|
7191
|
+
/** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
|
|
7192
|
+
placementPass = null;
|
|
7193
|
+
/** The backstop: re-evaluate every tracked-or-configured device. */
|
|
7194
|
+
reconcilePass = null;
|
|
6604
7195
|
stopped = false;
|
|
6605
7196
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6606
7197
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6607
7198
|
paused = false;
|
|
7199
|
+
/**
|
|
7200
|
+
* The ADAPTIVE attach bound. A transient `spawn ffmpeg` failure (EAGAIN) is
|
|
7201
|
+
* the host saying it cannot fork right now, and the answer is to attach
|
|
7202
|
+
* FEWER cameras at once — not to serialise them, which `e2fa17a2b` removed
|
|
7203
|
+
* for good reason. Every bounded-parallel attach in this file reads it.
|
|
7204
|
+
*/
|
|
7205
|
+
spawnPressure;
|
|
6608
7206
|
constructor(deps) {
|
|
6609
7207
|
this.deps = deps;
|
|
7208
|
+
this.spawnPressure = new SpawnPressureGovernor({
|
|
7209
|
+
maxConcurrency: 8,
|
|
7210
|
+
now: () => this.now(),
|
|
7211
|
+
logger: deps.logger
|
|
7212
|
+
});
|
|
7213
|
+
this.wakeups = new DeviceWakeups({
|
|
7214
|
+
logger: deps.logger,
|
|
7215
|
+
now: () => this.now(),
|
|
7216
|
+
fire: (deviceId) => this.onWakeup(deviceId)
|
|
7217
|
+
});
|
|
6610
7218
|
}
|
|
6611
7219
|
now() {
|
|
6612
7220
|
return this.deps.now?.() ?? Date.now();
|
|
6613
7221
|
}
|
|
6614
7222
|
/**
|
|
7223
|
+
* A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
|
|
7224
|
+
* cancels) the next one as part of deciding, so the chain is self-sustaining.
|
|
7225
|
+
*
|
|
7226
|
+
* A throw here breaks that chain — the device keeps whatever state it had and
|
|
7227
|
+
* has no timer behind it — so it is NAMED, and the reconcile is what picks it
|
|
7228
|
+
* up. Silence would read as "the window never closed".
|
|
7229
|
+
*/
|
|
7230
|
+
onWakeup(deviceId) {
|
|
7231
|
+
this.evaluateDevice(deviceId).catch((err) => {
|
|
7232
|
+
this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
|
|
7233
|
+
tags: { deviceId },
|
|
7234
|
+
meta: { error: errMsg(err) }
|
|
7235
|
+
});
|
|
7236
|
+
});
|
|
7237
|
+
}
|
|
7238
|
+
/**
|
|
7239
|
+
* Arm (or cancel) this device's next wake-up from the config it was just
|
|
7240
|
+
* evaluated against.
|
|
7241
|
+
*
|
|
7242
|
+
* Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
|
|
7243
|
+
* attach chain can throw (broker not ready, stream unpublished) and the
|
|
7244
|
+
* schedule does not depend on whether it succeeded. Arming first means a
|
|
7245
|
+
* failed attach still gets its band edge, and only a failure to READ the
|
|
7246
|
+
* config can lose a wake-up.
|
|
7247
|
+
*/
|
|
7248
|
+
scheduleNextDecision(deviceId, config) {
|
|
7249
|
+
if (this.stopped || this.paused) {
|
|
7250
|
+
this.wakeups.cancel(deviceId);
|
|
7251
|
+
return;
|
|
7252
|
+
}
|
|
7253
|
+
const next = nextDecisionChangeAt({
|
|
7254
|
+
config,
|
|
7255
|
+
triggers: this.triggersFor(deviceId),
|
|
7256
|
+
nowMs: this.now()
|
|
7257
|
+
});
|
|
7258
|
+
if (next === null) {
|
|
7259
|
+
this.wakeups.cancel(deviceId);
|
|
7260
|
+
return;
|
|
7261
|
+
}
|
|
7262
|
+
this.wakeups.arm(deviceId, next.atMs, next.reason);
|
|
7263
|
+
}
|
|
7264
|
+
/**
|
|
6615
7265
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
6616
7266
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
6617
|
-
* periodic
|
|
7267
|
+
* periodic passes. Idempotent — safe to call once after the index is hydrated.
|
|
6618
7268
|
*/
|
|
6619
7269
|
async start() {
|
|
6620
7270
|
if (this.stopped) return;
|
|
@@ -6627,6 +7277,7 @@ var RecordingController = class {
|
|
|
6627
7277
|
}
|
|
6628
7278
|
this.restore = new ReadinessRestore({
|
|
6629
7279
|
subscribeReady: (handler) => this.deps.onBrokerReady(handler),
|
|
7280
|
+
concurrency: () => this.spawnPressure.concurrency,
|
|
6630
7281
|
attempt: async (deviceId) => {
|
|
6631
7282
|
await this.evaluateDevice(deviceId);
|
|
6632
7283
|
},
|
|
@@ -6635,45 +7286,83 @@ var RecordingController = class {
|
|
|
6635
7286
|
meta: { error: errMsg(err) }
|
|
6636
7287
|
})
|
|
6637
7288
|
});
|
|
6638
|
-
if (this.
|
|
7289
|
+
if (this.reconcilePass === null) {
|
|
6639
7290
|
this.livenessPass = startPeriodicPass({
|
|
6640
7291
|
name: "liveness",
|
|
6641
|
-
intervalMs:
|
|
6642
|
-
slowAfterMs:
|
|
7292
|
+
intervalMs: LIVENESS_TICK_MS,
|
|
7293
|
+
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
6643
7294
|
logger: this.deps.logger,
|
|
6644
7295
|
now: () => this.now(),
|
|
6645
7296
|
run: () => this.checkIdleWriters()
|
|
6646
7297
|
});
|
|
6647
|
-
this.
|
|
6648
|
-
name: "
|
|
6649
|
-
intervalMs:
|
|
6650
|
-
slowAfterMs:
|
|
7298
|
+
this.placementPass = startPeriodicPass({
|
|
7299
|
+
name: "placement",
|
|
7300
|
+
intervalMs: PLACEMENT_TICK_MS,
|
|
7301
|
+
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
7302
|
+
logger: this.deps.logger,
|
|
7303
|
+
now: () => this.now(),
|
|
7304
|
+
run: () => this.replanPlacement()
|
|
7305
|
+
});
|
|
7306
|
+
this.reconcilePass = startPeriodicPass({
|
|
7307
|
+
name: "reconcile",
|
|
7308
|
+
intervalMs: RECONCILE_TICK_MS,
|
|
7309
|
+
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
6651
7310
|
logger: this.deps.logger,
|
|
6652
7311
|
now: () => this.now(),
|
|
6653
|
-
run: () => this.
|
|
7312
|
+
run: () => this.reconcile()
|
|
6654
7313
|
});
|
|
6655
7314
|
}
|
|
6656
7315
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
6657
7316
|
}
|
|
6658
|
-
/**
|
|
6659
|
-
|
|
6660
|
-
|
|
7317
|
+
/**
|
|
7318
|
+
* Every device the recorder is responsible for right now: those with writers
|
|
7319
|
+
* attached, plus every one with a persisted config. A failure to enumerate the
|
|
7320
|
+
* persisted set degrades to "the active ones" rather than aborting the pass.
|
|
7321
|
+
*/
|
|
7322
|
+
async trackedDeviceIds() {
|
|
6661
7323
|
const ids = new Set(this.active.keys());
|
|
6662
|
-
let persisted = [];
|
|
6663
7324
|
try {
|
|
6664
|
-
|
|
6665
|
-
} catch {
|
|
6666
|
-
|
|
7325
|
+
for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
|
|
7326
|
+
} catch (err) {
|
|
7327
|
+
this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
|
|
7328
|
+
error: errMsg(err),
|
|
7329
|
+
activeDevices: this.active.size
|
|
7330
|
+
} });
|
|
7331
|
+
}
|
|
7332
|
+
return [...ids];
|
|
7333
|
+
}
|
|
7334
|
+
/**
|
|
7335
|
+
* Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
|
|
7336
|
+
* beyond a statfs per location), and it must not be starved by an attach that
|
|
7337
|
+
* is waiting on a 20 s broker timeout. It changes the PLAN only — a running
|
|
7338
|
+
* writer keeps its root until it next attaches, which is the boundary rule.
|
|
7339
|
+
*/
|
|
7340
|
+
async replanPlacement() {
|
|
7341
|
+
if (this.stopped || this.paused) return;
|
|
7342
|
+
if (this.deps.onPlacementTick === void 0) return;
|
|
7343
|
+
const ids = await this.trackedDeviceIds();
|
|
6667
7344
|
try {
|
|
6668
|
-
await this.deps.onPlacementTick
|
|
7345
|
+
await this.deps.onPlacementTick(ids);
|
|
6669
7346
|
} catch (err) {
|
|
6670
7347
|
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: errMsg(err) } });
|
|
6671
7348
|
}
|
|
6672
|
-
|
|
7349
|
+
}
|
|
7350
|
+
/**
|
|
7351
|
+
* The backstop pass: re-evaluate every tracked-or-configured device.
|
|
7352
|
+
*
|
|
7353
|
+
* Nothing here is the normal path any more — a window close, a band edge, a
|
|
7354
|
+
* config change and a broker recovery all reach `evaluateDevice` on their own.
|
|
7355
|
+
* This exists for the four things that cannot: a wake-up the event path threw
|
|
7356
|
+
* before arming, an attach that failed with no later readiness transition to
|
|
7357
|
+
* ride, a timer lost to a runner respawn, and clock drift.
|
|
7358
|
+
*/
|
|
7359
|
+
async reconcile() {
|
|
7360
|
+
if (this.stopped) return;
|
|
7361
|
+
await runBounded(await this.trackedDeviceIds(), this.spawnPressure.concurrency, async (id) => {
|
|
6673
7362
|
try {
|
|
6674
7363
|
await this.evaluateDevice(id);
|
|
6675
7364
|
} catch (err) {
|
|
6676
|
-
this.deps.logger.warn("recorder controller:
|
|
7365
|
+
this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
|
|
6677
7366
|
tags: { deviceId: id },
|
|
6678
7367
|
meta: { error: errMsg(err) }
|
|
6679
7368
|
});
|
|
@@ -6697,8 +7386,9 @@ var RecordingController = class {
|
|
|
6697
7386
|
* idle source can re-dial cleanly), then re-evaluate: if the band still demands
|
|
6698
7387
|
* recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
|
|
6699
7388
|
* the broker is not ready, the re-attach throws and the device stays on the
|
|
6700
|
-
* readiness queue (retried on the next broker ready) — and the periodic
|
|
6701
|
-
* the backstop. Bounded OUTER retry (
|
|
7389
|
+
* readiness queue (retried on the next broker ready) — and the periodic
|
|
7390
|
+
* reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
|
|
7391
|
+
* never a tight loop.
|
|
6702
7392
|
*/
|
|
6703
7393
|
async recoverWriterGaveUp(deviceId) {
|
|
6704
7394
|
if (this.stopped) return;
|
|
@@ -6708,7 +7398,7 @@ var RecordingController = class {
|
|
|
6708
7398
|
await this.evaluateDevice(deviceId);
|
|
6709
7399
|
} catch (err) {
|
|
6710
7400
|
this.restore?.add(deviceId, true);
|
|
6711
|
-
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/
|
|
7401
|
+
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
|
|
6712
7402
|
tags: { deviceId },
|
|
6713
7403
|
meta: { error: errMsg(err) }
|
|
6714
7404
|
});
|
|
@@ -6759,10 +7449,16 @@ var RecordingController = class {
|
|
|
6759
7449
|
}
|
|
6760
7450
|
/**
|
|
6761
7451
|
* Record a qualifying trigger (motion/audio) for a device and converge: an
|
|
6762
|
-
* `events` band whose window this opens attaches now
|
|
6763
|
-
*
|
|
6764
|
-
*
|
|
6765
|
-
*
|
|
7452
|
+
* `events` band whose window this opens attaches now, and the same
|
|
7453
|
+
* `evaluateDevice` arms the wake-up that will CLOSE it at
|
|
7454
|
+
* `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
|
|
7455
|
+
* The qualification test (motion detected / audio over threshold) is done
|
|
7456
|
+
* upstream — by the time we're here, it qualified.
|
|
7457
|
+
*
|
|
7458
|
+
* A fresher trigger re-arms the wake-up later, which is exactly how the
|
|
7459
|
+
* sliding window already behaved: continuous motion stays ONE recording,
|
|
7460
|
+
* because `evaluateDevice` short-circuits at "already recording — converged"
|
|
7461
|
+
* and only the timer moves.
|
|
6766
7462
|
*/
|
|
6767
7463
|
async onTrigger(deviceId, source, atMs) {
|
|
6768
7464
|
if (this.stopped) return;
|
|
@@ -6774,7 +7470,7 @@ var RecordingController = class {
|
|
|
6774
7470
|
try {
|
|
6775
7471
|
await this.evaluateDevice(deviceId);
|
|
6776
7472
|
} catch (err) {
|
|
6777
|
-
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/
|
|
7473
|
+
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
|
|
6778
7474
|
tags: { deviceId },
|
|
6779
7475
|
meta: {
|
|
6780
7476
|
source,
|
|
@@ -6795,7 +7491,9 @@ var RecordingController = class {
|
|
|
6795
7491
|
if (this.stopped || this.paused) return;
|
|
6796
7492
|
const config = await this.deps.loadConfig(deviceId);
|
|
6797
7493
|
if (this.stopped || this.paused) return;
|
|
6798
|
-
|
|
7494
|
+
const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
|
|
7495
|
+
this.scheduleNextDecision(deviceId, config);
|
|
7496
|
+
if (!wantRecording) {
|
|
6799
7497
|
this.reportUnrecordable(deviceId, config);
|
|
6800
7498
|
await this.detachDevice(deviceId);
|
|
6801
7499
|
this.restore?.remove(deviceId);
|
|
@@ -6854,6 +7552,15 @@ var RecordingController = class {
|
|
|
6854
7552
|
});
|
|
6855
7553
|
}
|
|
6856
7554
|
/**
|
|
7555
|
+
* Throw before spending a single broker RPC when the registry already says the
|
|
7556
|
+
* `stream-broker` on the owner node is not ready. See
|
|
7557
|
+
* {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
|
|
7558
|
+
*/
|
|
7559
|
+
failFastIfBrokerCold(deviceId) {
|
|
7560
|
+
if (this.deps.brokerReady?.() !== false) return;
|
|
7561
|
+
throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
|
|
7562
|
+
}
|
|
7563
|
+
/**
|
|
6857
7564
|
* The attach body — extracted so the {@link attaching} in-flight guard in
|
|
6858
7565
|
* `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
|
|
6859
7566
|
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
@@ -6865,6 +7572,7 @@ var RecordingController = class {
|
|
|
6865
7572
|
*/
|
|
6866
7573
|
async performAttach(deviceId, config) {
|
|
6867
7574
|
this.restore?.add(deviceId, true);
|
|
7575
|
+
this.failFastIfBrokerCold(deviceId);
|
|
6868
7576
|
const profiles = await this.resolveProfiles(deviceId, config.profiles);
|
|
6869
7577
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
6870
7578
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
@@ -6922,6 +7630,16 @@ var RecordingController = class {
|
|
|
6922
7630
|
}
|
|
6923
7631
|
}
|
|
6924
7632
|
async runFillMissingProfiles(deviceId, config, current, skipped) {
|
|
7633
|
+
if (this.deps.brokerReady?.() === false) {
|
|
7634
|
+
this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
|
|
7635
|
+
tags: { deviceId },
|
|
7636
|
+
meta: {
|
|
7637
|
+
profiles: [...skipped],
|
|
7638
|
+
ownerNodeId: this.deps.ownerNodeId
|
|
7639
|
+
}
|
|
7640
|
+
});
|
|
7641
|
+
return;
|
|
7642
|
+
}
|
|
6925
7643
|
const have = new Set(current.map((r) => r.profile));
|
|
6926
7644
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
6927
7645
|
const added = [];
|
|
@@ -6999,7 +7717,8 @@ var RecordingController = class {
|
|
|
6999
7717
|
logger: deviceLog,
|
|
7000
7718
|
onGaveUp: () => {
|
|
7001
7719
|
this.recoverWriterGaveUp(deviceId);
|
|
7002
|
-
}
|
|
7720
|
+
},
|
|
7721
|
+
onResourcePressure: () => this.spawnPressure.report(deviceId)
|
|
7003
7722
|
});
|
|
7004
7723
|
const writerStartedMs = this.now();
|
|
7005
7724
|
writer.start();
|
|
@@ -7014,7 +7733,10 @@ var RecordingController = class {
|
|
|
7014
7733
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
7015
7734
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
7016
7735
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
7017
|
-
await this.handleFinalizedSegment(deviceId, profile, placement,
|
|
7736
|
+
await this.handleFinalizedSegment(deviceId, profile, placement, {
|
|
7737
|
+
attachedAtMs: writerStartedMs,
|
|
7738
|
+
segmentSeconds
|
|
7739
|
+
}, startMs, durMs, flatAbsPath);
|
|
7018
7740
|
},
|
|
7019
7741
|
logger: deviceLog
|
|
7020
7742
|
}),
|
|
@@ -7031,13 +7753,16 @@ var RecordingController = class {
|
|
|
7031
7753
|
* keep/discard gate touches `events`-mode segments only, so the continuous
|
|
7032
7754
|
* path is unchanged.
|
|
7033
7755
|
*/
|
|
7034
|
-
async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
|
|
7756
|
+
async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
|
|
7035
7757
|
try {
|
|
7036
7758
|
const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
|
|
7037
|
-
|
|
7759
|
+
const triggers = this.triggersFor(deviceId);
|
|
7760
|
+
if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
|
|
7761
|
+
this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
|
|
7038
7762
|
await promises.unlink(flatAbsPath).catch(() => {});
|
|
7039
7763
|
return;
|
|
7040
7764
|
}
|
|
7765
|
+
if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
|
|
7041
7766
|
await this.deps.segmentStore.onFinalized({
|
|
7042
7767
|
deviceId,
|
|
7043
7768
|
profile,
|
|
@@ -7058,6 +7783,47 @@ var RecordingController = class {
|
|
|
7058
7783
|
}
|
|
7059
7784
|
}
|
|
7060
7785
|
/**
|
|
7786
|
+
* Name a discarded `events`-band segment. This branch deletes recorded
|
|
7787
|
+
* footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
|
|
7788
|
+
* then `unpinned` with an empty timeline in between and no line anywhere said
|
|
7789
|
+
* why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
|
|
7790
|
+
* every segment it produced.
|
|
7791
|
+
*
|
|
7792
|
+
* Level is chosen, not defaulted. A discard at the TAIL of a window is the
|
|
7793
|
+
* feature working (`info`) — the writer keeps running until the next converge
|
|
7794
|
+
* tick and those trailing segments were never asked for. A discard from an
|
|
7795
|
+
* attach that has retained NOTHING is a fault (`warn`): the whole pull was
|
|
7796
|
+
* wasted, which is exactly the shape of the 1439 loss and of any future
|
|
7797
|
+
* regression in the attach allowance.
|
|
7798
|
+
*/
|
|
7799
|
+
reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
|
|
7800
|
+
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
7801
|
+
const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
|
|
7802
|
+
const meta = {
|
|
7803
|
+
profile,
|
|
7804
|
+
startMs,
|
|
7805
|
+
durMs,
|
|
7806
|
+
lastMotionMs: triggers.lastMotionMs,
|
|
7807
|
+
lastAudioMs: triggers.lastAudioMs,
|
|
7808
|
+
postMs,
|
|
7809
|
+
preMs,
|
|
7810
|
+
attachedAtMs: attach.attachedAtMs,
|
|
7811
|
+
missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
|
|
7812
|
+
keptAnythingFromThisAttach: keptAnything
|
|
7813
|
+
};
|
|
7814
|
+
if (keptAnything) {
|
|
7815
|
+
this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
|
|
7816
|
+
tags: { deviceId },
|
|
7817
|
+
meta
|
|
7818
|
+
});
|
|
7819
|
+
return;
|
|
7820
|
+
}
|
|
7821
|
+
this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
|
|
7822
|
+
tags: { deviceId },
|
|
7823
|
+
meta
|
|
7824
|
+
});
|
|
7825
|
+
}
|
|
7826
|
+
/**
|
|
7061
7827
|
* Resolve the storage location a (device, profile) writes to, at an attach
|
|
7062
7828
|
* boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
|
|
7063
7829
|
* when the addon supplied one; otherwise to the pure `resolvePlacement` in
|
|
@@ -7099,7 +7865,10 @@ var RecordingController = class {
|
|
|
7099
7865
|
if (!recordings) return;
|
|
7100
7866
|
this.active.delete(deviceId);
|
|
7101
7867
|
this.skippedProfiles.delete(deviceId);
|
|
7102
|
-
for (const r of recordings)
|
|
7868
|
+
for (const r of recordings) {
|
|
7869
|
+
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
7870
|
+
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
7871
|
+
}
|
|
7103
7872
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
7104
7873
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
7105
7874
|
tags: { deviceId },
|
|
@@ -7110,6 +7879,7 @@ var RecordingController = class {
|
|
|
7110
7879
|
async pause() {
|
|
7111
7880
|
if (this.stopped || this.paused) return;
|
|
7112
7881
|
this.paused = true;
|
|
7882
|
+
this.wakeups.cancelAll();
|
|
7113
7883
|
while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
|
|
7114
7884
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
7115
7885
|
}
|
|
@@ -7125,13 +7895,16 @@ var RecordingController = class {
|
|
|
7125
7895
|
}
|
|
7126
7896
|
for (const deviceId of ids) await this.evaluateDevice(deviceId);
|
|
7127
7897
|
}
|
|
7128
|
-
/** Tear EVERYTHING down:
|
|
7898
|
+
/** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
|
|
7129
7899
|
async stop() {
|
|
7130
7900
|
this.stopped = true;
|
|
7901
|
+
this.wakeups.cancelAll();
|
|
7131
7902
|
this.livenessPass?.stop();
|
|
7132
7903
|
this.livenessPass = null;
|
|
7133
|
-
this.
|
|
7134
|
-
this.
|
|
7904
|
+
this.placementPass?.stop();
|
|
7905
|
+
this.placementPass = null;
|
|
7906
|
+
this.reconcilePass?.stop();
|
|
7907
|
+
this.reconcilePass = null;
|
|
7135
7908
|
this.restore?.stop();
|
|
7136
7909
|
this.restore = null;
|
|
7137
7910
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
@@ -8420,6 +9193,23 @@ var PLAYBACK_PREFIX = "playback";
|
|
|
8420
9193
|
/** Data-plane prefix for on-demand single stills →
|
|
8421
9194
|
* `/addon/recorder/still/<deviceId>/<epochSec>`. */
|
|
8422
9195
|
var STILL_PREFIX = "still";
|
|
9196
|
+
/**
|
|
9197
|
+
* Bound on pending hour-ledger writes (one per finalized segment, an RPC to
|
|
9198
|
+
* the hub store). Steady state completes in milliseconds and holds ~0–2; under
|
|
9199
|
+
* a store stall the fleet-wide finalize rate (~3.4/s) fills 256 in ~75s of
|
|
9200
|
+
* grace before refusals begin. Each pending write holds one hour-row copy
|
|
9201
|
+
* (≤360 paths ≈ ~25KB), so the lane's worst-case heap is single-digit MB —
|
|
9202
|
+
* a dropped write is one silent re-seed from the archive walk (D148).
|
|
9203
|
+
*/
|
|
9204
|
+
var LEDGER_GATE_LIMIT = 256;
|
|
9205
|
+
/**
|
|
9206
|
+
* Bound on pending mfra tail reads (one per finalized RECENT segment; each is
|
|
9207
|
+
* an open + two reads on the same 4-thread libuv pool the live writers use,
|
|
9208
|
+
* holding read buffers while pending). 64 bounds both the held memory and the
|
|
9209
|
+
* pool contention; a dropped capture costs that segment's client one extra
|
|
9210
|
+
* round trip to the file's tail on the first seek.
|
|
9211
|
+
*/
|
|
9212
|
+
var MFRA_GATE_LIMIT = 64;
|
|
8423
9213
|
/** Data-plane prefix for export MP4 downloads → `/addon/recorder/exports/<id>.mp4`. */
|
|
8424
9214
|
var EXPORTS_PREFIX = "exports";
|
|
8425
9215
|
/** How often the export janitor sweeps expired exports. */
|
|
@@ -8496,6 +9286,25 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8496
9286
|
/** Sync-sample tables captured at finalize, served by the directory so a
|
|
8497
9287
|
* playback seek needs no mfra tail fetch. RAM-only, TTL-bounded. */
|
|
8498
9288
|
mfraTables = new MfraTableStore();
|
|
9289
|
+
/** Bound on the per-finalize fire-and-forget lanes (item 3, 2026-08-18): a
|
|
9290
|
+
* disk/store stall must throttle this work, never balloon live heap — the
|
|
9291
|
+
* unbounded pending set is what fed the two `Ineffective mark-compacts`
|
|
9292
|
+
* fatals. Saturation is reported loudly but throttled; every individual
|
|
9293
|
+
* drop is logged with its deviceId at the call site. */
|
|
9294
|
+
ledgerWriteGate = new PendingGate({
|
|
9295
|
+
label: "hour-ledger",
|
|
9296
|
+
limit: LEDGER_GATE_LIMIT,
|
|
9297
|
+
onSaturated: (report) => {
|
|
9298
|
+
this.reportGateSaturated(report);
|
|
9299
|
+
}
|
|
9300
|
+
});
|
|
9301
|
+
mfraCaptureGate = new PendingGate({
|
|
9302
|
+
label: "mfra-capture",
|
|
9303
|
+
limit: MFRA_GATE_LIMIT,
|
|
9304
|
+
onSaturated: (report) => {
|
|
9305
|
+
this.reportGateSaturated(report);
|
|
9306
|
+
}
|
|
9307
|
+
});
|
|
8499
9308
|
/** Continuous + events band write-path controller (ffmpeg writers + watchers). */
|
|
8500
9309
|
controller = null;
|
|
8501
9310
|
/** Per-device motion/audio markers for playback (B3 events feed the controller). */
|
|
@@ -8533,6 +9342,20 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8533
9342
|
constructor() {
|
|
8534
9343
|
super({ ...DEFAULT_CONFIG });
|
|
8535
9344
|
}
|
|
9345
|
+
/**
|
|
9346
|
+
* One throttled line per saturated lane per report interval. Lane-level (no
|
|
9347
|
+
* deviceId — the lane aggregates every camera); the per-drop debug lines at
|
|
9348
|
+
* the call sites carry the tag. WARN because a saturated lane means the
|
|
9349
|
+
* store or the disk is stalling under live finalize load — the line that
|
|
9350
|
+
* explains the missing ledger hours and mfra misses an operator will see.
|
|
9351
|
+
*/
|
|
9352
|
+
reportGateSaturated(report) {
|
|
9353
|
+
this.ctx.logger.warn("recorder: fire-and-forget lane saturated — dropping over the bound", { meta: {
|
|
9354
|
+
lane: report.label,
|
|
9355
|
+
dropped: report.dropped,
|
|
9356
|
+
pending: report.pending
|
|
9357
|
+
} });
|
|
9358
|
+
}
|
|
8536
9359
|
async onInitialize() {
|
|
8537
9360
|
const raw = this.ctx.kernel.localNodeId ?? this.ctx.id;
|
|
8538
9361
|
this.nodeId = raw.includes("/") ? raw.split("/")[0] : raw;
|
|
@@ -8596,18 +9419,28 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8596
9419
|
this.segmentHours?.dropSegments(rows);
|
|
8597
9420
|
},
|
|
8598
9421
|
onIndexed: (row, absPath) => {
|
|
8599
|
-
this.segmentHours
|
|
9422
|
+
const ledger = this.segmentHours;
|
|
9423
|
+
if (ledger && !this.ledgerWriteGate.run(() => ledger.recordSegment(row))) this.ctx.logger.debug("recorder: hour-ledger write dropped — lane saturated; the archive walk reconciles this hour", {
|
|
9424
|
+
tags: { deviceId: row.deviceId },
|
|
9425
|
+
meta: { path: row.path }
|
|
9426
|
+
});
|
|
8600
9427
|
if (!isMfraTableServable(row.startMs, Date.now())) return;
|
|
8601
|
-
|
|
8602
|
-
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
9428
|
+
if (!this.mfraCaptureGate.run(async () => {
|
|
9429
|
+
try {
|
|
9430
|
+
const table = await readMfraTable(absPath, row.bytes);
|
|
9431
|
+
this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
|
|
9432
|
+
} catch (err) {
|
|
9433
|
+
this.ctx.logger.debug("mfra capture failed", {
|
|
9434
|
+
tags: { deviceId: row.deviceId },
|
|
9435
|
+
meta: {
|
|
9436
|
+
path: row.path,
|
|
9437
|
+
error: errMsg(err)
|
|
9438
|
+
}
|
|
9439
|
+
});
|
|
9440
|
+
}
|
|
9441
|
+
})) this.ctx.logger.debug("recorder: mfra capture dropped — lane saturated; first seek pays one tail fetch", {
|
|
9442
|
+
tags: { deviceId: row.deviceId },
|
|
9443
|
+
meta: { path: row.path }
|
|
8611
9444
|
});
|
|
8612
9445
|
},
|
|
8613
9446
|
removeDirIfEmpty: async (locationId, relDir) => {
|
|
@@ -8665,7 +9498,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8665
9498
|
onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
|
|
8666
9499
|
if (state === "ready") handler();
|
|
8667
9500
|
}),
|
|
8668
|
-
brokerCall: makeBrokerCall(brokerHandle)
|
|
9501
|
+
brokerCall: makeBrokerCall(brokerHandle),
|
|
9502
|
+
brokerReady: () => brokerHandle.isReady
|
|
8669
9503
|
});
|
|
8670
9504
|
const relocateEngine = new RelocateEngine({
|
|
8671
9505
|
index: this.index,
|