@camstack/addon-pipeline 1.2.90 → 1.2.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-yc-chXuC.js → addon-utils-DnUCCZVx.js} +1 -1
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +1 -1
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +2 -2
- package/dist/{dist-n4Nbnxx5.js → dist-BbRv3bM2.js} +139 -22
- package/dist/{dist-BJYwWEZ9.mjs → dist-XbrYiMV3.mjs} +134 -23
- package/dist/{event-loop-stall-monitor-BiPffk0D.mjs → event-loop-stall-monitor-CFWrOZ6G.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-DG7QY00k.js → event-loop-stall-monitor-D9hqbc68.js} +1 -1
- package/dist/{lazy-sharp-EoSOHgoj.js → lazy-sharp-U0EtN7_C.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/recorder/index.js +655 -84
- package/dist/recorder/index.mjs +654 -83
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CsrMZYLB.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BNOPhQ-y.mjs} +2 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DMyOrQHq.mjs +26 -0
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DgivrPsH.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DsIi4G5Q.mjs} +1 -1
- package/dist/stream-broker/{hostInit-DoFovQmc.mjs → hostInit-DCV2-mSi.mjs} +2 -2
- package/dist/stream-broker/index.js +194 -30
- package/dist/stream-broker/index.mjs +194 -30
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-DjuxhGo3.js → worker-protocol-CNC-ZcU3.js} +1 -1
- package/dist/{worker-protocol-XZ1e36rM.mjs → worker-protocol-D0ewr67U.mjs} +1 -1
- package/package.json +3 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DNSELPGi.mjs +0 -26
package/dist/recorder/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const require_dist = require("../dist-
|
|
1
|
+
const require_dist = require("../dist-BbRv3bM2.js");
|
|
2
2
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
3
|
-
const require_addon_utils = require("../addon-utils-
|
|
3
|
+
const require_addon_utils = require("../addon-utils-DnUCCZVx.js");
|
|
4
4
|
let node_crypto = require("node:crypto");
|
|
5
5
|
let node_child_process = require("node:child_process");
|
|
6
6
|
let node_path = require("node:path");
|
|
@@ -1069,6 +1069,7 @@ var RelocateEngine = class {
|
|
|
1069
1069
|
deviceId: input.deviceId ?? null,
|
|
1070
1070
|
profiles: input.profiles ? [...input.profiles] : [],
|
|
1071
1071
|
footageClass: input.footageClass,
|
|
1072
|
+
sinceMs: input.sinceMs ?? null,
|
|
1072
1073
|
throttleMbps: input.throttleMbps ?? DEFAULT_THROTTLE_MBPS,
|
|
1073
1074
|
entities: input.entities ? [...input.entities] : ["segments"],
|
|
1074
1075
|
filesMoved: 0,
|
|
@@ -1129,7 +1130,7 @@ var RelocateEngine = class {
|
|
|
1129
1130
|
const bytesPerMs = job.throttleMbps * 1024 * 1024 / 1e3;
|
|
1130
1131
|
try {
|
|
1131
1132
|
const samePhysicalRoot = await rootsPhysicallySame(from.root, to.root);
|
|
1132
|
-
const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).sort((a, b) => a.startMs - b.startMs);
|
|
1133
|
+
const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).filter((r) => job.sinceMs === null || r.startMs >= job.sinceMs).sort((a, b) => a.startMs - b.startMs);
|
|
1133
1134
|
job.filesTotal = segRows.length;
|
|
1134
1135
|
for (const row of segRows) {
|
|
1135
1136
|
if (job.cancelRequested) break;
|
|
@@ -1165,9 +1166,26 @@ var RelocateEngine = class {
|
|
|
1165
1166
|
}
|
|
1166
1167
|
}
|
|
1167
1168
|
/** Move one segment file and re-point its index row to the target location
|
|
1168
|
-
* (same location-relative path — the layout is identical on every root).
|
|
1169
|
+
* (same location-relative path — the layout is identical on every root).
|
|
1170
|
+
* A source that is already gone (evicted, never flushed, ENOSPC) is dropped
|
|
1171
|
+
* from the index and skipped — one ghost file must not abort a drain. */
|
|
1169
1172
|
async moveSegment(row, from, to, samePhysicalRoot) {
|
|
1170
|
-
if (!samePhysicalRoot)
|
|
1173
|
+
if (!samePhysicalRoot) try {
|
|
1174
|
+
await moveFile(node_path.default.join(from.root, row.path), node_path.default.join(to.root, row.path), row.bytes);
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
if (isEnoent(err)) {
|
|
1177
|
+
this.deps.index.removeSegments([row.path]);
|
|
1178
|
+
this.deps.logger.warn("relocate skipped missing source", {
|
|
1179
|
+
tags: { deviceId: row.deviceId },
|
|
1180
|
+
meta: {
|
|
1181
|
+
path: row.path,
|
|
1182
|
+
locationId: from.id
|
|
1183
|
+
}
|
|
1184
|
+
});
|
|
1185
|
+
return;
|
|
1186
|
+
}
|
|
1187
|
+
throw err;
|
|
1188
|
+
}
|
|
1171
1189
|
this.deps.index.removeSegments([row.path]);
|
|
1172
1190
|
this.deps.index.addSegment({
|
|
1173
1191
|
...row,
|
|
@@ -1175,6 +1193,9 @@ var RelocateEngine = class {
|
|
|
1175
1193
|
});
|
|
1176
1194
|
}
|
|
1177
1195
|
};
|
|
1196
|
+
function isEnoent(err) {
|
|
1197
|
+
return typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT";
|
|
1198
|
+
}
|
|
1178
1199
|
/** Logical migration classes are profile-derived, never location-derived:
|
|
1179
1200
|
* high+mid are `recordings`; low is `recordingsLow`. */
|
|
1180
1201
|
var profileBelongsToClass = (profile, footageClass) => {
|
|
@@ -4213,6 +4234,7 @@ function buildRecordingProvider(deps) {
|
|
|
4213
4234
|
rescanStorage: async ({ deviceId }) => {
|
|
4214
4235
|
const locations = await deps.refreshLocations();
|
|
4215
4236
|
await deps.hydrateDevice(deviceId, locations);
|
|
4237
|
+
await deps.reconcileHourLedger?.(deviceId);
|
|
4216
4238
|
const config = await loadDeviceConfig(deps.configStore, deviceId);
|
|
4217
4239
|
await deps.opsLog?.append({
|
|
4218
4240
|
op: "rescan",
|
|
@@ -5076,7 +5098,11 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
5076
5098
|
try {
|
|
5077
5099
|
entries = await node_fs.promises.readdir(deviceDir, { recursive: true });
|
|
5078
5100
|
} catch (err) {
|
|
5079
|
-
if (err.code
|
|
5101
|
+
if (err.code === "ENOENT") {
|
|
5102
|
+
for (const location of aliases) index.hydrateDevice(deviceId, location.id, []);
|
|
5103
|
+
continue;
|
|
5104
|
+
}
|
|
5105
|
+
logger.warn("recorder: hydrateDevice walk failed for location", {
|
|
5080
5106
|
tags: { deviceId },
|
|
5081
5107
|
meta: {
|
|
5082
5108
|
locationIds: aliases.map((location) => location.id),
|
|
@@ -5104,6 +5130,7 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
5104
5130
|
handled += 1;
|
|
5105
5131
|
if (handled % chunkSize === 0) await yieldBetween();
|
|
5106
5132
|
}
|
|
5133
|
+
for (const location of aliases) if (!pathsByLocation.has(location.id)) pathsByLocation.set(location.id, []);
|
|
5107
5134
|
let indexed = 0;
|
|
5108
5135
|
for (const [locationId, relPaths] of pathsByLocation) {
|
|
5109
5136
|
index.hydrateDevice(deviceId, locationId, relPaths);
|
|
@@ -5811,18 +5838,232 @@ function shouldRecordAt(config, triggers, at) {
|
|
|
5811
5838
|
return eventsBandDemanded(band, triggers, at.getTime());
|
|
5812
5839
|
}
|
|
5813
5840
|
/**
|
|
5814
|
-
*
|
|
5815
|
-
*
|
|
5816
|
-
*
|
|
5817
|
-
*
|
|
5841
|
+
* How late the FIRST segment of an attach may be NAMED, relative to the writer
|
|
5842
|
+
* spawn, and still count as "the segment this attach was spawned to produce".
|
|
5843
|
+
*
|
|
5844
|
+
* ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
|
|
5845
|
+
* first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
|
|
5846
|
+
* spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
|
|
5847
|
+
* 3.2 s. 5 s carries that with margin while staying far below the writer's own
|
|
5848
|
+
* "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
|
|
5849
|
+
* slow is not mistaken for one that is producing useful footage.
|
|
5818
5850
|
*/
|
|
5819
|
-
|
|
5851
|
+
var FIRST_SEGMENT_GRACE_MS = 5e3;
|
|
5852
|
+
/**
|
|
5853
|
+
* Latest segment START this attach may still produce and have kept, regardless
|
|
5854
|
+
* of where the operator's post-buffer ended: one full segment (nothing can be
|
|
5855
|
+
* finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
|
|
5856
|
+
* latency.
|
|
5857
|
+
*/
|
|
5858
|
+
function attachRetainDeadlineMs(attach) {
|
|
5859
|
+
return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
|
|
5860
|
+
}
|
|
5861
|
+
/**
|
|
5862
|
+
* Should a finalized segment `[segStartMs, segEndMs]` recorded under an
|
|
5863
|
+
* `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
|
|
5864
|
+
* trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
|
|
5865
|
+
* this is where `preBufferSec` retroactively retains pre-trigger footage — OR
|
|
5866
|
+
* it is the footage `attach` was spawned to produce (below).
|
|
5867
|
+
*
|
|
5868
|
+
* ## Why the attach matters here
|
|
5869
|
+
*
|
|
5870
|
+
* The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
|
|
5871
|
+
* gate is evaluated on segment timestamps. They shared one window, and a writer
|
|
5872
|
+
* physically cannot finalize anything before `spawn + segmentSeconds + dial`.
|
|
5873
|
+
* So the last ~`segmentSeconds` of every demand window was a zone where the
|
|
5874
|
+
* recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
|
|
5875
|
+
* result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
|
|
5876
|
+
* post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
|
|
5877
|
+
* finalized segments were unlinked.
|
|
5878
|
+
*
|
|
5879
|
+
* The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
|
|
5880
|
+
* `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
|
|
5881
|
+
* inside `trigger + postMs` and retains exactly what it retained before. It
|
|
5882
|
+
* only ever extends the tail of a LATE attach, by at most one segment plus the
|
|
5883
|
+
* grace, and only for a trigger whose live window was still open AT ATTACH — the
|
|
5884
|
+
* precise statement of "if the live gate said yes, the keep gate honours the
|
|
5885
|
+
* first segment that decision produces". A stale trigger extends nothing.
|
|
5886
|
+
*/
|
|
5887
|
+
function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
|
|
5820
5888
|
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
5821
|
-
const
|
|
5889
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
5890
|
+
const overlaps = (lastMs) => {
|
|
5891
|
+
if (lastMs == null) return false;
|
|
5892
|
+
if (segEndMs < lastMs - preMs) return false;
|
|
5893
|
+
if (segStartMs <= lastMs + postMs) return true;
|
|
5894
|
+
return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
|
|
5895
|
+
};
|
|
5822
5896
|
if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
|
|
5823
5897
|
if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
|
|
5824
5898
|
return false;
|
|
5825
5899
|
}
|
|
5900
|
+
/**
|
|
5901
|
+
* By how many ms a discarded segment missed being kept — its start past the
|
|
5902
|
+
* latest start that WOULD have been retained, mirroring
|
|
5903
|
+
* {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
|
|
5904
|
+
* the attach allowance for a trigger that was still live at attach). Null when
|
|
5905
|
+
* no qualifying trigger exists at all, which is a different fact and says so.
|
|
5906
|
+
*
|
|
5907
|
+
* Negative means the segment was dropped on the PRE-buffer side (it ended
|
|
5908
|
+
* before `trigger - preBufferSec`), not for being late.
|
|
5909
|
+
*
|
|
5910
|
+
* Reporting-only. The discard branch has to name a number: "outside the window"
|
|
5911
|
+
* with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
|
|
5912
|
+
*/
|
|
5913
|
+
function segmentMissedByMs(segStartMs, band, triggers, attach) {
|
|
5914
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
5915
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
5916
|
+
const latestKeptStart = (lastMs) => {
|
|
5917
|
+
if (lastMs == null) return null;
|
|
5918
|
+
const windowEndMs = lastMs + postMs;
|
|
5919
|
+
return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
|
|
5920
|
+
};
|
|
5921
|
+
const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
|
|
5922
|
+
if (bounds.length === 0) return null;
|
|
5923
|
+
return segStartMs - Math.max(...bounds);
|
|
5924
|
+
}
|
|
5925
|
+
/** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
|
|
5926
|
+
function localInstantMs(year, month, day, hhmm) {
|
|
5927
|
+
const [hours, minutes] = hhmm.split(":");
|
|
5928
|
+
return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
|
|
5929
|
+
}
|
|
5930
|
+
/**
|
|
5931
|
+
* Every instant inside the horizon at which `bandActiveAt` COULD change value:
|
|
5932
|
+
* each band's start and end on each local day, plus each local midnight (the
|
|
5933
|
+
* `days` membership boundary). A superset — cheap to compute (a few dozen
|
|
5934
|
+
* entries) and filtered against `activeBandAt` below.
|
|
5935
|
+
*/
|
|
5936
|
+
function candidateEdgesMs(bands, nowMs) {
|
|
5937
|
+
const base = new Date(nowMs);
|
|
5938
|
+
const year = base.getFullYear();
|
|
5939
|
+
const month = base.getMonth();
|
|
5940
|
+
const day = base.getDate();
|
|
5941
|
+
const out = [];
|
|
5942
|
+
for (let offset = 0; offset <= 8; offset++) {
|
|
5943
|
+
out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
|
|
5944
|
+
for (const band of bands) {
|
|
5945
|
+
out.push(localInstantMs(year, month, day + offset, band.start));
|
|
5946
|
+
out.push(localInstantMs(year, month, day + offset, band.end));
|
|
5947
|
+
}
|
|
5948
|
+
}
|
|
5949
|
+
return out;
|
|
5950
|
+
}
|
|
5951
|
+
/**
|
|
5952
|
+
* The next instant at which a DIFFERENT band (or no band) covers the clock, or
|
|
5953
|
+
* `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
|
|
5954
|
+
* band (`start === end`, empty `days`) being the common case that legitimately
|
|
5955
|
+
* has no edge at all.
|
|
5956
|
+
*
|
|
5957
|
+
* Identity is by band REFERENCE, which is what makes this exact: two distinct
|
|
5958
|
+
* band objects with the same fields yield an edge that changes nothing, which
|
|
5959
|
+
* costs one wasted wake-up and never costs footage.
|
|
5960
|
+
*/
|
|
5961
|
+
function nextBandEdgeMs(config, nowMs) {
|
|
5962
|
+
const bands = config.bands;
|
|
5963
|
+
if (!bands || bands.length === 0) return null;
|
|
5964
|
+
const current = activeBandAt({ bands }, new Date(nowMs));
|
|
5965
|
+
const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
|
|
5966
|
+
for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
|
|
5967
|
+
return null;
|
|
5968
|
+
}
|
|
5969
|
+
/**
|
|
5970
|
+
* The first instant an `events` band's demand window is CLOSED, given the
|
|
5971
|
+
* triggers it listens for, or `null` when no window is open at `nowMs`.
|
|
5972
|
+
*
|
|
5973
|
+
* Demand is an OR across the listened trigger kinds, so an open window ends
|
|
5974
|
+
* with the LATEST of them — a motion trigger followed by an audio trigger 5 s
|
|
5975
|
+
* later keeps the camera attached until the audio one expires, exactly as the
|
|
5976
|
+
* sliding window already behaved.
|
|
5977
|
+
*/
|
|
5978
|
+
function eventsWindowCloseMs(band, triggers, nowMs) {
|
|
5979
|
+
if (band.mode !== "events") return null;
|
|
5980
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
5981
|
+
const ends = [];
|
|
5982
|
+
if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
|
|
5983
|
+
if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
|
|
5984
|
+
const open = ends.filter((ms) => ms >= nowMs);
|
|
5985
|
+
if (open.length === 0) return null;
|
|
5986
|
+
return Math.max(...open) + 1;
|
|
5987
|
+
}
|
|
5988
|
+
/**
|
|
5989
|
+
* THE next instant the controller must re-evaluate this device, or `null` when
|
|
5990
|
+
* nothing can change without an event the controller already receives (a
|
|
5991
|
+
* trigger or a config write).
|
|
5992
|
+
*
|
|
5993
|
+
* Whichever of the two sources comes first wins, and the reason is carried so
|
|
5994
|
+
* the wake-up is attributable in the log without correlating timestamps.
|
|
5995
|
+
*/
|
|
5996
|
+
function nextDecisionChangeAt(input) {
|
|
5997
|
+
const { config, triggers, nowMs } = input;
|
|
5998
|
+
if (!config.enabled) return null;
|
|
5999
|
+
const bands = config.bands;
|
|
6000
|
+
if (!bands || bands.length === 0) return null;
|
|
6001
|
+
const band = activeBandAt({ bands }, new Date(nowMs));
|
|
6002
|
+
const edgeMs = nextBandEdgeMs(config, nowMs);
|
|
6003
|
+
const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
|
|
6004
|
+
if (closeMs === null) return edgeMs === null ? null : {
|
|
6005
|
+
atMs: edgeMs,
|
|
6006
|
+
reason: "band-edge"
|
|
6007
|
+
};
|
|
6008
|
+
if (edgeMs === null || closeMs <= edgeMs) return {
|
|
6009
|
+
atMs: closeMs,
|
|
6010
|
+
reason: "window-close"
|
|
6011
|
+
};
|
|
6012
|
+
return {
|
|
6013
|
+
atMs: edgeMs,
|
|
6014
|
+
reason: "band-edge"
|
|
6015
|
+
};
|
|
6016
|
+
}
|
|
6017
|
+
//#endregion
|
|
6018
|
+
//#region src/recorder/addon/device-wakeups.ts
|
|
6019
|
+
/** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
|
|
6020
|
+
var MAX_TIMER_DELAY_MS = 2147483647;
|
|
6021
|
+
var DeviceWakeups = class {
|
|
6022
|
+
deps;
|
|
6023
|
+
timers = /* @__PURE__ */ new Map();
|
|
6024
|
+
constructor(deps) {
|
|
6025
|
+
this.deps = deps;
|
|
6026
|
+
}
|
|
6027
|
+
/**
|
|
6028
|
+
* Wake this device at `atMs`, replacing any wake-up already armed for it.
|
|
6029
|
+
* `reason` is carried into the log line so a wake is attributable without
|
|
6030
|
+
* correlating it against the band model by hand.
|
|
6031
|
+
*/
|
|
6032
|
+
arm(deviceId, atMs, reason) {
|
|
6033
|
+
this.cancel(deviceId);
|
|
6034
|
+
const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
|
|
6035
|
+
const timer = setTimeout(() => {
|
|
6036
|
+
this.timers.delete(deviceId);
|
|
6037
|
+
this.deps.fire(deviceId);
|
|
6038
|
+
}, delayMs);
|
|
6039
|
+
timer.unref?.();
|
|
6040
|
+
this.timers.set(deviceId, timer);
|
|
6041
|
+
this.deps.logger.debug("recorder: device wake-up armed", {
|
|
6042
|
+
tags: { deviceId },
|
|
6043
|
+
meta: {
|
|
6044
|
+
atMs,
|
|
6045
|
+
delayMs,
|
|
6046
|
+
reason
|
|
6047
|
+
}
|
|
6048
|
+
});
|
|
6049
|
+
}
|
|
6050
|
+
/** Drop this device's pending wake-up, if any. Idempotent. */
|
|
6051
|
+
cancel(deviceId) {
|
|
6052
|
+
const timer = this.timers.get(deviceId);
|
|
6053
|
+
if (timer === void 0) return;
|
|
6054
|
+
clearTimeout(timer);
|
|
6055
|
+
this.timers.delete(deviceId);
|
|
6056
|
+
}
|
|
6057
|
+
/** Drop every pending wake-up — shutdown and maintenance pause. */
|
|
6058
|
+
cancelAll() {
|
|
6059
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
6060
|
+
this.timers.clear();
|
|
6061
|
+
}
|
|
6062
|
+
/** Devices with a wake-up pending — for tests and for shutdown assertions. */
|
|
6063
|
+
get pendingCount() {
|
|
6064
|
+
return this.timers.size;
|
|
6065
|
+
}
|
|
6066
|
+
};
|
|
5826
6067
|
//#endregion
|
|
5827
6068
|
//#region src/recorder/addon/periodic-pass.ts
|
|
5828
6069
|
function startPeriodicPass(deps) {
|
|
@@ -6406,19 +6647,37 @@ var SegmentWriter = class {
|
|
|
6406
6647
|
* - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
|
|
6407
6648
|
* the v2 `SegmentStore.onFinalized` (stat + relocate + index).
|
|
6408
6649
|
*
|
|
6409
|
-
* Per device it decides whether
|
|
6410
|
-
* (`
|
|
6411
|
-
*
|
|
6412
|
-
* 1. `setDeviceConfig` (operator changed the bands / enable),
|
|
6413
|
-
* 2. a
|
|
6414
|
-
* 3. a
|
|
6415
|
-
*
|
|
6416
|
-
*
|
|
6417
|
-
*
|
|
6418
|
-
*
|
|
6419
|
-
*
|
|
6420
|
-
*
|
|
6421
|
-
*
|
|
6650
|
+
* Per device it decides whether the active band demands recording right now
|
|
6651
|
+
* (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
|
|
6652
|
+
* is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
|
|
6653
|
+
* 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
|
|
6654
|
+
* 2. a qualifying motion/audio trigger (`onTrigger`) — opens a window,
|
|
6655
|
+
* 3. a per-device WAKE-UP armed for the exact instant the decision could
|
|
6656
|
+
* change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
|
|
6657
|
+
* `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
|
|
6658
|
+
* 4. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`),
|
|
6659
|
+
* 5. the periodic RECONCILE (below), for everything the four above can lose.
|
|
6660
|
+
*
|
|
6661
|
+
* (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
|
|
6662
|
+
* demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
|
|
6663
|
+
* trigger, so nothing called `evaluateDevice` at the moment it closed and the
|
|
6664
|
+
* detach waited for the next pass. A 4K camera stayed dialled, decoded and
|
|
6665
|
+
* written for up to a whole tick past the operator's `postBufferSec`.
|
|
6666
|
+
*
|
|
6667
|
+
* There are THREE periodic passes, each on its own timer and each non-overlapping
|
|
6668
|
+
* (`periodic-pass.ts`):
|
|
6669
|
+
* - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
|
|
6670
|
+
* producing nothing?
|
|
6671
|
+
* - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
|
|
6672
|
+
* deliberately NOT on the attach beat, so a cold broker cannot delay it.
|
|
6673
|
+
* - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
|
|
6674
|
+
* broker, and the only backstop for a lost wake-up or a failed attach.
|
|
6675
|
+
*
|
|
6676
|
+
* They were ONE pass until 2026-08-13, when a convergence sweep wedged on
|
|
6677
|
+
* unbounded broker RPCs and took the liveness watchdog down with it for 9-27
|
|
6678
|
+
* minutes at a time; placement was split out of the second on 2026-08-18, after
|
|
6679
|
+
* a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
|
|
6680
|
+
* cold-dialling ~40 RTSP sources.
|
|
6422
6681
|
*
|
|
6423
6682
|
* EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
|
|
6424
6683
|
* trigger-gating. The enable intent is persisted in durable-state by the
|
|
@@ -6431,8 +6690,45 @@ var SegmentWriter = class {
|
|
|
6431
6690
|
* `ReadinessRegistry` is the single source of truth.
|
|
6432
6691
|
*/
|
|
6433
6692
|
var DEFAULT_WATCH_INTERVAL_MS = 2e3;
|
|
6434
|
-
/**
|
|
6435
|
-
|
|
6693
|
+
/**
|
|
6694
|
+
* Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
|
|
6695
|
+
* question is an in-memory scan over `active`, it costs nothing, and its bound
|
|
6696
|
+
* (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
|
|
6697
|
+
*/
|
|
6698
|
+
var LIVENESS_TICK_MS = 3e4;
|
|
6699
|
+
/**
|
|
6700
|
+
* Placement cadence — unchanged from the beat it used to share with
|
|
6701
|
+
* convergence, because nothing about placement wanted to be rarer. What changed
|
|
6702
|
+
* is that it no longer rides a pass that can block on broker RPCs: it is one
|
|
6703
|
+
* `statfs` per location and it re-plans only, so a plan reaches a camera at its
|
|
6704
|
+
* NEXT attach.
|
|
6705
|
+
*/
|
|
6706
|
+
var PLACEMENT_TICK_MS = 3e4;
|
|
6707
|
+
/**
|
|
6708
|
+
* Reconcile cadence — the SAFETY NET, not the mechanism.
|
|
6709
|
+
*
|
|
6710
|
+
* It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
|
|
6711
|
+
* window or crossed a band edge, so the loop had to keep asking. Both are now
|
|
6712
|
+
* armed for their exact instant (`decision-schedule.ts`), a config change is an
|
|
6713
|
+
* RPC, and a broker recovery is a readiness transition — so this pass exists
|
|
6714
|
+
* only for what those cannot cover: a wake-up lost because the event path threw
|
|
6715
|
+
* before it could arm one, an attach that failed with no later ready transition
|
|
6716
|
+
* to ride, a timer lost to a runner respawn, and clock drift.
|
|
6717
|
+
*
|
|
6718
|
+
* 2 minutes, and the number is chosen against a bound this recorder already
|
|
6719
|
+
* accepts. The liveness watchdog tolerates a pinned-but-silent writer for
|
|
6720
|
+
* `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
|
|
6721
|
+
* recording but is detached is the same class of fault — footage is not being
|
|
6722
|
+
* written and only a periodic check will notice — so its recovery is put on the
|
|
6723
|
+
* same order rather than an order slower. Four times rarer than the beat that
|
|
6724
|
+
* blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
|
|
6725
|
+
* ~2 minutes of footage for a camera whose event path failed entirely.
|
|
6726
|
+
*
|
|
6727
|
+
* Going rarer than this trades real footage for CPU that the pass no longer
|
|
6728
|
+
* spends anyway: with the broker-readiness pre-check below, a reconcile over a
|
|
6729
|
+
* healthy fleet is N config reads and no RPC at all.
|
|
6730
|
+
*/
|
|
6731
|
+
var RECONCILE_TICK_MS = 12e4;
|
|
6436
6732
|
/**
|
|
6437
6733
|
* Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
|
|
6438
6734
|
* teardown. The UDS request default is 60s, but the runner supervisor's
|
|
@@ -6471,11 +6767,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
|
|
|
6471
6767
|
*/
|
|
6472
6768
|
var ATTACH_RPC_TIMEOUT_MS = 2e4;
|
|
6473
6769
|
/**
|
|
6474
|
-
* A pass longer than this is reported with its duration.
|
|
6475
|
-
* cadence: a pass that cannot finish inside its own interval is
|
|
6476
|
-
* that used to go unlogged for half an hour.
|
|
6770
|
+
* A pass longer than this is reported with its duration. Each pass uses its OWN
|
|
6771
|
+
* cadence as the budget: a pass that cannot finish inside its own interval is
|
|
6772
|
+
* the condition that used to go unlogged for half an hour. Keeping it per-pass
|
|
6773
|
+
* matters now that the three cadences differ — a 40 s reconcile is healthy
|
|
6774
|
+
* against a 120 s beat and was a red flag against a 30 s one.
|
|
6477
6775
|
*/
|
|
6478
|
-
|
|
6776
|
+
function passSlowAfterMs(intervalMs) {
|
|
6777
|
+
return intervalMs;
|
|
6778
|
+
}
|
|
6479
6779
|
/**
|
|
6480
6780
|
* Race `promise` against a deadline. Handlers stay attached to the losing
|
|
6481
6781
|
* promise, so a post-deadline settlement can never surface as a process-level
|
|
@@ -6558,30 +6858,100 @@ var RecordingController = class {
|
|
|
6558
6858
|
*/
|
|
6559
6859
|
lastSegmentAt = /* @__PURE__ */ new Map();
|
|
6560
6860
|
/**
|
|
6861
|
+
* Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
|
|
6862
|
+
* at least one `events` segment. Read only to LEVEL the discard log: an
|
|
6863
|
+
* attach that kept nothing is a wasted pull (warn), a later discard from one
|
|
6864
|
+
* that kept something is the tail of the window (info). Bounded by
|
|
6865
|
+
* device×profile like {@link lastSegmentAt}, and cleared with it on detach.
|
|
6866
|
+
*/
|
|
6867
|
+
attachRetained = /* @__PURE__ */ new Map();
|
|
6868
|
+
/**
|
|
6561
6869
|
* Devices already named by {@link reportUnrecordable} — one warn per device
|
|
6562
6870
|
* per "enabled but can never record" episode, re-armed as soon as the device
|
|
6563
6871
|
* records again.
|
|
6564
6872
|
*/
|
|
6565
6873
|
unrecordableReported = /* @__PURE__ */ new Set();
|
|
6874
|
+
/**
|
|
6875
|
+
* Profiles the last attach skipped (stream not published / getStream threw).
|
|
6876
|
+
* `evaluateDevice` retries only these on the converge tick so a dead `low`
|
|
6877
|
+
* cannot keep `high` dark, without a `listAllProfileSlots` RPC on every
|
|
6878
|
+
* healthy device every 30s.
|
|
6879
|
+
*/
|
|
6880
|
+
skippedProfiles = /* @__PURE__ */ new Map();
|
|
6566
6881
|
restore = null;
|
|
6882
|
+
/**
|
|
6883
|
+
* One pending wake-up per device, armed for the exact instant its recording
|
|
6884
|
+
* decision could change on its own — an `events` window closing or a band
|
|
6885
|
+
* edge. THE reason a detach is punctual instead of up to a pass late.
|
|
6886
|
+
*/
|
|
6887
|
+
wakeups;
|
|
6567
6888
|
/** Cheap, never-delayed: is any pinned writer producing nothing? */
|
|
6568
6889
|
livenessPass = null;
|
|
6569
|
-
/**
|
|
6570
|
-
|
|
6890
|
+
/** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
|
|
6891
|
+
placementPass = null;
|
|
6892
|
+
/** The backstop: re-evaluate every tracked-or-configured device. */
|
|
6893
|
+
reconcilePass = null;
|
|
6571
6894
|
stopped = false;
|
|
6572
6895
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6573
6896
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6574
6897
|
paused = false;
|
|
6575
6898
|
constructor(deps) {
|
|
6576
6899
|
this.deps = deps;
|
|
6900
|
+
this.wakeups = new DeviceWakeups({
|
|
6901
|
+
logger: deps.logger,
|
|
6902
|
+
now: () => this.now(),
|
|
6903
|
+
fire: (deviceId) => this.onWakeup(deviceId)
|
|
6904
|
+
});
|
|
6577
6905
|
}
|
|
6578
6906
|
now() {
|
|
6579
6907
|
return this.deps.now?.() ?? Date.now();
|
|
6580
6908
|
}
|
|
6581
6909
|
/**
|
|
6910
|
+
* A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
|
|
6911
|
+
* cancels) the next one as part of deciding, so the chain is self-sustaining.
|
|
6912
|
+
*
|
|
6913
|
+
* A throw here breaks that chain — the device keeps whatever state it had and
|
|
6914
|
+
* has no timer behind it — so it is NAMED, and the reconcile is what picks it
|
|
6915
|
+
* up. Silence would read as "the window never closed".
|
|
6916
|
+
*/
|
|
6917
|
+
onWakeup(deviceId) {
|
|
6918
|
+
this.evaluateDevice(deviceId).catch((err) => {
|
|
6919
|
+
this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
|
|
6920
|
+
tags: { deviceId },
|
|
6921
|
+
meta: { error: require_dist.errMsg(err) }
|
|
6922
|
+
});
|
|
6923
|
+
});
|
|
6924
|
+
}
|
|
6925
|
+
/**
|
|
6926
|
+
* Arm (or cancel) this device's next wake-up from the config it was just
|
|
6927
|
+
* evaluated against.
|
|
6928
|
+
*
|
|
6929
|
+
* Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
|
|
6930
|
+
* attach chain can throw (broker not ready, stream unpublished) and the
|
|
6931
|
+
* schedule does not depend on whether it succeeded. Arming first means a
|
|
6932
|
+
* failed attach still gets its band edge, and only a failure to READ the
|
|
6933
|
+
* config can lose a wake-up.
|
|
6934
|
+
*/
|
|
6935
|
+
scheduleNextDecision(deviceId, config) {
|
|
6936
|
+
if (this.stopped || this.paused) {
|
|
6937
|
+
this.wakeups.cancel(deviceId);
|
|
6938
|
+
return;
|
|
6939
|
+
}
|
|
6940
|
+
const next = nextDecisionChangeAt({
|
|
6941
|
+
config,
|
|
6942
|
+
triggers: this.triggersFor(deviceId),
|
|
6943
|
+
nowMs: this.now()
|
|
6944
|
+
});
|
|
6945
|
+
if (next === null) {
|
|
6946
|
+
this.wakeups.cancel(deviceId);
|
|
6947
|
+
return;
|
|
6948
|
+
}
|
|
6949
|
+
this.wakeups.arm(deviceId, next.atMs, next.reason);
|
|
6950
|
+
}
|
|
6951
|
+
/**
|
|
6582
6952
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
6583
6953
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
6584
|
-
* periodic
|
|
6954
|
+
* periodic passes. Idempotent — safe to call once after the index is hydrated.
|
|
6585
6955
|
*/
|
|
6586
6956
|
async start() {
|
|
6587
6957
|
if (this.stopped) return;
|
|
@@ -6602,45 +6972,83 @@ var RecordingController = class {
|
|
|
6602
6972
|
meta: { error: require_dist.errMsg(err) }
|
|
6603
6973
|
})
|
|
6604
6974
|
});
|
|
6605
|
-
if (this.
|
|
6975
|
+
if (this.reconcilePass === null) {
|
|
6606
6976
|
this.livenessPass = startPeriodicPass({
|
|
6607
6977
|
name: "liveness",
|
|
6608
|
-
intervalMs:
|
|
6609
|
-
slowAfterMs:
|
|
6978
|
+
intervalMs: LIVENESS_TICK_MS,
|
|
6979
|
+
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
6610
6980
|
logger: this.deps.logger,
|
|
6611
6981
|
now: () => this.now(),
|
|
6612
6982
|
run: () => this.checkIdleWriters()
|
|
6613
6983
|
});
|
|
6614
|
-
this.
|
|
6615
|
-
name: "
|
|
6616
|
-
intervalMs:
|
|
6617
|
-
slowAfterMs:
|
|
6984
|
+
this.placementPass = startPeriodicPass({
|
|
6985
|
+
name: "placement",
|
|
6986
|
+
intervalMs: PLACEMENT_TICK_MS,
|
|
6987
|
+
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
6988
|
+
logger: this.deps.logger,
|
|
6989
|
+
now: () => this.now(),
|
|
6990
|
+
run: () => this.replanPlacement()
|
|
6991
|
+
});
|
|
6992
|
+
this.reconcilePass = startPeriodicPass({
|
|
6993
|
+
name: "reconcile",
|
|
6994
|
+
intervalMs: RECONCILE_TICK_MS,
|
|
6995
|
+
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
6618
6996
|
logger: this.deps.logger,
|
|
6619
6997
|
now: () => this.now(),
|
|
6620
|
-
run: () => this.
|
|
6998
|
+
run: () => this.reconcile()
|
|
6621
6999
|
});
|
|
6622
7000
|
}
|
|
6623
7001
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
6624
7002
|
}
|
|
6625
|
-
/**
|
|
6626
|
-
|
|
6627
|
-
|
|
7003
|
+
/**
|
|
7004
|
+
* Every device the recorder is responsible for right now: those with writers
|
|
7005
|
+
* attached, plus every one with a persisted config. A failure to enumerate the
|
|
7006
|
+
* persisted set degrades to "the active ones" rather than aborting the pass.
|
|
7007
|
+
*/
|
|
7008
|
+
async trackedDeviceIds() {
|
|
6628
7009
|
const ids = new Set(this.active.keys());
|
|
6629
|
-
let persisted = [];
|
|
6630
7010
|
try {
|
|
6631
|
-
|
|
6632
|
-
} catch {
|
|
6633
|
-
|
|
7011
|
+
for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
|
|
7012
|
+
} catch (err) {
|
|
7013
|
+
this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
|
|
7014
|
+
error: require_dist.errMsg(err),
|
|
7015
|
+
activeDevices: this.active.size
|
|
7016
|
+
} });
|
|
7017
|
+
}
|
|
7018
|
+
return [...ids];
|
|
7019
|
+
}
|
|
7020
|
+
/**
|
|
7021
|
+
* Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
|
|
7022
|
+
* beyond a statfs per location), and it must not be starved by an attach that
|
|
7023
|
+
* is waiting on a 20 s broker timeout. It changes the PLAN only — a running
|
|
7024
|
+
* writer keeps its root until it next attaches, which is the boundary rule.
|
|
7025
|
+
*/
|
|
7026
|
+
async replanPlacement() {
|
|
7027
|
+
if (this.stopped || this.paused) return;
|
|
7028
|
+
if (this.deps.onPlacementTick === void 0) return;
|
|
7029
|
+
const ids = await this.trackedDeviceIds();
|
|
6634
7030
|
try {
|
|
6635
|
-
await this.deps.onPlacementTick
|
|
7031
|
+
await this.deps.onPlacementTick(ids);
|
|
6636
7032
|
} catch (err) {
|
|
6637
7033
|
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: require_dist.errMsg(err) } });
|
|
6638
7034
|
}
|
|
6639
|
-
|
|
7035
|
+
}
|
|
7036
|
+
/**
|
|
7037
|
+
* The backstop pass: re-evaluate every tracked-or-configured device.
|
|
7038
|
+
*
|
|
7039
|
+
* Nothing here is the normal path any more — a window close, a band edge, a
|
|
7040
|
+
* config change and a broker recovery all reach `evaluateDevice` on their own.
|
|
7041
|
+
* This exists for the four things that cannot: a wake-up the event path threw
|
|
7042
|
+
* before arming, an attach that failed with no later readiness transition to
|
|
7043
|
+
* ride, a timer lost to a runner respawn, and clock drift.
|
|
7044
|
+
*/
|
|
7045
|
+
async reconcile() {
|
|
7046
|
+
if (this.stopped) return;
|
|
7047
|
+
await runBounded(await this.trackedDeviceIds(), 8, async (id) => {
|
|
6640
7048
|
try {
|
|
6641
7049
|
await this.evaluateDevice(id);
|
|
6642
7050
|
} catch (err) {
|
|
6643
|
-
this.deps.logger.warn("recorder controller:
|
|
7051
|
+
this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
|
|
6644
7052
|
tags: { deviceId: id },
|
|
6645
7053
|
meta: { error: require_dist.errMsg(err) }
|
|
6646
7054
|
});
|
|
@@ -6664,8 +7072,9 @@ var RecordingController = class {
|
|
|
6664
7072
|
* idle source can re-dial cleanly), then re-evaluate: if the band still demands
|
|
6665
7073
|
* recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
|
|
6666
7074
|
* the broker is not ready, the re-attach throws and the device stays on the
|
|
6667
|
-
* readiness queue (retried on the next broker ready) — and the periodic
|
|
6668
|
-
* the backstop. Bounded OUTER retry (
|
|
7075
|
+
* readiness queue (retried on the next broker ready) — and the periodic
|
|
7076
|
+
* reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
|
|
7077
|
+
* never a tight loop.
|
|
6669
7078
|
*/
|
|
6670
7079
|
async recoverWriterGaveUp(deviceId) {
|
|
6671
7080
|
if (this.stopped) return;
|
|
@@ -6675,7 +7084,7 @@ var RecordingController = class {
|
|
|
6675
7084
|
await this.evaluateDevice(deviceId);
|
|
6676
7085
|
} catch (err) {
|
|
6677
7086
|
this.restore?.add(deviceId, true);
|
|
6678
|
-
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/
|
|
7087
|
+
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
|
|
6679
7088
|
tags: { deviceId },
|
|
6680
7089
|
meta: { error: require_dist.errMsg(err) }
|
|
6681
7090
|
});
|
|
@@ -6726,10 +7135,16 @@ var RecordingController = class {
|
|
|
6726
7135
|
}
|
|
6727
7136
|
/**
|
|
6728
7137
|
* Record a qualifying trigger (motion/audio) for a device and converge: an
|
|
6729
|
-
* `events` band whose window this opens attaches now
|
|
6730
|
-
*
|
|
6731
|
-
*
|
|
6732
|
-
*
|
|
7138
|
+
* `events` band whose window this opens attaches now, and the same
|
|
7139
|
+
* `evaluateDevice` arms the wake-up that will CLOSE it at
|
|
7140
|
+
* `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
|
|
7141
|
+
* The qualification test (motion detected / audio over threshold) is done
|
|
7142
|
+
* upstream — by the time we're here, it qualified.
|
|
7143
|
+
*
|
|
7144
|
+
* A fresher trigger re-arms the wake-up later, which is exactly how the
|
|
7145
|
+
* sliding window already behaved: continuous motion stays ONE recording,
|
|
7146
|
+
* because `evaluateDevice` short-circuits at "already recording — converged"
|
|
7147
|
+
* and only the timer moves.
|
|
6733
7148
|
*/
|
|
6734
7149
|
async onTrigger(deviceId, source, atMs) {
|
|
6735
7150
|
if (this.stopped) return;
|
|
@@ -6741,7 +7156,7 @@ var RecordingController = class {
|
|
|
6741
7156
|
try {
|
|
6742
7157
|
await this.evaluateDevice(deviceId);
|
|
6743
7158
|
} catch (err) {
|
|
6744
|
-
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/
|
|
7159
|
+
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
|
|
6745
7160
|
tags: { deviceId },
|
|
6746
7161
|
meta: {
|
|
6747
7162
|
source,
|
|
@@ -6762,14 +7177,19 @@ var RecordingController = class {
|
|
|
6762
7177
|
if (this.stopped || this.paused) return;
|
|
6763
7178
|
const config = await this.deps.loadConfig(deviceId);
|
|
6764
7179
|
if (this.stopped || this.paused) return;
|
|
6765
|
-
|
|
7180
|
+
const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
|
|
7181
|
+
this.scheduleNextDecision(deviceId, config);
|
|
7182
|
+
if (!wantRecording) {
|
|
6766
7183
|
this.reportUnrecordable(deviceId, config);
|
|
6767
7184
|
await this.detachDevice(deviceId);
|
|
6768
7185
|
this.restore?.remove(deviceId);
|
|
6769
7186
|
return;
|
|
6770
7187
|
}
|
|
6771
7188
|
this.unrecordableReported.delete(deviceId);
|
|
6772
|
-
if (this.active.has(deviceId))
|
|
7189
|
+
if (this.active.has(deviceId)) {
|
|
7190
|
+
if (this.skippedProfiles.has(deviceId)) await this.fillMissingProfiles(deviceId, config);
|
|
7191
|
+
return;
|
|
7192
|
+
}
|
|
6773
7193
|
if (this.attaching.has(deviceId)) return;
|
|
6774
7194
|
this.attaching.add(deviceId);
|
|
6775
7195
|
const attaching = this.performAttach(deviceId, config);
|
|
@@ -6818,49 +7238,141 @@ var RecordingController = class {
|
|
|
6818
7238
|
});
|
|
6819
7239
|
}
|
|
6820
7240
|
/**
|
|
7241
|
+
* Throw before spending a single broker RPC when the registry already says the
|
|
7242
|
+
* `stream-broker` on the owner node is not ready. See
|
|
7243
|
+
* {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
|
|
7244
|
+
*/
|
|
7245
|
+
failFastIfBrokerCold(deviceId) {
|
|
7246
|
+
if (this.deps.brokerReady?.() !== false) return;
|
|
7247
|
+
throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
|
|
7248
|
+
}
|
|
7249
|
+
/**
|
|
6821
7250
|
* The attach body — extracted so the {@link attaching} in-flight guard in
|
|
6822
7251
|
* `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
|
|
6823
|
-
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
6824
|
-
*
|
|
7252
|
+
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
7253
|
+
*
|
|
7254
|
+
* Per-profile: a throw from one slot is logged and skipped. Rolling back the
|
|
7255
|
+
* whole device when a single stream was dead (618 `derived` on `low`) left
|
|
7256
|
+
* ZERO footage for hours. If every profile fails, throw so the restore queue
|
|
7257
|
+
* retries the device.
|
|
6825
7258
|
*/
|
|
6826
7259
|
async performAttach(deviceId, config) {
|
|
6827
7260
|
this.restore?.add(deviceId, true);
|
|
7261
|
+
this.failFastIfBrokerCold(deviceId);
|
|
6828
7262
|
const profiles = await this.resolveProfiles(deviceId, config.profiles);
|
|
6829
7263
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
6830
7264
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
6831
7265
|
const recordings = [];
|
|
6832
|
-
|
|
6833
|
-
|
|
7266
|
+
const failed = [];
|
|
7267
|
+
for (const profile of profiles) try {
|
|
7268
|
+
recordings.push(await this.attachProfile(deviceId, profile, segmentSeconds));
|
|
6834
7269
|
} catch (err) {
|
|
6835
|
-
|
|
6836
|
-
|
|
7270
|
+
failed.push(profile);
|
|
7271
|
+
this.deps.logger.warn("recorder: profile attach failed — recording the remaining profiles", {
|
|
7272
|
+
tags: { deviceId },
|
|
7273
|
+
meta: {
|
|
7274
|
+
profile,
|
|
7275
|
+
error: require_dist.errMsg(err)
|
|
7276
|
+
}
|
|
7277
|
+
});
|
|
6837
7278
|
}
|
|
7279
|
+
if (recordings.length === 0) throw new Error(`recorder controller: no profile attached for device ${deviceId}` + (failed.length > 0 ? ` (failed: ${failed.join(",")})` : ""));
|
|
6838
7280
|
if (this.paused) {
|
|
6839
7281
|
await Promise.all(recordings.map((recording) => this.teardownProfile(deviceId, recording)));
|
|
6840
7282
|
return;
|
|
6841
7283
|
}
|
|
6842
7284
|
this.active.set(deviceId, recordings);
|
|
7285
|
+
if (failed.length > 0) this.skippedProfiles.set(deviceId, new Set(failed));
|
|
7286
|
+
else this.skippedProfiles.delete(deviceId);
|
|
6843
7287
|
this.deps.logger.info("recorder pinned broker source(s) for recording", {
|
|
6844
7288
|
tags: { deviceId },
|
|
6845
7289
|
meta: {
|
|
6846
|
-
profiles,
|
|
7290
|
+
profiles: recordings.map((r) => r.profile),
|
|
7291
|
+
failedProfiles: failed,
|
|
6847
7292
|
segmentSeconds,
|
|
6848
7293
|
pipelineKeys: recordings.map((r) => r.pipelineKey)
|
|
6849
7294
|
}
|
|
6850
7295
|
});
|
|
6851
7296
|
}
|
|
6852
7297
|
/**
|
|
7298
|
+
* Retry profiles the last attach skipped, without tearing down writers that
|
|
7299
|
+
* are already producing. Called from `evaluateDevice` when the device is
|
|
7300
|
+
* already in `active` and {@link skippedProfiles} is non-empty. Failures stay
|
|
7301
|
+
* logged; the next converge tick retries.
|
|
7302
|
+
*/
|
|
7303
|
+
async fillMissingProfiles(deviceId, config) {
|
|
7304
|
+
if (this.attaching.has(deviceId)) return;
|
|
7305
|
+
const current = this.active.get(deviceId);
|
|
7306
|
+
const skipped = this.skippedProfiles.get(deviceId);
|
|
7307
|
+
if (!current || !skipped || skipped.size === 0) return;
|
|
7308
|
+
this.attaching.add(deviceId);
|
|
7309
|
+
const filling = this.runFillMissingProfiles(deviceId, config, current, skipped);
|
|
7310
|
+
this.attachingTasks.set(deviceId, filling);
|
|
7311
|
+
try {
|
|
7312
|
+
await filling;
|
|
7313
|
+
} finally {
|
|
7314
|
+
this.attachingTasks.delete(deviceId);
|
|
7315
|
+
this.attaching.delete(deviceId);
|
|
7316
|
+
}
|
|
7317
|
+
}
|
|
7318
|
+
async runFillMissingProfiles(deviceId, config, current, skipped) {
|
|
7319
|
+
if (this.deps.brokerReady?.() === false) {
|
|
7320
|
+
this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
|
|
7321
|
+
tags: { deviceId },
|
|
7322
|
+
meta: {
|
|
7323
|
+
profiles: [...skipped],
|
|
7324
|
+
ownerNodeId: this.deps.ownerNodeId
|
|
7325
|
+
}
|
|
7326
|
+
});
|
|
7327
|
+
return;
|
|
7328
|
+
}
|
|
7329
|
+
const have = new Set(current.map((r) => r.profile));
|
|
7330
|
+
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
7331
|
+
const added = [];
|
|
7332
|
+
for (const profile of [...skipped]) {
|
|
7333
|
+
if (have.has(profile)) {
|
|
7334
|
+
skipped.delete(profile);
|
|
7335
|
+
continue;
|
|
7336
|
+
}
|
|
7337
|
+
try {
|
|
7338
|
+
added.push(await this.attachProfile(deviceId, profile, segmentSeconds));
|
|
7339
|
+
skipped.delete(profile);
|
|
7340
|
+
} catch (err) {
|
|
7341
|
+
this.deps.logger.warn("recorder: profile attach retry failed — keeping the healthy profiles", {
|
|
7342
|
+
tags: { deviceId },
|
|
7343
|
+
meta: {
|
|
7344
|
+
profile,
|
|
7345
|
+
error: require_dist.errMsg(err)
|
|
7346
|
+
}
|
|
7347
|
+
});
|
|
7348
|
+
}
|
|
7349
|
+
}
|
|
7350
|
+
if (skipped.size === 0) this.skippedProfiles.delete(deviceId);
|
|
7351
|
+
if (added.length === 0) return;
|
|
7352
|
+
const existing = this.active.get(deviceId);
|
|
7353
|
+
if (!existing || this.paused || this.stopped) {
|
|
7354
|
+
await Promise.all(added.map((recording) => this.teardownProfile(deviceId, recording)));
|
|
7355
|
+
return;
|
|
7356
|
+
}
|
|
7357
|
+
this.active.set(deviceId, [...existing, ...added]);
|
|
7358
|
+
this.deps.logger.info("recorder pinned additional broker source(s) for recording", {
|
|
7359
|
+
tags: { deviceId },
|
|
7360
|
+
meta: {
|
|
7361
|
+
profiles: added.map((r) => r.profile),
|
|
7362
|
+
pipelineKeys: added.map((r) => r.pipelineKey)
|
|
7363
|
+
}
|
|
7364
|
+
});
|
|
7365
|
+
}
|
|
7366
|
+
/**
|
|
6853
7367
|
* Pick which profiles to record. The broker enumerates assigned profile slots,
|
|
6854
7368
|
* deduped by physical source (`selectAssignedProfileSlots`) so the same camera
|
|
6855
|
-
* encoder is never recorded twice.
|
|
6856
|
-
*
|
|
6857
|
-
*
|
|
7369
|
+
* encoder is never recorded twice. Default is high+low among those slots
|
|
7370
|
+
* (`resolveRecordingProfiles`); an operator `override` DISABLES the
|
|
7371
|
+
* unselected profiles. A selection matching none of the assigned sources
|
|
7372
|
+
* falls back to the default/assigned set (minimum of 1 when anything is assigned).
|
|
6858
7373
|
*/
|
|
6859
7374
|
async resolveProfiles(deviceId, override) {
|
|
6860
|
-
|
|
6861
|
-
if (!override || override.length === 0) return assigned;
|
|
6862
|
-
const selected = assigned.filter((p) => override.includes(p));
|
|
6863
|
-
return selected.length > 0 ? selected : assigned;
|
|
7375
|
+
return require_dist.resolveRecordingProfiles(require_dist.selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, require_dist.nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile), override);
|
|
6864
7376
|
}
|
|
6865
7377
|
/**
|
|
6866
7378
|
* Attach one (device, profile): acquire the broker source, mkdir the staging
|
|
@@ -6906,7 +7418,10 @@ var RecordingController = class {
|
|
|
6906
7418
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
6907
7419
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
6908
7420
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
6909
|
-
await this.handleFinalizedSegment(deviceId, profile, placement,
|
|
7421
|
+
await this.handleFinalizedSegment(deviceId, profile, placement, {
|
|
7422
|
+
attachedAtMs: writerStartedMs,
|
|
7423
|
+
segmentSeconds
|
|
7424
|
+
}, startMs, durMs, flatAbsPath);
|
|
6910
7425
|
},
|
|
6911
7426
|
logger: deviceLog
|
|
6912
7427
|
}),
|
|
@@ -6923,13 +7438,16 @@ var RecordingController = class {
|
|
|
6923
7438
|
* keep/discard gate touches `events`-mode segments only, so the continuous
|
|
6924
7439
|
* path is unchanged.
|
|
6925
7440
|
*/
|
|
6926
|
-
async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
|
|
7441
|
+
async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
|
|
6927
7442
|
try {
|
|
6928
7443
|
const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
|
|
6929
|
-
|
|
7444
|
+
const triggers = this.triggersFor(deviceId);
|
|
7445
|
+
if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
|
|
7446
|
+
this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
|
|
6930
7447
|
await node_fs.promises.unlink(flatAbsPath).catch(() => {});
|
|
6931
7448
|
return;
|
|
6932
7449
|
}
|
|
7450
|
+
if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
|
|
6933
7451
|
await this.deps.segmentStore.onFinalized({
|
|
6934
7452
|
deviceId,
|
|
6935
7453
|
profile,
|
|
@@ -6950,6 +7468,47 @@ var RecordingController = class {
|
|
|
6950
7468
|
}
|
|
6951
7469
|
}
|
|
6952
7470
|
/**
|
|
7471
|
+
* Name a discarded `events`-band segment. This branch deletes recorded
|
|
7472
|
+
* footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
|
|
7473
|
+
* then `unpinned` with an empty timeline in between and no line anywhere said
|
|
7474
|
+
* why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
|
|
7475
|
+
* every segment it produced.
|
|
7476
|
+
*
|
|
7477
|
+
* Level is chosen, not defaulted. A discard at the TAIL of a window is the
|
|
7478
|
+
* feature working (`info`) — the writer keeps running until the next converge
|
|
7479
|
+
* tick and those trailing segments were never asked for. A discard from an
|
|
7480
|
+
* attach that has retained NOTHING is a fault (`warn`): the whole pull was
|
|
7481
|
+
* wasted, which is exactly the shape of the 1439 loss and of any future
|
|
7482
|
+
* regression in the attach allowance.
|
|
7483
|
+
*/
|
|
7484
|
+
reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
|
|
7485
|
+
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
7486
|
+
const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
|
|
7487
|
+
const meta = {
|
|
7488
|
+
profile,
|
|
7489
|
+
startMs,
|
|
7490
|
+
durMs,
|
|
7491
|
+
lastMotionMs: triggers.lastMotionMs,
|
|
7492
|
+
lastAudioMs: triggers.lastAudioMs,
|
|
7493
|
+
postMs,
|
|
7494
|
+
preMs,
|
|
7495
|
+
attachedAtMs: attach.attachedAtMs,
|
|
7496
|
+
missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
|
|
7497
|
+
keptAnythingFromThisAttach: keptAnything
|
|
7498
|
+
};
|
|
7499
|
+
if (keptAnything) {
|
|
7500
|
+
this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
|
|
7501
|
+
tags: { deviceId },
|
|
7502
|
+
meta
|
|
7503
|
+
});
|
|
7504
|
+
return;
|
|
7505
|
+
}
|
|
7506
|
+
this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
|
|
7507
|
+
tags: { deviceId },
|
|
7508
|
+
meta
|
|
7509
|
+
});
|
|
7510
|
+
}
|
|
7511
|
+
/**
|
|
6953
7512
|
* Resolve the storage location a (device, profile) writes to, at an attach
|
|
6954
7513
|
* boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
|
|
6955
7514
|
* when the addon supplied one; otherwise to the pure `resolvePlacement` in
|
|
@@ -6990,7 +7549,11 @@ var RecordingController = class {
|
|
|
6990
7549
|
const recordings = this.active.get(deviceId);
|
|
6991
7550
|
if (!recordings) return;
|
|
6992
7551
|
this.active.delete(deviceId);
|
|
6993
|
-
|
|
7552
|
+
this.skippedProfiles.delete(deviceId);
|
|
7553
|
+
for (const r of recordings) {
|
|
7554
|
+
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
7555
|
+
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
7556
|
+
}
|
|
6994
7557
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
6995
7558
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
6996
7559
|
tags: { deviceId },
|
|
@@ -7001,6 +7564,7 @@ var RecordingController = class {
|
|
|
7001
7564
|
async pause() {
|
|
7002
7565
|
if (this.stopped || this.paused) return;
|
|
7003
7566
|
this.paused = true;
|
|
7567
|
+
this.wakeups.cancelAll();
|
|
7004
7568
|
while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
|
|
7005
7569
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
7006
7570
|
}
|
|
@@ -7016,13 +7580,16 @@ var RecordingController = class {
|
|
|
7016
7580
|
}
|
|
7017
7581
|
for (const deviceId of ids) await this.evaluateDevice(deviceId);
|
|
7018
7582
|
}
|
|
7019
|
-
/** Tear EVERYTHING down:
|
|
7583
|
+
/** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
|
|
7020
7584
|
async stop() {
|
|
7021
7585
|
this.stopped = true;
|
|
7586
|
+
this.wakeups.cancelAll();
|
|
7022
7587
|
this.livenessPass?.stop();
|
|
7023
7588
|
this.livenessPass = null;
|
|
7024
|
-
this.
|
|
7025
|
-
this.
|
|
7589
|
+
this.placementPass?.stop();
|
|
7590
|
+
this.placementPass = null;
|
|
7591
|
+
this.reconcilePass?.stop();
|
|
7592
|
+
this.reconcilePass = null;
|
|
7026
7593
|
this.restore?.stop();
|
|
7027
7594
|
this.restore = null;
|
|
7028
7595
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
@@ -8556,7 +9123,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8556
9123
|
onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
|
|
8557
9124
|
if (state === "ready") handler();
|
|
8558
9125
|
}),
|
|
8559
|
-
brokerCall: makeBrokerCall(brokerHandle)
|
|
9126
|
+
brokerCall: makeBrokerCall(brokerHandle),
|
|
9127
|
+
brokerReady: () => brokerHandle.isReady
|
|
8560
9128
|
});
|
|
8561
9129
|
const relocateEngine = new RelocateEngine({
|
|
8562
9130
|
index: this.index,
|
|
@@ -8606,6 +9174,9 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
|
|
|
8606
9174
|
refreshLocations: () => this.refreshLocations(),
|
|
8607
9175
|
capacity: (root) => this.locationCapacity(root),
|
|
8608
9176
|
hydrateDevice: (deviceId, locations) => hydrateDeviceFromStorage(this.ctx.api, this.index, deviceId, locations, this.ctx.logger),
|
|
9177
|
+
reconcileHourLedger: async (deviceId) => {
|
|
9178
|
+
await this.segmentHours?.reconcileFromIndex(this.index, [deviceId], Date.now());
|
|
9179
|
+
},
|
|
8609
9180
|
calendar: this.calendar,
|
|
8610
9181
|
mfraFor: (deviceId, profile, startMs) => this.mfraTables.get(deviceId, profile, startMs),
|
|
8611
9182
|
hydrateWindow: (deviceId, fromMs, toMs) => hydrateWindowFromStorage(this.index, deviceId, fromMs, toMs, this.resolvedLocations, this.ctx.logger),
|