@camstack/addon-pipeline 1.2.90 → 1.2.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{addon-utils-yc-chXuC.js → addon-utils-DnUCCZVx.js} +1 -1
- package/dist/audio-analyzer/index.js +2 -2
- package/dist/audio-analyzer/index.mjs +1 -1
- package/dist/detection-pipeline/index.js +4 -4
- package/dist/detection-pipeline/index.mjs +2 -2
- package/dist/{dist-n4Nbnxx5.js → dist-BbRv3bM2.js} +139 -22
- package/dist/{dist-BJYwWEZ9.mjs → dist-XbrYiMV3.mjs} +134 -23
- package/dist/{event-loop-stall-monitor-BiPffk0D.mjs → event-loop-stall-monitor-CFWrOZ6G.mjs} +1 -1
- package/dist/{event-loop-stall-monitor-DG7QY00k.js → event-loop-stall-monitor-D9hqbc68.js} +1 -1
- package/dist/{lazy-sharp-EoSOHgoj.js → lazy-sharp-U0EtN7_C.js} +1 -1
- package/dist/motion-wasm/index.js +2 -2
- package/dist/motion-wasm/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +195 -14
- package/dist/pipeline-runner/index.mjs +194 -13
- package/dist/recorder/index.js +655 -84
- package/dist/recorder/index.mjs +654 -83
- package/dist/session-decode/decode-worker-child.js +4 -4
- package/dist/session-decode/decode-worker-child.mjs +3 -3
- package/dist/stream-broker/_stub.js +2 -2
- package/dist/stream-broker/{_virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-CsrMZYLB.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_stream_broker_widgets-BNOPhQ-y.mjs} +2 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DMyOrQHq.mjs +26 -0
- package/dist/stream-broker/{_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DgivrPsH.mjs → _virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DsIi4G5Q.mjs} +1 -1
- package/dist/stream-broker/{hostInit-DoFovQmc.mjs → hostInit-DCV2-mSi.mjs} +2 -2
- package/dist/stream-broker/index.js +194 -30
- package/dist/stream-broker/index.mjs +194 -30
- package/dist/stream-broker/remoteEntry.js +1 -1
- package/dist/{worker-protocol-DjuxhGo3.js → worker-protocol-CNC-ZcU3.js} +1 -1
- package/dist/{worker-protocol-XZ1e36rM.mjs → worker-protocol-D0ewr67U.mjs} +1 -1
- package/package.json +3 -2
- package/dist/stream-broker/_virtual_mf___mfe_internal__addon_stream_broker_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-DNSELPGi.mjs +0 -26
package/dist/recorder/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $ as resolveRecordingProfiles, Ct as selectAssignedProfileSlots, Et as array, I as deriveRecordingMode, It as EventCategory, Mt as object, Nt as record, Pt as string, S as RecordingConfigSchema, X as recordingCapability, Z as recordingExportCapability, at as errMsg, bt as nodePin, dt as BaseAddon, f as EVENT_PAD_MS, gt as hydrateSchema, jt as number, l as DEFAULT_EVENTS_BAND_BUFFER_SEC, m as ExportRecordSchema, mt as DeviceType, tt as storageEvictableCapability, v as OpsLogEntrySchema, x as RECORDING_EXPORT_MAX_READ_BYTES } from "../dist-XbrYiMV3.mjs";
|
|
2
2
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
3
3
|
import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-A2S9D7pu.mjs";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
@@ -1068,6 +1068,7 @@ var RelocateEngine = class {
|
|
|
1068
1068
|
deviceId: input.deviceId ?? null,
|
|
1069
1069
|
profiles: input.profiles ? [...input.profiles] : [],
|
|
1070
1070
|
footageClass: input.footageClass,
|
|
1071
|
+
sinceMs: input.sinceMs ?? null,
|
|
1071
1072
|
throttleMbps: input.throttleMbps ?? DEFAULT_THROTTLE_MBPS,
|
|
1072
1073
|
entities: input.entities ? [...input.entities] : ["segments"],
|
|
1073
1074
|
filesMoved: 0,
|
|
@@ -1128,7 +1129,7 @@ var RelocateEngine = class {
|
|
|
1128
1129
|
const bytesPerMs = job.throttleMbps * 1024 * 1024 / 1e3;
|
|
1129
1130
|
try {
|
|
1130
1131
|
const samePhysicalRoot = await rootsPhysicallySame(from.root, to.root);
|
|
1131
|
-
const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).sort((a, b) => a.startMs - b.startMs);
|
|
1132
|
+
const segRows = job.entities.includes("segments") === false ? [] : this.deps.index.segmentsOnLocation(from.id).filter((r) => job.deviceId === null || r.deviceId === job.deviceId).filter((r) => job.profiles.length === 0 || job.profiles.includes(r.profile)).filter((r) => profileBelongsToClass(r.profile, job.footageClass)).filter((r) => job.sinceMs === null || r.startMs >= job.sinceMs).sort((a, b) => a.startMs - b.startMs);
|
|
1132
1133
|
job.filesTotal = segRows.length;
|
|
1133
1134
|
for (const row of segRows) {
|
|
1134
1135
|
if (job.cancelRequested) break;
|
|
@@ -1164,9 +1165,26 @@ var RelocateEngine = class {
|
|
|
1164
1165
|
}
|
|
1165
1166
|
}
|
|
1166
1167
|
/** Move one segment file and re-point its index row to the target location
|
|
1167
|
-
* (same location-relative path — the layout is identical on every root).
|
|
1168
|
+
* (same location-relative path — the layout is identical on every root).
|
|
1169
|
+
* A source that is already gone (evicted, never flushed, ENOSPC) is dropped
|
|
1170
|
+
* from the index and skipped — one ghost file must not abort a drain. */
|
|
1168
1171
|
async moveSegment(row, from, to, samePhysicalRoot) {
|
|
1169
|
-
if (!samePhysicalRoot)
|
|
1172
|
+
if (!samePhysicalRoot) try {
|
|
1173
|
+
await moveFile(path.join(from.root, row.path), path.join(to.root, row.path), row.bytes);
|
|
1174
|
+
} catch (err) {
|
|
1175
|
+
if (isEnoent(err)) {
|
|
1176
|
+
this.deps.index.removeSegments([row.path]);
|
|
1177
|
+
this.deps.logger.warn("relocate skipped missing source", {
|
|
1178
|
+
tags: { deviceId: row.deviceId },
|
|
1179
|
+
meta: {
|
|
1180
|
+
path: row.path,
|
|
1181
|
+
locationId: from.id
|
|
1182
|
+
}
|
|
1183
|
+
});
|
|
1184
|
+
return;
|
|
1185
|
+
}
|
|
1186
|
+
throw err;
|
|
1187
|
+
}
|
|
1170
1188
|
this.deps.index.removeSegments([row.path]);
|
|
1171
1189
|
this.deps.index.addSegment({
|
|
1172
1190
|
...row,
|
|
@@ -1174,6 +1192,9 @@ var RelocateEngine = class {
|
|
|
1174
1192
|
});
|
|
1175
1193
|
}
|
|
1176
1194
|
};
|
|
1195
|
+
function isEnoent(err) {
|
|
1196
|
+
return typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT";
|
|
1197
|
+
}
|
|
1177
1198
|
/** Logical migration classes are profile-derived, never location-derived:
|
|
1178
1199
|
* high+mid are `recordings`; low is `recordingsLow`. */
|
|
1179
1200
|
var profileBelongsToClass = (profile, footageClass) => {
|
|
@@ -4212,6 +4233,7 @@ function buildRecordingProvider(deps) {
|
|
|
4212
4233
|
rescanStorage: async ({ deviceId }) => {
|
|
4213
4234
|
const locations = await deps.refreshLocations();
|
|
4214
4235
|
await deps.hydrateDevice(deviceId, locations);
|
|
4236
|
+
await deps.reconcileHourLedger?.(deviceId);
|
|
4215
4237
|
const config = await loadDeviceConfig(deps.configStore, deviceId);
|
|
4216
4238
|
await deps.opsLog?.append({
|
|
4217
4239
|
op: "rescan",
|
|
@@ -5075,7 +5097,11 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
5075
5097
|
try {
|
|
5076
5098
|
entries = await promises.readdir(deviceDir, { recursive: true });
|
|
5077
5099
|
} catch (err) {
|
|
5078
|
-
if (err.code
|
|
5100
|
+
if (err.code === "ENOENT") {
|
|
5101
|
+
for (const location of aliases) index.hydrateDevice(deviceId, location.id, []);
|
|
5102
|
+
continue;
|
|
5103
|
+
}
|
|
5104
|
+
logger.warn("recorder: hydrateDevice walk failed for location", {
|
|
5079
5105
|
tags: { deviceId },
|
|
5080
5106
|
meta: {
|
|
5081
5107
|
locationIds: aliases.map((location) => location.id),
|
|
@@ -5103,6 +5129,7 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
|
|
|
5103
5129
|
handled += 1;
|
|
5104
5130
|
if (handled % chunkSize === 0) await yieldBetween();
|
|
5105
5131
|
}
|
|
5132
|
+
for (const location of aliases) if (!pathsByLocation.has(location.id)) pathsByLocation.set(location.id, []);
|
|
5106
5133
|
let indexed = 0;
|
|
5107
5134
|
for (const [locationId, relPaths] of pathsByLocation) {
|
|
5108
5135
|
index.hydrateDevice(deviceId, locationId, relPaths);
|
|
@@ -5810,18 +5837,232 @@ function shouldRecordAt(config, triggers, at) {
|
|
|
5810
5837
|
return eventsBandDemanded(band, triggers, at.getTime());
|
|
5811
5838
|
}
|
|
5812
5839
|
/**
|
|
5813
|
-
*
|
|
5814
|
-
*
|
|
5815
|
-
*
|
|
5816
|
-
*
|
|
5840
|
+
* How late the FIRST segment of an attach may be NAMED, relative to the writer
|
|
5841
|
+
* spawn, and still count as "the segment this attach was spawned to produce".
|
|
5842
|
+
*
|
|
5843
|
+
* ffmpeg names a segment when it OPENS it, which is after the RTSP dial and the
|
|
5844
|
+
* first keyframe. Measured on camera 1439 (4K, H.265) on 2026-08-18: writer
|
|
5845
|
+
* spawned at trigger+29.5 s, first `high` segment named at trigger+32.7 s —
|
|
5846
|
+
* 3.2 s. 5 s carries that with margin while staying far below the writer's own
|
|
5847
|
+
* "this is dead" bound (`WRITER_IDLE_MIN_MS`, 90 s), so a writer that is merely
|
|
5848
|
+
* slow is not mistaken for one that is producing useful footage.
|
|
5817
5849
|
*/
|
|
5818
|
-
|
|
5850
|
+
var FIRST_SEGMENT_GRACE_MS = 5e3;
|
|
5851
|
+
/**
|
|
5852
|
+
* Latest segment START this attach may still produce and have kept, regardless
|
|
5853
|
+
* of where the operator's post-buffer ended: one full segment (nothing can be
|
|
5854
|
+
* finalized sooner) plus {@link FIRST_SEGMENT_GRACE_MS} of spawn→first-name
|
|
5855
|
+
* latency.
|
|
5856
|
+
*/
|
|
5857
|
+
function attachRetainDeadlineMs(attach) {
|
|
5858
|
+
return attach.attachedAtMs + attach.segmentSeconds * 1e3 + FIRST_SEGMENT_GRACE_MS;
|
|
5859
|
+
}
|
|
5860
|
+
/**
|
|
5861
|
+
* Should a finalized segment `[segStartMs, segEndMs]` recorded under an
|
|
5862
|
+
* `events` band be KEPT (else discarded)? Kept iff it overlaps a qualifying
|
|
5863
|
+
* trigger's full window `[trigger - preBufferSec, trigger + postBufferSec]` —
|
|
5864
|
+
* this is where `preBufferSec` retroactively retains pre-trigger footage — OR
|
|
5865
|
+
* it is the footage `attach` was spawned to produce (below).
|
|
5866
|
+
*
|
|
5867
|
+
* ## Why the attach matters here
|
|
5868
|
+
*
|
|
5869
|
+
* The LIVE gate ({@link shouldRecordAt}) is evaluated at wall clock; this KEEP
|
|
5870
|
+
* gate is evaluated on segment timestamps. They shared one window, and a writer
|
|
5871
|
+
* physically cannot finalize anything before `spawn + segmentSeconds + dial`.
|
|
5872
|
+
* So the last ~`segmentSeconds` of every demand window was a zone where the
|
|
5873
|
+
* recorder was GUARANTEED to attach, pull the camera, and delete 100% of the
|
|
5874
|
+
* result: on camera 1439 a trigger reached the recorder 29.5 s into its 30 s
|
|
5875
|
+
* post-buffer (hub overload, `busLagMs=142173`), ffmpeg ran 31 s, and all six
|
|
5876
|
+
* finalized segments were unlinked.
|
|
5877
|
+
*
|
|
5878
|
+
* The allowance is deliberately NOT a wider `postBufferSec`: it is anchored to
|
|
5879
|
+
* `attachedAtMs`, so a PUNCTUAL attach (the normal case) has a deadline far
|
|
5880
|
+
* inside `trigger + postMs` and retains exactly what it retained before. It
|
|
5881
|
+
* only ever extends the tail of a LATE attach, by at most one segment plus the
|
|
5882
|
+
* grace, and only for a trigger whose live window was still open AT ATTACH — the
|
|
5883
|
+
* precise statement of "if the live gate said yes, the keep gate honours the
|
|
5884
|
+
* first segment that decision produces". A stale trigger extends nothing.
|
|
5885
|
+
*/
|
|
5886
|
+
function segmentRetainedForEvents(segStartMs, segEndMs, band, triggers, attach) {
|
|
5819
5887
|
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
5820
|
-
const
|
|
5888
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
5889
|
+
const overlaps = (lastMs) => {
|
|
5890
|
+
if (lastMs == null) return false;
|
|
5891
|
+
if (segEndMs < lastMs - preMs) return false;
|
|
5892
|
+
if (segStartMs <= lastMs + postMs) return true;
|
|
5893
|
+
return attach.attachedAtMs <= lastMs + postMs && segStartMs <= deadlineMs;
|
|
5894
|
+
};
|
|
5821
5895
|
if (band.triggers?.motion && overlaps(triggers.lastMotionMs)) return true;
|
|
5822
5896
|
if (band.triggers?.audioThresholdDbfs != null && overlaps(triggers.lastAudioMs)) return true;
|
|
5823
5897
|
return false;
|
|
5824
5898
|
}
|
|
5899
|
+
/**
|
|
5900
|
+
* By how many ms a discarded segment missed being kept — its start past the
|
|
5901
|
+
* latest start that WOULD have been retained, mirroring
|
|
5902
|
+
* {@link segmentRetainedForEvents}'s upper bound exactly (post-buffer end, or
|
|
5903
|
+
* the attach allowance for a trigger that was still live at attach). Null when
|
|
5904
|
+
* no qualifying trigger exists at all, which is a different fact and says so.
|
|
5905
|
+
*
|
|
5906
|
+
* Negative means the segment was dropped on the PRE-buffer side (it ended
|
|
5907
|
+
* before `trigger - preBufferSec`), not for being late.
|
|
5908
|
+
*
|
|
5909
|
+
* Reporting-only. The discard branch has to name a number: "outside the window"
|
|
5910
|
+
* with no magnitude made a 2.5 s miss and a 10-minute miss indistinguishable.
|
|
5911
|
+
*/
|
|
5912
|
+
function segmentMissedByMs(segStartMs, band, triggers, attach) {
|
|
5913
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
5914
|
+
const deadlineMs = attachRetainDeadlineMs(attach);
|
|
5915
|
+
const latestKeptStart = (lastMs) => {
|
|
5916
|
+
if (lastMs == null) return null;
|
|
5917
|
+
const windowEndMs = lastMs + postMs;
|
|
5918
|
+
return attach.attachedAtMs <= windowEndMs ? Math.max(windowEndMs, deadlineMs) : windowEndMs;
|
|
5919
|
+
};
|
|
5920
|
+
const bounds = [band.triggers?.motion === true ? latestKeptStart(triggers.lastMotionMs) : null, band.triggers?.audioThresholdDbfs != null ? latestKeptStart(triggers.lastAudioMs) : null].filter((ms) => ms != null);
|
|
5921
|
+
if (bounds.length === 0) return null;
|
|
5922
|
+
return segStartMs - Math.max(...bounds);
|
|
5923
|
+
}
|
|
5924
|
+
/** Local-calendar instant for `HH:MM` on the given local Y/M/D (day may overflow). */
|
|
5925
|
+
function localInstantMs(year, month, day, hhmm) {
|
|
5926
|
+
const [hours, minutes] = hhmm.split(":");
|
|
5927
|
+
return new Date(year, month, day, Number(hours), Number(minutes), 0, 0).getTime();
|
|
5928
|
+
}
|
|
5929
|
+
/**
|
|
5930
|
+
* Every instant inside the horizon at which `bandActiveAt` COULD change value:
|
|
5931
|
+
* each band's start and end on each local day, plus each local midnight (the
|
|
5932
|
+
* `days` membership boundary). A superset — cheap to compute (a few dozen
|
|
5933
|
+
* entries) and filtered against `activeBandAt` below.
|
|
5934
|
+
*/
|
|
5935
|
+
function candidateEdgesMs(bands, nowMs) {
|
|
5936
|
+
const base = new Date(nowMs);
|
|
5937
|
+
const year = base.getFullYear();
|
|
5938
|
+
const month = base.getMonth();
|
|
5939
|
+
const day = base.getDate();
|
|
5940
|
+
const out = [];
|
|
5941
|
+
for (let offset = 0; offset <= 8; offset++) {
|
|
5942
|
+
out.push(new Date(year, month, day + offset, 0, 0, 0, 0).getTime());
|
|
5943
|
+
for (const band of bands) {
|
|
5944
|
+
out.push(localInstantMs(year, month, day + offset, band.start));
|
|
5945
|
+
out.push(localInstantMs(year, month, day + offset, band.end));
|
|
5946
|
+
}
|
|
5947
|
+
}
|
|
5948
|
+
return out;
|
|
5949
|
+
}
|
|
5950
|
+
/**
|
|
5951
|
+
* The next instant at which a DIFFERENT band (or no band) covers the clock, or
|
|
5952
|
+
* `null` when none exists inside {@link BAND_EDGE_HORIZON_DAYS} — the always-on
|
|
5953
|
+
* band (`start === end`, empty `days`) being the common case that legitimately
|
|
5954
|
+
* has no edge at all.
|
|
5955
|
+
*
|
|
5956
|
+
* Identity is by band REFERENCE, which is what makes this exact: two distinct
|
|
5957
|
+
* band objects with the same fields yield an edge that changes nothing, which
|
|
5958
|
+
* costs one wasted wake-up and never costs footage.
|
|
5959
|
+
*/
|
|
5960
|
+
function nextBandEdgeMs(config, nowMs) {
|
|
5961
|
+
const bands = config.bands;
|
|
5962
|
+
if (!bands || bands.length === 0) return null;
|
|
5963
|
+
const current = activeBandAt({ bands }, new Date(nowMs));
|
|
5964
|
+
const candidates = [...new Set(candidateEdgesMs(bands, nowMs))].filter((ms) => ms > nowMs).toSorted((a, b) => a - b);
|
|
5965
|
+
for (const ms of candidates) if (activeBandAt({ bands }, new Date(ms)) !== current) return ms;
|
|
5966
|
+
return null;
|
|
5967
|
+
}
|
|
5968
|
+
/**
|
|
5969
|
+
* The first instant an `events` band's demand window is CLOSED, given the
|
|
5970
|
+
* triggers it listens for, or `null` when no window is open at `nowMs`.
|
|
5971
|
+
*
|
|
5972
|
+
* Demand is an OR across the listened trigger kinds, so an open window ends
|
|
5973
|
+
* with the LATEST of them — a motion trigger followed by an audio trigger 5 s
|
|
5974
|
+
* later keeps the camera attached until the audio one expires, exactly as the
|
|
5975
|
+
* sliding window already behaved.
|
|
5976
|
+
*/
|
|
5977
|
+
function eventsWindowCloseMs(band, triggers, nowMs) {
|
|
5978
|
+
if (band.mode !== "events") return null;
|
|
5979
|
+
const { postMs } = resolveBandBufferMs(band);
|
|
5980
|
+
const ends = [];
|
|
5981
|
+
if (band.triggers?.motion === true && triggers.lastMotionMs != null) ends.push(triggers.lastMotionMs + postMs);
|
|
5982
|
+
if (band.triggers?.audioThresholdDbfs != null && triggers.lastAudioMs != null) ends.push(triggers.lastAudioMs + postMs);
|
|
5983
|
+
const open = ends.filter((ms) => ms >= nowMs);
|
|
5984
|
+
if (open.length === 0) return null;
|
|
5985
|
+
return Math.max(...open) + 1;
|
|
5986
|
+
}
|
|
5987
|
+
/**
|
|
5988
|
+
* THE next instant the controller must re-evaluate this device, or `null` when
|
|
5989
|
+
* nothing can change without an event the controller already receives (a
|
|
5990
|
+
* trigger or a config write).
|
|
5991
|
+
*
|
|
5992
|
+
* Whichever of the two sources comes first wins, and the reason is carried so
|
|
5993
|
+
* the wake-up is attributable in the log without correlating timestamps.
|
|
5994
|
+
*/
|
|
5995
|
+
function nextDecisionChangeAt(input) {
|
|
5996
|
+
const { config, triggers, nowMs } = input;
|
|
5997
|
+
if (!config.enabled) return null;
|
|
5998
|
+
const bands = config.bands;
|
|
5999
|
+
if (!bands || bands.length === 0) return null;
|
|
6000
|
+
const band = activeBandAt({ bands }, new Date(nowMs));
|
|
6001
|
+
const edgeMs = nextBandEdgeMs(config, nowMs);
|
|
6002
|
+
const closeMs = band === null ? null : eventsWindowCloseMs(band, triggers, nowMs);
|
|
6003
|
+
if (closeMs === null) return edgeMs === null ? null : {
|
|
6004
|
+
atMs: edgeMs,
|
|
6005
|
+
reason: "band-edge"
|
|
6006
|
+
};
|
|
6007
|
+
if (edgeMs === null || closeMs <= edgeMs) return {
|
|
6008
|
+
atMs: closeMs,
|
|
6009
|
+
reason: "window-close"
|
|
6010
|
+
};
|
|
6011
|
+
return {
|
|
6012
|
+
atMs: edgeMs,
|
|
6013
|
+
reason: "band-edge"
|
|
6014
|
+
};
|
|
6015
|
+
}
|
|
6016
|
+
//#endregion
|
|
6017
|
+
//#region src/recorder/addon/device-wakeups.ts
|
|
6018
|
+
/** Node truncates a `setTimeout` delay past this to 1 ms — clamp instead. */
|
|
6019
|
+
var MAX_TIMER_DELAY_MS = 2147483647;
|
|
6020
|
+
var DeviceWakeups = class {
|
|
6021
|
+
deps;
|
|
6022
|
+
timers = /* @__PURE__ */ new Map();
|
|
6023
|
+
constructor(deps) {
|
|
6024
|
+
this.deps = deps;
|
|
6025
|
+
}
|
|
6026
|
+
/**
|
|
6027
|
+
* Wake this device at `atMs`, replacing any wake-up already armed for it.
|
|
6028
|
+
* `reason` is carried into the log line so a wake is attributable without
|
|
6029
|
+
* correlating it against the band model by hand.
|
|
6030
|
+
*/
|
|
6031
|
+
arm(deviceId, atMs, reason) {
|
|
6032
|
+
this.cancel(deviceId);
|
|
6033
|
+
const delayMs = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, atMs - this.deps.now()));
|
|
6034
|
+
const timer = setTimeout(() => {
|
|
6035
|
+
this.timers.delete(deviceId);
|
|
6036
|
+
this.deps.fire(deviceId);
|
|
6037
|
+
}, delayMs);
|
|
6038
|
+
timer.unref?.();
|
|
6039
|
+
this.timers.set(deviceId, timer);
|
|
6040
|
+
this.deps.logger.debug("recorder: device wake-up armed", {
|
|
6041
|
+
tags: { deviceId },
|
|
6042
|
+
meta: {
|
|
6043
|
+
atMs,
|
|
6044
|
+
delayMs,
|
|
6045
|
+
reason
|
|
6046
|
+
}
|
|
6047
|
+
});
|
|
6048
|
+
}
|
|
6049
|
+
/** Drop this device's pending wake-up, if any. Idempotent. */
|
|
6050
|
+
cancel(deviceId) {
|
|
6051
|
+
const timer = this.timers.get(deviceId);
|
|
6052
|
+
if (timer === void 0) return;
|
|
6053
|
+
clearTimeout(timer);
|
|
6054
|
+
this.timers.delete(deviceId);
|
|
6055
|
+
}
|
|
6056
|
+
/** Drop every pending wake-up — shutdown and maintenance pause. */
|
|
6057
|
+
cancelAll() {
|
|
6058
|
+
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
6059
|
+
this.timers.clear();
|
|
6060
|
+
}
|
|
6061
|
+
/** Devices with a wake-up pending — for tests and for shutdown assertions. */
|
|
6062
|
+
get pendingCount() {
|
|
6063
|
+
return this.timers.size;
|
|
6064
|
+
}
|
|
6065
|
+
};
|
|
5825
6066
|
//#endregion
|
|
5826
6067
|
//#region src/recorder/addon/periodic-pass.ts
|
|
5827
6068
|
function startPeriodicPass(deps) {
|
|
@@ -6405,19 +6646,37 @@ var SegmentWriter = class {
|
|
|
6405
6646
|
* - a `live.m3u8` `SegmentWatcher` that hands each finalized flat segment to
|
|
6406
6647
|
* the v2 `SegmentStore.onFinalized` (stat + relocate + index).
|
|
6407
6648
|
*
|
|
6408
|
-
* Per device it decides whether
|
|
6409
|
-
* (`
|
|
6410
|
-
*
|
|
6411
|
-
* 1. `setDeviceConfig` (operator changed the bands / enable),
|
|
6412
|
-
* 2. a
|
|
6413
|
-
* 3. a
|
|
6414
|
-
*
|
|
6415
|
-
*
|
|
6416
|
-
*
|
|
6417
|
-
*
|
|
6418
|
-
*
|
|
6419
|
-
*
|
|
6420
|
-
*
|
|
6649
|
+
* Per device it decides whether the active band demands recording right now
|
|
6650
|
+
* (`shouldRecordAt`) and ensures the writer set is running iff so. Convergence
|
|
6651
|
+
* is EVENT-DRIVEN; the periodic pass is a backstop, not the mechanism:
|
|
6652
|
+
* 1. `setDeviceConfig` (operator changed the bands / enable) — an RPC,
|
|
6653
|
+
* 2. a qualifying motion/audio trigger (`onTrigger`) — opens a window,
|
|
6654
|
+
* 3. a per-device WAKE-UP armed for the exact instant the decision could
|
|
6655
|
+
* change on its own (`decision-schedule.ts` + `device-wakeups.ts`): an
|
|
6656
|
+
* `events` window closing at `lastTrigger + postBufferSec`, or a band edge,
|
|
6657
|
+
* 4. a `stream-broker` ready transition (boot restore — via `ReadinessRestore`),
|
|
6658
|
+
* 5. the periodic RECONCILE (below), for everything the four above can lose.
|
|
6659
|
+
*
|
|
6660
|
+
* (3) is why a detach is now punctual. A trigger only ever OPENED a window — the
|
|
6661
|
+
* demand test is a sliding `atMs - lastMotionMs <= postMs` anchored on the LAST
|
|
6662
|
+
* trigger, so nothing called `evaluateDevice` at the moment it closed and the
|
|
6663
|
+
* detach waited for the next pass. A 4K camera stayed dialled, decoded and
|
|
6664
|
+
* written for up to a whole tick past the operator's `postBufferSec`.
|
|
6665
|
+
*
|
|
6666
|
+
* There are THREE periodic passes, each on its own timer and each non-overlapping
|
|
6667
|
+
* (`periodic-pass.ts`):
|
|
6668
|
+
* - LIVENESS ({@link LIVENESS_TICK_MS}) — cheap, in-memory: is a pinned writer
|
|
6669
|
+
* producing nothing?
|
|
6670
|
+
* - PLACEMENT ({@link PLACEMENT_TICK_MS}) — cheap (a statfs per location), and
|
|
6671
|
+
* deliberately NOT on the attach beat, so a cold broker cannot delay it.
|
|
6672
|
+
* - RECONCILE ({@link RECONCILE_TICK_MS}) — the only one that can touch the
|
|
6673
|
+
* broker, and the only backstop for a lost wake-up or a failed attach.
|
|
6674
|
+
*
|
|
6675
|
+
* They were ONE pass until 2026-08-13, when a convergence sweep wedged on
|
|
6676
|
+
* unbounded broker RPCs and took the liveness watchdog down with it for 9-27
|
|
6677
|
+
* minutes at a time; placement was split out of the second on 2026-08-18, after
|
|
6678
|
+
* a converge pass blocked for 60 001 ms (3 x {@link ATTACH_RPC_TIMEOUT_MS})
|
|
6679
|
+
* cold-dialling ~40 RTSP sources.
|
|
6421
6680
|
*
|
|
6422
6681
|
* EVENTS bands are treated as "not recording continuously" in B2 — B3 adds
|
|
6423
6682
|
* trigger-gating. The enable intent is persisted in durable-state by the
|
|
@@ -6430,8 +6689,45 @@ var SegmentWriter = class {
|
|
|
6430
6689
|
* `ReadinessRegistry` is the single source of truth.
|
|
6431
6690
|
*/
|
|
6432
6691
|
var DEFAULT_WATCH_INTERVAL_MS = 2e3;
|
|
6433
|
-
/**
|
|
6434
|
-
|
|
6692
|
+
/**
|
|
6693
|
+
* Liveness cadence — "is a pinned writer producing nothing?". Unchanged: the
|
|
6694
|
+
* question is an in-memory scan over `active`, it costs nothing, and its bound
|
|
6695
|
+
* (`writerIdleBoundMs`, floor 90 s) is sized against this beat.
|
|
6696
|
+
*/
|
|
6697
|
+
var LIVENESS_TICK_MS = 3e4;
|
|
6698
|
+
/**
|
|
6699
|
+
* Placement cadence — unchanged from the beat it used to share with
|
|
6700
|
+
* convergence, because nothing about placement wanted to be rarer. What changed
|
|
6701
|
+
* is that it no longer rides a pass that can block on broker RPCs: it is one
|
|
6702
|
+
* `statfs` per location and it re-plans only, so a plan reaches a camera at its
|
|
6703
|
+
* NEXT attach.
|
|
6704
|
+
*/
|
|
6705
|
+
var PLACEMENT_TICK_MS = 3e4;
|
|
6706
|
+
/**
|
|
6707
|
+
* Reconcile cadence — the SAFETY NET, not the mechanism.
|
|
6708
|
+
*
|
|
6709
|
+
* It was 30 s because it WAS the mechanism: nothing else ever closed an `events`
|
|
6710
|
+
* window or crossed a band edge, so the loop had to keep asking. Both are now
|
|
6711
|
+
* armed for their exact instant (`decision-schedule.ts`), a config change is an
|
|
6712
|
+
* RPC, and a broker recovery is a readiness transition — so this pass exists
|
|
6713
|
+
* only for what those cannot cover: a wake-up lost because the event path threw
|
|
6714
|
+
* before it could arm one, an attach that failed with no later ready transition
|
|
6715
|
+
* to ride, a timer lost to a runner respawn, and clock drift.
|
|
6716
|
+
*
|
|
6717
|
+
* 2 minutes, and the number is chosen against a bound this recorder already
|
|
6718
|
+
* accepts. The liveness watchdog tolerates a pinned-but-silent writer for
|
|
6719
|
+
* `WRITER_IDLE_MIN_MS` (90 s) before it recovers it; a device that WANTS
|
|
6720
|
+
* recording but is detached is the same class of fault — footage is not being
|
|
6721
|
+
* written and only a periodic check will notice — so its recovery is put on the
|
|
6722
|
+
* same order rather than an order slower. Four times rarer than the beat that
|
|
6723
|
+
* blocked for 60 001 ms on 2026-08-18, while still bounding the worst case at
|
|
6724
|
+
* ~2 minutes of footage for a camera whose event path failed entirely.
|
|
6725
|
+
*
|
|
6726
|
+
* Going rarer than this trades real footage for CPU that the pass no longer
|
|
6727
|
+
* spends anyway: with the broker-readiness pre-check below, a reconcile over a
|
|
6728
|
+
* healthy fleet is N config reads and no RPC at all.
|
|
6729
|
+
*/
|
|
6730
|
+
var RECONCILE_TICK_MS = 12e4;
|
|
6435
6731
|
/**
|
|
6436
6732
|
* Bound on the broker lease-release RPC (`releaseStreamWithCodec`) during
|
|
6437
6733
|
* teardown. The UDS request default is 60s, but the runner supervisor's
|
|
@@ -6470,11 +6766,15 @@ var RELEASE_RPC_TIMEOUT_MS = 2e3;
|
|
|
6470
6766
|
*/
|
|
6471
6767
|
var ATTACH_RPC_TIMEOUT_MS = 2e4;
|
|
6472
6768
|
/**
|
|
6473
|
-
* A pass longer than this is reported with its duration.
|
|
6474
|
-
* cadence: a pass that cannot finish inside its own interval is
|
|
6475
|
-
* that used to go unlogged for half an hour.
|
|
6769
|
+
* A pass longer than this is reported with its duration. Each pass uses its OWN
|
|
6770
|
+
* cadence as the budget: a pass that cannot finish inside its own interval is
|
|
6771
|
+
* the condition that used to go unlogged for half an hour. Keeping it per-pass
|
|
6772
|
+
* matters now that the three cadences differ — a 40 s reconcile is healthy
|
|
6773
|
+
* against a 120 s beat and was a red flag against a 30 s one.
|
|
6476
6774
|
*/
|
|
6477
|
-
|
|
6775
|
+
function passSlowAfterMs(intervalMs) {
|
|
6776
|
+
return intervalMs;
|
|
6777
|
+
}
|
|
6478
6778
|
/**
|
|
6479
6779
|
* Race `promise` against a deadline. Handlers stay attached to the losing
|
|
6480
6780
|
* promise, so a post-deadline settlement can never surface as a process-level
|
|
@@ -6557,30 +6857,100 @@ var RecordingController = class {
|
|
|
6557
6857
|
*/
|
|
6558
6858
|
lastSegmentAt = /* @__PURE__ */ new Map();
|
|
6559
6859
|
/**
|
|
6860
|
+
* Per `deviceId:profile`, the `attachedAtMs` of the attach that has retained
|
|
6861
|
+
* at least one `events` segment. Read only to LEVEL the discard log: an
|
|
6862
|
+
* attach that kept nothing is a wasted pull (warn), a later discard from one
|
|
6863
|
+
* that kept something is the tail of the window (info). Bounded by
|
|
6864
|
+
* device×profile like {@link lastSegmentAt}, and cleared with it on detach.
|
|
6865
|
+
*/
|
|
6866
|
+
attachRetained = /* @__PURE__ */ new Map();
|
|
6867
|
+
/**
|
|
6560
6868
|
* Devices already named by {@link reportUnrecordable} — one warn per device
|
|
6561
6869
|
* per "enabled but can never record" episode, re-armed as soon as the device
|
|
6562
6870
|
* records again.
|
|
6563
6871
|
*/
|
|
6564
6872
|
unrecordableReported = /* @__PURE__ */ new Set();
|
|
6873
|
+
/**
|
|
6874
|
+
* Profiles the last attach skipped (stream not published / getStream threw).
|
|
6875
|
+
* `evaluateDevice` retries only these on the converge tick so a dead `low`
|
|
6876
|
+
* cannot keep `high` dark, without a `listAllProfileSlots` RPC on every
|
|
6877
|
+
* healthy device every 30s.
|
|
6878
|
+
*/
|
|
6879
|
+
skippedProfiles = /* @__PURE__ */ new Map();
|
|
6565
6880
|
restore = null;
|
|
6881
|
+
/**
|
|
6882
|
+
* One pending wake-up per device, armed for the exact instant its recording
|
|
6883
|
+
* decision could change on its own — an `events` window closing or a band
|
|
6884
|
+
* edge. THE reason a detach is punctual instead of up to a pass late.
|
|
6885
|
+
*/
|
|
6886
|
+
wakeups;
|
|
6566
6887
|
/** Cheap, never-delayed: is any pinned writer producing nothing? */
|
|
6567
6888
|
livenessPass = null;
|
|
6568
|
-
/**
|
|
6569
|
-
|
|
6889
|
+
/** Cheap, and off the attach beat: re-plan where each (device, profile) writes. */
|
|
6890
|
+
placementPass = null;
|
|
6891
|
+
/** The backstop: re-evaluate every tracked-or-configured device. */
|
|
6892
|
+
reconcilePass = null;
|
|
6570
6893
|
stopped = false;
|
|
6571
6894
|
/** A reversible maintenance lease. Unlike `stopped`, it never changes
|
|
6572
6895
|
* persisted recording intent and `resume()` restarts normal convergence. */
|
|
6573
6896
|
paused = false;
|
|
6574
6897
|
constructor(deps) {
|
|
6575
6898
|
this.deps = deps;
|
|
6899
|
+
this.wakeups = new DeviceWakeups({
|
|
6900
|
+
logger: deps.logger,
|
|
6901
|
+
now: () => this.now(),
|
|
6902
|
+
fire: (deviceId) => this.onWakeup(deviceId)
|
|
6903
|
+
});
|
|
6576
6904
|
}
|
|
6577
6905
|
now() {
|
|
6578
6906
|
return this.deps.now?.() ?? Date.now();
|
|
6579
6907
|
}
|
|
6580
6908
|
/**
|
|
6909
|
+
* A per-device wake-up came due: re-decide. `evaluateDevice` re-arms (or
|
|
6910
|
+
* cancels) the next one as part of deciding, so the chain is self-sustaining.
|
|
6911
|
+
*
|
|
6912
|
+
* A throw here breaks that chain — the device keeps whatever state it had and
|
|
6913
|
+
* has no timer behind it — so it is NAMED, and the reconcile is what picks it
|
|
6914
|
+
* up. Silence would read as "the window never closed".
|
|
6915
|
+
*/
|
|
6916
|
+
onWakeup(deviceId) {
|
|
6917
|
+
this.evaluateDevice(deviceId).catch((err) => {
|
|
6918
|
+
this.deps.logger.warn("recorder: scheduled re-evaluation failed — no wake-up re-armed (the reconcile is the backstop)", {
|
|
6919
|
+
tags: { deviceId },
|
|
6920
|
+
meta: { error: errMsg(err) }
|
|
6921
|
+
});
|
|
6922
|
+
});
|
|
6923
|
+
}
|
|
6924
|
+
/**
|
|
6925
|
+
* Arm (or cancel) this device's next wake-up from the config it was just
|
|
6926
|
+
* evaluated against.
|
|
6927
|
+
*
|
|
6928
|
+
* Called from `evaluateDevice` BEFORE the attach/detach work, not after: the
|
|
6929
|
+
* attach chain can throw (broker not ready, stream unpublished) and the
|
|
6930
|
+
* schedule does not depend on whether it succeeded. Arming first means a
|
|
6931
|
+
* failed attach still gets its band edge, and only a failure to READ the
|
|
6932
|
+
* config can lose a wake-up.
|
|
6933
|
+
*/
|
|
6934
|
+
scheduleNextDecision(deviceId, config) {
|
|
6935
|
+
if (this.stopped || this.paused) {
|
|
6936
|
+
this.wakeups.cancel(deviceId);
|
|
6937
|
+
return;
|
|
6938
|
+
}
|
|
6939
|
+
const next = nextDecisionChangeAt({
|
|
6940
|
+
config,
|
|
6941
|
+
triggers: this.triggersFor(deviceId),
|
|
6942
|
+
nowMs: this.now()
|
|
6943
|
+
});
|
|
6944
|
+
if (next === null) {
|
|
6945
|
+
this.wakeups.cancel(deviceId);
|
|
6946
|
+
return;
|
|
6947
|
+
}
|
|
6948
|
+
this.wakeups.arm(deviceId, next.atMs, next.reason);
|
|
6949
|
+
}
|
|
6950
|
+
/**
|
|
6581
6951
|
* Boot restore: seed the readiness-gated queue with every persisted device and
|
|
6582
6952
|
* (re)evaluate each on every `stream-broker` ready transition. Also arms the
|
|
6583
|
-
* periodic
|
|
6953
|
+
* periodic passes. Idempotent — safe to call once after the index is hydrated.
|
|
6584
6954
|
*/
|
|
6585
6955
|
async start() {
|
|
6586
6956
|
if (this.stopped) return;
|
|
@@ -6601,45 +6971,83 @@ var RecordingController = class {
|
|
|
6601
6971
|
meta: { error: errMsg(err) }
|
|
6602
6972
|
})
|
|
6603
6973
|
});
|
|
6604
|
-
if (this.
|
|
6974
|
+
if (this.reconcilePass === null) {
|
|
6605
6975
|
this.livenessPass = startPeriodicPass({
|
|
6606
6976
|
name: "liveness",
|
|
6607
|
-
intervalMs:
|
|
6608
|
-
slowAfterMs:
|
|
6977
|
+
intervalMs: LIVENESS_TICK_MS,
|
|
6978
|
+
slowAfterMs: passSlowAfterMs(LIVENESS_TICK_MS),
|
|
6609
6979
|
logger: this.deps.logger,
|
|
6610
6980
|
now: () => this.now(),
|
|
6611
6981
|
run: () => this.checkIdleWriters()
|
|
6612
6982
|
});
|
|
6613
|
-
this.
|
|
6614
|
-
name: "
|
|
6615
|
-
intervalMs:
|
|
6616
|
-
slowAfterMs:
|
|
6983
|
+
this.placementPass = startPeriodicPass({
|
|
6984
|
+
name: "placement",
|
|
6985
|
+
intervalMs: PLACEMENT_TICK_MS,
|
|
6986
|
+
slowAfterMs: passSlowAfterMs(PLACEMENT_TICK_MS),
|
|
6987
|
+
logger: this.deps.logger,
|
|
6988
|
+
now: () => this.now(),
|
|
6989
|
+
run: () => this.replanPlacement()
|
|
6990
|
+
});
|
|
6991
|
+
this.reconcilePass = startPeriodicPass({
|
|
6992
|
+
name: "reconcile",
|
|
6993
|
+
intervalMs: RECONCILE_TICK_MS,
|
|
6994
|
+
slowAfterMs: passSlowAfterMs(RECONCILE_TICK_MS),
|
|
6617
6995
|
logger: this.deps.logger,
|
|
6618
6996
|
now: () => this.now(),
|
|
6619
|
-
run: () => this.
|
|
6997
|
+
run: () => this.reconcile()
|
|
6620
6998
|
});
|
|
6621
6999
|
}
|
|
6622
7000
|
await this.restore.start(ids.map((id) => [id, true]));
|
|
6623
7001
|
}
|
|
6624
|
-
/**
|
|
6625
|
-
|
|
6626
|
-
|
|
7002
|
+
/**
|
|
7003
|
+
* Every device the recorder is responsible for right now: those with writers
|
|
7004
|
+
* attached, plus every one with a persisted config. A failure to enumerate the
|
|
7005
|
+
* persisted set degrades to "the active ones" rather than aborting the pass.
|
|
7006
|
+
*/
|
|
7007
|
+
async trackedDeviceIds() {
|
|
6627
7008
|
const ids = new Set(this.active.keys());
|
|
6628
|
-
let persisted = [];
|
|
6629
7009
|
try {
|
|
6630
|
-
|
|
6631
|
-
} catch {
|
|
6632
|
-
|
|
7010
|
+
for (const id of await this.deps.enabledDeviceIds()) ids.add(id);
|
|
7011
|
+
} catch (err) {
|
|
7012
|
+
this.deps.logger.warn("recorder controller: could not enumerate persisted devices — this pass covers the active ones only", { meta: {
|
|
7013
|
+
error: errMsg(err),
|
|
7014
|
+
activeDevices: this.active.size
|
|
7015
|
+
} });
|
|
7016
|
+
}
|
|
7017
|
+
return [...ids];
|
|
7018
|
+
}
|
|
7019
|
+
/**
|
|
7020
|
+
* Re-plan where each (device, profile) writes. Its OWN pass: cheap (no I/O
|
|
7021
|
+
* beyond a statfs per location), and it must not be starved by an attach that
|
|
7022
|
+
* is waiting on a 20 s broker timeout. It changes the PLAN only — a running
|
|
7023
|
+
* writer keeps its root until it next attaches, which is the boundary rule.
|
|
7024
|
+
*/
|
|
7025
|
+
async replanPlacement() {
|
|
7026
|
+
if (this.stopped || this.paused) return;
|
|
7027
|
+
if (this.deps.onPlacementTick === void 0) return;
|
|
7028
|
+
const ids = await this.trackedDeviceIds();
|
|
6633
7029
|
try {
|
|
6634
|
-
await this.deps.onPlacementTick
|
|
7030
|
+
await this.deps.onPlacementTick(ids);
|
|
6635
7031
|
} catch (err) {
|
|
6636
7032
|
this.deps.logger.warn("recorder controller: placement recompute failed", { meta: { error: errMsg(err) } });
|
|
6637
7033
|
}
|
|
6638
|
-
|
|
7034
|
+
}
|
|
7035
|
+
/**
|
|
7036
|
+
* The backstop pass: re-evaluate every tracked-or-configured device.
|
|
7037
|
+
*
|
|
7038
|
+
* Nothing here is the normal path any more — a window close, a band edge, a
|
|
7039
|
+
* config change and a broker recovery all reach `evaluateDevice` on their own.
|
|
7040
|
+
* This exists for the four things that cannot: a wake-up the event path threw
|
|
7041
|
+
* before arming, an attach that failed with no later readiness transition to
|
|
7042
|
+
* ride, a timer lost to a runner respawn, and clock drift.
|
|
7043
|
+
*/
|
|
7044
|
+
async reconcile() {
|
|
7045
|
+
if (this.stopped) return;
|
|
7046
|
+
await runBounded(await this.trackedDeviceIds(), 8, async (id) => {
|
|
6639
7047
|
try {
|
|
6640
7048
|
await this.evaluateDevice(id);
|
|
6641
7049
|
} catch (err) {
|
|
6642
|
-
this.deps.logger.warn("recorder controller:
|
|
7050
|
+
this.deps.logger.warn("recorder controller: reconcile evaluate failed", {
|
|
6643
7051
|
tags: { deviceId: id },
|
|
6644
7052
|
meta: { error: errMsg(err) }
|
|
6645
7053
|
});
|
|
@@ -6663,8 +7071,9 @@ var RecordingController = class {
|
|
|
6663
7071
|
* idle source can re-dial cleanly), then re-evaluate: if the band still demands
|
|
6664
7072
|
* recording, `evaluateDevice` re-queues + re-attaches a fresh writer set. If
|
|
6665
7073
|
* the broker is not ready, the re-attach throws and the device stays on the
|
|
6666
|
-
* readiness queue (retried on the next broker ready) — and the periodic
|
|
6667
|
-
* the backstop. Bounded OUTER retry (
|
|
7074
|
+
* readiness queue (retried on the next broker ready) — and the periodic
|
|
7075
|
+
* reconcile is the backstop. Bounded OUTER retry (RECONCILE_TICK_MS cadence),
|
|
7076
|
+
* never a tight loop.
|
|
6668
7077
|
*/
|
|
6669
7078
|
async recoverWriterGaveUp(deviceId) {
|
|
6670
7079
|
if (this.stopped) return;
|
|
@@ -6674,7 +7083,7 @@ var RecordingController = class {
|
|
|
6674
7083
|
await this.evaluateDevice(deviceId);
|
|
6675
7084
|
} catch (err) {
|
|
6676
7085
|
this.restore?.add(deviceId, true);
|
|
6677
|
-
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/
|
|
7086
|
+
this.deps.logger.warn("recorder: writer give-up recovery deferred (retries on next broker ready/reconcile)", {
|
|
6678
7087
|
tags: { deviceId },
|
|
6679
7088
|
meta: { error: errMsg(err) }
|
|
6680
7089
|
});
|
|
@@ -6725,10 +7134,16 @@ var RecordingController = class {
|
|
|
6725
7134
|
}
|
|
6726
7135
|
/**
|
|
6727
7136
|
* Record a qualifying trigger (motion/audio) for a device and converge: an
|
|
6728
|
-
* `events` band whose window this opens attaches now
|
|
6729
|
-
*
|
|
6730
|
-
*
|
|
6731
|
-
*
|
|
7137
|
+
* `events` band whose window this opens attaches now, and the same
|
|
7138
|
+
* `evaluateDevice` arms the wake-up that will CLOSE it at
|
|
7139
|
+
* `atMs + postBufferSec`. Called from the addon's EventCapture subscription.
|
|
7140
|
+
* The qualification test (motion detected / audio over threshold) is done
|
|
7141
|
+
* upstream — by the time we're here, it qualified.
|
|
7142
|
+
*
|
|
7143
|
+
* A fresher trigger re-arms the wake-up later, which is exactly how the
|
|
7144
|
+
* sliding window already behaved: continuous motion stays ONE recording,
|
|
7145
|
+
* because `evaluateDevice` short-circuits at "already recording — converged"
|
|
7146
|
+
* and only the timer moves.
|
|
6732
7147
|
*/
|
|
6733
7148
|
async onTrigger(deviceId, source, atMs) {
|
|
6734
7149
|
if (this.stopped) return;
|
|
@@ -6740,7 +7155,7 @@ var RecordingController = class {
|
|
|
6740
7155
|
try {
|
|
6741
7156
|
await this.evaluateDevice(deviceId);
|
|
6742
7157
|
} catch (err) {
|
|
6743
|
-
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/
|
|
7158
|
+
this.deps.logger.warn("recorder: trigger attach deferred — stream not available yet (retries on next broker ready/reconcile)", {
|
|
6744
7159
|
tags: { deviceId },
|
|
6745
7160
|
meta: {
|
|
6746
7161
|
source,
|
|
@@ -6761,14 +7176,19 @@ var RecordingController = class {
|
|
|
6761
7176
|
if (this.stopped || this.paused) return;
|
|
6762
7177
|
const config = await this.deps.loadConfig(deviceId);
|
|
6763
7178
|
if (this.stopped || this.paused) return;
|
|
6764
|
-
|
|
7179
|
+
const wantRecording = shouldRecordAt(config, this.triggersFor(deviceId), new Date(this.now()));
|
|
7180
|
+
this.scheduleNextDecision(deviceId, config);
|
|
7181
|
+
if (!wantRecording) {
|
|
6765
7182
|
this.reportUnrecordable(deviceId, config);
|
|
6766
7183
|
await this.detachDevice(deviceId);
|
|
6767
7184
|
this.restore?.remove(deviceId);
|
|
6768
7185
|
return;
|
|
6769
7186
|
}
|
|
6770
7187
|
this.unrecordableReported.delete(deviceId);
|
|
6771
|
-
if (this.active.has(deviceId))
|
|
7188
|
+
if (this.active.has(deviceId)) {
|
|
7189
|
+
if (this.skippedProfiles.has(deviceId)) await this.fillMissingProfiles(deviceId, config);
|
|
7190
|
+
return;
|
|
7191
|
+
}
|
|
6772
7192
|
if (this.attaching.has(deviceId)) return;
|
|
6773
7193
|
this.attaching.add(deviceId);
|
|
6774
7194
|
const attaching = this.performAttach(deviceId, config);
|
|
@@ -6817,49 +7237,141 @@ var RecordingController = class {
|
|
|
6817
7237
|
});
|
|
6818
7238
|
}
|
|
6819
7239
|
/**
|
|
7240
|
+
* Throw before spending a single broker RPC when the registry already says the
|
|
7241
|
+
* `stream-broker` on the owner node is not ready. See
|
|
7242
|
+
* {@link RecordingControllerDeps.brokerReady}; absent → assume ready.
|
|
7243
|
+
*/
|
|
7244
|
+
failFastIfBrokerCold(deviceId) {
|
|
7245
|
+
if (this.deps.brokerReady?.() !== false) return;
|
|
7246
|
+
throw new Error(`recorder controller: stream-broker not ready on ${this.deps.ownerNodeId} — attach for device ${deviceId} deferred to the next ready transition`);
|
|
7247
|
+
}
|
|
7248
|
+
/**
|
|
6820
7249
|
* The attach body — extracted so the {@link attaching} in-flight guard in
|
|
6821
7250
|
* `evaluateDevice` wraps it cleanly. Resolves the profiles, spawns a
|
|
6822
|
-
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
6823
|
-
*
|
|
7251
|
+
* SegmentWriter + watcher per profile, and records the set in `active`.
|
|
7252
|
+
*
|
|
7253
|
+
* Per-profile: a throw from one slot is logged and skipped. Rolling back the
|
|
7254
|
+
* whole device when a single stream was dead (618 `derived` on `low`) left
|
|
7255
|
+
* ZERO footage for hours. If every profile fails, throw so the restore queue
|
|
7256
|
+
* retries the device.
|
|
6824
7257
|
*/
|
|
6825
7258
|
async performAttach(deviceId, config) {
|
|
6826
7259
|
this.restore?.add(deviceId, true);
|
|
7260
|
+
this.failFastIfBrokerCold(deviceId);
|
|
6827
7261
|
const profiles = await this.resolveProfiles(deviceId, config.profiles);
|
|
6828
7262
|
if (profiles.length === 0) throw new Error(`recorder controller: no assigned broker sources for device ${deviceId}`);
|
|
6829
7263
|
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
6830
7264
|
const recordings = [];
|
|
6831
|
-
|
|
6832
|
-
|
|
7265
|
+
const failed = [];
|
|
7266
|
+
for (const profile of profiles) try {
|
|
7267
|
+
recordings.push(await this.attachProfile(deviceId, profile, segmentSeconds));
|
|
6833
7268
|
} catch (err) {
|
|
6834
|
-
|
|
6835
|
-
|
|
7269
|
+
failed.push(profile);
|
|
7270
|
+
this.deps.logger.warn("recorder: profile attach failed — recording the remaining profiles", {
|
|
7271
|
+
tags: { deviceId },
|
|
7272
|
+
meta: {
|
|
7273
|
+
profile,
|
|
7274
|
+
error: errMsg(err)
|
|
7275
|
+
}
|
|
7276
|
+
});
|
|
6836
7277
|
}
|
|
7278
|
+
if (recordings.length === 0) throw new Error(`recorder controller: no profile attached for device ${deviceId}` + (failed.length > 0 ? ` (failed: ${failed.join(",")})` : ""));
|
|
6837
7279
|
if (this.paused) {
|
|
6838
7280
|
await Promise.all(recordings.map((recording) => this.teardownProfile(deviceId, recording)));
|
|
6839
7281
|
return;
|
|
6840
7282
|
}
|
|
6841
7283
|
this.active.set(deviceId, recordings);
|
|
7284
|
+
if (failed.length > 0) this.skippedProfiles.set(deviceId, new Set(failed));
|
|
7285
|
+
else this.skippedProfiles.delete(deviceId);
|
|
6842
7286
|
this.deps.logger.info("recorder pinned broker source(s) for recording", {
|
|
6843
7287
|
tags: { deviceId },
|
|
6844
7288
|
meta: {
|
|
6845
|
-
profiles,
|
|
7289
|
+
profiles: recordings.map((r) => r.profile),
|
|
7290
|
+
failedProfiles: failed,
|
|
6846
7291
|
segmentSeconds,
|
|
6847
7292
|
pipelineKeys: recordings.map((r) => r.pipelineKey)
|
|
6848
7293
|
}
|
|
6849
7294
|
});
|
|
6850
7295
|
}
|
|
6851
7296
|
/**
|
|
7297
|
+
* Retry profiles the last attach skipped, without tearing down writers that
|
|
7298
|
+
* are already producing. Called from `evaluateDevice` when the device is
|
|
7299
|
+
* already in `active` and {@link skippedProfiles} is non-empty. Failures stay
|
|
7300
|
+
* logged; the next converge tick retries.
|
|
7301
|
+
*/
|
|
7302
|
+
async fillMissingProfiles(deviceId, config) {
|
|
7303
|
+
if (this.attaching.has(deviceId)) return;
|
|
7304
|
+
const current = this.active.get(deviceId);
|
|
7305
|
+
const skipped = this.skippedProfiles.get(deviceId);
|
|
7306
|
+
if (!current || !skipped || skipped.size === 0) return;
|
|
7307
|
+
this.attaching.add(deviceId);
|
|
7308
|
+
const filling = this.runFillMissingProfiles(deviceId, config, current, skipped);
|
|
7309
|
+
this.attachingTasks.set(deviceId, filling);
|
|
7310
|
+
try {
|
|
7311
|
+
await filling;
|
|
7312
|
+
} finally {
|
|
7313
|
+
this.attachingTasks.delete(deviceId);
|
|
7314
|
+
this.attaching.delete(deviceId);
|
|
7315
|
+
}
|
|
7316
|
+
}
|
|
7317
|
+
async runFillMissingProfiles(deviceId, config, current, skipped) {
|
|
7318
|
+
if (this.deps.brokerReady?.() === false) {
|
|
7319
|
+
this.deps.logger.info("recorder: skipped-profile retry deferred — stream-broker not ready (healthy profiles keep recording)", {
|
|
7320
|
+
tags: { deviceId },
|
|
7321
|
+
meta: {
|
|
7322
|
+
profiles: [...skipped],
|
|
7323
|
+
ownerNodeId: this.deps.ownerNodeId
|
|
7324
|
+
}
|
|
7325
|
+
});
|
|
7326
|
+
return;
|
|
7327
|
+
}
|
|
7328
|
+
const have = new Set(current.map((r) => r.profile));
|
|
7329
|
+
const segmentSeconds = config.segmentSeconds ?? this.deps.segmentSeconds;
|
|
7330
|
+
const added = [];
|
|
7331
|
+
for (const profile of [...skipped]) {
|
|
7332
|
+
if (have.has(profile)) {
|
|
7333
|
+
skipped.delete(profile);
|
|
7334
|
+
continue;
|
|
7335
|
+
}
|
|
7336
|
+
try {
|
|
7337
|
+
added.push(await this.attachProfile(deviceId, profile, segmentSeconds));
|
|
7338
|
+
skipped.delete(profile);
|
|
7339
|
+
} catch (err) {
|
|
7340
|
+
this.deps.logger.warn("recorder: profile attach retry failed — keeping the healthy profiles", {
|
|
7341
|
+
tags: { deviceId },
|
|
7342
|
+
meta: {
|
|
7343
|
+
profile,
|
|
7344
|
+
error: errMsg(err)
|
|
7345
|
+
}
|
|
7346
|
+
});
|
|
7347
|
+
}
|
|
7348
|
+
}
|
|
7349
|
+
if (skipped.size === 0) this.skippedProfiles.delete(deviceId);
|
|
7350
|
+
if (added.length === 0) return;
|
|
7351
|
+
const existing = this.active.get(deviceId);
|
|
7352
|
+
if (!existing || this.paused || this.stopped) {
|
|
7353
|
+
await Promise.all(added.map((recording) => this.teardownProfile(deviceId, recording)));
|
|
7354
|
+
return;
|
|
7355
|
+
}
|
|
7356
|
+
this.active.set(deviceId, [...existing, ...added]);
|
|
7357
|
+
this.deps.logger.info("recorder pinned additional broker source(s) for recording", {
|
|
7358
|
+
tags: { deviceId },
|
|
7359
|
+
meta: {
|
|
7360
|
+
profiles: added.map((r) => r.profile),
|
|
7361
|
+
pipelineKeys: added.map((r) => r.pipelineKey)
|
|
7362
|
+
}
|
|
7363
|
+
});
|
|
7364
|
+
}
|
|
7365
|
+
/**
|
|
6852
7366
|
* Pick which profiles to record. The broker enumerates assigned profile slots,
|
|
6853
7367
|
* deduped by physical source (`selectAssignedProfileSlots`) so the same camera
|
|
6854
|
-
* encoder is never recorded twice.
|
|
6855
|
-
*
|
|
6856
|
-
*
|
|
7368
|
+
* encoder is never recorded twice. Default is high+low among those slots
|
|
7369
|
+
* (`resolveRecordingProfiles`); an operator `override` DISABLES the
|
|
7370
|
+
* unselected profiles. A selection matching none of the assigned sources
|
|
7371
|
+
* falls back to the default/assigned set (minimum of 1 when anything is assigned).
|
|
6857
7372
|
*/
|
|
6858
7373
|
async resolveProfiles(deviceId, override) {
|
|
6859
|
-
|
|
6860
|
-
if (!override || override.length === 0) return assigned;
|
|
6861
|
-
const selected = assigned.filter((p) => override.includes(p));
|
|
6862
|
-
return selected.length > 0 ? selected : assigned;
|
|
7374
|
+
return resolveRecordingProfiles(selectAssignedProfileSlots(await this.deps.brokerCall(() => withDeadline(this.deps.api.streamBroker.listAllProfileSlots.query(void 0, nodePin(this.deps.ownerNodeId)), ATTACH_RPC_TIMEOUT_MS, "listAllProfileSlots")), deviceId).map((slot) => slot.profile), override);
|
|
6863
7375
|
}
|
|
6864
7376
|
/**
|
|
6865
7377
|
* Attach one (device, profile): acquire the broker source, mkdir the staging
|
|
@@ -6905,7 +7417,10 @@ var RecordingController = class {
|
|
|
6905
7417
|
intervalMs: this.deps.watchIntervalMs > 0 ? this.deps.watchIntervalMs : DEFAULT_WATCH_INTERVAL_MS,
|
|
6906
7418
|
onFinalized: async (startMs, durMs, flatAbsPath) => {
|
|
6907
7419
|
this.lastSegmentAt.set(idleKey(deviceId, profile), this.now());
|
|
6908
|
-
await this.handleFinalizedSegment(deviceId, profile, placement,
|
|
7420
|
+
await this.handleFinalizedSegment(deviceId, profile, placement, {
|
|
7421
|
+
attachedAtMs: writerStartedMs,
|
|
7422
|
+
segmentSeconds
|
|
7423
|
+
}, startMs, durMs, flatAbsPath);
|
|
6909
7424
|
},
|
|
6910
7425
|
logger: deviceLog
|
|
6911
7426
|
}),
|
|
@@ -6922,13 +7437,16 @@ var RecordingController = class {
|
|
|
6922
7437
|
* keep/discard gate touches `events`-mode segments only, so the continuous
|
|
6923
7438
|
* path is unchanged.
|
|
6924
7439
|
*/
|
|
6925
|
-
async handleFinalizedSegment(deviceId, profile, placement, startMs, durMs, flatAbsPath) {
|
|
7440
|
+
async handleFinalizedSegment(deviceId, profile, placement, attach, startMs, durMs, flatAbsPath) {
|
|
6926
7441
|
try {
|
|
6927
7442
|
const band = activeBandAt({ bands: (await this.deps.loadConfig(deviceId)).bands ?? [] }, new Date(startMs + durMs));
|
|
6928
|
-
|
|
7443
|
+
const triggers = this.triggersFor(deviceId);
|
|
7444
|
+
if (band?.mode === "events" && !segmentRetainedForEvents(startMs, startMs + durMs, band, triggers, attach)) {
|
|
7445
|
+
this.reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs);
|
|
6929
7446
|
await promises.unlink(flatAbsPath).catch(() => {});
|
|
6930
7447
|
return;
|
|
6931
7448
|
}
|
|
7449
|
+
if (band?.mode === "events") this.attachRetained.set(idleKey(deviceId, profile), attach.attachedAtMs);
|
|
6932
7450
|
await this.deps.segmentStore.onFinalized({
|
|
6933
7451
|
deviceId,
|
|
6934
7452
|
profile,
|
|
@@ -6949,6 +7467,47 @@ var RecordingController = class {
|
|
|
6949
7467
|
}
|
|
6950
7468
|
}
|
|
6951
7469
|
/**
|
|
7470
|
+
* Name a discarded `events`-band segment. This branch deletes recorded
|
|
7471
|
+
* footage and used to write NOTHING: on camera 1439 the operator saw `pinned`
|
|
7472
|
+
* then `unpinned` with an empty timeline in between and no line anywhere said
|
|
7473
|
+
* why — the recorder had spawned ffmpeg, pulled 4K RTSP for 31 s and unlinked
|
|
7474
|
+
* every segment it produced.
|
|
7475
|
+
*
|
|
7476
|
+
* Level is chosen, not defaulted. A discard at the TAIL of a window is the
|
|
7477
|
+
* feature working (`info`) — the writer keeps running until the next converge
|
|
7478
|
+
* tick and those trailing segments were never asked for. A discard from an
|
|
7479
|
+
* attach that has retained NOTHING is a fault (`warn`): the whole pull was
|
|
7480
|
+
* wasted, which is exactly the shape of the 1439 loss and of any future
|
|
7481
|
+
* regression in the attach allowance.
|
|
7482
|
+
*/
|
|
7483
|
+
reportDiscardedSegment(deviceId, profile, attach, band, triggers, startMs, durMs) {
|
|
7484
|
+
const { preMs, postMs } = resolveBandBufferMs(band);
|
|
7485
|
+
const keptAnything = this.attachRetained.get(idleKey(deviceId, profile)) === attach.attachedAtMs;
|
|
7486
|
+
const meta = {
|
|
7487
|
+
profile,
|
|
7488
|
+
startMs,
|
|
7489
|
+
durMs,
|
|
7490
|
+
lastMotionMs: triggers.lastMotionMs,
|
|
7491
|
+
lastAudioMs: triggers.lastAudioMs,
|
|
7492
|
+
postMs,
|
|
7493
|
+
preMs,
|
|
7494
|
+
attachedAtMs: attach.attachedAtMs,
|
|
7495
|
+
missedByMs: segmentMissedByMs(startMs, band, triggers, attach),
|
|
7496
|
+
keptAnythingFromThisAttach: keptAnything
|
|
7497
|
+
};
|
|
7498
|
+
if (keptAnything) {
|
|
7499
|
+
this.deps.logger.info("recorder: discarding events segment outside every trigger window (tail of the window)", {
|
|
7500
|
+
tags: { deviceId },
|
|
7501
|
+
meta
|
|
7502
|
+
});
|
|
7503
|
+
return;
|
|
7504
|
+
}
|
|
7505
|
+
this.deps.logger.warn("recorder: discarding events segment — this attach has retained NOTHING, the whole pull is wasted", {
|
|
7506
|
+
tags: { deviceId },
|
|
7507
|
+
meta
|
|
7508
|
+
});
|
|
7509
|
+
}
|
|
7510
|
+
/**
|
|
6952
7511
|
* Resolve the storage location a (device, profile) writes to, at an attach
|
|
6953
7512
|
* boundary. Delegates to the injected {@link RecordingControllerDeps.placeProfile}
|
|
6954
7513
|
* when the addon supplied one; otherwise to the pure `resolvePlacement` in
|
|
@@ -6989,7 +7548,11 @@ var RecordingController = class {
|
|
|
6989
7548
|
const recordings = this.active.get(deviceId);
|
|
6990
7549
|
if (!recordings) return;
|
|
6991
7550
|
this.active.delete(deviceId);
|
|
6992
|
-
|
|
7551
|
+
this.skippedProfiles.delete(deviceId);
|
|
7552
|
+
for (const r of recordings) {
|
|
7553
|
+
this.lastSegmentAt.delete(idleKey(deviceId, r.profile));
|
|
7554
|
+
this.attachRetained.delete(idleKey(deviceId, r.profile));
|
|
7555
|
+
}
|
|
6993
7556
|
await Promise.all(recordings.map((r) => this.teardownProfile(deviceId, r)));
|
|
6994
7557
|
this.deps.logger.info("recorder unpinned broker source(s) — recording stopped", {
|
|
6995
7558
|
tags: { deviceId },
|
|
@@ -7000,6 +7563,7 @@ var RecordingController = class {
|
|
|
7000
7563
|
async pause() {
|
|
7001
7564
|
if (this.stopped || this.paused) return;
|
|
7002
7565
|
this.paused = true;
|
|
7566
|
+
this.wakeups.cancelAll();
|
|
7003
7567
|
while (this.attachingTasks.size > 0) await Promise.allSettled([...this.attachingTasks.values()]);
|
|
7004
7568
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
7005
7569
|
}
|
|
@@ -7015,13 +7579,16 @@ var RecordingController = class {
|
|
|
7015
7579
|
}
|
|
7016
7580
|
for (const deviceId of ids) await this.evaluateDevice(deviceId);
|
|
7017
7581
|
}
|
|
7018
|
-
/** Tear EVERYTHING down:
|
|
7582
|
+
/** Tear EVERYTHING down: timers, restore subscription, every writer/watcher/lease. */
|
|
7019
7583
|
async stop() {
|
|
7020
7584
|
this.stopped = true;
|
|
7585
|
+
this.wakeups.cancelAll();
|
|
7021
7586
|
this.livenessPass?.stop();
|
|
7022
7587
|
this.livenessPass = null;
|
|
7023
|
-
this.
|
|
7024
|
-
this.
|
|
7588
|
+
this.placementPass?.stop();
|
|
7589
|
+
this.placementPass = null;
|
|
7590
|
+
this.reconcilePass?.stop();
|
|
7591
|
+
this.reconcilePass = null;
|
|
7025
7592
|
this.restore?.stop();
|
|
7026
7593
|
this.restore = null;
|
|
7027
7594
|
await Promise.all(Array.from(this.active.keys()).map((deviceId) => this.detachDevice(deviceId)));
|
|
@@ -8555,7 +9122,8 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8555
9122
|
onBrokerReady: (handler) => this.ctx.onCapabilityStateChange("stream-broker", brokerScope(ingestOwner.ownerNodeId), (state) => {
|
|
8556
9123
|
if (state === "ready") handler();
|
|
8557
9124
|
}),
|
|
8558
|
-
brokerCall: makeBrokerCall(brokerHandle)
|
|
9125
|
+
brokerCall: makeBrokerCall(brokerHandle),
|
|
9126
|
+
brokerReady: () => brokerHandle.isReady
|
|
8559
9127
|
});
|
|
8560
9128
|
const relocateEngine = new RelocateEngine({
|
|
8561
9129
|
index: this.index,
|
|
@@ -8605,6 +9173,9 @@ var RecorderV2Addon = class extends BaseAddon {
|
|
|
8605
9173
|
refreshLocations: () => this.refreshLocations(),
|
|
8606
9174
|
capacity: (root) => this.locationCapacity(root),
|
|
8607
9175
|
hydrateDevice: (deviceId, locations) => hydrateDeviceFromStorage(this.ctx.api, this.index, deviceId, locations, this.ctx.logger),
|
|
9176
|
+
reconcileHourLedger: async (deviceId) => {
|
|
9177
|
+
await this.segmentHours?.reconcileFromIndex(this.index, [deviceId], Date.now());
|
|
9178
|
+
},
|
|
8608
9179
|
calendar: this.calendar,
|
|
8609
9180
|
mfraFor: (deviceId, profile, startMs) => this.mfraTables.get(deviceId, profile, startMs),
|
|
8610
9181
|
hydrateWindow: (deviceId, fromMs, toMs) => hydrateWindowFromStorage(this.index, deviceId, fromMs, toMs, this.resolvedLocations, this.ctx.logger),
|