@camstack/addon-pipeline 1.2.135 → 1.2.137

package/dist/recorder/index.js

@@ -1,13 +1,14 @@
-const require_dist = require("../dist-BzLiQ8Ue.js");
+const require_dist = require("../dist-4EMKvxCp.js");
+const require_node = require("../node-BOHn8gpk.js");
 const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
 const require_restream_intent = require("../restream-intent-Cv9x3jmu.js");
-const require_addon_utils = require("../addon-utils-CKf8kbwh.js");
+const require_addon_utils = require("../addon-utils-C6b5Tnia.js");
 const require_retire_root_keys = require("../retire-root-keys-KE6D6Xh_.js");
 let node_crypto = require("node:crypto");
-let node_child_process = require("node:child_process");
+let node_fs = require("node:fs");
 let node_path = require("node:path");
 node_path = require_dist.__toESM(node_path);
-let node_fs = require("node:fs");
+let node_child_process = require("node:child_process");
 let node_fs_promises = require("node:fs/promises");
 //#region src/recorder/event-capture.ts
 function markerFromMotion(data, pad) {
@@ -6225,6 +6226,7 @@ Object.freeze({
 		"listProfiles",
 		"listRuntimeNodes"
 	],
+	"load-contribution": ["list"],
 	"log-channels": ["list"],
 	"log-destination": ["query"],
 	"login-method": ["getLoginMethods"],
@@ -8734,6 +8736,38 @@ var RESTART_MAX_MS = 1e4;
 /** A process that ran at least this long is "stable" → reset the restart count,
 *  so sporadic blips over a long lifetime never accumulate into a give-up. */
 var STABLE_RUN_MS = 3e4;
+/**
+* The window the SECOND budget counts over — and it counts EVERY restart,
+* whatever that run lasted.
+*
+* `STABLE_RUN_MS` above is a reset, and a reset is a blind spot: measured on
+* 2026-08-27, `SegmentWriter restarting` fired **1 814 times in 12 h** and
+* almost every `ranMs` was over 30 s (122 620 / 165 255 / 345 420 / 394 522 ms
+* are four consecutive real samples), so `attempt` never left 1, `onGaveUp`
+* never fired, and `recoverWriterGaveUp` never ran. The storm could have lasted
+* forever without a single line saying anything was wrong.
+*
+* A window cannot be zeroed by a long run, which is the whole point. Modelled
+* on `CrashSupervisor` (kernel, D6): timestamps age out of `windowMs`, and the
+* count inside it is the budget.
+*/
+var RESTART_WINDOW_MS = 60 * 6e4;
+/**
+* Restarts inside {@link RESTART_WINDOW_MS} before the writer stops respawning
+* and escalates to the controller.
+*
+* 15/hour, from the measured distribution. Per writer over the same 12 h:
+* 1441/high 549 (**45.8/h**), 615/high 241 (20.1/h), 618/high 217 (18.1/h),
+* 592/high 211 (17.6/h), 590/high 195 (16.3/h) — every one of them a storm and
+* every one of them silent — against `low` profiles at 4–15 per 12 h (≤1.3/h),
+* which is what a healthy writer looks like. 15 separates them with room for a
+* camera that blips hourly and is otherwise fine.
+*
+* It never shadows {@link MAX_RESTARTS}: 10 CONSECUTIVE rapid failures happen
+* inside ~90 s, so the rapid breaker still trips first on a hard failure. This
+* one exists for the slow storm the rapid breaker structurally cannot see.
+*/
+var MAX_RESTARTS_IN_WINDOW = 15;
 /** Grace between the SIGTERM in stop() and the escalated SIGKILL. */
 var KILL_GRACE_MS = 500;
 /** Supervises one passthrough ffmpeg for a single (camera, profile). */
@@ -8743,10 +8777,22 @@ var SegmentWriter = class {
 	proc = null;
 	stopped = false;
 	restarts = 0;
+	/**
+	* Epoch-ms of every termination still inside {@link RESTART_WINDOW_MS}.
+	* Replaced, never mutated in place, so a reader can never observe a
+	* half-pruned window.
+	*/
+	restartWindow = [];
 	restartTimer = null;
 	startedAt = 0;
 	exitWait = null;
 	resolveExit = null;
+	/**
+	* The cost claim for the CURRENT ffmpeg generation. Taken in `start()` and
+	* released in `onTermination()` — the two edges of one process, so a restart
+	* storm cannot accumulate claims for children that are already gone.
+	*/
+	costClaim = require_node.NO_COST_CLAIM;
 	constructor(cfg, deps) {
 		this.cfg = cfg;
 		this.deps = deps;
@@ -8760,6 +8806,7 @@ var SegmentWriter = class {
 			this.resolveExit = resolve;
 		});
 		this.startedAt = Date.now();
+		this.costClaim = this.deps.claimCost?.(proc.pid) ?? require_node.NO_COST_CLAIM;
 		const stderrTail = [];
 		proc.stderr?.on("data", (chunk) => {
 			const text = chunk instanceof Buffer ? chunk.toString("utf8") : String(chunk);
@@ -8776,10 +8823,11 @@ var SegmentWriter = class {
 			settled = true;
 			this.onTermination(termination, stderrTail);
 		};
-		proc.on("exit", (code) => {
+		proc.on("exit", (code, signal) => {
 			terminate({
 				reason: "exit",
-				exitCode: typeof code === "number" ? code : null
+				exitCode: typeof code === "number" ? code : null,
+				signal: typeof signal === "string" ? signal : null
 			});
 		});
 		proc.on("error", (raw) => {
@@ -8795,32 +8843,53 @@ var SegmentWriter = class {
 	/**
 	* The ONE restart policy, reached from both `exit` and `error`.
 	*
-	* `deps.logger` is the device-scoped child the controller binds
-	* (`logger.withTags({ deviceId })`), so every line below carries
-	* `tags: { deviceId }` — "why is 617 worse than 615" is always asked per
-	* camera, and a writer that stops recording one camera must be greppable by
-	* that camera.
+	* Every line below carries `tags: { deviceId }` from `deps.deviceId` — "why
+	* is 617 worse than 615" is always asked per camera, and a writer that stops
+	* recording one camera must be greppable by that camera.
 	*/
 	onTermination(termination, stderrTail) {
+		this.costClaim.release();
+		this.costClaim = require_node.NO_COST_CLAIM;
 		this.resolveExit?.();
 		this.resolveExit = null;
 		this.proc = null;
 		if (this.stopped) return;
-		const ranMs = Date.now() - this.startedAt;
+		const now = Date.now();
+		const ranMs = now - this.startedAt;
 		if (ranMs >= STABLE_RUN_MS) this.restarts = 0;
-		if (termination.reason === "spawn-error") {
-			this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", { meta: {
+		this.restartWindow = [...this.restartWindow.filter((at) => at > now - RESTART_WINDOW_MS), now];
+		const restartsInWindow = this.restartWindow.length;
+		if (termination.reason === "exit") this.deps.logger.warn("SegmentWriter ffmpeg exited", {
+			tags: { deviceId: this.deps.deviceId },
+			meta: {
 				outDir: this.cfg.outDir,
-				errorCode: termination.errorCode,
-				error: termination.message,
-				permanent: termination.permanent
-			} });
-			if (termination.permanent) {
-				this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", { meta: {
+				exitCode: termination.exitCode,
+				signal: termination.signal,
+				ranMs,
+				restartsInWindow,
+				windowMs: RESTART_WINDOW_MS,
+				stderrTail: stderrTail.join(" | ")
+			}
+		});
+		if (termination.reason === "spawn-error") {
+			this.deps.logger.warn("SegmentWriter ffmpeg spawn failed", {
+				tags: { deviceId: this.deps.deviceId },
+				meta: {
 					outDir: this.cfg.outDir,
 					errorCode: termination.errorCode,
-					error: termination.message
-				} });
+					error: termination.message,
+					permanent: termination.permanent
+				}
+			});
+			if (termination.permanent) {
+				this.deps.logger.error("SegmentWriter giving up: ffmpeg cannot be executed on this node", {
+					tags: { deviceId: this.deps.deviceId },
+					meta: {
+						outDir: this.cfg.outDir,
+						errorCode: termination.errorCode,
+						error: termination.message
+					}
+				});
 				this.stopped = true;
 				this.deps.onGaveUp?.();
 				return;
@@ -8828,25 +8897,51 @@ var SegmentWriter = class {
 			this.deps.onResourcePressure?.();
 		}
 		if (this.restarts >= MAX_RESTARTS) {
-			this.deps.logger.warn("SegmentWriter giving up after max restarts", { meta: {
-				outDir: this.cfg.outDir,
-				reason: termination.reason,
-				code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
-				stderrTail: stderrTail.join(" | ")
-			} });
+			this.deps.logger.warn("SegmentWriter giving up after max restarts", {
+				tags: { deviceId: this.deps.deviceId },
+				meta: {
+					outDir: this.cfg.outDir,
+					reason: termination.reason,
+					code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
+					stderrTail: stderrTail.join(" | ")
+				}
+			});
+			this.stopped = true;
+			this.deps.onGaveUp?.();
+			return;
+		}
+		if (restartsInWindow >= MAX_RESTARTS_IN_WINDOW) {
+			this.deps.logger.warn("SegmentWriter giving up: restart storm", {
+				tags: { deviceId: this.deps.deviceId },
+				meta: {
+					outDir: this.cfg.outDir,
+					restartsInWindow,
+					windowMs: RESTART_WINDOW_MS,
+					budget: MAX_RESTARTS_IN_WINDOW,
+					attempt: this.restarts,
+					ranMs,
+					reason: termination.reason,
+					code: termination.reason === "exit" ? termination.exitCode : termination.errorCode,
+					stderrTail: stderrTail.join(" | ")
+				}
+			});
 			this.stopped = true;
 			this.deps.onGaveUp?.();
 			return;
 		}
 		this.restarts++;
 		const delayMs = Math.min(RESTART_BASE_MS * 2 ** (this.restarts - 1), RESTART_MAX_MS);
-		this.deps.logger.info("SegmentWriter restarting", { meta: {
-			outDir: this.cfg.outDir,
-			attempt: this.restarts,
-			delayMs,
-			ranMs,
-			reason: termination.reason
-		} });
+		this.deps.logger.info("SegmentWriter restarting", {
+			tags: { deviceId: this.deps.deviceId },
+			meta: {
+				outDir: this.cfg.outDir,
+				attempt: this.restarts,
+				restartsInWindow,
+				delayMs,
+				ranMs,
+				reason: termination.reason
+			}
+		});
 		this.restartTimer = setTimeout(() => {
 			this.restartTimer = null;
 			this.start();
@@ -10110,6 +10205,13 @@ var RecordingController = class {
 		}, {
 			spawn: this.deps.spawn,
 			logger: deviceLog,
+			deviceId,
+			claimCost: (pid) => this.deps.costRegistry?.claim({
+				role: "recording",
+				deviceId,
+				unit: `${deviceId}/${profile}`,
+				pid
+			}) ?? require_node.NO_COST_CLAIM,
 			onGaveUp: () => {
 				this.recoverWriterGaveUp(deviceId, profile);
 			},
@@ -10533,6 +10635,209 @@ function buildRecordingExportProvider(deps) {
 */
 var resolveRecordingHubHostname = require_hub_hostname.resolveHubHostname;
 //#endregion
+//#region src/recorder/redundant-segments.ts
+/**
+* Return the indices (into `segs`) of segments safe to delete: those fully
+* contained within a kept segment `[coverStart, coverEnd]`. Sorting by start
+* ascending, then by end DESCENDING, guarantees the containing (longer) segment
+* is seen first and kept, and any shorter segment nested inside it is removed.
+*/
+function selectRedundantSegments(segs) {
+	if (segs.length < 2) return [];
+	const order = segs.map((s, i) => ({
+		start: s.startMs,
+		end: s.startMs + s.durMs,
+		i
+	})).toSorted((a, b) => a.start - b.start || b.end - a.end);
+	const remove = [];
+	let coverStart = Number.NEGATIVE_INFINITY;
+	let coverEnd = Number.NEGATIVE_INFINITY;
+	for (const seg of order) if (seg.start >= coverStart && seg.end <= coverEnd && seg.end > seg.start) remove.push(seg.i);
+	else {
+		coverStart = seg.start;
+		coverEnd = seg.end;
+	}
+	return remove;
+}
+//#endregion
+//#region src/recorder/addon/redundancy-sweep.ts
+/**
+* Periodic redundancy sweep — the only thing that removes duplicated footage.
+*
+* WHY IT EXISTS AGAIN
+* ───────────────────
+* Every `SegmentWriter` restart re-dials the broker with `withRecordingIntent`,
+* and the broker answers a RECORDING dial with its pre-roll ring (≈10 s). So a
+* restart does not resume at the live edge: it re-writes ~10 s of media that is
+* already on disk. At the 2026-08-27 measurement — **1 814 restarts in 12 h** —
+* that is ~5 hours of duplicated `high`-profile footage per day, on a fuse
+* share that was already at 90 % full.
+*
+* The pruner that used to remove it, `runRedundancyJanitor`, was reachable ONLY
+* from `runFullArchiveWalk`, which was reachable only from
+* `scheduleDeferredFullWalk`, which had no caller at all — so on 2026-08-27
+* (commit `9308428b4`) all three were deleted together as dead code. That was
+* correct about the walk and left the system with **no duplicate pruner
+* whatsoever**. The precedent for letting this class of waste run unattended is
+* the 179 GB of stranded staging segments in `staging-reconcile.ts`.
+*
+* WHAT IS DIFFERENT FROM THE ONE THAT WAS DELETED
+* ───────────────────────────────────────────────
+* The old janitor ran once, after a walk of the WHOLE archive, and asked the
+* in-RAM index for `segments(deviceId)` — a full copy plus a sort of every row
+* the device owns. At the live shape (7.1 M rows) that copy alone was measured
+* at ~560 MB of transient heap, which is precisely the cost D248 had just
+* finished removing from the pressure sweep. Reconnecting it in that shape
+* would have traded one regression for another.
+*
+* So this one is bounded by a LOOKBACK WINDOW, not by the archive: duplicates
+* are made by a restart, restarts are now, and `segmentsStartingIn` slices the
+* sorted view instead of copying it. Cost grows with footage RECORDED in the
+* window — the invariant the recorder is supposed to satisfy — never with rows
+* already indexed.
+*
+* WHAT IT WILL NOT DO
+* ───────────────────
+* It never deletes a segment that contributes unique coverage:
+* {@link selectRedundantSegments} selects only segments fully contained inside
+* another segment that is KEPT, and staggered partial overlaps are always kept.
+* The delete itself is `SegmentStore.evict` — the same audited path
+* disk-pressure eviction uses, so a removed segment is de-indexed exactly as an
+* evicted one is. A failure for one (device, profile, location) is logged and
+* never aborts the pass.
+*/
+/** The profiles a recorder writes. Same set `staging-reconcile` sweeps. */
+var SWEPT_PROFILES = [
+	"high",
+	"mid",
+	"low"
+];
+/** Group rows by the location that owns them. `evict` is per-location. */
+function byLocation(rows) {
+	const out = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const found = out.get(row.locationId);
+		if (found) found.push(row);
+		else out.set(row.locationId, [row]);
+	}
+	return out;
+}
+/**
+* Owns its timer and a single-flight guard, the same shape the retention sweep
+* and the export janitor already use — a slow pass is never overlapped by the
+* next tick.
+*/
+var RedundancySweeper = class {
+	deps;
+	timer = null;
+	sweeping = false;
+	constructor(deps) {
+		this.deps = deps;
+	}
+	/**
+	* Arm the sweep on its interval. Idempotent.
+	*
+	* Deliberately NOT run immediately: boot is when the writers are attaching,
+	* the staging reconcile is relocating thousands of files over shfs and the
+	* first viewer is painting. A duplicate that has waited an hour can wait
+	* another one; the write path cannot.
+	*/
+	start() {
+		if (this.timer !== null) return;
+		this.timer = setInterval(() => {
+			this.sweep();
+		}, this.deps.intervalMs);
+		this.timer.unref?.();
+	}
+	/** Stop the timer. Idempotent. An in-flight pass still completes. */
+	stop() {
+		if (this.timer !== null) {
+			clearInterval(this.timer);
+			this.timer = null;
+		}
+	}
+	/**
+	* One pass. Never throws: a device-list read failure aborts THIS pass
+	* (logged), a per-location evict failure is logged and the pass continues.
+	*/
+	async sweep() {
+		const empty = {
+			files: 0,
+			bytes: 0,
+			devices: 0
+		};
+		if (this.sweeping) {
+			this.deps.logger.debug("recorder: redundancy sweep already running — skipping this tick");
+			return empty;
+		}
+		this.sweeping = true;
+		try {
+			let deviceIds;
+			try {
+				deviceIds = await this.deps.deviceIds();
+			} catch (err) {
+				this.deps.logger.warn("recorder: redundancy sweep could not read the device list", { meta: { error: require_dist.errMsg(err) } });
+				return empty;
+			}
+			const toMs = (this.deps.now ?? Date.now)();
+			const fromMs = toMs - this.deps.lookbackMs;
+			let files = 0;
+			let bytes = 0;
+			let devices = 0;
+			for (const deviceId of deviceIds) {
+				let removedHere = 0;
+				for (const profile of SWEPT_PROFILES) {
+					const rows = this.deps.segmentsInWindow(deviceId, profile, fromMs, toMs);
+					const redundant = selectRedundantSegments(rows);
+					if (redundant.length === 0) continue;
+					const victims = redundant.map((i) => rows[i]).filter((row) => row !== void 0);
+					for (const [locationId, group] of byLocation(victims)) try {
+						const reclaimed = await this.deps.evict(locationId, group);
+						files += group.length;
+						bytes += reclaimed;
+						removedHere += group.length;
+					} catch (err) {
+						this.deps.logger.warn("recorder: redundancy sweep could not evict duplicates", {
+							tags: { deviceId },
+							meta: {
+								profile,
+								locationId,
+								count: group.length,
+								error: require_dist.errMsg(err)
+							}
+						});
+					}
+				}
+				if (removedHere > 0) {
+					devices += 1;
+					this.deps.logger.info("recorder: removed duplicated segments for device", {
+						tags: { deviceId },
+						meta: {
+							files: removedHere,
+							fromMs,
+							toMs
+						}
+					});
+				}
+			}
+			this.deps.logger.info("recorder: redundancy sweep complete", { meta: {
+				devices,
+				files,
+				bytes,
+				lookbackMs: this.deps.lookbackMs,
+				deviceCount: deviceIds.length
+			} });
+			return {
+				files,
+				bytes,
+				devices
+			};
+		} finally {
+			this.sweeping = false;
+		}
+	}
+};
+//#endregion
 //#region src/recorder/addon/retention-sweep.ts
 /**
 * Periodic footage-retention sweep (recording-spec §6).
@@ -12093,6 +12398,18 @@ var EXPORT_SWEEP_INTERVAL_MS = 5 * 6e4;
 *  hourly cadence keeps the storage.list/evict cost negligible while honouring
 *  the recording-spec §6 periodic retention. */
 var RETENTION_SWEEP_INTERVAL_MS = 60 * 6e4;
+/** How often the redundancy sweep removes footage a writer restart duplicated. */
+var REDUNDANCY_SWEEP_INTERVAL_MS = 60 * 6e4;
+/**
+* How far back each redundancy pass looks.
+*
+* Twice the interval, so a pass that is skipped (single-flight) or delayed by a
+* slow predecessor still covers the window its predecessor was supposed to.
+* It is deliberately NOT "the whole archive": duplicates are made by a writer
+* restart, restarts are recent, and a full-archive selection is the ~560 MB
+* transient copy D248 removed. See `redundancy-sweep.ts`.
+*/
+var REDUNDANCY_SWEEP_LOOKBACK_MS = 2 * REDUNDANCY_SWEEP_INTERVAL_MS;
 /** What the per-volume usage row reports when the hour ledger is unavailable:
 *  nothing measured, rather than a total counted off a partial RAM index. */
 var EMPTY_ARCHIVE_ACCOUNTING = {
@@ -12118,6 +12435,16 @@ function isSegmentStoreLogExtras(extras) {
 }
 var RecorderV2Addon = class extends require_dist.BaseAddon {
 	nodeId = "hub";
+	/**
+	* The children this addon started, claimed at the spawn site and released at
+	* their exit. It is what `load-contribution` reports: the recorder knows it
+	* ran an ffmpeg for camera 615 profile `high`, so nobody else has to work it
+	* out from a process table.
+	*
+	* One registry per PROCESS, and it dies with the process — which is correct,
+	* because those children were this process's children.
+	*/
+	costRegistry = new require_node.ChildCostRegistry();
 	/** True when THIS node is the designated recording node (`recordingNodeId`).
 	*  Every other node stays fully inert — registers no providers, serves no
 	*  playback, runs no controller. Resolved in `onInitialize`. */
@@ -12245,6 +12572,9 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 	/** Periodic footage-retention sweep (owns its timer + single-flight guard).
 	*  Constructed + started in `onHubReachable`, stopped on shutdown. */
 	retentionSweeper = null;
+	/** Periodic duplicate-footage sweep — the ONLY pruner of media a writer
+	*  restart re-wrote. Constructed + started in `onHubReachable`. */
+	redundancySweeper = null;
 	/** Reversible migration pause lease; never reflected in RecordingConfig. */
 	storageMigrationLeaseId = null;
 	/**
@@ -12461,6 +12791,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 			placeProfile: (deviceId, profile) => this.placeProfile(deviceId, profile),
 			onPlacementTick: (deviceIds) => this.recomputePlacement(deviceIds),
 			spawn: node_child_process.spawn,
+			costRegistry: this.costRegistry,
 			segmentSeconds: this.config.segmentSeconds,
 			watchIntervalMs: this.config.watchIntervalMs,
 			...consumerHostname !== void 0 ? { consumerHostname } : {},
@@ -12709,6 +13040,10 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 				{
 					capability: require_dist.recordingExportCapability,
 					provider: exportProvider
+				},
+				{
+					capability: require_dist.loadContributionCapability,
+					provider: { list: () => this.costRegistry.contributions() }
 				}
 			],
 			customActions: recorderDebugActions,
@@ -12733,6 +13068,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 		}
 		this.retentionSweeper?.stop();
 		this.retentionSweeper = null;
+		this.redundancySweeper?.stop();
+		this.redundancySweeper = null;
 		this.exportEngine = null;
 		this.recordingProvider = null;
 	}
@@ -12872,6 +13209,16 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 			intervalMs: RETENTION_SWEEP_INTERVAL_MS
 		});
 		this.retentionSweeper?.start();
+		const store = this.segmentStore;
+		if (store && this.redundancySweeper === null) this.redundancySweeper = new RedundancySweeper({
+			deviceIds: async () => [...(await readDeviceConfigs(this.configStore())).keys()],
+			segmentsInWindow: (deviceId, profile, fromMs, toMs) => this.index.segmentsStartingIn(deviceId, profile, fromMs, toMs),
+			evict: (locationId, rows) => store.evict(locationId, rows),
+			logger: this.ctx.logger,
+			intervalMs: REDUNDANCY_SWEEP_INTERVAL_MS,
+			lookbackMs: REDUNDANCY_SWEEP_LOOKBACK_MS
+		});
+		this.redundancySweeper?.start();
 	}
 	/**
 	* Rebuild the sensor trigger's reverse index from the persisted configs.