@camstack/addon-pipeline 1.2.75 → 1.2.77

package/dist/recorder/index.js

@@ -1,6 +1,6 @@
-const require_dist = require("../dist-6ZJCV0jY.js");
+const require_dist = require("../dist-DQZzL0Ri.js");
 const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
-const require_addon_utils = require("../addon-utils-a_36HH_N.js");
+const require_addon_utils = require("../addon-utils-bZcwLaVW.js");
 let node_crypto = require("node:crypto");
 let node_child_process = require("node:child_process");
 let node_path = require("node:path");
@@ -1710,7 +1710,7 @@ async function availabilityProfile(root, deviceId) {
 *  NOT profiles (`events/`, plus the `strips/` trees the retired filmstrip tier
 *  left on disk); a walker that trusts readdir would offer them as playable
 *  profiles. */
-var KNOWN_PROFILES = new Set([
+var KNOWN_PROFILES$1 = new Set([
 	"high",
 	"mid",
 	"low"
@@ -1718,7 +1718,7 @@ var KNOWN_PROFILES = new Set([
 /** `<root>/<deviceId>/` PROFILE subdirectories. Empty when absent. */
 async function profilesOf(root, deviceId) {
 	try {
-		return (await node_fs.promises.readdir(node_path.default.join(root, String(deviceId)))).filter((e) => KNOWN_PROFILES.has(e));
+		return (await node_fs.promises.readdir(node_path.default.join(root, String(deviceId)))).filter((e) => KNOWN_PROFILES$1.has(e));
 	} catch {
 		return [];
 	}
@@ -5665,26 +5665,29 @@ var PENDING_GRACE_MS = 3e4;
 * segment of every recording would be classified as a previous run's leftover
 * and dropped.
 */
-var PREVIOUS_RUN_SKEW_MS = 1e3;
+var PREVIOUS_RUN_SKEW_MS$1 = 1e3;
 /**
 * Decide whether a missing flat file is a late flush or a previous run's entry.
 *
-* ffmpeg's `-segment_list` file is not cleared between runs and a fresh watcher
-* starts at `processed = 0`, so the first passes after every attach replay the
-* PREVIOUS run's playlist. Those entries were relocated into the bucket tree
-* long ago; their absence is the system working. Over 2026-08-10..13 they were
-* 83% of all retries (median age 1 615 s, max 17.4 h) and produced 68 of the
-* 122 `never landed` warnings — a warning that reads like data loss for a file
-* that was correctly filed hours earlier.
+* ffmpeg opens `-segment_list` in TRUNCATE mode when it spawns, but not
+* instantly: a fresh watcher starts at `processed = 0` and, in the window
+* between spawn and that truncation, reads the PREVIOUS run's playlist body.
+* Those entries were relocated into the bucket tree long ago; their absence is
+* the system working. Over 2026-08-10..13 they were 83% of all retries (median
+* age 1 615 s, max 17.4 h) and produced 68 of the 122 `never landed` warnings —
+* a warning that reads like data loss for a file that was correctly filed hours
+* earlier.
 *
-* Note what this deliberately does NOT do: it does not skip a previous run's
-* entry whose file is STILL THERE. That case is real crash recovery — an
-* unclean stop leaves un-relocated segments in `.rec-tmp` and the playlist is
-* the only record of their durations — so the caller stats first and only asks
-* this question once the file is known to be absent.
+* What this is NOT is crash recovery, and an earlier version of this comment
+* claimed it was. Once ffmpeg truncates, the previous run's un-relocated
+* segments are named by NOTHING, so no amount of care in this function can
+* reach them — on 2026-08-14 that had stranded 179 GB across the fleet. They
+* are recovered from the DIRECTORY instead, by `staging-reconcile.ts`, which
+* the addon runs once at boot. This function only has to be right about the
+* entries a playlist does name.
 */
 function classifyMissingEntry(input) {
-	return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS ? "stale-previous-run" : "pending";
+	return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS$1 ? "stale-previous-run" : "pending";
 }
 /** Whether a pending entry has outlived its grace. Pure — the caller clocks it. */
 function shouldGiveUpOnPending(input) {
@@ -6979,6 +6982,312 @@ var RetentionSweeper = class {
 	}
 };
 //#endregion
+//#region src/recorder/addon/staging-reconcile.ts
+/**
+* Recover `.rec-tmp` segments that a writer restart orphaned.
+*
+* WHY THIS EXISTS
+* ───────────────
+* The segment watcher's ONLY record of what it still owes is ffmpeg's
+* `-segment_list live.m3u8`. ffmpeg opens that file in TRUNCATE mode on every
+* spawn, so a writer respawn destroys it. Every segment written but not yet
+* relocated at that instant becomes permanently unreachable:
+*
+*   - no playlist names it, so no watcher will ever finalize it;
+*   - `parseSegmentPath` cannot see it — staging is flat and dot-prefixed, kept
+*     deliberately out of the hydrate walk — so retention never counts it;
+*   - the one `.rec-tmp` sweep that exists (`sweepUnassignedStaging`) touches
+*     only locations with ZERO placement assignments, never the live one.
+*
+* So the footage is not deleted and not kept: it is stranded, and it accrues
+* for as long as the install runs.
+*
+* `segment-watcher.ts` carries a comment asserting the opposite — that the
+* playlist "is NOT cleared between runs", which is what its `stale-previous-run`
+* classification and its documented "that is the crash recovery" path are built
+* on. The live filesystem refutes it, and a leftover that describes the right
+* design reads as verification: on 2026-08-14 `/recordings/.rec-tmp/590/high/`
+* held a `live.m3u8` of 50 entries, every one named at or after that writer's
+* 20:28:31 start, beside **5 863 older `.m4s` files referenced by nobody** —
+* the oldest from 2026-07-04. Fleet total: 179 GB, on an array at 90% full.
+*
+* WHAT THIS DOES NOT DO
+* ─────────────────────
+* It never deletes. Recovery is the same `SegmentStore.onFinalized` the live
+* path uses — stat, relocate into the UTC bucket, index — so a recovered
+* segment is indistinguishable from one that was never orphaned, and a segment
+* it cannot place stays exactly where it is for the next attempt.
+*/
+/**
+* How far before the writer's own start a segment may be named and still count
+* as belonging to THIS run.
+*
+* Mirrors `segment-watcher.PREVIOUS_RUN_SKEW_MS`, and for the same reason:
+* ffmpeg names segments with whole-second `%s`, so the first segment of a run
+* carries the second already in progress when the process spawned. Without the
+* tolerance a recovery pass would reach for the live writer's opening segment —
+* the one file in the directory that is still being written.
+*/
+var PREVIOUS_RUN_SKEW_MS = 1e3;
+/**
+* How much longer than nominal an inter-segment gap may be and still be read as
+* one segment's duration.
+*
+* ffmpeg cuts on keyframes, so a real segment overshoots `-segment_time` by up
+* to a GOP; a gap of twice nominal is still plausibly one segment. Anything
+* wider is a RUN BOUNDARY — the writer was down in between — and the honest
+* duration is the nominal one. Trusting the gap there would paint a solid
+* timeline bar across hours of footage that does not exist, which is a worse
+* failure than under-reporting by a few seconds.
+*/
+var MAX_GAP_FACTOR = 2;
+/**
+* Decide which staged files belong to a previous run, and how long each ran.
+*
+* Pure — the caller does the I/O. The gate is the file's own name against the
+* writer's start: a segment named before this ffmpeg spawned cannot be one this
+* ffmpeg is writing, whatever the playlist says.
+*/
+function planStagingRecovery(input) {
+	const nominalMs = input.segmentSeconds * 1e3;
+	const cutoffMs = input.writerStartedMs - PREVIOUS_RUN_SKEW_MS;
+	const owned = input.playlistNames;
+	const candidates = [];
+	for (const name of input.names) {
+		const startMs = parseEpochStartMs(name);
+		if (startMs === null) continue;
+		if (startMs >= cutoffMs) continue;
+		if (owned?.has(name) === true) continue;
+		candidates.push({
+			name,
+			startMs
+		});
+	}
+	candidates.sort((a, b) => a.startMs - b.startMs);
+	const maxGapMs = nominalMs * MAX_GAP_FACTOR;
+	return candidates.map((c, i) => {
+		const next = candidates[i + 1];
+		const gapMs = next === void 0 ? 0 : next.startMs - c.startMs;
+		const durMs = gapMs > 0 && gapMs <= maxGapMs ? gapMs : nominalMs;
+		return {
+			name: c.name,
+			startMs: c.startMs,
+			durMs
+		};
+	});
+}
+/** The real filesystem. The only production value of {@link StagingIo}. */
+var nodeStagingIo = {
+	listDir: (dir) => node_fs.promises.readdir(dir),
+	readPlaylist: (dir) => node_fs.promises.readFile(node_path.default.join(dir, "live.m3u8"), "utf8").catch(() => null)
+};
+/** The only `<deviceId>/` subdirectories of staging that hold segments. */
+var KNOWN_PROFILES = new Set([
+	"high",
+	"mid",
+	"low"
+]);
+/** Default orphans per chunk. 590/high alone had 5 863 of them. */
+var DEFAULT_CHUNK_SIZE = 100;
+var macrotask = () => new Promise((resolve) => {
+	setTimeout(resolve, 0);
+});
+/**
+* Relocate every previous-run segment left in one staging dir.
+*
+* Chunked with a yield between batches: this runs alongside a live recorder and
+* a backlog is measured in thousands of files over shfs/FUSE — a single
+* uninterrupted loop is exactly the event-loop stall the recorder has already
+* paid for once (D128).
+*
+* A failure is per-segment and never aborts the pass: the file stays in staging
+* and the next attach retries it. Every drop is logged with the segment name —
+* silence reads as "never happened", and this whole class of loss went
+* unnoticed for six weeks precisely because nothing said anything.
+*/
+async function recoverStagingOrphans(dir, input, deps) {
+	let names;
+	try {
+		names = await deps.listDir(dir);
+	} catch (err) {
+		deps.logger.warn("recorder: staging recovery could not list the staging dir", { meta: {
+			dir,
+			error: err instanceof Error ? err.message : String(err)
+		} });
+		return {
+			found: 0,
+			recovered: 0,
+			failed: 0,
+			aborted: false
+		};
+	}
+	let playlistNames;
+	const body = await deps.readPlaylist(dir).catch(() => null);
+	if (body !== null) playlistNames = new Set(parseLivePlaylist(body).map((e) => e.segPath.replace(/^.*\//, "")));
+	const plan = planStagingRecovery({
+		...input,
+		names,
+		playlistNames
+	});
+	if (plan.length === 0) return {
+		found: 0,
+		recovered: 0,
+		failed: 0,
+		aborted: false
+	};
+	const chunkSize = deps.chunkSize ?? DEFAULT_CHUNK_SIZE;
+	const yieldBetween = deps.yieldBetween ?? macrotask;
+	let recovered = 0;
+	let failed = 0;
+	let aborted = false;
+	for (let i = 0; i < plan.length; i++) {
+		const orphan = plan[i];
+		if (deps.shouldStop?.() === true) {
+			aborted = true;
+			deps.logger.warn("recorder: staging recovery stopped short by shutdown", { meta: {
+				dir,
+				recovered,
+				failed,
+				remaining: plan.length - i
+			} });
+			break;
+		}
+		try {
+			await deps.finalize(orphan, `${dir}/${orphan.name}`);
+			recovered += 1;
+		} catch (err) {
+			failed += 1;
+			deps.logger.warn("recorder: staging recovery could not relocate an orphaned segment", { meta: {
+				dir,
+				seg: orphan.name,
+				startMs: orphan.startMs,
+				durMs: orphan.durMs,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+		if ((i + 1) % chunkSize === 0 && i + 1 < plan.length) await yieldBetween();
+	}
+	deps.logger.info("recorder: recovered segments a writer restart had orphaned in staging", { meta: {
+		dir,
+		found: plan.length,
+		recovered,
+		failed,
+		aborted,
+		oldestMs: plan[0]?.startMs,
+		newestMs: plan[plan.length - 1]?.startMs
+	} });
+	return {
+		found: plan.length,
+		recovered,
+		failed,
+		aborted
+	};
+}
+/**
+* Boot reconcile: relocate every previous-run segment stranded in staging, on
+* every location, for every camera — attached or not.
+*
+* WHY BOOT, AND NOT ATTACH
+* ────────────────────────
+* Orphans are created by a writer restart, so by construction they all predate
+* the runner that finds them; boot is when the whole backlog is visible at
+* once. Attach looked like the natural hook and is the wrong one twice over: it
+* only ever reaches cameras that are currently recording (device 1438 had a
+* staging backlog while attached to nothing), and it puts unbounded filesystem
+* work on the path that must start recording NOW — measured, it kept microtasks
+* in flight across `RecordingController.stop()` and made the runner's 5 s
+* SIGTERM-grace test fail under load.
+*
+* `bootMs` is the cutoff, and it is what makes this safe to run while writers
+* are spawning: a live writer's segments are all named after boot, so the plan
+* cannot select one. Nothing is deleted, and a segment that cannot be placed is
+* left exactly where it is for the next boot.
+*/
+async function recoverAllStagedOrphans(deps) {
+	let dirs = 0;
+	let found = 0;
+	let recovered = 0;
+	let failed = 0;
+	let aborted = false;
+	for (const location of deps.locations()) {
+		const stagingRoot = `${location.root}/${STAGING_DIR_NAME}`;
+		let deviceDirs;
+		try {
+			deviceDirs = await deps.listDir(stagingRoot);
+		} catch {
+			continue;
+		}
+		for (const deviceDir of deviceDirs) {
+			if (aborted) break;
+			const deviceId = Number(deviceDir);
+			if (!Number.isInteger(deviceId) || deviceId <= 0) continue;
+			let profileDirs;
+			try {
+				profileDirs = await deps.listDir(`${stagingRoot}/${deviceDir}`);
+			} catch {
+				continue;
+			}
+			const segmentSeconds = await deps.segmentSecondsFor(deviceId);
+			for (const profile of profileDirs) {
+				if (aborted) break;
+				if (!KNOWN_PROFILES.has(profile)) continue;
+				const dir = `${stagingRoot}/${deviceDir}/${profile}`;
+				dirs += 1;
+				const report = await recoverStagingOrphans(dir, {
+					writerStartedMs: deps.bootMs,
+					segmentSeconds
+				}, {
+					listDir: deps.listDir,
+					readPlaylist: deps.readPlaylist,
+					finalize: (orphan, flatAbsPath) => deps.finalize({
+						deviceId,
+						profile,
+						locationId: location.id,
+						locationRoot: location.root,
+						flatAbsPath,
+						startMs: orphan.startMs,
+						durMs: orphan.durMs
+					}),
+					logger: withDeviceTag(deps.logger, deviceId),
+					chunkSize: deps.chunkSize,
+					yieldBetween: deps.yieldBetween,
+					shouldStop: deps.shouldStop
+				});
+				found += report.found;
+				recovered += report.recovered;
+				failed += report.failed;
+				if (report.aborted) aborted = true;
+			}
+		}
+	}
+	if (found > 0 || failed > 0) deps.logger.info("recorder: staging boot reconcile complete", { meta: {
+		dirs,
+		found,
+		recovered,
+		failed,
+		aborted
+	} });
+	return {
+		dirs,
+		found,
+		recovered,
+		failed,
+		aborted
+	};
+}
+/** Bind `tags: { deviceId }` onto every line a per-dir pass writes. */
+function withDeviceTag(logger, deviceId) {
+	const tag = (extras) => ({
+		...typeof extras === "object" && extras !== null ? extras : {},
+		tags: { deviceId }
+	});
+	return {
+		info: (message, extras) => logger.info(message, tag(extras)),
+		warn: (message, extras) => logger.warn(message, tag(extras)),
+		debug: (message, extras) => logger.debug(message, tag(extras))
+	};
+}
+//#endregion
 //#region src/recorder/addon/index.ts
 /**
 * recorder addon SHELL — recording + storage-evictable providers on the
@@ -7516,11 +7825,13 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 			force: true
 		}).catch(() => {});
 		await this.sweepUnassignedStaging();
+		const stagingBootMs = Date.now();
 		try {
 			await this.controller?.start();
 		} catch (err) {
 			this.ctx.logger.warn("recorder: controller start failed", { meta: { error: require_dist.errMsg(err) } });
 		}
+		this.recoverStagedOrphans(stagingBootMs);
 		if (this.eventUnsub === null) this.eventUnsub = subscribeEventCapture({
 			eventBus: this.ctx.eventBus,
 			map: this.eventMap,
@@ -7825,6 +8136,45 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 		}
 	}
 	/**
+	* Boot reconcile for `.rec-tmp`: relocate every segment a previous runner's
+	* writers left stranded, on every location, for every camera.
+	*
+	* The counterpart to {@link sweepUnassignedStaging} above, which only ever
+	* looks at locations nothing is assigned to — so the LIVE root, the one that
+	* actually accumulates, had no pass at all (D148). Nothing here deletes: each
+	* orphan goes through the same `SegmentStore.onFinalized` a live segment
+	* does, and one it cannot place is left on disk for the next boot.
+	*/
+	async recoverStagedOrphans(bootMs) {
+		try {
+			await recoverAllStagedOrphans({
+				locations: () => this.resolvedLocations.map((l) => ({
+					id: l.id,
+					root: l.root
+				})),
+				listDir: nodeStagingIo.listDir,
+				readPlaylist: nodeStagingIo.readPlaylist,
+				finalize: async (input) => {
+					const store = this.segmentStore;
+					if (store === null) throw new Error("recorder is shutting down");
+					await store.onFinalized(input);
+				},
+				logger: this.ctx.logger,
+				bootMs,
+				segmentSecondsFor: async (deviceId) => {
+					try {
+						return (await loadDeviceConfig(this.configStore(), deviceId)).segmentSeconds ?? this.config.segmentSeconds;
+					} catch {
+						return this.config.segmentSeconds;
+					}
+				},
+				shouldStop: () => this.segmentStore === null
+			});
+		} catch (err) {
+			this.ctx.logger.warn("recorder: staging boot reconcile failed", { meta: { error: require_dist.errMsg(err) } });
+		}
+	}
+	/**
 	* Surface a storage-placement fault on the liveness/alert surface. STRICTLY
 	* best-effort: the alerts cap is a nice-to-have and an unmounted disk must
 	* still be shouted about in the log even when it is unreachable.