@camstack/addon-pipeline 1.2.76 → 1.2.78

package/dist/recorder/index.mjs

@@ -1,4 +1,4 @@
-import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-zksfWnEA.mjs";
+import { At as string, Dt as number, J as recordingExportCapability, Mt as EventCategory, N as deriveRecordingMode, Ot as object, St as array, Z as storageEvictableCapability, _ as OpsLogEntrySchema, b as RECORDING_EXPORT_MAX_READ_BYTES, c as DEFAULT_EVENTS_BAND_BUFFER_SEC, d as EVENT_PAD_MS, ft as hydrateSchema, gt as nodePin, kt as record, p as ExportRecordSchema, q as recordingCapability, st as BaseAddon, tt as errMsg, ut as DeviceType, x as RecordingConfigSchema, yt as selectAssignedProfileSlots } from "../dist-CsP_DikG.mjs";
 import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
 import { n as createFileDataPlaneHandler, s as parseRangeHeader, t as contentTypeFor } from "../addon-utils-A2S9D7pu.mjs";
 import { randomUUID } from "node:crypto";
@@ -389,7 +389,21 @@ async function readMfraTable(absPath, fileBytes) {
 /** How long a table stays served. Directory asks are for recent footage far
 *  more often than not, and a bounded window keeps this store O(day), not
 *  O(uptime): ~30 bytes a segment ⇒ a day across 20 cameras is a few MB. */
-var RETAIN_MS = 26 * 36e5;
+var MFRA_RETAIN_MS = 26 * 36e5;
+/**
+* Whether a segment is recent enough for a captured table to ever be served.
+*
+* The gate the FINALIZE path must ask BEFORE reading the file. A live segment
+* always passes; a recovered one almost never does, and capturing it is pure
+* loss twice over — the table is unservable, and the read is a real `open` +
+* two `read`s on the libuv threadpool. The D148 boot reconcile fired 122 400 of
+* them, unawaited and unbounded, behind a 4-thread pool that the live
+* SegmentWriters' own finalize I/O has to share: that is the "segment file
+* never landed" (>30 s) on four cameras at once.
+*/
+function isMfraTableServable(startMs, nowMs) {
+	return startMs >= nowMs - MFRA_RETAIN_MS;
+}
 /**
 * The RAM store the directory reads. Keys are `(deviceId, profile, startMs)`
 * — exactly how a directory row identifies a segment. Insertion is
@@ -405,7 +419,16 @@ var MfraTableStore = class {
 	key(deviceId, profile, startMs) {
 		return `${deviceId}:${profile}:${startMs}`;
 	}
+	/**
+	* Keep one table, unless it is already past the serve window.
+	*
+	* The out-of-window REJECT is not belt-and-braces: `prune` walks the
+	* insertion-ordered prefix and stops at the first live entry, so an expired
+	* entry appended AFTER a live one is never reachable again. A bulk recovery
+	* appends 122 400 of exactly that shape and the store grows without bound.
+	*/
 	record(deviceId, profile, startMs, table) {
+		if (!isMfraTableServable(startMs, this.now())) return;
 		this.tables.set(this.key(deviceId, profile, startMs), {
 			startMs,
 			table
@@ -426,7 +449,7 @@ var MfraTableStore = class {
 	}
 	/** Insertion order is time order, so the expired prefix is contiguous. */
 	prune() {
-		const cutoff = this.now() - RETAIN_MS;
+		const cutoff = this.now() - MFRA_RETAIN_MS;
 		for (const [key, entry] of this.tables) {
 			if (entry.startMs >= cutoff) break;
 			this.tables.delete(key);
@@ -625,6 +648,38 @@ var RecordingIndex = class {
 	segments(deviceId, profile) {
 		return [...this.sortedSegments(deviceId, profile)];
 	}
+	/** True when `relPath` is currently indexed for `deviceId`. O(1). */
+	hasSegment(deviceId, relPath) {
+		return this.byDevice.get(deviceId)?.has(relPath) === true;
+	}
+	/**
+	* Indexed-segment count per HOUR-BUCKET directory (`<dev>/<profile>/Y/M/D/H`)
+	* for one (device, profile), in ONE pass over the device's rows.
+	*
+	* Exists because `SegmentStore.evict` has to answer "does this eviction cover
+	* every segment the bucket holds?" for every bucket in the round, and it used
+	* to answer it with `segments(deviceId, profile)` — a full COPY of the
+	* device-profile archive plus a full `startsWith` scan, PER BUCKET. That is
+	* O(buckets x archive), and D148 is what made it bite: before it, a retention
+	* round touched a handful of buckets at the retention edge; after it, the
+	* recovered backlog made ~8 900 sparse buckets across six weeks evictable in
+	* ONE round. Measured on that shape (1.63 M live rows + 122 400 victims):
+	* 182 596 ms of uninterrupted main-thread CPU, with no log line, which is the
+	* silent 100%-CPU phase the operator saw. One pass answers all of them.
+	*/
+	bucketCounts(deviceId, profile) {
+		const out = /* @__PURE__ */ new Map();
+		const m = this.byDevice.get(deviceId);
+		if (!m) return out;
+		for (const s of m.values()) {
+			if (s.profile !== profile) continue;
+			const cut = s.path.lastIndexOf("/");
+			if (cut <= 0) continue;
+			const bucketDir = s.path.slice(0, cut);
+			out.set(bucketDir, (out.get(bucketDir) ?? 0) + 1);
+		}
+		return out;
+	}
 	/** Aggregate accounting for a device across ALL its locations (use `accountingForLocation` for one location). */
 	accounting(deviceId) {
 		const m = this.byDevice.get(deviceId);
@@ -1257,6 +1312,7 @@ var SegmentStore = class SegmentStore {
 	async evict(location, rows) {
 		let reclaimed = 0;
 		const removed = [];
+		const startedMs = this.deps.now?.() ?? Date.now();
 		const byBucket = /* @__PURE__ */ new Map();
 		for (const r of rows) {
 			const cut = r.path.lastIndexOf("/");
@@ -1265,13 +1321,29 @@ var SegmentStore = class SegmentStore {
 			if (group) group.push(r);
 			else byBucket.set(bucketDir, [r]);
 		}
+		const census = /* @__PURE__ */ new Map();
+		const countsFor = (deviceId, profile) => {
+			const key = `${deviceId}:${profile}`;
+			const cached = census.get(key);
+			if (cached) return cached;
+			const counts = this.deps.index.bucketCounts(deviceId, profile);
+			census.set(key, counts);
+			return counts;
+		};
+		const chunkSize = this.deps.evictChunkSize ?? SegmentStore.DEFAULT_EVICT_CHUNK;
+		const yieldBetween = this.deps.yieldBetween ?? macrotask$2;
+		let bucketsRemoved = 0;
+		let handled = 0;
 		for (const [bucketDir, bucketRows] of byBucket) {
-			if (bucketDir !== "" && this.bucketFullyEvicted(bucketDir, bucketRows)) try {
+			if (bucketDir !== "" && this.bucketFullyEvicted(bucketDir, bucketRows, countsFor)) try {
 				await this.deps.removeDir(location, bucketDir);
+				bucketsRemoved += 1;
 				for (const r of bucketRows) {
 					removed.push(r.path);
 					reclaimed += r.bytes;
 				}
+				handled += 1;
+				if (handled % chunkSize === 0) await yieldBetween();
 				continue;
 			} catch (err) {
 				this.deps.logger.warn("bucket removeDir failed — falling back to per-file deletes", {
@@ -1292,15 +1364,47 @@ var SegmentStore = class SegmentStore {
 					error: err
 				});
 			}
+			handled += 1;
+			if (handled % chunkSize === 0) await yieldBetween();
 		}
 		if (removed.length > 0) {
 			this.deps.index.removeSegments(removed);
-			for (const r of rows) this.deps.onEvicted?.(r.deviceId, r.startMs);
+			this.notifyEvicted(rows);
 		}
 		await this.pruneEmptyAncestors(location, [...byBucket.keys()]);
+		this.deps.logger.info("recorder: eviction round complete", { meta: {
+			location,
+			rows: rows.length,
+			buckets: byBucket.size,
+			bucketsRemoved,
+			removed: removed.length,
+			reclaimed,
+			ms: (this.deps.now?.() ?? Date.now()) - startedMs
+		} });
 		return reclaimed;
 	}
 	/**
+	* Tell the calendar cache what went, ONCE PER (device, UTC day).
+	*
+	* `CalendarIndex.invalidate` drops a whole UTC day, so a per-ROW call repeats
+	* the same work for every segment of that day — and its hour-list sweep is
+	* O(cached hours), in one uninterrupted synchronous block with no await in
+	* it. Measured over a 122 400-row round with 5 000 cached hours: 15 285 ms of
+	* frozen event loop, scaling linearly with how long the session has been
+	* scrubbing. Deduped it is ~190 calls and unmeasurable.
+	*/
+	notifyEvicted(rows) {
+		const notify = this.deps.onEvicted;
+		if (!notify) return;
+		const seen = /* @__PURE__ */ new Set();
+		for (const r of rows) {
+			const key = `${r.deviceId}:${Math.floor(r.startMs / SegmentStore.DAY_MS)}`;
+			if (seen.has(key)) continue;
+			seen.add(key);
+			notify(r.deviceId, r.startMs);
+		}
+	}
+	/**
 	* Walk up from each emptied hour directory removing ancestors that are now
 	* empty, so that **a directory's existence means footage**.
 	*
@@ -1335,22 +1439,37 @@ var SegmentStore = class SegmentStore {
 	}
 	/** Hour of one UTC hour-bucket in ms. */
 	static HOUR_MS = 36e5;
+	static DAY_MS = 864e5;
+	/** Buckets between yields. Small enough that one chunk is milliseconds of
+	*  work now the per-bucket archive scan is gone. */
+	static DEFAULT_EVICT_CHUNK = 25;
 	/**
 	* True when `victims` cover EVERY indexed segment of `bucketDir` AND the
 	* bucket's hour has fully elapsed. The current-hour guard matters because
 	* `onFinalized` relocates new segments INTO the current bucket concurrently —
 	* an rm racing that rename would orphan a just-indexed segment.
+	*
+	* The cover test is a COUNT against the round's one-pass census plus an O(1)
+	* indexed-ness check per victim, never a scan of the device archive: a
+	* bucketDir is `<dev>/<profile>/Y/M/D/H`, so it names exactly one
+	* (device, profile), and the victims of a bucket are unique paths. Equal
+	* counts with every victim indexed therefore means the same set.
 	*/
-	bucketFullyEvicted(bucketDir, victims) {
+	bucketFullyEvicted(bucketDir, victims, countsFor) {
 		const first = victims[0];
 		if (!first) return false;
 		const bucketStartMs = Math.floor(first.startMs / SegmentStore.HOUR_MS) * SegmentStore.HOUR_MS;
 		if ((this.deps.now?.() ?? Date.now()) < bucketStartMs + SegmentStore.HOUR_MS) return false;
-		const victimPaths = new Set(victims.map((v) => v.path));
-		const indexed = this.deps.index.segments(first.deviceId, first.profile).filter((s) => s.path.startsWith(`${bucketDir}/`));
-		return indexed.length === victimPaths.size && indexed.every((s) => victimPaths.has(s.path));
+		const indexedInBucket = countsFor(first.deviceId, first.profile).get(bucketDir) ?? 0;
+		if (indexedInBucket === 0) return false;
+		const indexedVictims = /* @__PURE__ */ new Set();
+		for (const v of victims) if (this.deps.index.hasSegment(v.deviceId, v.path)) indexedVictims.add(v.path);
+		return indexedInBucket === indexedVictims.size;
 	}
 };
+var macrotask$2 = () => new Promise((resolve) => {
+	setImmediate(resolve);
+});
 //#endregion
 //#region src/recorder/still/still-source.ts
 /**
@@ -1709,7 +1828,7 @@ async function availabilityProfile(root, deviceId) {
 *  NOT profiles (`events/`, plus the `strips/` trees the retired filmstrip tier
 *  left on disk); a walker that trusts readdir would offer them as playable
 *  profiles. */
-var KNOWN_PROFILES = new Set([
+var KNOWN_PROFILES$1 = new Set([
 	"high",
 	"mid",
 	"low"
@@ -1717,7 +1836,7 @@ var KNOWN_PROFILES = new Set([
 /** `<root>/<deviceId>/` PROFILE subdirectories. Empty when absent. */
 async function profilesOf(root, deviceId) {
 	try {
-		return (await promises.readdir(path.join(root, String(deviceId)))).filter((e) => KNOWN_PROFILES.has(e));
+		return (await promises.readdir(path.join(root, String(deviceId)))).filter((e) => KNOWN_PROFILES$1.has(e));
 	} catch {
 		return [];
 	}
@@ -2011,7 +2130,7 @@ var CalendarIndex = class {
 		const day = utcDayOf(atMs);
 		this.byDevice.get(deviceId)?.delete(day);
 		const prefix = `${deviceId}:`;
-		for (const key of [...this.hourSegs.keys()]) {
+		for (const key of this.hourSegs.keys()) {
 			if (!key.startsWith(prefix)) continue;
 			if (utcDayOf(Number(key.slice(key.lastIndexOf(":") + 1))) === day) this.hourSegs.delete(key);
 		}
@@ -4838,9 +4957,12 @@ async function resolveRecordingsLocations(api, logger) {
 * (no footage yet) and any per-location failure are warned + skipped
 * (best-effort), so one bad volume never blocks the rest of the hydrate.
 */
-async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger) {
+async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger, options = {}) {
+	const yieldBetween = options.yieldBetween ?? macrotask$1;
+	const chunkSize = options.chunkSize ?? HYDRATE_CHUNK;
 	for (const [root, aliases] of aliasesByRoot(locations)) {
 		const deviceDir = path.join(root, String(deviceId));
+		const startedMs = Date.now();
 		let entries;
 		try {
 			entries = await promises.readdir(deviceDir, { recursive: true });
@@ -4854,17 +4976,48 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
 			});
 			continue;
 		}
+		const walkedMs = Date.now();
+		const locationForProfile = /* @__PURE__ */ new Map();
 		const pathsByLocation = /* @__PURE__ */ new Map();
-		for (const entry of entries.filter((entry) => entry.endsWith(".m4s"))) {
+		let handled = 0;
+		for (const entry of entries) {
+			if (!entry.endsWith(".m4s")) continue;
 			const relative = `${deviceId}/${entry.split(path.sep).join("/")}`;
-			const location = locationForHydratedProfile(aliases, relative.split("/")[1] ?? "");
-			const paths = pathsByLocation.get(location.id);
+			const profile = relative.split("/")[1] ?? "";
+			let locationId = locationForProfile.get(profile);
+			if (locationId === void 0) {
+				locationId = locationForHydratedProfile(aliases, profile).id;
+				locationForProfile.set(profile, locationId);
+			}
+			const paths = pathsByLocation.get(locationId);
 			if (paths) paths.push(relative);
-			else pathsByLocation.set(location.id, [relative]);
+			else pathsByLocation.set(locationId, [relative]);
+			handled += 1;
+			if (handled % chunkSize === 0) await yieldBetween();
+		}
+		let indexed = 0;
+		for (const [locationId, relPaths] of pathsByLocation) {
+			index.hydrateDevice(deviceId, locationId, relPaths);
+			indexed += relPaths.length;
 		}
-		for (const [locationId, relPaths] of pathsByLocation) index.hydrateDevice(deviceId, locationId, relPaths);
+		logger.info("recorder: hydrated a device from one recordings root", {
+			tags: { deviceId },
+			meta: {
+				root,
+				entries: entries.length,
+				indexed,
+				locationIds: [...pathsByLocation.keys()],
+				walkMs: walkedMs - startedMs,
+				ms: Date.now() - startedMs
+			}
+		});
 	}
 }
+/** Entries classified between yields in {@link hydrateDeviceFromStorage}. */
+var HYDRATE_CHUNK = 2e4;
+var macrotask$1 = () => new Promise((resolve) => {
+	setImmediate(resolve);
+});
 /** `<deviceId>/<profile>/YYYY/MM/DD/HH` for an hour bucket (paths are UTC). */
 function hourDirRelPath(deviceId, profile, hourStartMs) {
 	const d = new Date(hourStartMs);
@@ -5664,26 +5817,29 @@ var PENDING_GRACE_MS = 3e4;
 * segment of every recording would be classified as a previous run's leftover
 * and dropped.
 */
-var PREVIOUS_RUN_SKEW_MS = 1e3;
+var PREVIOUS_RUN_SKEW_MS$1 = 1e3;
 /**
 * Decide whether a missing flat file is a late flush or a previous run's entry.
 *
-* ffmpeg's `-segment_list` file is not cleared between runs and a fresh watcher
-* starts at `processed = 0`, so the first passes after every attach replay the
-* PREVIOUS run's playlist. Those entries were relocated into the bucket tree
-* long ago; their absence is the system working. Over 2026-08-10..13 they were
-* 83% of all retries (median age 1 615 s, max 17.4 h) and produced 68 of the
-* 122 `never landed` warnings — a warning that reads like data loss for a file
-* that was correctly filed hours earlier.
+* ffmpeg opens `-segment_list` in TRUNCATE mode when it spawns, but not
+* instantly: a fresh watcher starts at `processed = 0` and, in the window
+* between spawn and that truncation, reads the PREVIOUS run's playlist body.
+* Those entries were relocated into the bucket tree long ago; their absence is
+* the system working. Over 2026-08-10..13 they were 83% of all retries (median
+* age 1 615 s, max 17.4 h) and produced 68 of the 122 `never landed` warnings —
+* a warning that reads like data loss for a file that was correctly filed hours
+* earlier.
 *
-* Note what this deliberately does NOT do: it does not skip a previous run's
-* entry whose file is STILL THERE. That case is real crash recovery — an
-* unclean stop leaves un-relocated segments in `.rec-tmp` and the playlist is
-* the only record of their durations — so the caller stats first and only asks
-* this question once the file is known to be absent.
+* What this is NOT is crash recovery, and an earlier version of this comment
+* claimed it was. Once ffmpeg truncates, the previous run's un-relocated
+* segments are named by NOTHING, so no amount of care in this function can
+* reach them — on 2026-08-14 that had stranded 179 GB across the fleet. They
+* are recovered from the DIRECTORY instead, by `staging-reconcile.ts`, which
+* the addon runs once at boot. This function only has to be right about the
+* entries a playlist does name.
 */
 function classifyMissingEntry(input) {
-	return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS ? "stale-previous-run" : "pending";
+	return input.segmentStartMs < input.writerStartedMs - PREVIOUS_RUN_SKEW_MS$1 ? "stale-previous-run" : "pending";
 }
 /** Whether a pending entry has outlived its grace. Pure — the caller clocks it. */
 function shouldGiveUpOnPending(input) {
@@ -6945,20 +7101,30 @@ var RetentionSweeper = class {
 				this.deps.logger.warn("recorder: retention sweep could not read device configs", { meta: { error: errMsg(err) } });
 				return;
 			}
+			const withPolicy = [...configs].filter(([, config]) => hasRetentionPolicy(config));
+			const devicesSkipped = configs.size - withPolicy.length;
+			this.deps.logger.info("recorder: retention sweep starting", { meta: {
+				devices: withPolicy.map(([deviceId]) => deviceId),
+				skipped: devicesSkipped
+			} });
 			let devicesSwept = 0;
-			let devicesSkipped = 0;
 			let bucketsPruned = 0;
 			let bytesReclaimed = 0;
-			for (const [deviceId, config] of configs) {
-				if (!hasRetentionPolicy(config)) {
-					devicesSkipped += 1;
-					continue;
-				}
+			for (const [deviceId] of withPolicy) {
+				const startedMs = Date.now();
 				try {
 					const result = await this.deps.prune(deviceId);
 					devicesSwept += 1;
 					bucketsPruned += result.deletedBuckets;
 					bytesReclaimed += result.reclaimedBytes;
+					this.deps.logger.info("recorder: retention prune complete for device", {
+						tags: { deviceId },
+						meta: {
+							deletedBuckets: result.deletedBuckets,
+							reclaimedBytes: result.reclaimedBytes,
+							ms: Date.now() - startedMs
+						}
+					});
 				} catch (err) {
 					this.deps.logger.warn("recorder: retention prune failed for device", {
 						tags: { deviceId },
@@ -6978,6 +7144,402 @@ var RetentionSweeper = class {
 	}
 };
 //#endregion
+//#region src/recorder/loop-pacer.ts
+var DEFAULT_BUSY_LAG_MS = 25;
+var DEFAULT_BACKOFF_STEP_MS = 25;
+var DEFAULT_MAX_BACKOFF_MS = 250;
+var LoopPacer = class {
+	deps;
+	consecutive = 0;
+	hops = 0;
+	contendedHops = 0;
+	pausedMs = 0;
+	maxLagMs = 0;
+	constructor(deps) {
+		this.deps = deps;
+	}
+	/**
+	* One paced yield. Always hands the loop back at least once; sleeps longer
+	* while the previous hop shows the loop is contended, and forgets the
+	* back-off as soon as one hop comes back clean.
+	*/
+	async hop() {
+		const busyLagMs = this.deps.busyLagMs ?? DEFAULT_BUSY_LAG_MS;
+		const stepMs = this.deps.backoffStepMs ?? DEFAULT_BACKOFF_STEP_MS;
+		const maxMs = this.deps.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+		const before = this.deps.now();
+		await this.deps.sleep(0);
+		const lagMs = this.deps.now() - before;
+		this.hops += 1;
+		if (lagMs > this.maxLagMs) this.maxLagMs = lagMs;
+		if (lagMs < busyLagMs) {
+			this.consecutive = 0;
+			return;
+		}
+		this.consecutive += 1;
+		this.contendedHops += 1;
+		const pauseMs = Math.min(stepMs * this.consecutive, maxMs);
+		this.pausedMs += pauseMs;
+		await this.deps.sleep(pauseMs);
+	}
+	stats() {
+		return {
+			hops: this.hops,
+			contendedHops: this.contendedHops,
+			pausedMs: this.pausedMs,
+			maxLagMs: this.maxLagMs
+		};
+	}
+};
+/** The production pacer: real clock, real timers. */
+function nodeLoopPacer() {
+	return new LoopPacer({
+		now: () => Date.now(),
+		sleep: (ms) => new Promise((resolve) => {
+			setTimeout(resolve, ms);
+		})
+	});
+}
+//#endregion
+//#region src/recorder/addon/staging-reconcile.ts
+/**
+* Recover `.rec-tmp` segments that a writer restart orphaned.
+*
+* WHY THIS EXISTS
+* ───────────────
+* The segment watcher's ONLY record of what it still owes is ffmpeg's
+* `-segment_list live.m3u8`. ffmpeg opens that file in TRUNCATE mode on every
+* spawn, so a writer respawn destroys it. Every segment written but not yet
+* relocated at that instant becomes permanently unreachable:
+*
+*   - no playlist names it, so no watcher will ever finalize it;
+*   - `parseSegmentPath` cannot see it — staging is flat and dot-prefixed, kept
+*     deliberately out of the hydrate walk — so retention never counts it;
+*   - the one `.rec-tmp` sweep that exists (`sweepUnassignedStaging`) touches
+*     only locations with ZERO placement assignments, never the live one.
+*
+* So the footage is not deleted and not kept: it is stranded, and it accrues
+* for as long as the install runs.
+*
+* `segment-watcher.ts` carries a comment asserting the opposite — that the
+* playlist "is NOT cleared between runs", which is what its `stale-previous-run`
+* classification and its documented "that is the crash recovery" path are built
+* on. The live filesystem refutes it, and a leftover that describes the right
+* design reads as verification: on 2026-08-14 `/recordings/.rec-tmp/590/high/`
+* held a `live.m3u8` of 50 entries, every one named at or after that writer's
+* 20:28:31 start, beside **5 863 older `.m4s` files referenced by nobody** —
+* the oldest from 2026-07-04. Fleet total: 179 GB, on an array at 90% full.
+*
+* WHAT THIS DOES NOT DO
+* ─────────────────────
+* It never deletes. Recovery is the same `SegmentStore.onFinalized` the live
+* path uses — stat, relocate into the UTC bucket, index — so a recovered
+* segment is indistinguishable from one that was never orphaned, and a segment
+* it cannot place stays exactly where it is for the next attempt.
+*/
+/**
+* How far before the writer's own start a segment may be named and still count
+* as belonging to THIS run.
+*
+* Mirrors `segment-watcher.PREVIOUS_RUN_SKEW_MS`, and for the same reason:
+* ffmpeg names segments with whole-second `%s`, so the first segment of a run
+* carries the second already in progress when the process spawned. Without the
+* tolerance a recovery pass would reach for the live writer's opening segment —
+* the one file in the directory that is still being written.
+*/
+var PREVIOUS_RUN_SKEW_MS = 1e3;
+/**
+* How much longer than nominal an inter-segment gap may be and still be read as
+* one segment's duration.
+*
+* ffmpeg cuts on keyframes, so a real segment overshoots `-segment_time` by up
+* to a GOP; a gap of twice nominal is still plausibly one segment. Anything
+* wider is a RUN BOUNDARY — the writer was down in between — and the honest
+* duration is the nominal one. Trusting the gap there would paint a solid
+* timeline bar across hours of footage that does not exist, which is a worse
+* failure than under-reporting by a few seconds.
+*/
+var MAX_GAP_FACTOR = 2;
+/**
+* Decide which staged files belong to a previous run, and how long each ran.
+*
+* Pure — the caller does the I/O. The gate is the file's own name against the
+* writer's start: a segment named before this ffmpeg spawned cannot be one this
+* ffmpeg is writing, whatever the playlist says.
+*/
+function planStagingRecovery(input) {
+	const nominalMs = input.segmentSeconds * 1e3;
+	const cutoffMs = input.writerStartedMs - PREVIOUS_RUN_SKEW_MS;
+	const owned = input.playlistNames;
+	const candidates = [];
+	for (const name of input.names) {
+		const startMs = parseEpochStartMs(name);
+		if (startMs === null) continue;
+		if (startMs >= cutoffMs) continue;
+		if (owned?.has(name) === true) continue;
+		candidates.push({
+			name,
+			startMs
+		});
+	}
+	candidates.sort((a, b) => a.startMs - b.startMs);
+	const maxGapMs = nominalMs * MAX_GAP_FACTOR;
+	return candidates.map((c, i) => {
+		const next = candidates[i + 1];
+		const gapMs = next === void 0 ? 0 : next.startMs - c.startMs;
+		const durMs = gapMs > 0 && gapMs <= maxGapMs ? gapMs : nominalMs;
+		return {
+			name: c.name,
+			startMs: c.startMs,
+			durMs
+		};
+	});
+}
+/** The real filesystem. The only production value of {@link StagingIo}. */
+var nodeStagingIo = {
+	listDir: (dir) => promises.readdir(dir),
+	readPlaylist: (dir) => promises.readFile(path.join(dir, "live.m3u8"), "utf8").catch(() => null)
+};
+/** The only `<deviceId>/` subdirectories of staging that hold segments. */
+var KNOWN_PROFILES = new Set([
+	"high",
+	"mid",
+	"low"
+]);
+/** Default orphans per chunk. 590/high alone had 5 863 of them. */
+var DEFAULT_CHUNK_SIZE = 100;
+/**
+* Orphans between progress lines.
+*
+* A directory of 5 863 files takes minutes over shfs and used to report only
+* when it finished, so "is it still going, and how far in" had no answer at
+* all — the exact shape of the blackout this pass was written to end.
+*/
+var PROGRESS_EVERY = 1e3;
+var macrotask = () => new Promise((resolve) => {
+	setTimeout(resolve, 0);
+});
+/**
+* Relocate every previous-run segment left in one staging dir.
+*
+* Chunked with a yield between batches: this runs alongside a live recorder and
+* a backlog is measured in thousands of files over shfs/FUSE — a single
+* uninterrupted loop is exactly the event-loop stall the recorder has already
+* paid for once (D128).
+*
+* A failure is per-segment and never aborts the pass: the file stays in staging
+* and the next attach retries it. Every drop is logged with the segment name —
+* silence reads as "never happened", and this whole class of loss went
+* unnoticed for six weeks precisely because nothing said anything.
+*/
+async function recoverStagingOrphans(dir, input, deps) {
+	let names;
+	try {
+		names = await deps.listDir(dir);
+	} catch (err) {
+		deps.logger.warn("recorder: staging recovery could not list the staging dir", { meta: {
+			dir,
+			error: err instanceof Error ? err.message : String(err)
+		} });
+		return {
+			found: 0,
+			recovered: 0,
+			failed: 0,
+			aborted: false
+		};
+	}
+	let playlistNames;
+	const body = await deps.readPlaylist(dir).catch(() => null);
+	if (body !== null) playlistNames = new Set(parseLivePlaylist(body).map((e) => e.segPath.replace(/^.*\//, "")));
+	const plan = planStagingRecovery({
+		...input,
+		names,
+		playlistNames
+	});
+	if (plan.length === 0) return {
+		found: 0,
+		recovered: 0,
+		failed: 0,
+		aborted: false
+	};
+	deps.logger.info("recorder: staging recovery starting on a directory", { meta: {
+		dir,
+		found: plan.length,
+		staged: names.length,
+		oldestMs: plan[0]?.startMs,
+		newestMs: plan[plan.length - 1]?.startMs
+	} });
+	const chunkSize = deps.chunkSize ?? DEFAULT_CHUNK_SIZE;
+	const yieldBetween = deps.yieldBetween ?? macrotask;
+	let recovered = 0;
+	let failed = 0;
+	let aborted = false;
+	for (let i = 0; i < plan.length; i++) {
+		const orphan = plan[i];
+		if (deps.shouldStop?.() === true) {
+			aborted = true;
+			deps.logger.warn("recorder: staging recovery stopped short by shutdown", { meta: {
+				dir,
+				recovered,
+				failed,
+				remaining: plan.length - i
+			} });
+			break;
+		}
+		try {
+			await deps.finalize(orphan, `${dir}/${orphan.name}`);
+			recovered += 1;
+		} catch (err) {
+			failed += 1;
+			deps.logger.warn("recorder: staging recovery could not relocate an orphaned segment", { meta: {
+				dir,
+				seg: orphan.name,
+				startMs: orphan.startMs,
+				durMs: orphan.durMs,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+		}
+		if ((i + 1) % PROGRESS_EVERY === 0 && i + 1 < plan.length) deps.logger.info("recorder: staging recovery progress", { meta: {
+			dir,
+			done: i + 1,
+			found: plan.length,
+			recovered,
+			failed
+		} });
+		if ((i + 1) % chunkSize === 0 && i + 1 < plan.length) await yieldBetween();
+	}
+	deps.logger.info("recorder: recovered segments a writer restart had orphaned in staging", { meta: {
+		dir,
+		found: plan.length,
+		recovered,
+		failed,
+		aborted,
+		oldestMs: plan[0]?.startMs,
+		newestMs: plan[plan.length - 1]?.startMs
+	} });
+	return {
+		found: plan.length,
+		recovered,
+		failed,
+		aborted
+	};
+}
+/**
+* Boot reconcile: relocate every previous-run segment stranded in staging, on
+* every location, for every camera — attached or not.
+*
+* WHY BOOT, AND NOT ATTACH
+* ────────────────────────
+* Orphans are created by a writer restart, so by construction they all predate
+* the runner that finds them; boot is when the whole backlog is visible at
+* once. Attach looked like the natural hook and is the wrong one twice over: it
+* only ever reaches cameras that are currently recording (device 1438 had a
+* staging backlog while attached to nothing), and it puts unbounded filesystem
+* work on the path that must start recording NOW — measured, it kept microtasks
+* in flight across `RecordingController.stop()` and made the runner's 5 s
+* SIGTERM-grace test fail under load.
+*
+* `bootMs` is the cutoff, and it is what makes this safe to run while writers
+* are spawning: a live writer's segments are all named after boot, so the plan
+* cannot select one. Nothing is deleted, and a segment that cannot be placed is
+* left exactly where it is for the next boot.
+*/
+async function recoverAllStagedOrphans(deps) {
+	let dirs = 0;
+	let found = 0;
+	let recovered = 0;
+	let failed = 0;
+	let aborted = false;
+	const startedMs = Date.now();
+	const injectedYield = deps.yieldBetween;
+	const pacer = injectedYield === void 0 ? nodeLoopPacer() : null;
+	const yieldBetween = injectedYield ?? (() => pacer === null ? macrotask() : pacer.hop());
+	const locations = deps.locations();
+	deps.logger.info("recorder: staging boot reconcile starting", { meta: {
+		locations: locations.map((l) => l.id),
+		bootMs: deps.bootMs
+	} });
+	for (const location of locations) {
+		const stagingRoot = `${location.root}/${STAGING_DIR_NAME}`;
+		let deviceDirs;
+		try {
+			deviceDirs = await deps.listDir(stagingRoot);
+		} catch {
+			continue;
+		}
+		for (const deviceDir of deviceDirs) {
+			if (aborted) break;
+			const deviceId = Number(deviceDir);
+			if (!Number.isInteger(deviceId) || deviceId <= 0) continue;
+			let profileDirs;
+			try {
+				profileDirs = await deps.listDir(`${stagingRoot}/${deviceDir}`);
+			} catch {
+				continue;
+			}
+			const segmentSeconds = await deps.segmentSecondsFor(deviceId);
+			for (const profile of profileDirs) {
+				if (aborted) break;
+				if (!KNOWN_PROFILES.has(profile)) continue;
+				const dir = `${stagingRoot}/${deviceDir}/${profile}`;
+				dirs += 1;
+				const report = await recoverStagingOrphans(dir, {
+					writerStartedMs: deps.bootMs,
+					segmentSeconds
+				}, {
+					listDir: deps.listDir,
+					readPlaylist: deps.readPlaylist,
+					finalize: (orphan, flatAbsPath) => deps.finalize({
+						deviceId,
+						profile,
+						locationId: location.id,
+						locationRoot: location.root,
+						flatAbsPath,
+						startMs: orphan.startMs,
+						durMs: orphan.durMs
+					}),
+					logger: withDeviceTag(deps.logger, deviceId),
+					chunkSize: deps.chunkSize,
+					yieldBetween,
+					shouldStop: deps.shouldStop
+				});
+				found += report.found;
+				recovered += report.recovered;
+				failed += report.failed;
+				if (report.aborted) aborted = true;
+			}
+		}
+	}
+	deps.logger.info("recorder: staging boot reconcile complete", { meta: {
+		dirs,
+		found,
+		recovered,
+		failed,
+		aborted,
+		ms: Date.now() - startedMs,
+		...pacer === null ? {} : { pacing: pacer.stats() }
+	} });
+	return {
+		dirs,
+		found,
+		recovered,
+		failed,
+		aborted
+	};
+}
+/** Bind `tags: { deviceId }` onto every line a per-dir pass writes. */
+function withDeviceTag(logger, deviceId) {
+	const tag = (extras) => ({
+		...typeof extras === "object" && extras !== null ? extras : {},
+		tags: { deviceId }
+	});
+	return {
+		info: (message, extras) => logger.info(message, tag(extras)),
+		warn: (message, extras) => logger.warn(message, tag(extras)),
+		debug: (message, extras) => logger.debug(message, tag(extras))
+	};
+}
+//#endregion
 //#region src/recorder/addon/index.ts
 /**
 * recorder addon SHELL — recording + storage-evictable providers on the
@@ -7165,6 +7727,7 @@ var RecorderV2Addon = class extends BaseAddon {
 				this.calendar.invalidate(deviceId, atMs);
 			},
 			onIndexed: (row, absPath) => {
+				if (!isMfraTableServable(row.startMs, Date.now())) return;
 				readMfraTable(absPath, row.bytes).then((table) => {
 					this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
 				}).catch((err) => {
@@ -7182,13 +7745,18 @@ var RecorderV2Addon = class extends BaseAddon {
 				if (!loc) throw new Error(`recorder: unknown recordings location ${locationId}`);
 				await promises.rmdir(path.join(loc.root, relDir));
 			},
-			logger: { warn: (message, extras) => {
-				if (isSegmentStoreLogExtras(extras)) this.ctx.logger.warn(message, {
-					tags: extras.tags,
-					meta: { error: extras.error }
-				});
-				else this.ctx.logger.warn(message, { meta: { extras } });
-			} }
+			logger: {
+				warn: (message, extras) => {
+					if (isSegmentStoreLogExtras(extras)) this.ctx.logger.warn(message, {
+						tags: extras.tags,
+						meta: { error: extras.error }
+					});
+					else this.ctx.logger.warn(message, { meta: { extras } });
+				},
+				info: (message, extras) => {
+					this.ctx.logger.info(message, { meta: extras.meta });
+				}
+			}
 		};
 		this.segmentStore = new SegmentStore(segmentStoreDeps);
 		const brokerHandle = this.ctx.useCapability("stream-broker", brokerScope(ingestOwner.ownerNodeId));
@@ -7515,11 +8083,13 @@ var RecorderV2Addon = class extends BaseAddon {
 			force: true
 		}).catch(() => {});
 		await this.sweepUnassignedStaging();
+		const stagingBootMs = Date.now();
 		try {
 			await this.controller?.start();
 		} catch (err) {
 			this.ctx.logger.warn("recorder: controller start failed", { meta: { error: errMsg(err) } });
 		}
+		this.recoverStagedOrphans(stagingBootMs);
 		if (this.eventUnsub === null) this.eventUnsub = subscribeEventCapture({
 			eventBus: this.ctx.eventBus,
 			map: this.eventMap,
@@ -7824,6 +8394,45 @@ var RecorderV2Addon = class extends BaseAddon {
 		}
 	}
 	/**
+	* Boot reconcile for `.rec-tmp`: relocate every segment a previous runner's
+	* writers left stranded, on every location, for every camera.
+	*
+	* The counterpart to {@link sweepUnassignedStaging} above, which only ever
+	* looks at locations nothing is assigned to — so the LIVE root, the one that
+	* actually accumulates, had no pass at all (D148). Nothing here deletes: each
+	* orphan goes through the same `SegmentStore.onFinalized` a live segment
+	* does, and one it cannot place is left on disk for the next boot.
+	*/
+	async recoverStagedOrphans(bootMs) {
+		try {
+			await recoverAllStagedOrphans({
+				locations: () => this.resolvedLocations.map((l) => ({
+					id: l.id,
+					root: l.root
+				})),
+				listDir: nodeStagingIo.listDir,
+				readPlaylist: nodeStagingIo.readPlaylist,
+				finalize: async (input) => {
+					const store = this.segmentStore;
+					if (store === null) throw new Error("recorder is shutting down");
+					await store.onFinalized(input);
+				},
+				logger: this.ctx.logger,
+				bootMs,
+				segmentSecondsFor: async (deviceId) => {
+					try {
+						return (await loadDeviceConfig(this.configStore(), deviceId)).segmentSeconds ?? this.config.segmentSeconds;
+					} catch {
+						return this.config.segmentSeconds;
+					}
+				},
+				shouldStop: () => this.segmentStore === null
+			});
+		} catch (err) {
+			this.ctx.logger.warn("recorder: staging boot reconcile failed", { meta: { error: errMsg(err) } });
+		}
+	}
+	/**
 	* Surface a storage-placement fault on the liveness/alert surface. STRICTLY
 	* best-effort: the alerts cap is a nice-to-have and an unmounted disk must
 	* still be shouted about in the log even when it is unreachable.