@camstack/addon-pipeline 1.2.96 → 1.2.98

package/dist/recorder/index.js

@@ -1,6 +1,6 @@
-const require_dist = require("../dist-w9QWDvWn.js");
+const require_dist = require("../dist-dm3t4BOt.js");
 const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
-const require_addon_utils = require("../addon-utils-B6usm-YO.js");
+const require_addon_utils = require("../addon-utils-eUS6n_Zj.js");
 let node_crypto = require("node:crypto");
 let node_child_process = require("node:child_process");
 let node_path = require("node:path");
@@ -4333,9 +4333,15 @@ function buildRecordingProvider(deps) {
 				const first = group[0];
 				const cap = await deps.capacity(first.root);
 				const ids = group.map((l) => l.id);
+				if (cap.error !== void 0) deps.logger.warn("recorder: volume capacity unavailable (statfs failed)", { meta: {
+					locationId: first.id,
+					root: first.root,
+					error: cap.error
+				} });
 				return {
 					locationId: first.id,
 					locationIds: ids,
+					root: first.root,
 					usedBytes: deps.index.accountingForLocations(new Set(ids)).bytes,
 					availableBytes: cap.availableBytes,
 					totalBytes: cap.totalBytes
@@ -5569,13 +5575,20 @@ async function probeLocationVolume(location, rememberedVolumeId, deps) {
 }
 /**
 * Where to write instead when a location fails its volume probe: the normal
-* placement chain over every OTHER location. With N locations this is a
-* sibling of the same type; with one it is the unusable location itself —
-* a shouting recorder beats a stopped camera, and refusing to record is the
-* one outcome no operator asked for.
-*/
-function failoverForUnusableLocation(unusable, locations, profile) {
-	const siblings = locations.filter((l) => l.id !== unusable.id);
+* placement chain over every location NOT already known unusable. With N
+* locations this is a healthy sibling of the same type; with none left it is
+* the unusable location itself — a shouting recorder beats a stopped camera,
+* and refusing to record is the one outcome no operator asked for.
+*
+* `alreadyUnusable` is not an optimisation. Without it the search happily
+* returned a sibling that had ALSO failed its probe, so with two broken disks
+* every attach handed back the other one and the stored assignment flipped
+* A→B→A→B forever — one durable write and one audit row per attach, per camera,
+* per profile. Observed live on 2026-08-19 (the UD disk did not come back after
+* a reboot): 716 EACCES/min, with the ops-log ring flooded by the flapping.
+*/
+function failoverForUnusableLocation(unusable, locations, profile, alreadyUnusable = /* @__PURE__ */ new Set()) {
+	const siblings = locations.filter((l) => l.id !== unusable.id && !alreadyUnusable.has(l.id));
 	try {
 		return resolvePlacement(siblings, profile);
 	} catch {
@@ -5626,6 +5639,13 @@ var PlacementService = class {
 	verified = /* @__PURE__ */ new Set();
 	/** Locations the probe declared unusable — re-probed after a change. */
 	unusable = /* @__PURE__ */ new Set();
+	/** Cameras that have already announced "every location is unusable". One line
+	*  per camera per episode: the fault is asked per-camera, and repeating it per
+	*  attach is the noise this state exists to end. */
+	allUnusableReported = /* @__PURE__ */ new Set();
+	/** When the current all-unusable episode last (re-)probed. `null` ⇒ no
+	*  episode in progress. */
+	allUnusableProbedAt = null;
 	/** The fair tie-break the PURE planner defers to (D37). Advisory state, and
 	*  consulted only for a tie the capacity signal cannot separate. */
 	rotation = new PlacementRotation();
@@ -5635,6 +5655,12 @@ var PlacementService = class {
 	get probeDeps() {
 		return this.deps.probe ?? fsVolumeProbeDeps;
 	}
+	get now() {
+		return this.deps.now?.() ?? Date.now();
+	}
+	get reprobeMs() {
+		return this.deps.reprobeMs ?? 6e4;
+	}
 	/**
 	* Forget every probe verdict. Called whenever the location set is re-resolved
 	* (rescan, storage migration) — a remount is exactly the event the probe
@@ -5643,6 +5669,8 @@ var PlacementService = class {
 	onLocationsChanged() {
 		this.verified.clear();
 		this.unusable.clear();
+		this.allUnusableReported.clear();
+		this.allUnusableProbedAt = null;
 	}
 	/**
 	* Resolve the location for one (camera, profile) at an attach boundary.
@@ -5651,12 +5679,92 @@ var PlacementService = class {
 	*/
 	async place(deviceId, profile) {
 		const pool = this.pool();
+		this.reprobeAfterBackoff(pool);
 		const resolved = await this.assignedLocation(deviceId, profile, pool) ?? resolvePlacement(pool, profile);
 		const usable = await this.ensureUsable(resolved, pool, profile, deviceId);
+		if (this.everyLocationUnusable(pool)) {
+			this.reportEveryLocationUnusable(deviceId, profile, resolved, pool);
+			return resolved;
+		}
+		this.allUnusableReported.delete(deviceId);
 		await this.recordDecision(deviceId, profile, usable, resolved.id === usable.id);
 		return usable;
 	}
 	/**
+	* Has the marker probe declared this location's volume NOT the one we last
+	* wrote to?
+	*
+	* Read by the segment-hour ledger: a walk over a root whose volume failed its
+	* probe found nothing because nothing is mounted there, and that must never be
+	* allowed to delete durable rows (D49). Unprobed ⇒ usable, which is honest —
+	* the probe runs at the first writer attach, so at boot it has usually not run
+	* at all, and the ledger's own magnitude guard covers that window.
+	*/
+	isLocationUsable(locationId) {
+		return !this.unusable.has(locationId);
+	}
+	/** True when the pool is non-empty and the probe has failed EVERY member. */
+	everyLocationUnusable(pool) {
+		return pool.length > 0 && pool.every((location) => this.unusable.has(location.id));
+	}
+	/**
+	* Announce the terminal state — ONCE per camera per episode.
+	*
+	* Before this, two unusable locations produced an infinite failover
+	* ping-pong: `failoverForUnusableLocation` did not consult the unusable set,
+	* so each attach handed back the other broken disk, `recordDecision` saw a
+	* changed assignment and wrote the store plus an audit row, and the writer
+	* hit EACCES on a path that was never going to work. Measured on the live hub
+	* on 2026-08-19 with the UD disk unmounted: 716 EACCES/min.
+	*
+	* Recording is NOT refused. With a single location this state is reached the
+	* moment that one location fails, and the recorder keeps attaching to it on
+	* purpose — a shouting recorder beats a stopped camera. What changes is that
+	* it shouts once instead of once per attach, and writes nothing durable while
+	* shouting.
+	*/
+	reportEveryLocationUnusable(deviceId, profile, target, pool) {
+		this.allUnusableProbedAt ??= this.now;
+		if (this.allUnusableReported.has(deviceId)) return;
+		this.allUnusableReported.add(deviceId);
+		this.deps.logger.warn("recorder placement: EVERY recordings location failed its volume probe — the assignment is FROZEN on its current target and nothing more is written until a disk comes back", {
+			tags: { deviceId },
+			meta: {
+				profile,
+				targetLocationId: target.id,
+				root: target.root,
+				locationIds: pool.map((location) => location.id),
+				reprobeMs: this.reprobeMs
+			}
+		});
+	}
+	/**
+	* While every location is unusable, re-probe them all — at most once per
+	* {@link ALL_UNUSABLE_REPROBE_MS}. This is the backoff: without it the probe
+	* memo would hold for the life of the process (only `onLocationsChanged`
+	* clears it) and a disk remounted by an operator would need an addon restart
+	* to be noticed.
+	*/
+	reprobeAfterBackoff(pool) {
+		if (!this.everyLocationUnusable(pool)) {
+			this.allUnusableProbedAt = null;
+			return;
+		}
+		const now = this.now;
+		if (this.allUnusableProbedAt === null) {
+			this.allUnusableProbedAt = now;
+			return;
+		}
+		if (now - this.allUnusableProbedAt < this.reprobeMs) return;
+		this.allUnusableProbedAt = now;
+		this.deps.logger.info("recorder placement: re-probing every storage location after backoff — a disk that came back is picked up here", { meta: {
+			locationIds: pool.map((location) => location.id),
+			backoffMs: this.reprobeMs
+		} });
+		this.unusable.clear();
+		this.verified.clear();
+	}
+	/**
 	* The locations placement may write to: operator-ENABLED and alias-guarded.
 	* A location the operator has not opted in is still read, swept and drained —
 	* it is simply never a write target. Creating a disk must not start writing
@@ -5805,13 +5913,13 @@ var PlacementService = class {
 	* ONCE per location per location-set generation — it is a claim, not a poll.
 	*/
 	async ensureUsable(chosen, locations, profile, deviceId) {
-		if (this.unusable.has(chosen.id)) return failoverForUnusableLocation(chosen, locations, profile);
+		if (this.unusable.has(chosen.id)) return failoverForUnusableLocation(chosen, locations, profile, this.unusable);
 		if (this.verified.has(chosen.id)) return chosen;
 		const result = await probeLocationVolume(chosen, (await readPlacementState(this.deps.state)).volumeIds[chosen.id] ?? null, this.probeDeps);
 		if (result.outcome === "mismatch") {
 			this.unusable.add(chosen.id);
 			this.reportMountFault(result, deviceId);
-			return failoverForUnusableLocation(chosen, locations, profile);
+			return failoverForUnusableLocation(chosen, locations, profile, this.unusable);
 		}
 		if (result.outcome === "unreadable") {
 			this.deps.logger.warn("recorder placement: volume probe unreadable — assignment unchanged", {
@@ -7925,6 +8033,34 @@ function makeBrokerCall(handle) {
 	return (fn) => handle.call(fn, BROKER_READY_TIMEOUT_MS);
 }
 //#endregion
+//#region src/recorder/addon/location-capacity.ts
+/** Human-readable failure text for a thrown value of unknown shape. */
+function reasonOf(err) {
+	return err instanceof Error ? err.message : String(err);
+}
+/**
+* Free + total bytes on the volume backing `root`.
+*
+* Never throws: an unreachable volume is one degraded ROW, not a failed
+* `getStorageUsage` — a NAS that stops answering must not blank the capacity of
+* every other disk on the node.
+*/
+async function readLocationCapacity(root, statfs) {
+	try {
+		const st = await statfs(root);
+		return {
+			availableBytes: st.bavail * st.bsize,
+			totalBytes: st.blocks * st.bsize
+		};
+	} catch (err) {
+		return {
+			availableBytes: null,
+			totalBytes: null,
+			error: reasonOf(err)
+		};
+	}
+}
+//#endregion
 //#region src/durable/durable-ledger.ts
 /** Default reseed cap — every current consumer's row set is installation-bounded. */
 var DEFAULT_LOAD_LIMIT = 1e5;
@@ -8212,6 +8348,17 @@ var RECORDING_SEGMENT_HOURS_INDEXES = [{
 	name: "idx_recorder_segment_hours_hour",
 	columns: ["hourStartMs"]
 }];
+/**
+* The largest share of a device's held hours a single walk may delete.
+*
+* On 2026-08-19 the recorder respawned while `/recordings` was an empty tmpfs —
+* the UD disk had not mounted — and one walk took the ledger from 873 hours to
+* 32 (a 96% drop). The ledger is precisely what lets the NEXT boot skip a
+* ~400 s recursive walk, so a single bad walk taxed every boot after it. A
+* retention sweep does not delete half a camera's archive in one pass; a
+* `readdir` over an unmounted root deletes all of it.
+*/
+var LEDGER_PRUNE_MAX_SHARE = .5;
 function hourStartMs(startMs) {
 	return Math.floor(startMs / HOUR_MS$1) * HOUR_MS$1;
 }
@@ -8285,8 +8432,10 @@ var SPEC = {
 var SegmentHourLedger = class {
 	ledger;
 	logger;
+	isLocationUsable;
 	constructor(deps) {
 		this.logger = deps.logger;
+		this.isLocationUsable = deps.isLocationUsable ?? (() => true);
 		this.ledger = new DurableLedger({
 			spec: SPEC,
 			store: deps.store,
@@ -8376,6 +8525,10 @@ var SegmentHourLedger = class {
 	*
 	* Devices not in `deviceIds` are left untouched (a partial walk must not
 	* prune cameras it did not look at).
+	*
+	* And a walk this ledger does not TRUST prunes nothing at all — see
+	* {@link refusedPrunes}. What such a walk FOUND is still real, so its paths
+	* are unioned in; what it did not find decides nothing (D49).
 	*/
 	async reconcileFromIndex(index, deviceIds, nowMs) {
 		const walked = new Set(deviceIds);
@@ -8397,15 +8550,17 @@ var SegmentHourLedger = class {
 				paths: [...existing.paths, seg.path]
 			});
 		}
+		const refused = this.refusedPrunes(walked, fromIndex);
 		for (const row of this.ledger.snapshot()) {
 			if (!walked.has(row.deviceId)) continue;
 			const disk = fromIndex.get(row.key);
+			const additiveOnly = row.hourStartMs === currentHour || refused.devices.has(row.deviceId) || refused.locations.has(locationKey(row.deviceId, row.locationId));
 			if (disk === void 0) {
-				if (row.hourStartMs === currentHour) continue;
+				if (additiveOnly) continue;
 				await this.ledger.forget(row.key);
 				continue;
 			}
-			if (row.hourStartMs === currentHour) {
+			if (additiveOnly) {
 				const union = uniquePaths([...disk.paths, ...row.paths]);
 				if (!samePaths(union, row.paths)) await this.ledger.put({
 					...row,
@@ -8420,7 +8575,85 @@ var SegmentHourLedger = class {
 			await this.ledger.put(disk);
 		}
 	}
+	/**
+	* Which of this walk's deletions are REFUSED, and why.
+	*
+	* Two independent reasons, because the live fault of 2026-08-19 would have
+	* escaped either one alone:
+	*
+	* - **the volume said no.** `PlacementService`'s marker probe knows the bytes
+	*   under a root are not the bytes we wrote. A walk of such a root found
+	*   nothing because nothing is mounted there, not because the footage is
+	*   gone.
+	* - **the magnitude said no.** At BOOT the probe has not run yet — it runs at
+	*   the first writer attach — so on the morning the recorder respawned onto an
+	*   empty tmpfs the volume was not flagged and the prune went through: 873
+	*   hours to 32. A walk that deletes more than
+	*   {@link LEDGER_PRUNE_MAX_SHARE} of a device's held hours is refused on its
+	*   size alone.
+	*
+	* A refusal is one line per device (or per device+location), carrying the
+	* delta, tagged with `deviceId` — a ledger that silently declined to prune and
+	* a ledger with nothing to prune must never read the same.
+	*/
+	refusedPrunes(walked, fromIndex) {
+		const heldByDevice = /* @__PURE__ */ new Map();
+		const unusable = /* @__PURE__ */ new Map();
+		for (const row of this.ledger.snapshot()) {
+			if (!walked.has(row.deviceId)) continue;
+			heldByDevice.set(row.deviceId, (heldByDevice.get(row.deviceId) ?? 0) + 1);
+			if (this.isLocationUsable(row.locationId)) continue;
+			const key = locationKey(row.deviceId, row.locationId);
+			const seen = unusable.get(key);
+			unusable.set(key, seen === void 0 ? {
+				deviceId: row.deviceId,
+				locationId: row.locationId,
+				held: 1
+			} : {
+				...seen,
+				held: seen.held + 1
+			});
+		}
+		const onDiskByDevice = /* @__PURE__ */ new Map();
+		for (const row of fromIndex.values()) onDiskByDevice.set(row.deviceId, (onDiskByDevice.get(row.deviceId) ?? 0) + 1);
+		const devices = /* @__PURE__ */ new Set();
+		for (const [deviceId, held] of heldByDevice) {
+			if (held < 8) continue;
+			const onDisk = onDiskByDevice.get(deviceId) ?? 0;
+			if (onDisk >= held * .5) continue;
+			devices.add(deviceId);
+			this.logger.warn("recorder: segment-hour ledger prune REFUSED — this walk would delete most of the camera archive, which is what an unmounted root looks like; keeping the previous ledger", {
+				tags: { deviceId },
+				meta: {
+					held,
+					onDisk,
+					dropped: held - onDisk,
+					maxShare: LEDGER_PRUNE_MAX_SHARE
+				}
+			});
+		}
+		const locations = /* @__PURE__ */ new Set();
+		for (const [key, entry] of unusable) {
+			if (devices.has(entry.deviceId)) continue;
+			locations.add(key);
+			this.logger.warn("recorder: segment-hour ledger prune REFUSED — the volume under this location failed its probe, so the walk found nothing because nothing is mounted there", {
+				tags: { deviceId: entry.deviceId },
+				meta: {
+					locationId: entry.locationId,
+					held: entry.held,
+					onDisk: onDiskByDevice.get(entry.deviceId) ?? 0
+				}
+			});
+		}
+		return {
+			devices,
+			locations
+		};
+	}
 };
+function locationKey(deviceId, locationId) {
+	return `${deviceId}|${locationId}`;
+}
 //#endregion
 //#region src/recorder/addon/recording-export-provider.ts
 /** Content type of every rendered export — the engine only ever writes MP4. */
@@ -8910,7 +9143,9 @@ var macrotask = () => new Promise((resolve) => {
 * unnoticed for six weeks precisely because nothing said anything.
 */
 async function recoverStagingOrphans(dir, input, deps) {
-	let names;
+	const chunkSize = deps.chunkSize ?? DEFAULT_CHUNK_SIZE;
+	const yieldBetween = deps.yieldBetween ?? macrotask;
+	let names = null;
 	try {
 		names = await deps.listDir(dir);
 	} catch (err) {
@@ -8918,13 +9153,14 @@ async function recoverStagingOrphans(dir, input, deps) {
 			dir,
 			error: err instanceof Error ? err.message : String(err)
 		} });
-		return {
-			found: 0,
-			recovered: 0,
-			failed: 0,
-			aborted: false
-		};
 	}
+	await yieldBetween();
+	if (names === null) return {
+		found: 0,
+		recovered: 0,
+		failed: 0,
+		aborted: false
+	};
 	let playlistNames;
 	const body = await deps.readPlaylist(dir).catch(() => null);
 	if (body !== null) playlistNames = new Set(parseLivePlaylist(body).map((e) => e.segPath.replace(/^.*\//, "")));
@@ -8946,8 +9182,6 @@ async function recoverStagingOrphans(dir, input, deps) {
 		oldestMs: plan[0]?.startMs,
 		newestMs: plan[plan.length - 1]?.startMs
 	} });
-	const chunkSize = deps.chunkSize ?? DEFAULT_CHUNK_SIZE;
-	const yieldBetween = deps.yieldBetween ?? macrotask;
 	let recovered = 0;
 	let failed = 0;
 	let aborted = false;
@@ -9047,24 +9281,41 @@ async function recoverAllStagedOrphans(deps) {
 		locations: locations.map((l) => l.id),
 		bootMs: deps.bootMs
 	} });
-	const work = [];
-	for (const location of locations) {
-		const stagingRoot = `${location.root}/${STAGING_DIR_NAME}`;
-		let deviceDirs;
+	/**
+	* One directory listing on the walk, paced through the pass's own pacer.
+	*
+	* "A handful of listings" is what this walk was costed as, and on a WARM
+	* share it is — 4.1 s for the whole thing. Cold, after a power-cycle, each
+	* one is a ~9.4 s FUSE round-trip and 58 of them summed to 547 s with the
+	* loop to themselves, while the controller re-pinned writers 118 times and
+	* the first recording landed at t+714 s.
+	*
+	* Paced per LISTING and not per entry, because that is where the latency
+	* is; and through the SAME `yieldBetween` the recovery uses, so there is one
+	* pacer, one back-off state and one set of counters for the whole pass —
+	* never a second scheduling authority (D167).
+	*/
+	const listPaced = async (dir) => {
+		let entries = null;
 		try {
-			deviceDirs = await deps.listDir(stagingRoot);
+			entries = await deps.listDir(dir);
 		} catch {
-			continue;
+			entries = null;
 		}
+		await yieldBetween();
+		return entries;
+	};
+	const walkStartedMs = Date.now();
+	const work = [];
+	for (const location of locations) {
+		const stagingRoot = `${location.root}/${STAGING_DIR_NAME}`;
+		const deviceDirs = await listPaced(stagingRoot);
+		if (deviceDirs === null) continue;
 		for (const deviceDir of deviceDirs) {
 			const deviceId = Number(deviceDir);
 			if (!Number.isInteger(deviceId) || deviceId <= 0) continue;
-			let profileDirs;
-			try {
-				profileDirs = await deps.listDir(`${stagingRoot}/${deviceDir}`);
-			} catch {
-				continue;
-			}
+			const profileDirs = await listPaced(`${stagingRoot}/${deviceDir}`);
+			if (profileDirs === null) continue;
 			const segmentSeconds = await deps.segmentSecondsFor(deviceId);
 			for (const profile of profileDirs) {
 				if (!KNOWN_PROFILES.has(profile)) continue;
@@ -9078,6 +9329,7 @@ async function recoverAllStagedOrphans(deps) {
 			}
 		}
 	}
+	const walkMs = Date.now() - walkStartedMs;
 	const queue = [...work];
 	const lane = async () => {
 		for (;;) {
@@ -9120,6 +9372,7 @@ async function recoverAllStagedOrphans(deps) {
 		failed,
 		aborted,
 		ms: Date.now() - startedMs,
+		walkMs,
 		...pacer === null ? {} : { pacing: pacer.stats() }
 	} });
 	return {
@@ -9372,7 +9625,8 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 		}
 		this.segmentHours = new SegmentHourLedger({
 			store: this.ctx.api.settingsStore,
-			logger: this.ctx.logger
+			logger: this.ctx.logger,
+			isLocationUsable: (locationId) => this.placement?.isLocationUsable(locationId) ?? true
 		});
 		try {
 			await this.segmentHours.declare();
@@ -10201,20 +10455,14 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 	playbackRoots() {
 		return [node_path.default.join(this.ctx.dataDir, "playback"), ...this.resolvedLocations.map((l) => l.root)];
 	}
-	/** Free + total bytes on a location's volume (`statfs`); null when unstattable. */
+	/**
+	* Free + total bytes on a location's volume (`statfs`); null plus a reason
+	* when the volume will not answer. The reason is NOT logged here — the caller
+	* (`getStorageUsage`) knows which location the root belongs to and logs it
+	* against that location.
+	*/
 	async locationCapacity(root) {
-		try {
-			const st = await node_fs.promises.statfs(root);
-			return {
-				availableBytes: st.bavail * st.bsize,
-				totalBytes: st.blocks * st.bsize
-			};
-		} catch {
-			return {
-				availableBytes: null,
-				totalBytes: null
-			};
-		}
+		return readLocationCapacity(root, (target) => node_fs.promises.statfs(target));
 	}
 	/**
 	* Global settings surfaced in the admin UI (and persisted to the addon