@camstack/addon-pipeline 1.2.77 → 1.2.79

package/dist/recorder/index.js

@@ -1,6 +1,6 @@
-const require_dist = require("../dist-DQZzL0Ri.js");
+const require_dist = require("../dist-BUIpqu3E.js");
 const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
-const require_addon_utils = require("../addon-utils-bZcwLaVW.js");
+const require_addon_utils = require("../addon-utils-BZQxAJ1G.js");
 let node_crypto = require("node:crypto");
 let node_child_process = require("node:child_process");
 let node_path = require("node:path");
@@ -390,7 +390,21 @@ async function readMfraTable(absPath, fileBytes) {
 /** How long a table stays served. Directory asks are for recent footage far
 *  more often than not, and a bounded window keeps this store O(day), not
 *  O(uptime): ~30 bytes a segment ⇒ a day across 20 cameras is a few MB. */
-var RETAIN_MS = 26 * 36e5;
+var MFRA_RETAIN_MS = 26 * 36e5;
+/**
+* Whether a segment is recent enough for a captured table to ever be served.
+*
+* The gate the FINALIZE path must ask BEFORE reading the file. A live segment
+* always passes; a recovered one almost never does, and capturing it is pure
+* loss twice over — the table is unservable, and the read is a real `open` +
+* two `read`s on the libuv threadpool. The D148 boot reconcile fired 122 400 of
+* them, unawaited and unbounded, behind a 4-thread pool that the live
+* SegmentWriters' own finalize I/O has to share: that is the "segment file
+* never landed" (>30 s) on four cameras at once.
+*/
+function isMfraTableServable(startMs, nowMs) {
+	return startMs >= nowMs - MFRA_RETAIN_MS;
+}
 /**
 * The RAM store the directory reads. Keys are `(deviceId, profile, startMs)`
 * — exactly how a directory row identifies a segment. Insertion is
@@ -406,7 +420,16 @@ var MfraTableStore = class {
 	key(deviceId, profile, startMs) {
 		return `${deviceId}:${profile}:${startMs}`;
 	}
+	/**
+	* Keep one table, unless it is already past the serve window.
+	*
+	* The out-of-window REJECT is not belt-and-braces: `prune` walks the
+	* insertion-ordered prefix and stops at the first live entry, so an expired
+	* entry appended AFTER a live one is never reachable again. A bulk recovery
+	* appends 122 400 of exactly that shape and the store grows without bound.
+	*/
 	record(deviceId, profile, startMs, table) {
+		if (!isMfraTableServable(startMs, this.now())) return;
 		this.tables.set(this.key(deviceId, profile, startMs), {
 			startMs,
 			table
@@ -427,7 +450,7 @@ var MfraTableStore = class {
 	}
 	/** Insertion order is time order, so the expired prefix is contiguous. */
 	prune() {
-		const cutoff = this.now() - RETAIN_MS;
+		const cutoff = this.now() - MFRA_RETAIN_MS;
 		for (const [key, entry] of this.tables) {
 			if (entry.startMs >= cutoff) break;
 			this.tables.delete(key);
@@ -626,6 +649,38 @@ var RecordingIndex = class {
 	segments(deviceId, profile) {
 		return [...this.sortedSegments(deviceId, profile)];
 	}
+	/** True when `relPath` is currently indexed for `deviceId`. O(1). */
+	hasSegment(deviceId, relPath) {
+		return this.byDevice.get(deviceId)?.has(relPath) === true;
+	}
+	/**
+	* Indexed-segment count per HOUR-BUCKET directory (`<dev>/<profile>/Y/M/D/H`)
+	* for one (device, profile), in ONE pass over the device's rows.
+	*
+	* Exists because `SegmentStore.evict` has to answer "does this eviction cover
+	* every segment the bucket holds?" for every bucket in the round, and it used
+	* to answer it with `segments(deviceId, profile)` — a full COPY of the
+	* device-profile archive plus a full `startsWith` scan, PER BUCKET. That is
+	* O(buckets x archive), and D148 is what made it bite: before it, a retention
+	* round touched a handful of buckets at the retention edge; after it, the
+	* recovered backlog made ~8 900 sparse buckets across six weeks evictable in
+	* ONE round. Measured on that shape (1.63 M live rows + 122 400 victims):
+	* 182 596 ms of uninterrupted main-thread CPU, with no log line, which is the
+	* silent 100%-CPU phase the operator saw. One pass answers all of them.
+	*/
+	bucketCounts(deviceId, profile) {
+		const out = /* @__PURE__ */ new Map();
+		const m = this.byDevice.get(deviceId);
+		if (!m) return out;
+		for (const s of m.values()) {
+			if (s.profile !== profile) continue;
+			const cut = s.path.lastIndexOf("/");
+			if (cut <= 0) continue;
+			const bucketDir = s.path.slice(0, cut);
+			out.set(bucketDir, (out.get(bucketDir) ?? 0) + 1);
+		}
+		return out;
+	}
 	/** Aggregate accounting for a device across ALL its locations (use `accountingForLocation` for one location). */
 	accounting(deviceId) {
 		const m = this.byDevice.get(deviceId);
@@ -1258,6 +1313,7 @@ var SegmentStore = class SegmentStore {
 	async evict(location, rows) {
 		let reclaimed = 0;
 		const removed = [];
+		const startedMs = this.deps.now?.() ?? Date.now();
 		const byBucket = /* @__PURE__ */ new Map();
 		for (const r of rows) {
 			const cut = r.path.lastIndexOf("/");
@@ -1266,13 +1322,29 @@ var SegmentStore = class SegmentStore {
 			if (group) group.push(r);
 			else byBucket.set(bucketDir, [r]);
 		}
+		const census = /* @__PURE__ */ new Map();
+		const countsFor = (deviceId, profile) => {
+			const key = `${deviceId}:${profile}`;
+			const cached = census.get(key);
+			if (cached) return cached;
+			const counts = this.deps.index.bucketCounts(deviceId, profile);
+			census.set(key, counts);
+			return counts;
+		};
+		const chunkSize = this.deps.evictChunkSize ?? SegmentStore.DEFAULT_EVICT_CHUNK;
+		const yieldBetween = this.deps.yieldBetween ?? macrotask$2;
+		let bucketsRemoved = 0;
+		let handled = 0;
 		for (const [bucketDir, bucketRows] of byBucket) {
-			if (bucketDir !== "" && this.bucketFullyEvicted(bucketDir, bucketRows)) try {
+			if (bucketDir !== "" && this.bucketFullyEvicted(bucketDir, bucketRows, countsFor)) try {
 				await this.deps.removeDir(location, bucketDir);
+				bucketsRemoved += 1;
 				for (const r of bucketRows) {
 					removed.push(r.path);
 					reclaimed += r.bytes;
 				}
+				handled += 1;
+				if (handled % chunkSize === 0) await yieldBetween();
 				continue;
 			} catch (err) {
 				this.deps.logger.warn("bucket removeDir failed — falling back to per-file deletes", {
@@ -1293,15 +1365,47 @@ var SegmentStore = class SegmentStore {
 					error: err
 				});
 			}
+			handled += 1;
+			if (handled % chunkSize === 0) await yieldBetween();
 		}
 		if (removed.length > 0) {
 			this.deps.index.removeSegments(removed);
-			for (const r of rows) this.deps.onEvicted?.(r.deviceId, r.startMs);
+			this.notifyEvicted(rows);
 		}
 		await this.pruneEmptyAncestors(location, [...byBucket.keys()]);
+		this.deps.logger.info("recorder: eviction round complete", { meta: {
+			location,
+			rows: rows.length,
+			buckets: byBucket.size,
+			bucketsRemoved,
+			removed: removed.length,
+			reclaimed,
+			ms: (this.deps.now?.() ?? Date.now()) - startedMs
+		} });
 		return reclaimed;
 	}
 	/**
+	* Tell the calendar cache what went, ONCE PER (device, UTC day).
+	*
+	* `CalendarIndex.invalidate` drops a whole UTC day, so a per-ROW call repeats
+	* the same work for every segment of that day — and its hour-list sweep is
+	* O(cached hours), in one uninterrupted synchronous block with no await in
+	* it. Measured over a 122 400-row round with 5 000 cached hours: 15 285 ms of
+	* frozen event loop, scaling linearly with how long the session has been
+	* scrubbing. Deduped it is ~190 calls and unmeasurable.
+	*/
+	notifyEvicted(rows) {
+		const notify = this.deps.onEvicted;
+		if (!notify) return;
+		const seen = /* @__PURE__ */ new Set();
+		for (const r of rows) {
+			const key = `${r.deviceId}:${Math.floor(r.startMs / SegmentStore.DAY_MS)}`;
+			if (seen.has(key)) continue;
+			seen.add(key);
+			notify(r.deviceId, r.startMs);
+		}
+	}
+	/**
 	* Walk up from each emptied hour directory removing ancestors that are now
 	* empty, so that **a directory's existence means footage**.
 	*
@@ -1336,22 +1440,37 @@ var SegmentStore = class SegmentStore {
 	}
 	/** Hour of one UTC hour-bucket in ms. */
 	static HOUR_MS = 36e5;
+	static DAY_MS = 864e5;
+	/** Buckets between yields. Small enough that one chunk is milliseconds of
+	*  work now the per-bucket archive scan is gone. */
+	static DEFAULT_EVICT_CHUNK = 25;
 	/**
 	* True when `victims` cover EVERY indexed segment of `bucketDir` AND the
 	* bucket's hour has fully elapsed. The current-hour guard matters because
 	* `onFinalized` relocates new segments INTO the current bucket concurrently —
 	* an rm racing that rename would orphan a just-indexed segment.
+	*
+	* The cover test is a COUNT against the round's one-pass census plus an O(1)
+	* indexed-ness check per victim, never a scan of the device archive: a
+	* bucketDir is `<dev>/<profile>/Y/M/D/H`, so it names exactly one
+	* (device, profile), and the victims of a bucket are unique paths. Equal
+	* counts with every victim indexed therefore means the same set.
 	*/
-	bucketFullyEvicted(bucketDir, victims) {
+	bucketFullyEvicted(bucketDir, victims, countsFor) {
 		const first = victims[0];
 		if (!first) return false;
 		const bucketStartMs = Math.floor(first.startMs / SegmentStore.HOUR_MS) * SegmentStore.HOUR_MS;
 		if ((this.deps.now?.() ?? Date.now()) < bucketStartMs + SegmentStore.HOUR_MS) return false;
-		const victimPaths = new Set(victims.map((v) => v.path));
-		const indexed = this.deps.index.segments(first.deviceId, first.profile).filter((s) => s.path.startsWith(`${bucketDir}/`));
-		return indexed.length === victimPaths.size && indexed.every((s) => victimPaths.has(s.path));
+		const indexedInBucket = countsFor(first.deviceId, first.profile).get(bucketDir) ?? 0;
+		if (indexedInBucket === 0) return false;
+		const indexedVictims = /* @__PURE__ */ new Set();
+		for (const v of victims) if (this.deps.index.hasSegment(v.deviceId, v.path)) indexedVictims.add(v.path);
+		return indexedInBucket === indexedVictims.size;
 	}
 };
+var macrotask$2 = () => new Promise((resolve) => {
+	setImmediate(resolve);
+});
 //#endregion
 //#region src/recorder/still/still-source.ts
 /**
@@ -2012,7 +2131,7 @@ var CalendarIndex = class {
 		const day = utcDayOf(atMs);
 		this.byDevice.get(deviceId)?.delete(day);
 		const prefix = `${deviceId}:`;
-		for (const key of [...this.hourSegs.keys()]) {
+		for (const key of this.hourSegs.keys()) {
 			if (!key.startsWith(prefix)) continue;
 			if (utcDayOf(Number(key.slice(key.lastIndexOf(":") + 1))) === day) this.hourSegs.delete(key);
 		}
@@ -4839,9 +4958,12 @@ async function resolveRecordingsLocations(api, logger) {
 * (no footage yet) and any per-location failure are warned + skipped
 * (best-effort), so one bad volume never blocks the rest of the hydrate.
 */
-async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger) {
+async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger, options = {}) {
+	const yieldBetween = options.yieldBetween ?? macrotask$1;
+	const chunkSize = options.chunkSize ?? HYDRATE_CHUNK;
 	for (const [root, aliases] of aliasesByRoot(locations)) {
 		const deviceDir = node_path.default.join(root, String(deviceId));
+		const startedMs = Date.now();
 		let entries;
 		try {
 			entries = await node_fs.promises.readdir(deviceDir, { recursive: true });
@@ -4855,17 +4977,48 @@ async function hydrateDeviceFromStorage(_api, index, deviceId, locations, logger
 			});
 			continue;
 		}
+		const walkedMs = Date.now();
+		const locationForProfile = /* @__PURE__ */ new Map();
 		const pathsByLocation = /* @__PURE__ */ new Map();
-		for (const entry of entries.filter((entry) => entry.endsWith(".m4s"))) {
+		let handled = 0;
+		for (const entry of entries) {
+			if (!entry.endsWith(".m4s")) continue;
 			const relative = `${deviceId}/${entry.split(node_path.default.sep).join("/")}`;
-			const location = locationForHydratedProfile(aliases, relative.split("/")[1] ?? "");
-			const paths = pathsByLocation.get(location.id);
+			const profile = relative.split("/")[1] ?? "";
+			let locationId = locationForProfile.get(profile);
+			if (locationId === void 0) {
+				locationId = locationForHydratedProfile(aliases, profile).id;
+				locationForProfile.set(profile, locationId);
+			}
+			const paths = pathsByLocation.get(locationId);
 			if (paths) paths.push(relative);
-			else pathsByLocation.set(location.id, [relative]);
+			else pathsByLocation.set(locationId, [relative]);
+			handled += 1;
+			if (handled % chunkSize === 0) await yieldBetween();
+		}
+		let indexed = 0;
+		for (const [locationId, relPaths] of pathsByLocation) {
+			index.hydrateDevice(deviceId, locationId, relPaths);
+			indexed += relPaths.length;
 		}
-		for (const [locationId, relPaths] of pathsByLocation) index.hydrateDevice(deviceId, locationId, relPaths);
+		logger.info("recorder: hydrated a device from one recordings root", {
+			tags: { deviceId },
+			meta: {
+				root,
+				entries: entries.length,
+				indexed,
+				locationIds: [...pathsByLocation.keys()],
+				walkMs: walkedMs - startedMs,
+				ms: Date.now() - startedMs
+			}
+		});
 	}
 }
+/** Entries classified between yields in {@link hydrateDeviceFromStorage}. */
+var HYDRATE_CHUNK = 2e4;
+var macrotask$1 = () => new Promise((resolve) => {
+	setImmediate(resolve);
+});
 /** `<deviceId>/<profile>/YYYY/MM/DD/HH` for an hour bucket (paths are UTC). */
 function hourDirRelPath(deviceId, profile, hourStartMs) {
 	const d = new Date(hourStartMs);
@@ -6949,20 +7102,30 @@ var RetentionSweeper = class {
 				this.deps.logger.warn("recorder: retention sweep could not read device configs", { meta: { error: require_dist.errMsg(err) } });
 				return;
 			}
+			const withPolicy = [...configs].filter(([, config]) => hasRetentionPolicy(config));
+			const devicesSkipped = configs.size - withPolicy.length;
+			this.deps.logger.info("recorder: retention sweep starting", { meta: {
+				devices: withPolicy.map(([deviceId]) => deviceId),
+				skipped: devicesSkipped
+			} });
 			let devicesSwept = 0;
-			let devicesSkipped = 0;
 			let bucketsPruned = 0;
 			let bytesReclaimed = 0;
-			for (const [deviceId, config] of configs) {
-				if (!hasRetentionPolicy(config)) {
-					devicesSkipped += 1;
-					continue;
-				}
+			for (const [deviceId] of withPolicy) {
+				const startedMs = Date.now();
 				try {
 					const result = await this.deps.prune(deviceId);
 					devicesSwept += 1;
 					bucketsPruned += result.deletedBuckets;
 					bytesReclaimed += result.reclaimedBytes;
+					this.deps.logger.info("recorder: retention prune complete for device", {
+						tags: { deviceId },
+						meta: {
+							deletedBuckets: result.deletedBuckets,
+							reclaimedBytes: result.reclaimedBytes,
+							ms: Date.now() - startedMs
+						}
+					});
 				} catch (err) {
 					this.deps.logger.warn("recorder: retention prune failed for device", {
 						tags: { deviceId },
@@ -6982,6 +7145,63 @@ var RetentionSweeper = class {
 	}
 };
 //#endregion
+//#region src/recorder/loop-pacer.ts
+var DEFAULT_BUSY_LAG_MS = 25;
+var DEFAULT_BACKOFF_STEP_MS = 25;
+var DEFAULT_MAX_BACKOFF_MS = 250;
+var LoopPacer = class {
+	deps;
+	consecutive = 0;
+	hops = 0;
+	contendedHops = 0;
+	pausedMs = 0;
+	maxLagMs = 0;
+	constructor(deps) {
+		this.deps = deps;
+	}
+	/**
+	* One paced yield. Always hands the loop back at least once; sleeps longer
+	* while the previous hop shows the loop is contended, and forgets the
+	* back-off as soon as one hop comes back clean.
+	*/
+	async hop() {
+		const busyLagMs = this.deps.busyLagMs ?? DEFAULT_BUSY_LAG_MS;
+		const stepMs = this.deps.backoffStepMs ?? DEFAULT_BACKOFF_STEP_MS;
+		const maxMs = this.deps.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+		const before = this.deps.now();
+		await this.deps.sleep(0);
+		const lagMs = this.deps.now() - before;
+		this.hops += 1;
+		if (lagMs > this.maxLagMs) this.maxLagMs = lagMs;
+		if (lagMs < busyLagMs) {
+			this.consecutive = 0;
+			return;
+		}
+		this.consecutive += 1;
+		this.contendedHops += 1;
+		const pauseMs = Math.min(stepMs * this.consecutive, maxMs);
+		this.pausedMs += pauseMs;
+		await this.deps.sleep(pauseMs);
+	}
+	stats() {
+		return {
+			hops: this.hops,
+			contendedHops: this.contendedHops,
+			pausedMs: this.pausedMs,
+			maxLagMs: this.maxLagMs
+		};
+	}
+};
+/** The production pacer: real clock, real timers. */
+function nodeLoopPacer() {
+	return new LoopPacer({
+		now: () => Date.now(),
+		sleep: (ms) => new Promise((resolve) => {
+			setTimeout(resolve, ms);
+		})
+	});
+}
+//#endregion
 //#region src/recorder/addon/staging-reconcile.ts
 /**
 * Recover `.rec-tmp` segments that a writer restart orphaned.
@@ -7089,6 +7309,14 @@ var KNOWN_PROFILES = new Set([
 ]);
 /** Default orphans per chunk. 590/high alone had 5 863 of them. */
 var DEFAULT_CHUNK_SIZE = 100;
+/**
+* Orphans between progress lines.
+*
+* A directory of 5 863 files takes minutes over shfs and used to report only
+* when it finished, so "is it still going, and how far in" had no answer at
+* all — the exact shape of the blackout this pass was written to end.
+*/
+var PROGRESS_EVERY = 1e3;
 var macrotask = () => new Promise((resolve) => {
 	setTimeout(resolve, 0);
 });
@@ -7135,6 +7363,13 @@ async function recoverStagingOrphans(dir, input, deps) {
 		failed: 0,
 		aborted: false
 	};
+	deps.logger.info("recorder: staging recovery starting on a directory", { meta: {
+		dir,
+		found: plan.length,
+		staged: names.length,
+		oldestMs: plan[0]?.startMs,
+		newestMs: plan[plan.length - 1]?.startMs
+	} });
 	const chunkSize = deps.chunkSize ?? DEFAULT_CHUNK_SIZE;
 	const yieldBetween = deps.yieldBetween ?? macrotask;
 	let recovered = 0;
@@ -7165,6 +7400,13 @@ async function recoverStagingOrphans(dir, input, deps) {
 				error: err instanceof Error ? err.message : String(err)
 			} });
 		}
+		if ((i + 1) % PROGRESS_EVERY === 0 && i + 1 < plan.length) deps.logger.info("recorder: staging recovery progress", { meta: {
+			dir,
+			done: i + 1,
+			found: plan.length,
+			recovered,
+			failed
+		} });
 		if ((i + 1) % chunkSize === 0 && i + 1 < plan.length) await yieldBetween();
 	}
 	deps.logger.info("recorder: recovered segments a writer restart had orphaned in staging", { meta: {
@@ -7209,7 +7451,16 @@ async function recoverAllStagedOrphans(deps) {
 	let recovered = 0;
 	let failed = 0;
 	let aborted = false;
-	for (const location of deps.locations()) {
+	const startedMs = Date.now();
+	const injectedYield = deps.yieldBetween;
+	const pacer = injectedYield === void 0 ? nodeLoopPacer() : null;
+	const yieldBetween = injectedYield ?? (() => pacer === null ? macrotask() : pacer.hop());
+	const locations = deps.locations();
+	deps.logger.info("recorder: staging boot reconcile starting", { meta: {
+		locations: locations.map((l) => l.id),
+		bootMs: deps.bootMs
+	} });
+	for (const location of locations) {
 		const stagingRoot = `${location.root}/${STAGING_DIR_NAME}`;
 		let deviceDirs;
 		try {
@@ -7250,7 +7501,7 @@ async function recoverAllStagedOrphans(deps) {
 					}),
 					logger: withDeviceTag(deps.logger, deviceId),
 					chunkSize: deps.chunkSize,
-					yieldBetween: deps.yieldBetween,
+					yieldBetween,
 					shouldStop: deps.shouldStop
 				});
 				found += report.found;
@@ -7260,12 +7511,14 @@ async function recoverAllStagedOrphans(deps) {
 			}
 		}
 	}
-	if (found > 0 || failed > 0) deps.logger.info("recorder: staging boot reconcile complete", { meta: {
+	deps.logger.info("recorder: staging boot reconcile complete", { meta: {
 		dirs,
 		found,
 		recovered,
 		failed,
-		aborted
+		aborted,
+		ms: Date.now() - startedMs,
+		...pacer === null ? {} : { pacing: pacer.stats() }
 	} });
 	return {
 		dirs,
@@ -7475,6 +7728,7 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 				this.calendar.invalidate(deviceId, atMs);
 			},
 			onIndexed: (row, absPath) => {
+				if (!isMfraTableServable(row.startMs, Date.now())) return;
 				readMfraTable(absPath, row.bytes).then((table) => {
 					this.mfraTables.record(row.deviceId, row.profile, row.startMs, table);
 				}).catch((err) => {
@@ -7492,13 +7746,18 @@ var RecorderV2Addon = class extends require_dist.BaseAddon {
 				if (!loc) throw new Error(`recorder: unknown recordings location ${locationId}`);
 				await node_fs.promises.rmdir(node_path.default.join(loc.root, relDir));
 			},
-			logger: { warn: (message, extras) => {
-				if (isSegmentStoreLogExtras(extras)) this.ctx.logger.warn(message, {
-					tags: extras.tags,
-					meta: { error: extras.error }
-				});
-				else this.ctx.logger.warn(message, { meta: { extras } });
-			} }
+			logger: {
+				warn: (message, extras) => {
+					if (isSegmentStoreLogExtras(extras)) this.ctx.logger.warn(message, {
+						tags: extras.tags,
+						meta: { error: extras.error }
+					});
+					else this.ctx.logger.warn(message, { meta: { extras } });
+				},
+				info: (message, extras) => {
+					this.ctx.logger.info(message, { meta: extras.meta });
+				}
+			}
 		};
 		this.segmentStore = new SegmentStore(segmentStoreDeps);
 		const brokerHandle = this.ctx.useCapability("stream-broker", brokerScope(ingestOwner.ownerNodeId));