@camstack/addon-post-analysis 1.2.144 → 1.2.147

package/dist/pipeline-analytics/index.js

@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_dist = require("../dist-C8RL0Hzy.js");
+const require_dist = require("../dist-1xYuXmDS.js");
 let node_fs = require("node:fs");
 node_fs = require_dist.__toESM(node_fs, 1);
 let node_path = require("node:path");
@@ -40213,9 +40213,6 @@ function readJpegSize(data) {
 *     table or the disk it consumes."
 */
 var RETRAIN_FRAMES_COLLECTION = "pipeline-analytics:retrain-frames";
-/** Storage location for retrain copies — the same default media root the event
-*  media lives on, under a prefix nothing else enumerates. */
-var RETRAIN_MEDIA_LOCATION = "eventMedia";
 /** Path prefix of every retrain copy. Never `<deviceId>/events/…`: the orphan
 *  audit resolves ownership from INDEX rows, but a future filesystem-side tool
 *  must be able to tell the two corpora apart by path alone. */
@@ -40308,6 +40305,7 @@ var RetrainFrameStore = class {
 	readSource;
 	now;
 	writeGate;
+	resolveEventMediaLocation;
 	constructor(deps) {
 		this.store = deps.store;
 		this.storage = deps.storage;
@@ -40315,6 +40313,29 @@ var RetrainFrameStore = class {
 		this.readSource = deps.readSource;
 		this.now = deps.now;
 		this.writeGate = deps.writeGate;
+		this.resolveEventMediaLocation = deps.resolveEventMediaLocation;
+	}
+	/**
+	* The location a NEW copy is written to, and the id its row NAMES.
+	*
+	* `null` = "no concrete location was resolved", which persists as the legacy
+	* NULL row. A resolver that throws must not refuse the copy — losing an
+	* operator's selection is worse than an unstamped row the seal can fix — but
+	* it says so, because that branch re-creates the exact population a
+	* non-blocking migration is gated on.
+	*/
+	async resolveWriteLocation(deviceId) {
+		if (!this.resolveEventMediaLocation) return null;
+		try {
+			const resolved = await this.resolveEventMediaLocation(deviceId);
+			return resolved === "eventMedia" ? null : resolved;
+		} catch (err) {
+			this.logger.warn("retrain copy could not resolve its eventMedia location — the row is written UNSTAMPED and a storage repoint would re-aim it at a disk that does not hold it", {
+				tags: { deviceId },
+				meta: { error: String(err) }
+			});
+			return null;
+		}
 	}
 	static async declare(store) {
 		await store.declareCollection.mutate({
@@ -40473,6 +40494,7 @@ var RetrainFrameStore = class {
 			};
 		}
 		const path = framePath(input.deviceId, input.trackId, frameId);
+		const locationId = await this.resolveWriteLocation(input.deviceId);
 		const frame = {
 			frameId,
 			deviceId: input.deviceId,
@@ -40483,11 +40505,11 @@ var RetrainFrameStore = class {
 			width: size.width,
 			height: size.height,
 			copiedAt: this.now(),
-			locationId: null
+			locationId
 		};
 		const persist = async () => {
 			await this.storage.write({
-				location: RETRAIN_MEDIA_LOCATION,
+				location: locationId ?? "eventMedia",
 				relativePath: path,
 				data: bytes
 			});
@@ -40504,7 +40526,7 @@ var RetrainFrameStore = class {
 					width: frame.width,
 					height: frame.height,
 					copiedAt: frame.copiedAt,
-					locationId: null
+					locationId: frame.locationId
 				}
 			});
 		};
@@ -43503,6 +43525,13 @@ function pickWritableMediaLocation(locations, currentDefaultId) {
 //#region src/pipeline-analytics/media-relocate-engine.ts
 var PAGE_SIZE = 200;
 var DEFAULT_THROTTLE_MBPS = 40;
+/** What a pass touches, reported on the job so an ops-log row says which of the
+*  three passes ran without anyone reading this file. */
+function entitiesFor(mode) {
+	if (mode === "seal") return ["mediaRows", "retrainFrameRows"];
+	if (mode === "gallery") return ["galleryMedia"];
+	return ["media", "retrainFrames"];
+}
 function snapshot(j) {
 	return {
 		jobId: j.jobId,
@@ -43510,7 +43539,7 @@ function snapshot(j) {
 		fromLocationId: j.fromLocationId,
 		toLocationId: j.toLocationId,
 		deviceId: j.deviceId,
-		entities: ["media", "retrainFrames"],
+		entities: [...entitiesFor(j.mode)],
 		filesMoved: j.filesMoved,
 		bytesMoved: j.bytesMoved,
 		filesTotal: null,
@@ -43519,6 +43548,22 @@ function snapshot(j) {
 		error: j.error
 	};
 }
+/**
+* True when the row names NO location — the population `seal` exists for.
+*
+* Deliberately NOT `mediaRowLocation(data) === DEFAULT_MEDIA_LOCATION`: that
+* helper collapses "unstamped" and "stamped with the bare type ref" into one
+* answer, and only the first is a row whose meaning changes under a repoint.
+*/
+function rowIsUnstamped(data) {
+	const id = data["locationId"];
+	return !(typeof id === "string" && id.length > 0);
+}
+/** True for the enrolled-gallery rows `move` skips and `gallery` selects. */
+function rowIsGallery(data) {
+	const ownerKind = String(data["ownerKind"] ?? "");
+	return isMediaOwnerKind(ownerKind) && isRetentionExemptOwnerKind(ownerKind);
+}
 var MediaRelocateEngine = class {
 	deps;
 	jobs = /* @__PURE__ */ new Map();
@@ -43534,11 +43579,34 @@ var MediaRelocateEngine = class {
 		job.cancelRequested = true;
 		return true;
 	}
+	/**
+	* How many rows still carry no `locationId`, per collection.
+	*
+	* Read-only, and the ONLY number the migration planner's non-blocking gate
+	* is allowed to trust: it is counted here, against the rows themselves,
+	* rather than inferred from "a seal job reported done" — a seal that failed
+	* some rows still reports a terminal state.
+	*/
+	async countUnstamped() {
+		const media = await this.countUnstampedIn(MEDIA_COLLECTION, "timestamp");
+		const retrainFrames = await this.countUnstampedIn(RETRAIN_FRAMES_COLLECTION, "copiedAt");
+		return {
+			media,
+			retrainFrames,
+			total: media + retrainFrames
+		};
+	}
+	async countUnstampedIn(collection, orderField) {
+		let unstamped = 0;
+		for await (const row of this.pageRows(collection, orderField)) if (rowIsUnstamped(row.data)) unstamped++;
+		return unstamped;
+	}
 	start(input) {
 		for (const j of this.jobs.values()) if (j.state === "running") throw new Error(`a media relocation is already running (${j.jobId})`);
 		const job = {
 			jobId: this.deps.newId(),
 			state: "running",
+			mode: input.mode ?? "move",
 			fromLocationId: "*",
 			toLocationId: input.toLocationId,
 			deviceId: null,
@@ -43553,130 +43621,104 @@ var MediaRelocateEngine = class {
 		this.run(job, input.throttleMbps ?? DEFAULT_THROTTLE_MBPS);
 		return job.jobId;
 	}
+	/**
+	* Page one collection oldest-first, yielding every row exactly once.
+	*
+	* The cursor is `(orderField, offset-within-ties)` rather than a plain
+	* offset: 201 rows sharing one timestamp is a real shape here (a burst of
+	* snapshots for one track), and a plain `offset` walk over a collection being
+	* mutated underneath is how rows get skipped. Rows only ever gain a
+	* `locationId` during a pass — never a new order key — so appending inserts
+	* land after the cursor and are picked up, and nothing is missed.
+	*/
+	async *pageRows(collection, orderField) {
+		let cursor = 0;
+		let offsetAtCursor = 0;
+		for (;;) {
+			const rows = await this.deps.store.query.query({
+				collection,
+				filter: {
+					whereBetween: { [orderField]: [cursor, Number.MAX_SAFE_INTEGER] },
+					orderBy: {
+						field: orderField,
+						direction: "asc"
+					},
+					limit: PAGE_SIZE,
+					offset: offsetAtCursor
+				}
+			});
+			if (rows.length === 0) return;
+			for (const row of rows) yield row;
+			const last = rows[rows.length - 1];
+			if (last === void 0) return;
+			const lastKey = Number(last.data[orderField] ?? cursor);
+			if (lastKey === cursor) offsetAtCursor += rows.length;
+			else {
+				cursor = lastKey;
+				offsetAtCursor = rows.filter((row) => Number(row.data[orderField]) === lastKey).length;
+			}
+			if (rows.length < PAGE_SIZE) return;
+		}
+	}
+	/** Does this pass act on this row at all? */
+	selects(job, data, isRetrain) {
+		if (job.mode === "seal") return rowIsUnstamped(data);
+		if (mediaRowLocation(data) === job.toLocationId) return false;
+		if (job.mode === "gallery") return !isRetrain && rowIsGallery(data);
+		return isRetrain || !rowIsGallery(data);
+	}
 	async run(job, throttleMbps) {
 		const sleep = this.deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
 		const bytesPerMs = throttleMbps * 1024 * 1024 / 1e3;
 		try {
 			await this.deps.resolveTargetRoot(job.toLocationId);
 			let failedRows = 0;
-			let cursor = 0;
-			let offsetAtCursor = 0;
-			for (;;) {
+			/** Selected rows that name no blob path. Counted and reported ONCE per
+			*  pass rather than per row: a row with no path names no bytes anywhere,
+			*  so there is nothing a repoint can orphan — but a pass that steps over
+			*  rows in silence reads as "there were none", which is how a migration
+			*  gets believed. */
+			let pathlessRows = 0;
+			const collections = job.mode === "gallery" ? [{
+				name: MEDIA_COLLECTION,
+				order: "timestamp"
+			}] : [{
+				name: MEDIA_COLLECTION,
+				order: "timestamp"
+			}, {
+				name: RETRAIN_FRAMES_COLLECTION,
+				order: "copiedAt"
+			}];
+			for (const collection of collections) {
 				if (job.cancelRequested) break;
-				const rows = await this.deps.store.query.query({
-					collection: MEDIA_COLLECTION,
-					filter: {
-						whereBetween: { timestamp: [cursor, Number.MAX_SAFE_INTEGER] },
-						orderBy: {
-							field: "timestamp",
-							direction: "asc"
-						},
-						limit: PAGE_SIZE,
-						offset: offsetAtCursor
-					}
-				});
-				if (rows.length === 0) break;
-				for (const row of rows) {
+				const isRetrain = collection.name === RETRAIN_FRAMES_COLLECTION;
+				for await (const row of this.pageRows(collection.name, collection.order)) {
 					if (job.cancelRequested) break;
 					const data = row.data;
-					const ownerKind = String(data["ownerKind"] ?? "");
-					if (isMediaOwnerKind(ownerKind) && isRetentionExemptOwnerKind(ownerKind)) continue;
-					const from = mediaRowLocation(data);
-					if (from === job.toLocationId) continue;
-					const relativePath = String(data["path"] ?? "");
-					if (relativePath.length === 0) continue;
-					try {
-						if (await this.samePhysicalLocation(from, job.toLocationId)) {
-							await this.deps.store.set.mutate({
-								collection: MEDIA_COLLECTION,
+					if (!this.selects(job, data, isRetrain)) continue;
+					if (job.mode === "seal") {
+						try {
+							await this.stamp(collection.name, row.id, data, job.toLocationId);
+							job.filesMoved++;
+						} catch (err) {
+							failedRows++;
+							this.deps.logger.debug("event-media seal row failed", { meta: {
+								collection: collection.name,
 								key: row.id,
-								value: {
-									...data,
-									locationId: job.toLocationId
-								}
-							});
-							continue;
+								error: String(err)
+							} });
 						}
-						const bytes = await this.deps.storage.read({
-							location: from,
-							relativePath
-						});
-						await this.deps.storage.write({
-							location: job.toLocationId,
-							relativePath,
-							data: bytes
-						});
-						if ((await this.deps.storage.read({
-							location: job.toLocationId,
-							relativePath
-						})).length !== bytes.length) throw new Error(`relocated media size mismatch for ${relativePath}`);
-						await this.deps.store.set.mutate({
-							collection: MEDIA_COLLECTION,
-							key: row.id,
-							value: {
-								...data,
-								locationId: job.toLocationId
-							}
-						});
-						await this.deps.storage.delete({
-							location: from,
-							relativePath
-						});
-						job.filesMoved++;
-						job.bytesMoved += bytes.length;
-						await sleep(bytes.length / bytesPerMs);
-					} catch (err) {
-						failedRows++;
-						this.deps.logger.debug("media relocate row failed", { meta: {
-							key: row.id,
-							error: String(err)
-						} });
-					}
-				}
-				const last = rows[rows.length - 1];
-				const lastTs = Number(last.data["timestamp"] ?? cursor);
-				if (lastTs === cursor) offsetAtCursor += rows.length;
-				else {
-					cursor = lastTs;
-					offsetAtCursor = rows.filter((row) => Number(row.data["timestamp"]) === lastTs).length;
-				}
-				if (rows.length < PAGE_SIZE) break;
-			}
-			let retrainOffset = 0;
-			for (;;) {
-				if (job.cancelRequested) break;
-				const rows = await this.deps.store.query.query({
-					collection: RETRAIN_FRAMES_COLLECTION,
-					filter: {
-						orderBy: {
-							field: "copiedAt",
-							direction: "asc"
-						},
-						limit: PAGE_SIZE,
-						offset: retrainOffset
+						continue;
 					}
-				});
-				if (rows.length === 0) break;
-				for (const row of rows) {
-					if (job.cancelRequested) break;
-					const data = row.data;
 					const from = mediaRowLocation(data);
-					if (from === job.toLocationId) continue;
 					const relativePath = String(data["path"] ?? "");
 					if (relativePath.length === 0) {
-						failedRows++;
+						pathlessRows++;
 						continue;
 					}
 					try {
 						if (await this.samePhysicalLocation(from, job.toLocationId)) {
-							await this.deps.store.set.mutate({
-								collection: RETRAIN_FRAMES_COLLECTION,
-								key: row.id,
-								value: {
-									...data,
-									locationId: job.toLocationId
-								}
-							});
+							await this.stamp(collection.name, row.id, data, job.toLocationId);
 							continue;
 						}
 						const bytes = await this.deps.storage.read({
@@ -43691,15 +43733,8 @@ var MediaRelocateEngine = class {
 						if ((await this.deps.storage.read({
 							location: job.toLocationId,
 							relativePath
-						})).length !== bytes.length) throw new Error(`relocated retrain frame size mismatch for ${relativePath}`);
-						await this.deps.store.set.mutate({
-							collection: RETRAIN_FRAMES_COLLECTION,
-							key: row.id,
-							value: {
-								...data,
-								locationId: job.toLocationId
-							}
-						});
+						})).length !== bytes.length) throw new Error(`relocated media size mismatch for ${relativePath}`);
+						await this.stamp(collection.name, row.id, data, job.toLocationId);
 						await this.deps.storage.delete({
 							location: from,
 							relativePath
@@ -43709,22 +43744,27 @@ var MediaRelocateEngine = class {
 						await sleep(bytes.length / bytesPerMs);
 					} catch (err) {
 						failedRows++;
-						this.deps.logger.debug("retrain frame relocate row failed", { meta: {
+						this.deps.logger.debug("media relocate row failed", { meta: {
+							collection: collection.name,
 							key: row.id,
 							error: String(err)
 						} });
 					}
 				}
-				retrainOffset += rows.length;
-				if (rows.length < PAGE_SIZE) break;
 			}
-			if (failedRows > 0) throw new Error(`event-media relocation left ${failedRows} row(s) unmoved`);
+			if (pathlessRows > 0) this.deps.logger.warn("media relocate stepped over rows that name no blob path", { meta: {
+				jobId: job.jobId,
+				mode: job.mode,
+				rows: pathlessRows
+			} });
+			if (failedRows > 0) throw new Error(`event-media ${job.mode} left ${failedRows} row(s) unmoved`);
 			job.state = job.cancelRequested ? "cancelled" : "done";
 		} catch (err) {
 			job.state = "failed";
 			job.error = err instanceof Error ? err.message : String(err);
 			this.deps.logger.warn("media relocate job failed", { meta: {
 				jobId: job.jobId,
+				mode: job.mode,
 				error: job.error
 			} });
 		} finally {
@@ -43732,6 +43772,16 @@ var MediaRelocateEngine = class {
 			this.deps.onFinished?.(snapshot(job));
 		}
 	}
+	async stamp(collection, key, data, locationId) {
+		await this.deps.store.set.mutate({
+			collection,
+			key,
+			value: {
+				...data,
+				locationId
+			}
+		});
+	}
 	async samePhysicalLocation(from, to) {
 		if (from === to) return true;
 		try {
@@ -62299,6 +62349,7 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_dist.B
 			logger: logger.child("RetrainFrames"),
 			now: () => Date.now(),
 			writeGate: this.mediaWriteGate,
+			resolveEventMediaLocation: (deviceId) => this.resolveEventMediaWriteLocation(api, logger, deviceId),
 			readSource: async (mediaKey) => {
 				const file = await mediaStore.getByKey(mediaKey);
 				return file === null ? null : Buffer.from(file.base64, "base64");
@@ -68975,10 +69026,22 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_dist.B
 		const jobId = this.mediaRelocate.start(input);
 		this.ctx.logger.info("pipeline-analytics: media relocate job enqueued", { meta: {
 			jobId,
+			mode: input.mode ?? "move",
 			to: input.toLocationId
 		} });
 		return { jobId };
 	}
+	/**
+	* How many media / retrain rows still carry no `locationId`.
+	*
+	* Counted against the rows, never inferred from a seal job's terminal state:
+	* the migration planner refuses a non-blocking `eventMedia` cutover on this
+	* number, and a seal that failed some rows still finishes.
+	*/
+	async countUnstampedEventMedia() {
+		if (!this.mediaRelocate) throw new Error("media relocation unavailable");
+		return this.mediaRelocate.countUnstamped();
+	}
 	/** Every relocate job this addon knows about, newest first (in RAM: the
 	*  move is resumable, so a lost list costs nothing but the display). */
 	async listRelocateMediaJobs() {