@camstack/addon-post-analysis 1.2.152 → 1.2.153

package/dist/pipeline-analytics/index.js

@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_dist = require("../dist-Bw6cidiJ.js");
+const require_dist = require("../dist-DeMgxNFC.js");
 let node_fs = require("node:fs");
 node_fs = require_dist.__toESM(node_fs, 1);
 let node_path = require("node:path");
@@ -25694,6 +25694,67 @@ var MEDIA_INDEXES = [
 	{
 		name: "idx_media_location",
 		columns: ["locationId"]
+	}),
+	(
+	/**
+	* The events-management footprint — `GROUP BY deviceId` and
+	* `SUM(sizeBytes) WHERE deviceId = ?`. See {@link MediaStore.footprintByDevice}.
+	*
+	* **Why not `idx_media_device_ts`.** That index already leads with
+	* `deviceId`, so it can find the range — but it does not carry `sizeBytes`,
+	* so the sum has to fetch the table row behind every index entry it visits.
+	* Over the whole collection that is one scattered read of the entire ~1.25 M
+	* row table (`path` alone is ~80 characters per row) on the volume the
+	* recorder is writing to. With `sizeBytes` in the index the planner never
+	* touches the table: `SCAN … USING COVERING INDEX idx_media_device_size`.
+	*
+	* It has to be a NEW index rather than a third column on the existing one.
+	* `declareCollection` emits `CREATE INDEX IF NOT EXISTS` keyed on the index
+	* NAME, so widening `idx_media_device_ts` in place would be a silent no-op on
+	* every hub that already has it — the declaration would read as done and the
+	* query would still hit the table.
+	*
+	* **What it costs.** ~19 bytes of index per row (two integers plus the
+	* rowid), so ~24 MB at 1.25 M rows, plus one more b-tree maintained by every
+	* insert into a table taking ~92 k rows/day. At ~1 insert/s that maintenance
+	* is noise beside the JPEG write the same call performs. The cost that is not
+	* noise is the one-off: the first `declareCollection` after this ships builds
+	* the index over every existing row. Do not deploy it beside a running drain
+	* or seal — the same caveat `idx_media_location` carries, for the same
+	* reason.
+	*/
+	{
+		name: "idx_media_device_size",
+		columns: ["deviceId", "sizeBytes"]
+	}),
+	(
+	/**
+	* The per-KIND footprint — `COUNT(*)`/`SUM(sizeBytes) WHERE kind = ?`, and
+	* the same scoped to one camera. See {@link MediaStore.footprintByKind}.
+	*
+	* `kind` leads no existing index, so without this each of the thirteen
+	* per-kind questions is a full table scan and the breakdown costs thirteen of
+	* them. With it, each kind is a contiguous covering range and the thirteen
+	* together scan the index once — the cost of the whole breakdown is one pass
+	* over ~36 MB of index instead of thirteen passes over the table.
+	*
+	* `deviceId` sits between `kind` and `sizeBytes` so the per-camera variant
+	* (`WHERE kind = ? AND deviceId = ?`) stays covering too, for ~3 bytes a row
+	* more than `(kind, sizeBytes)` would cost. Without it that query finds its
+	* range from the index and then reads the table for every row in it.
+	*
+	* ~29 bytes of index per row, so ~36 MB at 1.25 M rows, and one more b-tree
+	* on every insert. Same one-off caveat as the two indexes above: the first
+	* declare builds it over every existing row, so do not ship it beside a
+	* running drain or seal.
+	*/
+	{
+		name: "idx_media_kind_device_size",
+		columns: [
+			"kind",
+			"deviceId",
+			"sizeBytes"
+		]
 	})
 ];
 /** The storage location of a media row/record: its stamped `locationId`, or
@@ -26348,47 +26409,136 @@ var MediaStore = class {
 		};
 	}
 	/**
-	* On-demand footprint aggregation for the events-management UI: sum
-	* `sizeBytes` and count rows per device across the whole media collection.
-	* Pages by `id` (offset-based) so a large collection never loads at once.
-	* Metadata rows are small (no blob) — the image bytes live on the storage
-	* provider and are counted via the persisted `sizeBytes` column.
+	* On-demand footprint aggregation for the events-management UI: rows and
+	* summed `sizeBytes` per camera, across the whole media collection.
+	*
+	* ## Why this is a histogram plus one aggregate per camera
+	*
+	* It used to page the ENTIRE collection — `ORDER BY id LIMIT 1000 OFFSET n`,
+	* no predicate — and add `sizeBytes` up in JS. SQLite honours an OFFSET by
+	* stepping over `n` entries, so the walk is quadratic in the table: at the
+	* live hub's 1.25 M rows that is ~1 250 round trips and ~780 M index steps,
+	* with every row serialised across an RPC boundary and rebuilt in V8 to be
+	* added to a number. It timed out at 60 s on every call it ever made, and it
+	* is the only thing that can size the surplus-media deletion it gates.
+	*
+	* The answer is at most a few dozen `(deviceId, rows, bytes)` triples.
+	*
+	*  - The DEVICE SET comes from a histogram on `deviceId` with `bucketSize: 1`
+	*    — `GROUP BY deviceId`, served from `idx_media_device_size` as a covering
+	*    scan, the same trick `TrackStore.listDeviceIds` already uses (D56).
+	*    `bucket` IS the deviceId: the column is INTEGER and the bucket
+	*    expression is `CAST((deviceId - 0) / 1 AS INTEGER)`.
+	*  - The NUMBERS come from one `aggregate` per camera — `COUNT(*)` and
+	*    `SUM(sizeBytes)` in a single statement over an indexed `deviceId` range.
+	*
+	* Both figures for a camera come from the SAME statement deliberately. Taking
+	* `rows` from the histogram and `bytes` from the aggregate would let a write
+	* landing between the two produce a footprint whose row count and byte count
+	* describe different populations — a disagreement an operator sizing a
+	* deletion has no way to detect.
+	*
+	* A read that fails THROWS. It must never degrade to a zero: "this camera
+	* holds nothing" is precisely the claim a failed measurement must not be
+	* allowed to make to something about to delete (D295, D312).
 	*/
 	async footprintByDevice() {
-		const PAGE = 1e3;
+		const buckets = await this.store.histogram.query({
+			collection: MEDIA_COLLECTION,
+			field: "deviceId",
+			bucketSize: 1,
+			origin: 0
+		});
 		const out = /* @__PURE__ */ new Map();
-		let offset = 0;
-		for (;;) {
-			const rows = await this.store.query.query({
+		for (const bucket of buckets) {
+			if (bucket.count === 0 || !Number.isFinite(bucket.bucket)) continue;
+			const deviceId = bucket.bucket;
+			const agg = await this.store.aggregate.query({
 				collection: MEDIA_COLLECTION,
-				filter: {
-					orderBy: {
-						field: "id",
-						direction: "asc"
-					},
-					limit: PAGE,
-					offset
-				}
+				fields: [{
+					as: "bytes",
+					field: "sizeBytes",
+					op: "sum"
+				}],
+				filter: { where: { deviceId } }
+			});
+			if (agg.count === 0) continue;
+			const bytes = agg.values["bytes"];
+			out.set(deviceId, {
+				bytes: bytes ?? 0,
+				rows: agg.count
 			});
-			if (rows.length === 0) break;
-			for (const row of rows) {
-				const data = row.data;
-				const deviceId = Number(data["deviceId"]);
-				const sizeBytes = Number(data["sizeBytes"]);
-				if (!Number.isFinite(deviceId)) continue;
-				const acc = out.get(deviceId) ?? {
-					bytes: 0,
-					rows: 0
-				};
-				acc.bytes += Number.isFinite(sizeBytes) ? sizeBytes : 0;
-				acc.rows += 1;
-				out.set(deviceId, acc);
-			}
-			if (rows.length < PAGE) break;
-			offset += PAGE;
 		}
 		return out;
 	}
+	/**
+	* The media footprint broken down by KIND — fleet-wide, or for one camera.
+	*
+	* `footprintByDevice` answers "how much is where"; this answers "how much is
+	* WHAT", which is the axis a deletion decision turns on. The deletable set
+	* (the periodic `snapshot` filmstrip, the surplus per-edge motion stills) and
+	* the keep set (`firstFrame`, rolling `lastFrame`, `thumbnail` /
+	* `thumbnailSmall`, `keyFrame` / `keyFrameSmall`, the face and plate buffers,
+	* gallery media, the CLIP `crop`) are told apart by `kind` and by nothing
+	* else. A byte total cannot size a deletion; this can.
+	*
+	* One `aggregate` per kind, each an indexed covering range on
+	* `idx_media_kind_device_size`, plus ONE unfiltered aggregate for the totals.
+	*
+	* That last statement is the point of the design and not a spare round trip.
+	* `MediaFileKindEnum.options` is the closed set the writers use, but the
+	* TABLE is not closed: a row written by a retired path, or by a version that
+	* knew a kind this one does not, matches no per-kind query. Adding the kinds
+	* up to produce the total would make such a row invisible and the denominator
+	* quietly too small — the failure mode that matters most here, because the
+	* caller is about to delete against it. So the total is measured
+	* independently and the difference is REPORTED as `unaccounted*` rather than
+	* being absorbed.
+	*/
+	async footprintByKind(deviceId) {
+		const scope = deviceId === void 0 ? void 0 : { where: { deviceId } };
+		const fields = [{
+			as: "bytes",
+			field: "sizeBytes",
+			op: "sum"
+		}];
+		const totals = await this.store.aggregate.query({
+			collection: MEDIA_COLLECTION,
+			fields: [...fields],
+			...scope === void 0 ? {} : { filter: scope }
+		});
+		const kinds = [];
+		let countedRows = 0;
+		let countedBytes = 0;
+		for (const kind of require_dist.MediaFileKindEnum.options) {
+			const agg = await this.store.aggregate.query({
+				collection: MEDIA_COLLECTION,
+				fields: [...fields],
+				filter: { where: {
+					kind,
+					...deviceId === void 0 ? {} : { deviceId }
+				} }
+			});
+			if (agg.count === 0) continue;
+			const bytes = agg.values["bytes"] ?? 0;
+			countedRows += agg.count;
+			countedBytes += bytes;
+			kinds.push({
+				kind,
+				rows: agg.count,
+				bytes
+			});
+		}
+		kinds.sort((a, b) => b.bytes - a.bytes);
+		const totalBytes = totals.values["bytes"] ?? 0;
+		return {
+			totalRows: totals.count,
+			totalBytes,
+			kinds,
+			unaccountedRows: totals.count - countedRows,
+			unaccountedBytes: totalBytes - countedBytes
+		};
+	}
 	/** Move a single media entry from one owner to another.
 	*  The new copy is written before the old one is removed.
 	*  The old-entry deletion is best-effort: on failure a warning is logged
@@ -37042,10 +37192,34 @@ var AnalyticsQueryFacade = class {
 	}
 	/**
 	* Durable event-store footprint for the events-management UI: event ROWS
-	* (motion + object + audio) counted per camera (indexed `count`) + total, and
-	* event-owned media BYTES on disk per camera (paged `sizeBytes` sum) + total.
-	* The device set is the union of cameras that have media OR persisted tracks;
-	* cameras with neither rows nor bytes are omitted.
+	* (motion + object + audio) per camera + total, and event-owned media BYTES
+	* on disk per camera + total. Cameras with neither rows nor bytes are
+	* omitted; the rest come back ordered by bytes, descending, which is the
+	* order the operator's table renders.
+	*
+	* ## The device set is a union of THREE sources
+	*
+	* It used to be `media ∪ tracks`, and a camera holding events but no media
+	* row and no persisted track was therefore invisible — not listed, and its
+	* events missing from `totalRows`. That combination is not exotic: media
+	* retention, a media deletion and `pruneTracksBefore` all run on clocks of
+	* their own, and every one of them can outlive the event rows. The number
+	* this method exists to produce is the denominator for a deletion, so an
+	* undercount is the one failure mode that matters, and it was silent.
+	*
+	* Events now contribute their own device set, so a camera is present if ANY
+	* of the three sources has ever heard of it.
+	*
+	* ## The cost is the answer, not the table
+	*
+	* Three grouped reads for the event rows (one per event collection, never
+	* three per camera), one grouped read plus one aggregate per camera for the
+	* media bytes, one for the track device set. The retired shape paid ~1 250
+	* round trips for the media half alone and timed out at 60 s on every call.
+	* See {@link MediaStore.footprintByDevice} and {@link EventStore.countByDevice}.
+	*
+	* A store that cannot answer THROWS rather than returning zeroes — see the
+	* same note on both.
 	*/
 	async getEventStoreFootprint() {
 		const eventStore = this.deps.eventStore();
@@ -37055,15 +37229,16 @@ var AnalyticsQueryFacade = class {
 			totalBytes: 0,
 			devices: []
 		};
-		const mediaByDevice = await mediaStore.footprintByDevice();
+		const [mediaByDevice, eventRowsByDevice] = await Promise.all([mediaStore.footprintByDevice(), eventStore.countByDevice()]);
 		const deviceIds = new Set(mediaByDevice.keys());
+		for (const id of eventRowsByDevice.keys()) deviceIds.add(id);
 		const trackStore = this.deps.trackStore();
 		if (trackStore) for (const id of await trackStore.listDeviceIds()) deviceIds.add(id);
 		const devices = [];
 		let totalRows = 0;
 		let totalBytes = 0;
 		for (const deviceId of deviceIds) {
-			const rows = await eventStore.countForDevice(deviceId);
+			const rows = eventRowsByDevice.get(deviceId) ?? 0;
 			const bytes = mediaByDevice.get(deviceId)?.bytes ?? 0;
 			if (rows === 0 && bytes === 0) continue;
 			totalRows += rows;
@@ -37082,6 +37257,26 @@ var AnalyticsQueryFacade = class {
 		};
 	}
 	/**
+	* The media footprint by KIND — the axis that says what is safe to delete.
+	* See {@link MediaStore.footprintByKind}.
+	*
+	* An absent media store is an empty breakdown, not a refusal: it is the same
+	* "the addon has not finished booting" branch every other method here takes,
+	* and the zeroes are accompanied by an empty `kinds` list, which no reader
+	* can mistake for a measured "there is nothing".
+	*/
+	async getEventMediaFootprintByKind(input) {
+		const mediaStore = this.deps.mediaStore();
+		if (!mediaStore) return {
+			totalRows: 0,
+			totalBytes: 0,
+			kinds: [],
+			unaccountedRows: 0,
+			unaccountedBytes: 0
+		};
+		return mediaStore.footprintByKind(input.deviceId);
+	}
+	/**
 	* Cluster-wide prune of events with `timestamp < olderThanMs` across every
 	* camera (via the device-agnostic `evictBefore`), deleting each pruned event's
 	* media in lockstep. Logs one global (`deviceId:null`) events ops-log row.
@@ -44750,6 +44945,15 @@ function snapshot(j) {
 */
 var LOCATION_ID_FIELD = "locationId";
 /**
+* Unstamped rows claimed per bulk seal statement.
+*
+* Bounds two things at once: the number of bound parameters in the `IN` list,
+* and how long one `UPDATE` holds the write lock on the database every process
+* in the cluster reads configuration through. Deliberately well under SQLite's
+* variable ceiling, and small enough that a cancel lands within one statement.
+*/
+var SEAL_BATCH_SIZE = 500;
+/**
 * True when the row names NO location — the population `seal` exists for.
 *
 * Deliberately NOT `mediaRowLocation(data) === DEFAULT_MEDIA_LOCATION`: that
@@ -44981,6 +45185,17 @@ var MediaRelocateEngine = class {
 	}
 	async countSelected(mode, toLocationId, stop) {
 		try {
+			if (mode === "seal") {
+				let sealable = 0;
+				for (const collection of collectionsFor(mode)) {
+					if (stop()) return null;
+					sealable += await this.deps.store.count.query({
+						collection: collection.name,
+						filter: { where: { [LOCATION_ID_FIELD]: null } }
+					});
+				}
+				return { rows: sealable };
+			}
 			let rows = 0;
 			for (const collection of collectionsFor(mode)) {
 				const isRetrain = collection.name === RETRAIN_FRAMES_COLLECTION;
@@ -45012,6 +45227,14 @@ var MediaRelocateEngine = class {
 			*  gets believed. */
 			let pathlessRows = 0;
 			job.filesTotal = (await this.countSelected(job.mode, job.toLocationId, () => job.cancelRequested))?.rows ?? null;
+			if (job.mode === "seal") {
+				for (const collection of collectionsFor(job.mode)) {
+					if (job.cancelRequested) break;
+					await this.sealCollection(collection.name, job);
+				}
+				job.state = job.cancelRequested ? "cancelled" : "done";
+				return;
+			}
 			for (const collection of collectionsFor(job.mode)) {
 				if (job.cancelRequested) break;
 				const isRetrain = collection.name === RETRAIN_FRAMES_COLLECTION;
@@ -45019,20 +45242,6 @@ var MediaRelocateEngine = class {
 					if (job.cancelRequested) break;
 					const data = row.data;
 					if (!selectsRow(job.mode, job.toLocationId, data, isRetrain)) continue;
-					if (job.mode === "seal") {
-						try {
-							await this.stamp(collection.name, row.id, data, job.toLocationId);
-							job.filesMoved++;
-						} catch (err) {
-							failedRows++;
-							this.deps.logger.debug("event-media seal row failed", { meta: {
-								collection: collection.name,
-								key: row.id,
-								error: String(err)
-							} });
-						}
-						continue;
-					}
 					const from = mediaRowLocation(data);
 					const relativePath = String(data["path"] ?? "");
 					if (relativePath.length === 0) {
@@ -45095,6 +45304,73 @@ var MediaRelocateEngine = class {
 			this.deps.onFinished?.(snapshot(job));
 		}
 	}
+	/**
+	* Stamp every unstamped row of one collection with the job's target, in
+	* bounded bulk statements.
+	*
+	* ## What this replaced
+	*
+	* A per-row loop: page the whole collection through {@link pageRows}, then
+	* one `store.set.mutate` per selected row — a full-row UPSERT rewriting every
+	* column to change one field, one RPC each, plus a second full walk before it
+	* to produce the denominator. Measured on the live hub against the 464 902
+	* unstamped rows: **2.3 rows/s, falling to 0.7 rows/s under a competing
+	* drain**, i.e. over two and a half days for a metadata stamp that reads no
+	* file and moves no byte. It was cancelled. This is the write-side twin of
+	* the read-side walk {@link countUnstamped} retired, and the fourth instance
+	* of the same pattern found in one day — work looped over the door instead of
+	* pushed into the statement.
+	*
+	* ## The shape
+	*
+	* Probe the indexed `IS NULL` range for a batch of ids, then ONE
+	* `updateWhere` over those ids. Two round trips per {@link SEAL_BATCH_SIZE}
+	* rows instead of one write per row.
+	*
+	* The probe carries no cursor and needs none: the predicate CONSUMES itself.
+	* A stamped row leaves the range, so the next probe returns the next batch,
+	* and a run that is interrupted anywhere simply resumes on what is left. That
+	* is also why the seal is idempotent (a second run probes an empty range and
+	* stamps nothing) and why it cannot repoint a row that already names a
+	* different location — `IS NULL` never selects one.
+	*
+	* A single unbounded `UPDATE … WHERE locationId IS NULL` would be one
+	* statement instead of ~1 860, and is not used: half a million row rewrites,
+	* each maintaining four indexes, is an unbounded statement holding the write
+	* lock on a database the whole cluster reads configuration through, under an
+	* RPC deadline that must not be raised. Batching bounds the statement, keeps
+	* `cancel` responsive between batches, and keeps `filesMoved` honest.
+	*
+	* ## The termination guard
+	*
+	* If the probe finds rows and the update reports **0**, this THROWS. That is
+	* not defensive noise: it is precisely what a backend compiling
+	* `locationId = NULL` does — matches nothing, reports success (D312) — and
+	* against it a self-consuming loop never terminates and a non-looping one
+	* reports a completed seal having stamped nothing. Zero unstamped rows is the
+	* single value the migration gate reads as "safe to cut over".
+	*/
+	async sealCollection(collection, job) {
+		for (;;) {
+			if (job.cancelRequested) return;
+			const batch = await this.deps.store.query.query({
+				collection,
+				filter: {
+					where: { [LOCATION_ID_FIELD]: null },
+					limit: SEAL_BATCH_SIZE
+				},
+				columns: [LOCATION_ID_FIELD]
+			});
+			if (batch.length === 0) return;
+			const { updated } = await this.deps.store.updateWhere.mutate({
+				collection,
+				filter: { whereIn: { id: batch.map((row) => row.id) } },
+				data: { [LOCATION_ID_FIELD]: job.toLocationId }
+			});
+			if (updated === 0) throw new Error(`event-media seal stamped 0 of ${batch.length} unstamped row(s) in "${collection}" — the bulk predicate matched nothing, so the population cannot be sealed`);
+			job.filesMoved += updated;
+		}
+	}
 	async stamp(collection, key, data, locationId) {
 		await this.deps.store.set.mutate({
 			collection,
@@ -51826,6 +52102,43 @@ var EventStore = class {
 		return motion + object + audio;
 	}
 	/**
+	* Persisted event rows (motion + object + audio) for EVERY device that has
+	* any — three round trips for the whole fleet.
+	*
+	* {@link countForDevice} answers one camera in three `COUNT(*)`s, which is
+	* right for one camera and wrong in a loop: the events-management footprint
+	* called it once per device, so the cost of "how many events does each camera
+	* hold" grew with the camera count for a result that is a few dozen integers.
+	*
+	* A histogram on `deviceId` with `bucketSize: 1` is `GROUP BY deviceId`, and
+	* every event collection already carries `idx_<kind>_device_ts`, which leads
+	* with `deviceId` — so the planner serves each of these from a covering index
+	* with no table access at all. `bucket` IS the deviceId (the column is
+	* INTEGER, the bucket expression is `CAST((deviceId - 0) / 1 AS INTEGER)`).
+	*
+	* Not best-effort: a failure propagates. The one caller is sizing a deletion,
+	* and a camera silently reported as holding zero events is the answer that
+	* must never come from a read that did not happen.
+	*/
+	async countByDevice() {
+		const perCollection = await Promise.all([
+			MOTION_EVENTS_COLLECTION,
+			OBJECT_EVENTS_COLLECTION,
+			AUDIO_EVENTS_COLLECTION
+		].map((collection) => this.store.histogram.query({
+			collection,
+			field: "deviceId",
+			bucketSize: 1,
+			origin: 0
+		})));
+		const out = /* @__PURE__ */ new Map();
+		for (const buckets of perCollection) for (const bucket of buckets) {
+			if (bucket.count === 0 || !Number.isFinite(bucket.bucket)) continue;
+			out.set(bucket.bucket, (out.get(bucket.bucket) ?? 0) + bucket.count);
+		}
+		return out;
+	}
+	/**
 	* Return per-kind event counts in equal-width time buckets.
 	*
 	* Each bucket maps to a `bucketStart = since + i * bucketMs`.
@@ -71549,6 +71862,9 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_dist.B
 	async getEventStoreFootprint() {
 		return this.queryFacade.getEventStoreFootprint();
 	}
+	async getEventMediaFootprintByKind(input) {
+		return this.queryFacade.getEventMediaFootprintByKind(input);
+	}
 	async pruneEvents(input) {
 		return this.queryFacade.pruneEvents(input);
 	}