@camstack/addon-post-analysis 1.2.22 → 1.2.24

package/dist/pipeline-analytics/index.js

@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-const require_dist = require("../dist-vIJhE1KT.js");
+const require_dist = require("../dist-MQmqxd7v.js");
 let node_fs = require("node:fs");
 let node_path = require("node:path");
 node_path = require_dist.__toESM(node_path);
@@ -10249,6 +10249,108 @@ var TrackStore = class {
 	}
 };
 //#endregion
+//#region src/pipeline-analytics/store/storage-pressure.ts
+/**
+* storage-pressure — make a full disk something the system REACTS to, not just
+* something it logs once per lost file.
+*
+* ## What happened
+*
+* On 2026-08-02 media writes started failing with
+* `ENOSPC: no space left on device, mkdir '/recordings/617/events/track/…'`.
+* For roughly five hours every new track was recorded with positions, a class,
+* a `bestEventId` and **zero snapshots** — a track that looks complete in the
+* list and has no picture behind it. The operator found it by noticing the
+* missing media, not from any alert.
+*
+* Two separate defects made that possible:
+*
+*  1. **Retention prunes by AGE only.** It has no feedback from the filesystem,
+*     so "the disk is full" is not an input it can respond to. It kept its
+*     configured window while writes failed.
+*  2. **The failure was logged per KEY.** Every lost frame produced its own
+*     `media put failed` WARN — dozens a minute, each true, none of them saying
+*     "this camera has stopped storing media". A per-item line is a receipt,
+*     not an alarm.
+*
+* This module is the missing feedback: it recognises an out-of-space failure,
+* counts it per device, and hands the caller ONE escalation to act on.
+*
+* ## Deliberately not a deleter
+*
+* It decides that pressure exists and says so. Freeing space is retention's
+* job, and giving a write-error handler the power to delete recordings is how
+* a transient fault becomes data loss. The trigger is separated from the act on
+* purpose.
+*/
+/** Errors that mean "the filesystem refused for want of room". */
+var OUT_OF_SPACE_CODES = ["ENOSPC", "EDQUOT"];
+/**
+* Is this the disk being full, rather than any other write failure?
+*
+* Matches the CODE, not a message substring: a permission error or a missing
+* parent directory must not trigger a retention response, and the message text
+* is not a contract.
+*/
+function isOutOfSpace(err) {
+	const code = typeof err === "object" && err !== null && "code" in err ? String(err.code) : "";
+	if (OUT_OF_SPACE_CODES.includes(code)) return true;
+	const msg = err instanceof Error ? err.message : String(err);
+	return OUT_OF_SPACE_CODES.some((c) => msg.includes(`${c}:`));
+}
+/** How long between escalations while pressure persists. */
+var PRESSURE_ESCALATION_INTERVAL_MS = 6e4;
+/**
+* Counts out-of-space failures and releases an escalation at most once per
+* interval — plus immediately on the FIRST one, because the first is the line
+* that should have existed five hours before anyone noticed.
+*
+* Clock injected so the interval is testable without waiting.
+*/
+var StoragePressureTracker = class {
+	failures = 0;
+	devices = /* @__PURE__ */ new Set();
+	lastEscalatedAt = null;
+	now;
+	intervalMs;
+	constructor(intervalMs = PRESSURE_ESCALATION_INTERVAL_MS, now = () => Date.now()) {
+		this.intervalMs = intervalMs;
+		this.now = now;
+	}
+	/**
+	* Record a failed write. Returns an escalation when the caller should act
+	* (alert loudly, ask retention to reclaim), or null to stay quiet.
+	*
+	* A non-space failure returns null and is not counted: this tracker exists
+	* for one condition, and diluting it with unrelated errors would make the
+	* escalation mean nothing.
+	*/
+	record(err, deviceId) {
+		if (!isOutOfSpace(err)) return null;
+		this.failures += 1;
+		if (typeof deviceId === "number") this.devices.add(deviceId);
+		const at = this.now();
+		const first = this.lastEscalatedAt === null;
+		if (!first && at - this.lastEscalatedAt < this.intervalMs) return null;
+		const out = {
+			failures: this.failures,
+			deviceIds: [...this.devices],
+			first
+		};
+		this.failures = 0;
+		this.devices.clear();
+		this.lastEscalatedAt = at;
+		return out;
+	}
+	/** Clear the state after a successful write — pressure that has passed must
+	*  not make the next isolated failure look like a continuing emergency. */
+	noteSuccess() {
+		if (this.failures === 0 && this.devices.size === 0) return;
+		this.failures = 0;
+		this.devices.clear();
+	}
+};
+//#endregion
 //#region src/pipeline-analytics/store/media-store.ts
 var MEDIA_COLLECTION = "pipeline-analytics:media";
 /** Owner-id prefix for a track's buffered FACE crop. Invariant established by
@@ -10358,6 +10460,14 @@ function buildPath(params) {
 	return isSingleInstanceKind(params.kind) ? `${base}/${params.kind}.jpg` : `${base}/${params.kind}-${params.timestamp}.jpg`;
 }
 var MediaStore = class {
+	/** Turns repeated out-of-space write failures into ONE escalation with a
+	*  count — see `storage-pressure.ts` for what five hours of silent media loss
+	*  cost. */
+	pressure = new StoragePressureTracker();
+	/** Optional hook so the owner can ask retention to reclaim space. Left
+	*  undefined by default: deciding there IS pressure and DELETING recordings
+	*  are separate powers on purpose. */
+	onStoragePressure;
 	storage;
 	store;
 	logger;
@@ -10406,6 +10516,7 @@ var MediaStore = class {
 					data: record
 				}
 			});
+			this.pressure.noteSuccess();
 			return key;
 		} catch (err) {
 			this.logger.warn("media put failed", {
@@ -10415,6 +10526,19 @@ var MediaStore = class {
 					error: String(err)
 				}
 			});
+			const pressure = this.pressure.record(err, params.deviceId);
+			if (pressure) {
+				this.logger.error("media storage is out of space — media is being LOST", {
+					tags: { deviceId: params.deviceId },
+					meta: {
+						failures: pressure.failures,
+						devices: pressure.deviceIds,
+						first: pressure.first,
+						hint: "retention prunes by age and cannot see a full disk — free space or shorten retention"
+					}
+				});
+				this.onStoragePressure?.(pressure);
+			}
 			throw err;
 		}
 	}
@@ -11024,6 +11148,54 @@ var EventStore = class {
 			});
 		}
 	}
+	/**
+	* Collect the ids matching `filter`, then delete every matching row in ONE
+	* statement. The shared body of every retention sweep here.
+	*
+	* The ids are still read because the callers delete each event's media in
+	* lockstep and a bulk delete returns only a count. What is gone is the
+	* per-row delete loop: ~7,200 round trips to prune a day of motion events,
+	* each carrying every column of the row it deleted.
+	*
+	* Three properties, each of which cost a red test to learn:
+	*
+	*  - **the filter is used for BOTH halves**, so what is reported and what is
+	*    removed cannot drift apart;
+	*  - **paging carries an explicit offset over ROWS READ.** Deleting each page
+	*    used to advance the cursor implicitly; reading without deleting re-reads
+	*    page one forever. And a row with a non-string id yields no id but still
+	*    occupies a place, so ids collected is the wrong cursor;
+	*  - **only an EMPTY page ends the scan.** The store caps pages on its own
+	*    (~500), so "fewer rows than requested" is normal, not the end — treating
+	*    it as the end silently deletes just the first page while reporting a
+	*    complete sweep;
+	*  - **a delete that removed nothing reports NO ids.** Callers remove media
+	*    against this list; claiming a deletion that did not happen would orphan
+	*    media in the opposite direction.
+	*/
+	async drainMatching(collection, filter) {
+		const ids = [];
+		let offset = 0;
+		for (;;) {
+			const rows = await this.store.query.query({
+				collection,
+				filter: {
+					...filter,
+					limit: 500,
+					offset
+				}
+			});
+			if (rows.length === 0) break;
+			offset += rows.length;
+			for (const row of rows) if (typeof row.id === "string") ids.push(row.id);
+		}
+		if (ids.length === 0) return ids;
+		const { deleted } = await this.store.deleteWhere.mutate({
+			collection,
+			filter
+		});
+		return deleted === 0 ? [] : ids;
+	}
 	buildFilter(q) {
 		const filter = {
 			where: { deviceId: q.deviceId },
@@ -11344,35 +11516,10 @@ var EventStore = class {
 	* page at a time (PRUNE_PAGE_SIZE) until no stale rows remain.
 	*/
 	async evictBefore(params) {
-		const batch = async (collection, cutoffMs) => {
-			const ids = [];
-			for (;;) {
-				const rows = await this.store.query.query({
-					collection,
-					filter: {
-						...params.deviceId !== void 0 ? { where: { deviceId: params.deviceId } } : {},
-						whereBetween: { timestamp: [0, cutoffMs] },
-						limit: 500
-					}
-				});
-				if (rows.length === 0) break;
-				let deletedInPage = 0;
-				for (const row of rows) {
-					const id = row.id;
-					if (typeof id !== "string") continue;
-					try {
-						await this.store.delete.mutate({
-							collection,
-							key: id
-						});
-						ids.push(id);
-						deletedInPage++;
-					} catch {}
-				}
-				if (deletedInPage === 0) break;
-			}
-			return ids;
-		};
+		const batch = async (collection, cutoffMs) => this.drainMatching(collection, {
+			...params.deviceId !== void 0 ? { where: { deviceId: params.deviceId } } : {},
+			whereBetween: { timestamp: [0, cutoffMs] }
+		});
 		return {
 			motion: await batch(MOTION_EVENTS_COLLECTION, params.motionCutoffMs),
 			object: await batch(OBJECT_EVENTS_COLLECTION, params.objectCutoffMs),
@@ -11395,35 +11542,16 @@ var EventStore = class {
 	* so ALL matching rows are deleted regardless of total count.
 	*/
 	async pruneBefore(input) {
-		const allForDevice = async (collection) => {
-			const ids = [];
-			for (;;) {
-				const rows = await this.store.query.query({
-					collection,
-					filter: {
-						where: { deviceId: input.deviceId },
-						whereBetween: { timestamp: [0, input.cutoffMs - 1] },
-						limit: 500
-					}
-				});
-				if (rows.length === 0) break;
-				let deletedInPage = 0;
-				for (const row of rows) {
-					const id = row.id;
-					if (typeof id !== "string") continue;
-					try {
-						await this.store.delete.mutate({
-							collection,
-							key: id
-						});
-						ids.push(id);
-						deletedInPage++;
-					} catch {}
-				}
-				if (deletedInPage === 0) break;
-			}
-			return ids;
+		/**
+		* The filter is written ONCE and used for both halves below, so the rows
+		* reported and the rows deleted cannot drift apart. Inclusive `[0, cutoff-1]`
+		* → selects timestamps strictly < `cutoffMs`.
+		*/
+		const staleForDevice = {
+			where: { deviceId: input.deviceId },
+			whereBetween: { timestamp: [0, input.cutoffMs - 1] }
 		};
+		const allForDevice = async (collection) => this.drainMatching(collection, staleForDevice);
 		const [motionIds, objectIds, audioIds] = await Promise.all([
 			allForDevice(MOTION_EVENTS_COLLECTION),
 			allForDevice(OBJECT_EVENTS_COLLECTION),
@@ -17546,6 +17674,22 @@ var OBJECT_EMBEDDING_COLUMNS = [
 		type: "TEXT"
 	}
 ];
+/**
+* Ceiling on the CANDIDATE pool a semantic search may load.
+*
+* `query()` was called with no limit at all, so a default search loaded the
+* entire collection — 86.5 MB across 7,375 rows on a live hub, every time,
+* before ranking anything.
+*
+* It cannot simply be the caller's result limit: search RANKS the pool and then
+* takes the top N, so limiting to N up front would return N arbitrary rows
+* instead of the N best. This bounds the pool well above any result count, and
+* `query()` orders by `timestamp` DESC whenever a limit applies so "the pool"
+* means the most recent rows rather than whatever the page order happened to
+* be. The caller logs when the cap is reached — a silently truncated search
+* reads as "nothing older matched".
+*/
+var EMBEDDING_SEARCH_CANDIDATE_CAP = 5e3;
 var ObjectEmbeddingStore = class {
 	store;
 	logger;
@@ -17620,7 +17764,13 @@ var ObjectEmbeddingStore = class {
 		if (input.modelId !== void 0) where["modelId"] = input.modelId;
 		const filter = { where };
 		if (input.since !== void 0 || input.until !== void 0) filter["whereBetween"] = { timestamp: [input.since ?? 0, input.until ?? Date.now()] };
-		if (input.limit !== void 0) filter["limit"] = input.limit;
+		if (input.limit !== void 0) {
+			filter["limit"] = input.limit;
+			filter["orderBy"] = {
+				field: "timestamp",
+				direction: "desc"
+			};
+		}
 		return (await this.store.query.query({
 			collection: OBJECT_EMBEDDINGS_COLLECTION,
 			filter
@@ -20735,7 +20885,7 @@ var PipelineAnalyticsAddon = class extends require_dist.BaseAddon {
 		let storage = this.ctx.kernel.storage;
 		const mediaRoot = process.env.CAMSTACK_MEDIA_ROOT?.trim();
 		if (mediaRoot) {
-			const { FilesystemStorageProvider } = await Promise.resolve().then(() => require("../node-C1I1svLc.js"));
+			const { FilesystemStorageProvider } = await Promise.resolve().then(() => require("../node-DrRsUzaw.js"));
 			storage = new FilesystemStorageProvider(mediaRoot, { eventMedia: mediaRoot });
 			logger.info("pipeline-analytics: event media rooted at CAMSTACK_MEDIA_ROOT", { meta: { mediaRoot } });
 		}
@@ -23670,7 +23820,17 @@ var PipelineAnalyticsAddon = class extends require_dist.BaseAddon {
 			deviceId: input.deviceId,
 			since: input.since,
 			until: input.until,
-			className: input.classFilter
+			className: input.classFilter,
+			limit: EMBEDDING_SEARCH_CANDIDATE_CAP
+		});
+		if (embeddingRows.length >= 5e3) this.ctx.logger.warn("searchObjectEvents: candidate pool truncated", {
+			meta: {
+				cap: EMBEDDING_SEARCH_CANDIDATE_CAP,
+				since: input.since,
+				until: input.until,
+				className: input.classFilter
+			},
+			tags: input.deviceId !== void 0 ? { deviceId: input.deviceId } : void 0
 		});
 		if (embeddingRows.length === 0) return [];
 		const embeddingMediaKeyByTrackId = /* @__PURE__ */ new Map();