@camstack/system 1.2.59 → 1.2.61

package/dist/builtins/sqlite-storage/sqlite-settings.addon.js

@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../../chunk-Cek0wNdY.js");
-const require_dist = require("../../dist-CV6XTApk.js");
+const require_dist = require("../../dist-BN681PgO.js");
 let node_crypto = require("node:crypto");
 let node_fs = require("node:fs");
 let node_module = require("node:module");
@@ -275,11 +275,191 @@ function compileFilter$1(filter, shape, mode, serialize) {
 	};
 }
 //#endregion
+//#region src/builtins/sqlite-storage/query-bounds.ts
+/**
+* Defensive row bounds for `data-store-provider.query`.
+*
+* **Why this file exists.** On 2026-08-07 hub-main died with
+* `FATAL ERROR: Reached heap limit` under `Data::GetValueJS` /
+* `Statement::JS_all`, four seconds after reporting `heapUsed=644MB
+* heapLimit=4288MB`. The heap sampler cannot see an allocation that begins and
+* ends between two samples, so the `[mem]` line said everything was fine right
+* up to the kill. The same stack had already killed the hub twice on 01/08 and
+* twice on 02/08.
+*
+* **It was NOT one 3.4 GB statement, and that correction matters.** The whole
+* database measures 153 MiB (134 MB in a backup taken 14 h earlier, with
+* `auto_vacuum=NONE` and `freelist_count=6` proving it had never been larger)
+* — no single `.all()` on it can produce 3.4 GB. What happened was MANY reads
+* live at once: a viewer resumed from background and requested a 12-hour
+* window over 16 cameras, twice, 33 ms apart. The final Mark-Compact reclaimed
+* 4 MB of 4 090 MB, i.e. every byte was still reachable. `Statement::JS_all` is
+* where the wall was hit, not the size of that one call. The caller-side bound
+* for that specific fan-out lives in `track-store.ts`
+* (`RECENT_FANOUT_ROW_BUDGET`); this file is the backstop under everything
+* else.
+*
+* So: a bound on the query, NOT a bigger `--max-old-space-size`. hub-main's
+* routine p95 heap is already 2 108 MB against a 4 288 MB ceiling; a larger
+* ceiling only moves the number the next unbounded read has to beat.
+*
+* **Why a cap and not a required `limit`.** 64 call sites across the repo
+* query a collection with no `limit`, and most of them are correct — a filter
+* narrow enough that the result set is a handful of rows (`where: { trackId }`,
+* `where: { ownerId }`). Making `limit` mandatory would be 64 edits that each
+* have to guess a number, and the 65th call site added next month would be
+* unbounded again. A cap in the ENGINE holds for every caller, including the
+* ones not written yet.
+*
+* **Why the cap is never silent.** A truncated read that looks like a complete
+* one is worse than the OOM: it turns "the hub died" into "the retention sweep
+* quietly stopped deleting". Every cap hit is logged at WARN naming the
+* collection and the bound, and the caller is expected to page.
+*/
+/**
+* Rows returned when the caller supplied no `limit` at all.
+*
+* Sized against the MEASURED hub database (2026-08-08), because a row cap is
+* meaningless without knowing what a row weighs:
+*
+* | table                              | rows    | bytes  | avg row |
+* | ---------------------------------- | ------- | ------ | ------- |
+* | `pipeline-analytics:tracks`        |   4 668 |  54 MB | 11.5 KB |
+* | `pipeline-analytics:media`         | 101 953 |  23 MB |   230 B |
+* | `pipeline-analytics:motion-events` | 148 738 |  12 MB |    78 B |
+* | `pipeline-analytics:audio-events`  |  99 608 |   8 MB |    80 B |
+* | `pipeline-analytics:faces`         |     372 |   4 MB | 10.8 KB |
+*
+* `tracks` is the pacing table: its `positions` JSON alone is 47 MB of the 54,
+* averaging **10 KB per row**, and it is parsed into JS objects on read. 2 000
+* such rows is ~23 MB of text before parsing — an order of magnitude under
+* hub-main's routine p95 heap (2 108 MB) rather than a multiple of it. The
+* same 2 000 rows of `motion-events` is 160 KB, so narrow tables (auth,
+* alerts, integrations, schedules) never notice the cap at all.
+*
+* An earlier draft sized this at 5 000 on the belief that `faces` was the
+* widest table. It is 372 rows and 4 MB — `pruneCapOverflow` keeps it that
+* way. The number was defensible; the stated evidence for it was wrong, which
+* is worse, because the next person reasons from the evidence.
+*
+* It is deliberately NOT tuned to "the largest legitimate result set" — a
+* caller that genuinely wants more says so with an explicit `limit`, which is
+* honoured up to {@link SETTINGS_QUERY_HARD_ROW_CAP}.
+*/
+var SETTINGS_QUERY_DEFAULT_ROW_CAP = 2e3;
+/**
+* Absolute ceiling, applied even to an explicit `limit`.
+*
+* The default cap can be opted out of; this one cannot, because the failure it
+* prevents is an OOM and there is no result set worth taking the process down
+* for. A caller needing more than 20 000 rows is describing a stream, not a
+* query, and must page (`limit` + `offset`) — every bulk consumer in the repo
+* already does. At the measured 11.5 KB/row of `tracks`, 20 000 rows is
+* ~230 MB: ruinous to serve, but survivable, which is the right place for a
+* ceiling whose only job is to stop the process dying.
+*/
+var SETTINGS_QUERY_HARD_ROW_CAP = 2e4;
+/**
+* Resolve the row bound a `query` will run under.
+*
+* Treats a non-finite, negative or non-integer `limit` as ABSENT rather than
+* as an error. That is not leniency for its own sake: SQLite reads a negative
+* `LIMIT` as "no limit at all", so `limit: -1` arriving over the wire — the
+* cap schema is `z.number().optional()`, which accepts it — would have been
+* the one input able to reproduce the OOM through a bound that looked applied.
+*
+* `limit: 0` is a real request for zero rows and is honoured.
+*/
+function resolveRowBound(requested) {
+	if (requested === void 0 || !Number.isFinite(requested) || requested < 0) return {
+		limit: SETTINGS_QUERY_DEFAULT_ROW_CAP,
+		source: "default-cap"
+	};
+	const asked = Math.floor(requested);
+	if (asked > 2e4) return {
+		limit: SETTINGS_QUERY_HARD_ROW_CAP,
+		source: "hard-cap"
+	};
+	return {
+		limit: asked,
+		source: "caller"
+	};
+}
+/**
+* Whether a bound being hit is something the CALLER did not ask for, and must
+* therefore be told about.
+*
+* A caller that asked for 500 rows and got 500 has not been truncated — it has
+* been served. Only a cap the engine imposed is news.
+*/
+function isImposedBound(source) {
+	return source !== "caller";
+}
+//#endregion
+//#region src/builtins/sqlite-storage/prefix-range.ts
+var MAX_ASCII = 127;
+/**
+* @returns the range for `prefix`, or `null` when the caller must fall back to
+*   a `LIKE` scan (empty prefix — which is the whole table and has no upper
+*   bound — or any byte outside ASCII).
+*/
+function prefixRange(prefix) {
+	if (prefix.length === 0) return null;
+	for (let i = 0; i < prefix.length; i++) {
+		const code = prefix.charCodeAt(i);
+		if (code === 0 || code > MAX_ASCII) return null;
+	}
+	const last = prefix.charCodeAt(prefix.length - 1);
+	if (last >= MAX_ASCII) return null;
+	return {
+		lo: prefix,
+		hi: prefix.slice(0, -1) + String.fromCharCode(last + 1)
+	};
+}
+//#endregion
 //#region src/builtins/sqlite-storage/sqlite-settings-backend.ts
 function parseRowData(raw) {
 	return require_dist.asJsonObject(require_dist.parseJsonUnknown(raw)) ?? {};
 }
 /**
+* `tags: { deviceId }` for a log line about a query, when the filter names one.
+*
+* Repo rule: every log line about a device carries the numeric id under that
+* exact key, because every question asked of these logs is per-camera ("why is
+* 617 truncating and 615 not?") and a line without the tag cannot be grouped.
+* Returns an empty object — not a `deviceId: undefined` — so a collection-wide
+* query does not log a tag that reads as "device unknown".
+*/
+function deviceTagOf(filter) {
+	const raw = filter?.where?.["deviceId"];
+	return typeof raw === "number" && Number.isFinite(raw) ? { tags: { deviceId: raw } } : {};
+}
+/**
+* Stable identity of a declared schema — every property that changes the DDL
+* or the set of columns a filter can name.
+*
+* Deliberately covers `indexes` too: an added index is as invisible to a
+* name-only memo as an added column, and `CREATE INDEX IF NOT EXISTS` is
+* cheap enough that re-running it costs nothing.
+*/
+function schemaFingerprint(schema) {
+	return JSON.stringify({
+		columns: schema.columns.map((c) => [
+			c.name,
+			c.type,
+			c.primaryKey ?? false,
+			c.notNull ?? false,
+			c.unique ?? false,
+			c.defaultValue ?? null
+		]),
+		indexes: (schema.indexes ?? []).map((i) => [
+			i.name,
+			[...i.columns],
+			i.unique ?? false
+		])
+	});
+}
+/**
 * SQLite implementation of ISettingsBackend.
 *
 * Every collection is structured: declared at boot (canonical KV
@@ -293,7 +473,24 @@ function parseRowData(raw) {
 var SqliteSettingsBackend = class SqliteSettingsBackend {
 	dbPath;
 	db = null;
-	structuredTables = /* @__PURE__ */ new Set();
+	/**
+	* Table name → fingerprint of the schema this process last materialised.
+	*
+	* Keyed by FINGERPRINT, not by name, and that is the whole point. An addon
+	* declares its schema at every boot, but the backend lives in hub-main,
+	* which outlives an addon respawn: `camstack deploy` restarts the addon
+	* runner and NOT hub-main. A name-only memo therefore made the second
+	* declaration of a table a no-op FOREVER — a redeployed addon whose schema
+	* had gained a column never got the `ALTER TABLE … ADD COLUMN`, and, because
+	* `compileFilter` silently drops a predicate naming a column the collection
+	* cannot express (select mode), the missing column read as "matches
+	* everything" instead of failing. That is how `pipeline-analytics:tracks`
+	* ran for a whole release with no `retrainStatus`: the retention sweep's
+	* `whereIn: { retrainStatus: ['none','trained'] }` pin evaporated, so every
+	* operator-staged track was eligible for deletion, and `countStaging`
+	* reported the entire history as staged.
+	*/
+	structuredTables = /* @__PURE__ */ new Map();
 	/** Map from scoped collection name → set of column names (non-id) that
 	*  the structured schema owns. Routes set/get/insert/update/query to
 	*  typed columns. Every collection MUST be declared here before use. */
@@ -316,9 +513,18 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 		"device-settings",
 		"alerts"
 	];
-	constructor(dbPath, runtimeDefaults) {
+	/**
+	* Optional, because the engine is constructed in places that have no addon
+	* context (migrations, one-shot tools, most specs). It is NOT optional in
+	* production: `sqlite-settings.addon.ts` passes one, and without it a row cap
+	* being hit would be silent — which is the failure mode the cap exists to
+	* avoid replacing.
+	*/
+	logger;
+	constructor(dbPath, runtimeDefaults, logger) {
 		this.dbPath = dbPath;
 		this.runtimeDefaults = runtimeDefaults ?? {};
+		this.logger = logger;
 	}
 	async initialize() {
 		const dir = this.dbPath.substring(0, this.dbPath.lastIndexOf("/"));
@@ -552,15 +758,47 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 				sql += ` ORDER BY ${expr} ${dir}`;
 			}
 		}
-		if (filter?.limit !== void 0) {
-			sql += ` LIMIT ?`;
-			params.push(filter.limit);
-		}
+		/**
+		* A `LIMIT` is ALWAYS appended — this is the fix for the 2026-08-07 OOM,
+		* where the absent one let `Statement.all()` materialise a whole table into
+		* V8 in a single call (`query-bounds.ts` has the evidence).
+		*
+		* One row MORE than the bound is fetched, deliberately. Without the
+		* sentinel, "the result set was exactly the cap" and "the result set was
+		* truncated at the cap" are the same observation, and the warning below
+		* would either cry wolf on every full page or stay silent on real
+		* truncation. The extra row is dropped before it reaches the caller.
+		*/
+		const bound = resolveRowBound(filter?.limit);
+		sql += ` LIMIT ?`;
+		params.push(bound.limit + 1);
 		if (filter?.offset !== void 0) {
 			sql += ` OFFSET ?`;
 			params.push(filter.offset);
 		}
-		return this.getDb().prepare(sql).all(...params).map((r) => {
+		const fetched = this.getDb().prepare(sql).all(...params);
+		const truncated = fetched.length > bound.limit;
+		const rows = truncated ? fetched.slice(0, bound.limit) : fetched;
+		if (truncated && isImposedBound(bound.source))
+ /**
+		* WARN, not debug: the caller believes it read the whole collection and
+		* is about to act on that belief. A retention sweep that sees 5 000 of
+		* 40 000 rows deletes 5 000 and reports success.
+		*/
+		this.logger?.warn("settings-store query truncated by a defensive row cap", {
+			...deviceTagOf(filter),
+			meta: {
+				collection: table,
+				rowCap: bound.limit,
+				boundSource: bound.source,
+				requestedLimit: filter?.limit ?? null,
+				hasWhere: filter?.where !== void 0,
+				hasWhereIn: filter?.whereIn !== void 0,
+				hasWhereBetween: filter?.whereBetween !== void 0,
+				remedy: "page with limit + offset, or narrow the filter"
+			}
+		});
+		return rows.map((r) => {
 			const id = String(r[decl.primaryKey] ?? "");
 			if (isKvShape) {
 				const v = r["data"];
@@ -626,10 +864,11 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 	setAllAddon(addonId, config) {
 		this.requireDeclared("addon-settings");
 		const db = this.getDb();
-		const deleteStmt = db.prepare("DELETE FROM \"addon-settings\" WHERE id LIKE ? || '%'");
+		const where = this.prefixWhere(`${addonId}.`);
+		const deleteStmt = db.prepare(`DELETE FROM "addon-settings" WHERE ${where.sql}`);
 		const insertStmt = db.prepare("INSERT INTO \"addon-settings\" (id, data) VALUES (?, ?)");
 		db.transaction(() => {
-			deleteStmt.run(`${addonId}.`);
+			deleteStmt.run(...where.params);
 			for (const [key, value] of Object.entries(config)) {
 				if (value === void 0) continue;
 				insertStmt.run(`${addonId}.${key}`, JSON.stringify({
@@ -663,11 +902,12 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 		this.requireDeclared("addon-device-settings");
 		const db = this.getDb();
 		const prefix = `${addonId}:${deviceId}.`;
-		const deleteStmt = db.prepare(`DELETE FROM "addon-device-settings" WHERE id LIKE ? || '%'`);
+		const where = this.prefixWhere(prefix);
+		const deleteStmt = db.prepare(`DELETE FROM "addon-device-settings" WHERE ${where.sql}`);
 		const insertStmt = db.prepare(`INSERT INTO "addon-device-settings" (id, data) VALUES (?, ?)
        ON CONFLICT(id) DO UPDATE SET data = excluded.data`);
 		db.transaction(() => {
-			deleteStmt.run(prefix);
+			deleteStmt.run(...where.params);
 			for (const [key, value] of Object.entries(values)) insertStmt.run(`${prefix}${key}`, JSON.stringify({
 				addonId,
 				deviceId,
@@ -678,8 +918,8 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 	}
 	clearAddonDevice(addonId, deviceId) {
 		this.requireDeclared("addon-device-settings");
-		const prefix = `${addonId}:${deviceId}.`;
-		this.getDb().prepare(`DELETE FROM "addon-device-settings" WHERE id LIKE ? || '%'`).run(prefix);
+		const where = this.prefixWhere(`${addonId}:${deviceId}.`);
+		this.getDb().prepare(`DELETE FROM "addon-device-settings" WHERE ${where.sql}`).run(...where.params);
 	}
 	/** Seed system-settings with runtime defaults (first boot) */
 	async seedDefaults() {
@@ -701,9 +941,31 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 		if (!this.db) throw new Error("SqliteSettingsBackend not initialized — call initialize() first");
 		return this.db;
 	}
+	/**
+	* `WHERE` fragment selecting every key that starts with `prefix`, plus its
+	* bound parameters.
+	*
+	* The range form is what makes the implicit PRIMARY KEY index usable — see
+	* `prefix-range.ts` for the `EXPLAIN QUERY PLAN` output and the measured
+	* 1.8x/19.4x. The `LIKE` fallback is the SQL that shipped before, kept
+	* verbatim for the prefixes a byte range cannot express, so a key shape this
+	* function refuses degrades in speed and never in correctness.
+	*/
+	prefixWhere(prefix) {
+		const range = prefixRange(prefix);
+		if (range === null) return {
+			sql: `id LIKE ? || '%'`,
+			params: [prefix]
+		};
+		return {
+			sql: "id >= ? AND id < ?",
+			params: [range.lo, range.hi]
+		};
+	}
 	getAllScoped(collection, scopeId) {
 		this.requireDeclared(collection);
-		const rows = this.getDb().prepare(`SELECT id, data FROM "${collection}" WHERE id LIKE ? || '.%'`).all(scopeId);
+		const where = this.prefixWhere(`${scopeId}.`);
+		const rows = this.getDb().prepare(`SELECT id, data FROM "${collection}" WHERE ${where.sql}`).all(...where.params);
 		const result = {};
 		for (const row of rows) {
 			const key = row.id.slice(scopeId.length + 1);
@@ -738,7 +1000,6 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 	}
 	async declareCollection(input) {
 		const table = this.scopedName(input.namespace, input.collection);
-		if (this.declaredCollections.has(table)) return;
 		const columns = input.columns.some((c) => c.primaryKey === true) ? input.columns : [{
 			name: "id",
 			type: "TEXT",
@@ -779,7 +1040,8 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 		return v;
 	}
 	async ensureTable(table, schema) {
-		if (this.structuredTables.has(table)) return;
+		const fingerprint = schemaFingerprint(schema);
+		if (this.structuredTables.get(table) === fingerprint) return;
 		const colDefs = schema.columns.map((col) => {
 			const parts = [`"${col.name}" ${col.type}`];
 			if (col.primaryKey) parts.push("PRIMARY KEY");
@@ -811,7 +1073,7 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 			const cols = idx.columns.map((c) => `"${c}"`).join(", ");
 			this.getDb().exec(`CREATE ${unique}INDEX IF NOT EXISTS "${idx.name}" ON "${table}" (${cols})`);
 		}
-		this.structuredTables.add(table);
+		this.structuredTables.set(table, fingerprint);
 	}
 	async tableInsert(table, row) {
 		const keys = Object.keys(row);
@@ -1416,7 +1678,7 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 			dbPath
 		} });
 		this.ctx.logger.info("DB file status", { meta: { dbExists } });
-		this.backend = new SqliteSettingsBackend(dbPath, { ...require_dist.RUNTIME_DEFAULTS });
+		this.backend = new SqliteSettingsBackend(dbPath, { ...require_dist.RUNTIME_DEFAULTS }, this.ctx.logger.child("Query"));
 		await this.backend.initialize();
 		try {
 			await purgeRetiredSettingsKeys(retiredKeyStoreOf(this.backend), this.ctx.logger.child("RetiredKeys"));