@camstack/system 1.2.57 → 1.2.59

package/dist/builtins/sqlite-storage/sqlite-settings.addon.js

@@ -3,11 +3,222 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../../chunk-Cek0wNdY.js");
-const require_dist = require("../../dist-BupxnZOi.js");
+const require_dist = require("../../dist-CV6XTApk.js");
 let node_crypto = require("node:crypto");
+let node_fs = require("node:fs");
 let node_module = require("node:module");
 let better_sqlite3 = require("better-sqlite3");
 better_sqlite3 = require_chunk.__toESM(better_sqlite3);
+//#region src/builtins/sqlite-storage/sqlite-pragmas.ts
+/**
+* Connection tuning for the hub's SQLite database.
+*
+* Every number here is sized against a MEASURED database, not a blog post.
+* On the live hub (2026-08-06), 20 Hz syscall sampling of hub-main put **210
+* of 2,561 samples inside `pread64`** — 161 on `camstack.db` and 49 on
+* `camstack.db-wal`, all 4 KiB page reads, i.e. page-cache misses. That is
+* 8.2% of wall-clock and ~20% during bursts, on the single thread that also
+* serves every tRPC request. fds 79/81 of hub-main were the two files, so the
+* cost is proven in-process rather than inferred.
+*
+* The database was **130 MB** with a **249 MB** WAL high-water mark. Its
+* tenants are not settings: the analytics event/track/media stores and the
+* sqlite-vec index all live in this one file.
+*/
+/**
+* Page cache, in KiB (SQLite reads a NEGATIVE `cache_size` as KiB rather than
+* as a page count, which is what makes it independent of `page_size`).
+*
+* -65536 = **64 MiB**. Sized from the 130 MB file: the hot working set is the
+* B-tree interior pages plus the indexes the analytics queries walk, which is
+* a fraction of the total — 64 MiB covers roughly half the file outright, so a
+* repeated query stops re-reading the same 4 KiB pages from disk. Going to the
+* full 130 MB would buy the tail of a cold table scan for twice the resident
+* cost; going smaller leaves the measured miss rate roughly where it is.
+*
+* Why the build default is not enough: better-sqlite3 compiles with
+* `SQLITE_DEFAULT_CACHE_SIZE=-16000`, so the starting point is ~16 MiB — 8x
+* short of the file, not the ~2 MiB stock SQLite default. This is a 4x
+* increase, not a 32x one.
+*
+* Cost: 64 MiB of malloc'd C memory, ONE connection's worth (better-sqlite3
+* holds a single handle and the vector index shares it). It is off-heap, so it
+* does not move V8's heap ceiling — the thing that produced the hub's previous
+* OOM.
+*/
+var SQLITE_PAGE_CACHE_KIB = -65536;
+/**
+* Memory-mapped I/O window, in bytes. 268435456 = **256 MiB**.
+*
+* Chosen to exceed the 130 MB database with room for it to roughly double, so
+* the whole main file can be mapped. Pages then arrive as minor faults against
+* the OS page cache instead of `pread64` syscalls — which is precisely the
+* 161-sample half of the measurement. It also means a page evicted from
+* SQLite's own cache is usually still resident in the OS cache, so the two
+* knobs compound rather than duplicate.
+*
+* Two honest caveats:
+*  - SQLite **never mmaps the WAL**. The 49 WAL samples are not addressed
+*    here; they are addressed by keeping the WAL small (see
+*    {@link SQLITE_JOURNAL_SIZE_LIMIT_BYTES} and `wal-maintenance.ts`).
+*  - With mmap, an I/O error on a mapped page surfaces as SIGBUS rather than a
+*    catchable SQLite error. The pragma is asserted in
+*    `__tests__/sqlite-pragmas.spec.ts` by reading it back, so a build that
+*    ignores it (`SQLITE_MAX_MMAP_SIZE=0`) fails loudly instead of quietly
+*    doing nothing.
+*/
+var SQLITE_MMAP_SIZE_BYTES = 268435456;
+/**
+* Upper bound on the WAL file left on disk after a checkpoint resets it, in
+* bytes. 67108864 = **64 MiB**.
+*
+* This is the passive half of the WAL fix and it carries no stall risk at all:
+* it takes no lock and blocks nothing. It only says "when a checkpoint has
+* already succeeded, truncate the file rather than leaving the space
+* allocated". Without it, a WAL that spiked once stays spiked — which is how
+* a 130 MB database ends up with a 249 MB sidecar.
+*
+* 64 MiB is deliberately well above the ~4 MiB `wal_autocheckpoint` threshold
+* (1000 pages), so ordinary operation never touches this ceiling and the file
+* is not being repeatedly truncated and re-grown.
+*/
+var SQLITE_JOURNAL_SIZE_LIMIT_BYTES = 67108864;
+//#endregion
+//#region src/builtins/sqlite-storage/retired-settings-keys.ts
+/**
+* Is THIS node the one whose settings store is the cluster's authority?
+*
+* Mirrors `resolveNodeRole` (server/backend `node-role.ts`) — unset or `'hub'`
+* means hub, which is the launcher's own default. It is duplicated rather than
+* imported because `@camstack/system` is a dependency OF the backend, not the
+* other way round; the parse is four lines and the env var is the contract.
+*
+* Anything else answers `false`. Skipping where the purge should have run is a
+* missed cleanup the next hub boot fixes; running where it should not have is a
+* write to a store this node does not own. Only one of those is recoverable.
+*/
+function settingsStoreIsAuthoritativeHere(env) {
+	const raw = (env["CAMSTACK_ROLE"] ?? "").trim().toLowerCase();
+	return raw === "" || raw === "hub";
+}
+/**
+* The dead keys, per row.
+*
+* `detection-pipeline` / `addon-settings` / `root` — every key in this row is
+* dead; the row itself is kept because the addon writes live keys into it
+* (`pipelineTemplates`, `videoPipelineSteps`).
+*
+*  - `pipelineSteps` / `pipelineEngine`: the persisted GLOBAL step + engine
+*    seed. Removed with `getGlobalSteps` becoming pure — dispatch builds steps
+*    from the orchestrator's `agentSettings` plus per-camera overrides, and
+*    never consulted the seed. See `addon-pipeline/src/detection-pipeline/provider.ts`.
+*  - `engineRuntime` / `engineBackend` / `engineDevice` / `probedBestEngine`:
+*    the per-node engine CASCADE, removed with the operator-facing engine
+*    election. Each node now runs a device-array of inference pools balanced
+*    per session, and the node-default pool derives live from the node's own
+*    hardware (`resolveAutoEngine`). See
+*    `addon-pipeline/src/detection-pipeline/engine-store-keys.ts`.
+*/
+var RETIRED_SETTINGS_KEYS = [{
+	namespace: "detection-pipeline",
+	collection: "addon-settings",
+	row: "root",
+	keys: ["pipelineSteps", "pipelineEngine"],
+	perNodeKeys: [
+		"engineRuntime",
+		"engineBackend",
+		"engineDevice",
+		"probedBestEngine"
+	],
+	reason: "global step/engine seed and the per-node engine cascade — both removed; the live per-camera dispatch path never read either"
+}];
+/**
+* Adapt the settings backend to {@link RetiredKeyStore}.
+*
+* `get` answering a non-object (missing row) is a MISS, not a fault: it maps to
+* `null` and the purge skips the row. A backend throw propagates, which is what
+* routes it to the "read failed, row left untouched" branch.
+*/
+function retiredKeyStoreOf(backend) {
+	return {
+		async read(spec) {
+			return require_dist.asJsonObject(await backend.get({
+				namespace: spec.namespace,
+				collection: spec.collection,
+				key: spec.row
+			}));
+		},
+		async write(spec, value) {
+			await backend.set({
+				namespace: spec.namespace,
+				collection: spec.collection,
+				key: spec.row,
+				value
+			});
+		}
+	};
+}
+/** Keys of `blob` that `spec` retires. Pure. */
+function planRetiredKeyPurge(blob, spec) {
+	const exact = new Set(spec.keys);
+	const found = [];
+	for (const key of Object.keys(blob)) {
+		if (exact.has(key)) {
+			found.push(key);
+			continue;
+		}
+		for (const perNode of spec.perNodeKeys) if (key === perNode || key.startsWith(`${perNode}@`)) {
+			found.push(key);
+			break;
+		}
+	}
+	return found;
+}
+/**
+* Delete every retired key present in the declared rows — on the hub.
+*
+* Off the authoritative node this is a no-op that touches nothing and says
+* nothing (see the module header). Returns one entry per row that actually
+* changed — empty when the store is already clean, which is the steady state
+* after the first boot that ran it.
+*/
+async function purgeRetiredSettingsKeys(store, logger, specs = RETIRED_SETTINGS_KEYS, env = process.env) {
+	if (!settingsStoreIsAuthoritativeHere(env)) return [];
+	const results = [];
+	for (const spec of specs) {
+		let blob;
+		try {
+			blob = await store.read(spec);
+		} catch (err) {
+			logger.warn("retired settings keys — read failed, row left untouched", { meta: {
+				addon: spec.namespace,
+				collection: spec.collection,
+				row: spec.row,
+				error: err instanceof Error ? err.message : String(err)
+			} });
+			continue;
+		}
+		if (blob === null) continue;
+		const doomed = planRetiredKeyPurge(blob, spec);
+		if (doomed.length === 0) continue;
+		const doomedSet = new Set(doomed);
+		const next = Object.fromEntries(Object.entries(blob).filter(([key]) => !doomedSet.has(key)));
+		await store.write(spec, next);
+		const result = {
+			addon: spec.namespace,
+			collection: spec.collection,
+			row: spec.row,
+			keys: doomed
+		};
+		results.push(result);
+		logger.info(`purged ${doomed.length} retired settings keys`, { meta: {
+			...result,
+			reason: spec.reason
+		} });
+	}
+	return results;
+}
+//#endregion
 //#region src/builtins/sqlite-storage/filter-compiler.ts
 /** Thrown by `mutate` mode. Distinct type so a caller can map it to a 400. */
 var UnsafeFilterError = class extends Error {
@@ -115,6 +326,9 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
 		this.db = new better_sqlite3.default(this.dbPath);
 		this.db.pragma("journal_mode = WAL");
 		this.db.pragma("foreign_keys = ON");
+		this.db.pragma(`cache_size = ${SQLITE_PAGE_CACHE_KIB}`);
+		this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
+		this.db.pragma(`journal_size_limit = ${SQLITE_JOURNAL_SIZE_LIMIT_BYTES}`);
 		for (const collection of SqliteSettingsBackend.CANONICAL_KV_COLLECTIONS) {
 			await this.ensureTable(collection, { columns: [{
 				name: "id",
@@ -957,6 +1171,209 @@ function rebuild(row) {
 	return out;
 }
 //#endregion
+//#region src/builtins/sqlite-storage/wal-maintenance.ts
+/**
+* Size at or above which the WAL is a candidate for a truncating checkpoint,
+* in bytes. 67108864 = **64 MiB**.
+*
+* Matched to `SQLITE_JOURNAL_SIZE_LIMIT_BYTES`: below this, the passive
+* mechanism (autocheckpoint + `journal_size_limit`) is doing its job and there
+* is nothing to reclaim. The live hub's WAL had reached 248,955,152 bytes —
+* roughly 2x the 130 MB database — which is what says the passive path alone
+* had stopped keeping up.
+*/
+var WAL_HIGH_WATER_BYTES = 67108864;
+/**
+* How long the WAL must have gone UNWRITTEN before a truncating checkpoint is
+* allowed, in ms.
+*
+* This is the stall guard and the reason this file exists. `wal_checkpoint
+* (TRUNCATE)` takes the writer lock, waits for readers, and copies the whole
+* WAL back into the database — synchronously, on the thread that serves every
+* request. An unconditional periodic one on a busy NVR would be the disease
+* (hub-main already spends 42 s per 2 minutes inside uninterrupted blocks),
+* not the cure.
+*
+* The idle signal is the WAL file's own mtime. It is one `stat`, and unlike a
+* write counter maintained in this class it covers EVERY writer on the file —
+* the settings backend, the sqlite-vec index sharing the handle, and the
+* short-lived second connection system-backup opens. On a genuinely busy NVR
+* this gate simply never opens, and `journal_size_limit` remains the only
+* thing bounding the file. That is the intended failure mode: doing nothing is
+* strictly better than a 249 MB copy in the middle of a live request.
+*/
+var WAL_IDLE_QUIET_MS = 3e4;
+/**
+* Ceiling on how long the checkpoint may wait for a lock, in ms.
+*
+* better-sqlite3 defaults `busy_timeout` to 5000. A checkpoint invokes the
+* busy handler while waiting for readers, so without this a "quiet" moment
+* that turns out not to be quiet could park the main thread for 5 s. 250 ms
+* means the checkpoint gives up and we try again on the next tick — the WAL
+* staying big for another minute costs nothing.
+*/
+var WAL_CHECKPOINT_BUSY_TIMEOUT_MS = 250;
+/** How often the gate is evaluated, in ms. Cheap: a `stat` unless it fires. */
+var WAL_MAINTENANCE_INTERVAL_MS = 6e4;
+/**
+* Multiple of the high-water mark past which a persistently un-checkpointable
+* WAL stops being a debug line and becomes a warning. At 4x (256 MiB) the WAL
+* is bigger than the database and something is holding a read transaction
+* open; that is worth seeing in Loki.
+*/
+var WAL_ALARM_MULTIPLE = 4;
+function isCheckpointRow(value) {
+	if (typeof value !== "object" || value === null) return false;
+	if (!("busy" in value) || !("log" in value) || !("checkpointed" in value)) return false;
+	return typeof value.busy === "number" && typeof value.log === "number" && typeof value.checkpointed === "number";
+}
+/** Narrow `PRAGMA wal_checkpoint` output without a cast; null when unreadable. */
+function readCheckpointResult(value) {
+	if (isCheckpointRow(value)) return value;
+	if (Array.isArray(value)) {
+		const [first] = value;
+		if (isCheckpointRow(first)) return first;
+	}
+	return null;
+}
+/**
+* Periodic, heavily gated `wal_checkpoint(TRUNCATE)`.
+*
+* The interesting behaviour is all refusal. A tick costs one `stat` in the
+* common case; it escalates to a lock-taking checkpoint only when the WAL is
+* over the high-water mark AND nothing has written to it for
+* {@link WAL_IDLE_QUIET_MS}, and even then the wait is bounded to
+* {@link WAL_CHECKPOINT_BUSY_TIMEOUT_MS}.
+*/
+var WalMaintenance = class {
+	db;
+	walPath;
+	logger;
+	highWaterBytes;
+	idleQuietMs;
+	intervalMs;
+	busyTimeoutMs;
+	timer = null;
+	constructor(options) {
+		this.db = options.db;
+		this.walPath = `${options.dbPath}-wal`;
+		this.logger = options.logger;
+		this.highWaterBytes = options.highWaterBytes ?? 67108864;
+		this.idleQuietMs = options.idleQuietMs ?? 3e4;
+		this.intervalMs = options.intervalMs ?? 6e4;
+		this.busyTimeoutMs = options.busyTimeoutMs ?? 250;
+	}
+	start() {
+		if (this.timer !== null) return;
+		const timer = setInterval(() => {
+			this.runOnce();
+		}, this.intervalMs);
+		timer.unref?.();
+		this.timer = timer;
+		this.logger.info("WAL maintenance armed", { meta: {
+			intervalMs: this.intervalMs,
+			highWaterBytes: this.highWaterBytes,
+			idleQuietMs: this.idleQuietMs
+		} });
+	}
+	stop() {
+		if (this.timer === null) return;
+		clearInterval(this.timer);
+		this.timer = null;
+	}
+	isRunning() {
+		return this.timer !== null;
+	}
+	/** Evaluate the gate once and act. Never throws. */
+	runOnce() {
+		try {
+			return this.evaluate();
+		} catch (err) {
+			const error = require_dist.errMsg(err);
+			this.logger.warn("WAL maintenance tick failed", { meta: {
+				error,
+				walPath: this.walPath
+			} });
+			return {
+				kind: "failed",
+				error
+			};
+		}
+	}
+	evaluate() {
+		const stat = this.statWal();
+		if (stat === null) return { kind: "no-wal" };
+		if (stat.size < this.highWaterBytes) return {
+			kind: "below-high-water",
+			walBytes: stat.size
+		};
+		const quietForMs = Date.now() - stat.mtimeMs;
+		if (quietForMs < this.idleQuietMs) {
+			const meta = {
+				walBytes: stat.size,
+				quietForMs,
+				idleQuietMs: this.idleQuietMs
+			};
+			if (stat.size >= this.highWaterBytes * WAL_ALARM_MULTIPLE) this.logger.warn("WAL checkpoint deferred: file far over high-water and still being written", { meta });
+			else this.logger.debug("WAL checkpoint deferred: database is not idle", { meta });
+			return {
+				kind: "deferred-not-idle",
+				walBytes: stat.size,
+				quietForMs
+			};
+		}
+		return this.checkpoint(stat.size);
+	}
+	/**
+	* Take the writer lock and fold the WAL back, with the wait bounded.
+	*
+	* The original `busy_timeout` is restored in a `finally`: this connection is
+	* shared with the settings backend and the vector index, and leaving them on
+	* a 250 ms timeout would turn an ordinary contended write into a spurious
+	* SQLITE_BUSY.
+	*/
+	checkpoint(beforeBytes) {
+		const previousTimeout = this.db.pragma("busy_timeout", { simple: true });
+		this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
+		let raw;
+		try {
+			raw = this.db.pragma("wal_checkpoint(TRUNCATE)");
+		} finally {
+			if (typeof previousTimeout === "number") this.db.pragma(`busy_timeout = ${previousTimeout}`);
+		}
+		const result = readCheckpointResult(raw);
+		const afterBytes = this.statWal()?.size ?? 0;
+		const outcome = {
+			kind: "checkpointed",
+			beforeBytes,
+			afterBytes,
+			busy: result !== null && result.busy !== 0,
+			walFrames: result?.log ?? -1,
+			checkpointedFrames: result?.checkpointed ?? -1
+		};
+		this.logger.info("WAL truncating checkpoint completed", { meta: {
+			beforeBytes,
+			afterBytes,
+			reclaimedBytes: beforeBytes - afterBytes,
+			busy: outcome.busy,
+			walFrames: outcome.walFrames,
+			checkpointedFrames: outcome.checkpointedFrames
+		} });
+		return outcome;
+	}
+	statWal() {
+		try {
+			const s = (0, node_fs.statSync)(this.walPath);
+			return {
+				size: s.size,
+				mtimeMs: s.mtimeMs
+			};
+		} catch {
+			return null;
+		}
+	}
+};
+//#endregion
 //#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
 /**
 * SQLite Settings addon — the relational ENGINE behind the data door.
@@ -977,11 +1394,12 @@ function rebuild(row) {
 var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 	backend = null;
 	vectorIndex = null;
+	walMaintenance = null;
 	constructor() {
 		super({});
 	}
 	async onInitialize() {
-		const addonId = this.ctx.id.replace("addon:", "");
+		const addonId = require_dist.bareAddonId(this.ctx.id);
 		const path = await import("node:path");
 		const dbPath = this.ctx.dataDir ? path.join(this.ctx.dataDir, "camstack.db") : path.join("camstack-data", "addons-data", addonId, "camstack.db");
 		const pathSource = this.ctx.dataDir ? "dataDir" : "fallback (hardcoded)";
@@ -1000,6 +1418,11 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 		this.ctx.logger.info("DB file status", { meta: { dbExists } });
 		this.backend = new SqliteSettingsBackend(dbPath, { ...require_dist.RUNTIME_DEFAULTS });
 		await this.backend.initialize();
+		try {
+			await purgeRetiredSettingsKeys(retiredKeyStoreOf(this.backend), this.ctx.logger.child("RetiredKeys"));
+		} catch (err) {
+			this.ctx.logger.warn("Retired-key purge failed", { meta: { error: require_dist.errMsg(err) } });
+		}
 		const vectorIndex = this.createVectorIndex(this.backend);
 		await vectorIndex.loadIndexRegistry();
 		this.vectorIndex = vectorIndex;
@@ -1020,6 +1443,16 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 			deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
 			stats: async (input) => vectorIndex.stats(input.index)
 		};
+		const db = this.backend.getDatabase();
+		if (db !== null) {
+			const maintenance = new WalMaintenance({
+				db,
+				dbPath,
+				logger: this.ctx.logger.child("WalMaintenance")
+			});
+			maintenance.start();
+			this.walMaintenance = maintenance;
+		}
 		this.ctx.logger.info("Initialized successfully");
 		return [{
 			capability: require_dist.dataStoreProviderCapability,
@@ -1030,6 +1463,8 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 		}];
 	}
 	async onShutdown() {
+		this.walMaintenance?.stop();
+		this.walMaintenance = null;
 		await this.backend?.shutdown();
 	}
 	getBackend() {
@@ -1067,6 +1502,14 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
 	}
 };
 //#endregion
+exports.SQLITE_JOURNAL_SIZE_LIMIT_BYTES = SQLITE_JOURNAL_SIZE_LIMIT_BYTES;
+exports.SQLITE_MMAP_SIZE_BYTES = SQLITE_MMAP_SIZE_BYTES;
+exports.SQLITE_PAGE_CACHE_KIB = SQLITE_PAGE_CACHE_KIB;
 exports.SqliteSettingsAddon = SqliteSettingsAddon;
 exports.default = SqliteSettingsAddon;
 exports.SqliteSettingsBackend = SqliteSettingsBackend;
+exports.WAL_CHECKPOINT_BUSY_TIMEOUT_MS = WAL_CHECKPOINT_BUSY_TIMEOUT_MS;
+exports.WAL_HIGH_WATER_BYTES = WAL_HIGH_WATER_BYTES;
+exports.WAL_IDLE_QUIET_MS = WAL_IDLE_QUIET_MS;
+exports.WAL_MAINTENANCE_INTERVAL_MS = WAL_MAINTENANCE_INTERVAL_MS;
+exports.WalMaintenance = WalMaintenance;