@camstack/addon-pipeline-orchestrator 1.2.181 → 1.2.182

package/dist/index.js

@@ -17097,6 +17097,7 @@ var NcSystemEventKindSchema = _enum([
 	"addon-crash-loop",
 	"addon-update-available",
 	"server-update-available",
+	"wrapper-update-available",
 	"alarm-triggered",
 	"alarm-armed",
 	"alarm-disarmed",
@@ -23355,6 +23356,53 @@ var ImageContractSchema = object({
 	/** One operator-grade sentence: this node runs image X; the contract says Y. */
 	message: string()
 });
+/**
+* The shell verdict (see {@link WrapperKindSchema}) with the proofs that
+* produced it.
+*
+* There is deliberately NO `canSelfUpdate` and no apply action: a wrapper
+* update is an ANNOUNCEMENT. The notification body carries the exact command
+* for a `docker` shell and says the app restarts into the new build for
+* `electron`, and that sentence is chosen from `kind` alone — a boolean
+* nothing reads is the defect this repo keeps paying for.
+*/
+var WrapperIdentitySchema = object({
+	kind: _enum([
+		"docker",
+		"electron",
+		"native",
+		"contradictory",
+		"unknown"
+	]),
+	/** Every probe that ran, in a stable order. Never empty. */
+	evidence: array(object({
+		/** Which probe this row reports. */
+		fact: _enum([
+			"dockerenv-file",
+			"proc-1-cgroup",
+			"platform",
+			"electron-app-version-env"
+		]),
+		/**
+		* `observed` — this process read it itself. `declared` — the shell told the
+		* process and nothing here can check it. Only the Electron app version is
+		* ever `declared`, and only because a child cannot see its parent.
+		*/
+		mode: _enum(["observed", "declared"]),
+		/** Did the probe support the fact it names? A `false` row is still evidence. */
+		holds: boolean(),
+		/** What was actually read, verbatim enough for an operator to argue with. */
+		detail: string()
+	})),
+	/**
+	* The version of the SHELL, when the shell has one it can state: the
+	* Electron app version it declared, or the baked seed closure that
+	* fingerprints a container / app bundle. `null` for `contradictory` and
+	* `unknown` — which version belongs to the shell is precisely what is not
+	* known there.
+	*/
+	currentVersion: string().nullable()
+});
 var ServerRollbackInfoSchema = object({
 	/** The version that failed (or was manually rolled back). */
 	fromVersion: string(),
@@ -23396,7 +23444,13 @@ var ServerPackageStatusSchema = object({
 	* Seed-vs-contract verdict (see {@link ImageContractSchema}). Optional for
 	* version skew: an older provider's payload simply omits it.
 	*/
-	imageContract: ImageContractSchema.optional()
+	imageContract: ImageContractSchema.optional(),
+	/**
+	* What this node runs INSIDE (see {@link WrapperIdentitySchema}). Optional
+	* for version skew: an older provider's payload simply omits it, and an
+	* absent wrapper is unknown — never `native`.
+	*/
+	wrapper: WrapperIdentitySchema.optional()
 });
 var ServerUpdateCheckResultSchema = object({
 	packageName: string(),
@@ -42925,6 +42979,52 @@ var LoadShedController = class LoadShedController {
 	}
 };
 //#endregion
+//#region src/dispatch-timing.ts
+/**
+* How long the placement census waits for ONE runner's `getLocalCameras`
+* before counting that node as not having reported. A node that did not
+* report never has its assignments dropped (`reconcilePlacement` says so), so
+* skipping a slow node costs one pass of orphan cleanup on that node, not
+* correctness. The census fans out in parallel, so this bounds the whole
+* census at one budget rather than one budget per node.
+*/
+var RECONCILE_NODE_BUDGET_MS = 1500;
+/**
+* Wait for `promise`, but never longer than `budgetMs`.
+*
+* The promise is NOT cancelled — it keeps running, and its eventual rejection
+* is observed here so an overrun never becomes an unhandled rejection. The
+* caller decides what an overrun means; this only says that it happened.
+*/
+function raceBudget(promise, budgetMs) {
+	return new Promise((resolve) => {
+		let done = false;
+		const timer = setTimeout(() => {
+			if (done) return;
+			done = true;
+			resolve({ kind: "budget-exceeded" });
+		}, budgetMs);
+		timer.unref?.();
+		promise.then((value) => {
+			if (done) return;
+			done = true;
+			clearTimeout(timer);
+			resolve({
+				kind: "settled",
+				value
+			});
+		}, (error) => {
+			if (done) return;
+			done = true;
+			clearTimeout(timer);
+			resolve({
+				kind: "rejected",
+				error
+			});
+		});
+	});
+}
+//#endregion
 //#region src/agent-load-service.ts
 var AgentLoadService = class AgentLoadService {
 	deps;
@@ -42962,28 +43062,69 @@ var AgentLoadService = class AgentLoadService {
 	* collects every known runner, which is what the UI wants.
 	*/
 	async collectAgentLoad(options) {
-		const loads = [];
 		const onlyEnabled = options?.onlyEnabled ?? false;
-		for (const nodeId of this.deps.topology.knownRunnerNodeIds()) {
-			if (onlyEnabled && !this.deps.topology.isNodeEnabled(nodeId)) {
-				this.deps.logger.debug("runner excluded by enabledNodes whitelist", { tags: { nodeId } });
-				continue;
-			}
-			if (!this.deps.api()) continue;
-			try {
-				const load = await this.queryLocalLoadBounded(nodeId);
-				loads.push(load);
-			} catch (err) {
-				const msg = errMsg(err);
-				this.deps.logger.debug("getLocalLoad failed", {
-					tags: { nodeId },
-					meta: { error: msg }
-				});
-			}
+		const nodeIds = this.deps.topology.knownRunnerNodeIds().filter((nodeId) => {
+			if (!onlyEnabled || this.deps.topology.isNodeEnabled(nodeId)) return true;
+			this.deps.logger.debug("runner excluded by enabledNodes whitelist", { tags: { nodeId } });
+			return false;
+		});
+		const budgetMs = this.agentLoadBudgetMs();
+		const entries = await Promise.all(nodeIds.map((nodeId) => this.censusNode(nodeId, budgetMs)));
+		const loads = entries.flatMap((e) => e.load === null ? [] : [e.load]);
+		const skipped = entries.filter((e) => e.load === null).map((e) => e.nodeId);
+		if (skipped.length > 0) {
+			const nodes = Object.fromEntries(entries.map((e) => [e.nodeId, {
+				ms: e.ms,
+				outcome: e.outcome,
+				...e.error === void 0 ? {} : { error: e.error }
+			}]));
+			this.deps.logger.warn("agent load census skipped a runner", { meta: {
+				budgetMs,
+				skipped,
+				nodes
+			} });
 		}
 		this.refreshAgentLoadCache(loads);
 		return loads;
 	}
+	/**
+	* One runner's load under `budgetMs`. A wedged runner (transport up enough
+	* to stay in the registry, but whose call neither resolves nor rejects) is
+	* counted as `budget-exceeded` and skipped exactly like an offline one; the
+	* in-flight call is left to settle on its own, as before.
+	*/
+	async censusNode(nodeId, budgetMs) {
+		const api = this.deps.api();
+		const startedAt = Date.now();
+		if (!api) return {
+			nodeId,
+			ms: 0,
+			outcome: "failed",
+			load: null,
+			error: "addon not initialized"
+		};
+		const outcome = await raceBudget(api.pipelineRunner.getLocalLoad.query({ nodeId }), budgetMs);
+		const ms = Date.now() - startedAt;
+		if (outcome.kind === "settled") return {
+			nodeId,
+			ms,
+			outcome: "reported",
+			load: outcome.value
+		};
+		if (outcome.kind === "rejected") return {
+			nodeId,
+			ms,
+			outcome: "failed",
+			load: null,
+			error: errMsg(outcome.error)
+		};
+		return {
+			nodeId,
+			ms,
+			outcome: "budget-exceeded",
+			load: null
+		};
+	}
 	refreshAgentLoadCache(loads) {
 		const next = /* @__PURE__ */ new Map();
 		for (const load of loads) next.set(load.nodeId, {
@@ -43030,25 +43171,6 @@ var AgentLoadService = class AgentLoadService {
 		const value = (this.deps.readGlobalSettings() ?? {}).agentLoadTimeoutMs;
 		return typeof value === "number" && value > 0 ? value : AgentLoadService.DEFAULT_AGENT_LOAD_TIMEOUT_MS;
 	}
-	/**
-	* `getLocalLoad` for one runner node, bounded by `agentLoadBudgetMs`. On
-	* timeout it rejects so `collectAgentLoad`'s catch treats the node exactly
-	* like an offline one (logged at debug, skipped).
-	*/
-	async queryLocalLoadBounded(nodeId) {
-		const api = this.deps.api();
-		if (!api) throw new Error("queryLocalLoadBounded: addon not initialized");
-		const budgetMs = this.agentLoadBudgetMs();
-		let timer;
-		const timeout = new Promise((_resolve, reject) => {
-			timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`getLocalLoad exceeded ${budgetMs}ms budget for ${nodeId}`)), budgetMs);
-		});
-		try {
-			return await Promise.race([api.pipelineRunner.getLocalLoad.query({ nodeId }), timeout]);
-		} finally {
-			if (timer) clearTimeout(timer);
-		}
-	}
 	/** Mirrors the former `onShutdown`'s `this.cachedAgentLoad.clear()`. */
 	dispose() {
 		this.cachedAgentLoad.clear();
@@ -50619,19 +50741,44 @@ var ReconcileController = class ReconcileController {
 	async runPlacementReconcile() {
 		const api = this.deps.api();
 		if (!api) return;
+		const censusStartedAt = Date.now();
 		const runners = [];
-		for (const nodeId of this.deps.topology.collectRunnerNodeIds()) try {
-			const attached = await api.pipelineRunner.getLocalCameras.query(void 0, nodePin(nodeId));
-			runners.push({
+		const census = [];
+		await Promise.all(this.deps.topology.collectRunnerNodeIds().map(async (nodeId) => {
+			const startedAt = Date.now();
+			const outcome = await raceBudget(api.pipelineRunner.getLocalCameras.query(void 0, nodePin(nodeId)), RECONCILE_NODE_BUDGET_MS);
+			const ms = Date.now() - startedAt;
+			if (outcome.kind === "settled") {
+				runners.push({
+					nodeId,
+					attached: outcome.value
+				});
+				census.push({
+					nodeId,
+					ms,
+					outcome: "reported"
+				});
+				return;
+			}
+			census.push({
 				nodeId,
-				attached
+				ms,
+				outcome: outcome.kind === "rejected" ? "failed" : "budget-exceeded"
 			});
-		} catch (err) {
 			this.deps.logger.debug("placement reconcile skipped node — getLocalCameras failed", {
 				tags: { nodeId },
-				meta: { error: errMsg(err) }
+				meta: {
+					error: outcome.kind === "rejected" ? errMsg(outcome.error) : "budget exceeded",
+					ms
+				}
 			});
-		}
+		}));
+		const censusMs = Date.now() - censusStartedAt;
+		if (censusMs >= 1e3 || census.some((entry) => entry.outcome !== "reported")) this.deps.logger.info("placement reconcile: runner census was slow", { meta: {
+			censusMs,
+			nodes: census,
+			budgetMs: RECONCILE_NODE_BUDGET_MS
+		} });
 		const { assignedNodeById, dispatching } = this.deps.ledger.snapshotForReconcile();
 		const idleSessionDeviceIds = new Set([...this.deps.activeDeviceIds()].filter((deviceId) => this.deps.isIdleSessionCamera(deviceId)));
 		const motionWatchAttachments = /* @__PURE__ */ new Set();
@@ -50645,7 +50792,7 @@ var ReconcileController = class ReconcileController {
 		});
 		if (detach.length > 0) {
 			this.deps.logger.info("placement reconcile: detaching orphans/duplicates", { meta: { detach } });
-			for (const { nodeId, deviceId } of detach) await this.deps.detachOn(nodeId, deviceId).catch((err) => {
+			await Promise.all(detach.map(({ nodeId, deviceId }) => this.deps.detachOn(nodeId, deviceId).catch((err) => {
 				this.deps.logger.warn("placement reconcile: detach failed", {
 					tags: {
 						nodeId,
@@ -50653,7 +50800,7 @@ var ReconcileController = class ReconcileController {
 					},
 					meta: { error: errMsg(err) }
 				});
-			});
+			})));
 		}
 		if (dropStaleAssignment.length > 0) {
 			this.deps.logger.info("placement reconcile: dropping stale assignments", { meta: { deviceIds: dropStaleAssignment } });
@@ -51043,6 +51190,13 @@ var TtlMemo = class {
 	* declared wrong.
 	*/
 	generations = /* @__PURE__ */ new Map();
+	/**
+	* The last value ANY read produced for a key, kept across
+	* {@link invalidate}. Never served by {@link get}; it is the bounded fallback
+	* of {@link getWithin} alone (D441) — a caller that would rather balance on a
+	* snapshot a few seconds old than wait seconds for the re-read.
+	*/
+	lastKnown = /* @__PURE__ */ new Map();
 	ttlMs;
 	now;
 	constructor(opts) {
@@ -51076,6 +51230,50 @@ var TtlMemo = class {
 		}
 	}
 	/**
+	* The value for `key` within `budgetMs`: from memory when fresh, from `load`
+	* when the read lands in time, and otherwise the LAST KNOWN value —
+	* invalidated or not — with its age (D441).
+	*
+	* Three properties, each pinned by a test:
+	*  - a read that overruns the budget is NOT abandoned: it keeps running and
+	*    stores its result for the next caller, so a stale answer is served at
+	*    most once per read;
+	*  - a COLD key waits for the read past the budget, because a cold cache must
+	*    never be the reason a camera fails to dispatch (the `get` contract);
+	*  - a read that FAILS with a last-known value in hand serves it as stale.
+	*/
+	async getWithin(key, budgetMs) {
+		const at = this.now();
+		const entry = this.entries.get(key);
+		if (entry) this.entries.set(key, {
+			...entry,
+			usedAt: at
+		});
+		if (entry && at - entry.readAt < this.ttlMs) return {
+			value: entry.value,
+			stale: false,
+			ageMs: at - entry.readAt
+		};
+		const fallback = this.lastKnown.get(key);
+		const pending = this.read(key);
+		if (fallback === void 0) return {
+			value: await pending,
+			stale: false,
+			ageMs: 0
+		};
+		const outcome = await raceBudget(pending, budgetMs);
+		if (outcome.kind === "settled") return {
+			value: outcome.value,
+			stale: false,
+			ageMs: 0
+		};
+		return {
+			value: fallback.value,
+			stale: true,
+			ageMs: this.now() - fallback.readAt
+		};
+	}
+	/**
 	* Declare the cached answer for `key` WRONG — not merely old.
 	*
 	* The caller has just changed the thing being cached (an attach or a detach
@@ -51120,6 +51318,7 @@ var TtlMemo = class {
 		this.entries.clear();
 		this.inFlight.clear();
 		this.generations.clear();
+		this.lastKnown.clear();
 	}
 	/** One shared in-flight read per key; stores the result on success. */
 	read(key) {
@@ -51127,14 +51326,15 @@ var TtlMemo = class {
 		if (existing) return existing;
 		const generation = this.generations.get(key) ?? 0;
 		const pending = this.opts.load(key).then((value) => {
-			if ((this.generations.get(key) ?? 0) !== generation) return value;
 			const at = this.now();
-			const prior = this.entries.get(key);
-			this.entries.set(key, {
+			const stored = {
 				value,
 				readAt: at,
-				usedAt: prior?.usedAt ?? at
-			});
+				usedAt: this.entries.get(key)?.usedAt ?? at
+			};
+			this.lastKnown.set(key, stored);
+			if ((this.generations.get(key) ?? 0) !== generation) return value;
+			this.entries.set(key, stored);
 			return value;
 		}).finally(() => {
 			if (this.inFlight.get(key) === pending) this.inFlight.delete(key);
@@ -51602,7 +51802,7 @@ var SessionDispatchController = class {
 					return;
 				}
 				this.activeRefireCountByDevice.delete(deviceId);
-				await this.dispatchDetectionSession(deviceId, cur, trigger);
+				const timings = await this.dispatchDetectionSession(deviceId, cur, trigger);
 				if (this.sessionRegistry.has(deviceId)) this.scheduleSessionTeardown(deviceId, cooldownMs);
 				const doneAt = Date.now();
 				this.deps.logger.info("session motion → attach latency", {
@@ -51611,6 +51811,7 @@ var SessionDispatchController = class {
 						...busLagMs !== void 0 ? { busLagMs } : {},
 						lockWaitMs: lockAcquiredAt - lockRequestedAt,
 						dispatchMs: doneAt - lockAcquiredAt,
+						...timings,
 						totalMs: doneAt - receivedAt,
 						...emittedAt !== void 0 ? { sinceMotionMs: doneAt - emittedAt } : {}
 					}
@@ -51643,9 +51844,32 @@ var SessionDispatchController = class {
 	*/
 	async dispatchDetectionSession(deviceId, config, trigger) {
 		const log = this.deps.logger.withTags({ deviceId });
-		await this.deps.reconcilePlacementFromRunners();
+		const reconcileStartedAt = Date.now();
+		const census = this.deps.reconcilePlacementFromRunners();
+		const censusOutcome = await raceBudget(census, 500);
+		const reconcileMs = Date.now() - reconcileStartedAt;
+		const reconcileBudgetExceeded = censusOutcome.kind === "budget-exceeded";
+		const dispatchGuard = this.deps.ledger.markDispatching(deviceId);
+		const releaseGuard = () => {
+			if (!reconcileBudgetExceeded) {
+				dispatchGuard.dispose();
+				return;
+			}
+			census.finally(() => dispatchGuard.dispose());
+		};
+		if (reconcileBudgetExceeded) log.warn("detection session: placement census overran its budget — placing without it", { meta: { budgetMs: 500 } });
+		else if (censusOutcome.kind === "rejected") log.warn("detection session: placement census failed — placing on the ledger as it is", { meta: {
+			error: errMsg(censusOutcome.error),
+			reconcileMs
+		} });
+		const balanceStartedAt = Date.now();
 		const preferredAgent = await this.deps.readPipelinePin(deviceId);
-		const loads = await this.agentLoads.get(CLUSTER_KEY);
+		const loadAnswer = await this.agentLoads.getWithin(CLUSTER_KEY, 500);
+		const loads = loadAnswer.value;
+		if (loadAnswer.stale) log.warn("detection session: agent load read overran its budget — balancing on the last known load", { meta: {
+			budgetMs: 500,
+			loadAgeMs: loadAnswer.ageMs
+		} });
 		const allEligible = this.deps.topology.detectionEligibleNodes(deviceId);
 		const usableEligible = allEligible.filter((n) => this.deps.isNodeInferenceUsable(n));
 		const eligible = usableEligible.length > 0 ? usableEligible : allEligible;
@@ -51659,9 +51883,11 @@ var SessionDispatchController = class {
 		});
 		if (!decision || decision.kind !== "assigned") {
 			log.warn("detection session: no eligible node — will retry on next motion", { meta: { reason: decision?.kind === "pending" ? decision.reason : "no-runners-online" } });
-			return;
+			releaseGuard();
+			return null;
 		}
 		const targetNodeId = decision.agentNodeId;
+		const balancedAt = Date.now();
 		const enabledDevices = await this.eligibleInferenceDevices.get(targetNodeId);
 		const targetLoad = loads.find((n) => n.nodeId === targetNodeId);
 		const deviceCaps = await this.inferenceDeviceCaps.get(targetNodeId);
@@ -51691,6 +51917,7 @@ var SessionDispatchController = class {
 				...pinnedDeviceKey ? { pin: pinnedDeviceKey } : {}
 			}
 		});
+		const deviceSelectedAt = Date.now();
 		const pipelineConfig = await this.deps.settingsStore.resolvePipelineForDevice(deviceId, targetNodeId, deviceKey);
 		const deviceBase = await this.deps.settingsStore.buildNodeInferenceDeviceSteps(targetNodeId, deviceKey ?? "");
 		const deviceOverride = deviceKey ? await this.deps.settingsStore.readCameraDeviceOverride(deviceId, targetNodeId, deviceKey) : {};
@@ -51726,9 +51953,11 @@ var SessionDispatchController = class {
 			preRollInferenceEnabled: config.preRollInferenceEnabled,
 			preRollInferenceFrames: config.preRollInferenceFrames
 		};
-		const dispatchGuard = this.deps.ledger.markDispatching(deviceId);
+		const configuredAt = Date.now();
+		let attachedAt = configuredAt;
 		try {
 			await this.deps.attachOn(targetNodeId, sessionConfig);
+			attachedAt = Date.now();
 			this.deps.recordAssignment(deviceId, targetNodeId, decision.reason, decision.reason === "manual");
 			this.sessionRegistry.register(deviceId, targetNodeId, Date.now(), deviceKey);
 			log.info("detection session: dispatched on motion", {
@@ -51744,9 +51973,18 @@ var SessionDispatchController = class {
 				}
 			});
 		} finally {
-			dispatchGuard.dispose();
+			releaseGuard();
 			this.notePlacementChanged();
 		}
+		return {
+			reconcileMs,
+			reconcileBudgetExceeded,
+			balanceMs: balancedAt - balanceStartedAt,
+			balanceLoadStale: loadAnswer.stale,
+			deviceSelectMs: deviceSelectedAt - balancedAt,
+			configMs: configuredAt - deviceSelectedAt,
+			attachMs: attachedAt - configuredAt
+		};
 	}
 	/**
 	* Schedule a session teardown after `cooldownMs` of no rising-edge motion.