@camstack/addon-pipeline-orchestrator 1.2.619 → 1.2.620

package/dist/index.js

@@ -77565,28 +77565,33 @@ var DETECTOR_PERF = {
 		},
 		"amd-igpu": {
 			"yolov9t-320": {
-				p50Ms: 188,
-				fps: 5,
+				p50Ms: 40,
+				fps: 25,
 				measured: true
 			},
 			"yolov9t-640": {
-				p50Ms: 671,
-				fps: 1,
+				p50Ms: 89,
+				fps: 11,
 				measured: true
 			},
 			"yolov9s-320": {
-				p50Ms: 401,
-				fps: 2,
+				p50Ms: 65,
+				fps: 15,
 				measured: true
 			},
 			"yolov9s-640": {
-				p50Ms: 1434,
-				fps: 1,
+				p50Ms: 125,
+				fps: 8,
 				measured: true
 			},
 			"yolov9m-320": {
-				p50Ms: 1076,
-				fps: 1,
+				p50Ms: 85,
+				fps: 12,
+				measured: true
+			},
+			"yolov9m-640": {
+				p50Ms: 228,
+				fps: 4,
 				measured: true
 			}
 		},
@@ -78113,6 +78118,55 @@ function resolveEffectiveShares(input) {
 		deviceWeights
 	};
 }
+/**
+* Automatic shares from CAPACITY (D755). Pure.
+*
+* A node's share is its capacity in parallel sessions over the cluster's; a
+* device's share inside its node follows its throughput. There is no load
+* input: nothing here depends on how many sessions ran, only on what each
+* node can carry.
+*
+* Two places a capacity can come from:
+*  - `hardware` — each enabled accelerator at its Auto tune model, from the
+*    measured latency table: ⌊fps × (1 − headroom) / camera fps⌋ sessions;
+*  - `benchmark` — the node's own measured max sessions per device, read
+*    through a {@link NodeCapacitySource}. A node whose results do not cover
+*    every enabled device it would be sized on is sized from `hardware`
+*    instead, and the answer says so per node — never a silent mix.
+*/
+/**
+* The measured capacity of `deviceKey` for the model it runs NOW, or
+* undefined. A cell measured for another model says nothing about this one
+* (a failed load of a heavier build is not this build's capacity), and a cell
+* that names no model cannot be matched. The ONE predicate behind both the
+* automatic shares and the automatic benchmark's "has results".
+*/
+function capacityForModel(node, deviceKey, modelId) {
+	if (node === void 0 || modelId === null) return void 0;
+	return node.devices.filter((d) => d.deviceKey === deviceKey && d.modelId === modelId).toSorted((a, b) => b.measuredAtMs - a.measuredAtMs)[0];
+}
+/**
+* Every measured (device × model) cell that no device runs now — what
+* {@link capacityForModel} skips, so the caller can say so once.
+*/
+function foreignCapacityCells(recommendations, measured) {
+	const out = [];
+	for (const rec of recommendations) {
+		const node = measured.get(rec.nodeId);
+		if (node === void 0) continue;
+		for (const cell of node.devices) {
+			const runs = rec.devices.find((d) => d.deviceKey === cell.deviceKey)?.chosen?.modelId ?? null;
+			if (cell.modelId === null || cell.modelId === runs) continue;
+			out.push({
+				nodeId: rec.nodeId,
+				deviceKey: cell.deviceKey,
+				measuredModelId: cell.modelId,
+				runsModelId: runs
+			});
+		}
+	}
+	return out;
+}
 /** Integer shares summing to 100 (largest remainder); ties broken by key. */
 function toPercentShares(weights) {
 	const total = [...weights.values()].reduce((a, b) => a + b, 0);
@@ -78164,7 +78218,11 @@ function sizeNode(rec, input) {
 	if (input.source === "hardware") return fromHardware(null);
 	if (!rec.reachable) return fromHardware(null);
 	const results = input.measured.get(rec.nodeId);
-	const byKey = new Map((results?.devices ?? []).map((d) => [d.deviceKey, d]));
+	const byKey = /* @__PURE__ */ new Map();
+	for (const d of rec.devices) {
+		const row = capacityForModel(results, d.deviceKey, d.chosen?.modelId ?? null);
+		if (row !== void 0) byKey.set(d.deviceKey, row);
+	}
 	const usable = rec.devices.filter((d) => d.enabled && d.available);
 	const mustCover = rec.devices.filter((d) => d.enabled && d.available && d.chosen !== null);
 	const covered = usable.filter((d) => byKey.has(d.deviceKey));
@@ -78414,16 +78472,34 @@ function measuredDetectorCells(rows, nodeId, ctx) {
 	return out;
 }
 /**
-* Per node, each device at its newest fresh `pipeline` measurement — the
-* `NodeCapacitySource` answer the automatic shares read.
+* Parallel sessions a `device`-phase cell stands for.
+*
+* Both numbers are "streams at the camera rate the hardware carried inside
+* the frame interval" (`kMax` for the detector alone, `maxSessions` for the
+* detector plus its enrichment steps) and the benchmark page already shows
+* `kMax` as "streams", so the unit is the same and the conversion is the
+* identity. It is an UPPER bound for a session that also runs enrichment —
+* which is why a `pipeline` cell for the same model always wins over it.
+*/
+function sessionsOfDeviceCell(kMax) {
+	return Math.max(0, Math.floor(kMax));
+}
+/**
+* Per node, every (device × model) at its best fresh measurement: the newest
+* `pipeline` cell, else the newest `device` cell read as
+* {@link sessionsOfDeviceCell}. Several models of one device are kept apart —
+* the caller picks the one the device runs NOW (a cell for another model says
+* nothing about it).
 */
 function measuredCapacityFromResults(rows, nodeIds, ctx) {
 	const byNode = /* @__PURE__ */ new Map();
 	for (const row of freshResults(rows, ctx)) {
-		if (!nodeIds.includes(row.nodeId) || row.cell.phase !== "pipeline") continue;
+		if (!nodeIds.includes(row.nodeId)) continue;
+		const candidate = capacityOfRow(row);
 		const devices = byNode.get(row.nodeId) ?? /* @__PURE__ */ new Map();
-		const held = devices.get(row.cell.deviceKey);
-		if (held === void 0 || row.measuredAtMs > held.measuredAtMs) devices.set(row.cell.deviceKey, row);
+		const key = `${candidate.deviceKey}\u0000${candidate.modelId ?? ""}`;
+		const held = devices.get(key);
+		if (held === void 0 || betterCapacityCell(candidate, held)) devices.set(key, candidate);
 		byNode.set(row.nodeId, devices);
 	}
 	const out = /* @__PURE__ */ new Map();
@@ -78432,16 +78508,32 @@ function measuredCapacityFromResults(rows, nodeIds, ctx) {
 		if (devices === void 0) continue;
 		out.set(nodeId, {
 			nodeId,
-			devices: [...devices.values()].toSorted((a, b) => a.cell.deviceKey.localeCompare(b.cell.deviceKey)).map((r) => ({
-				deviceKey: r.cell.deviceKey,
-				maxSessions: r.cell.phase === "pipeline" ? r.cell.maxSessions : 0,
-				modelId: r.cell.modelId,
-				measuredAtMs: r.measuredAtMs
-			}))
+			devices: [...devices.values()].toSorted((a, b) => a.deviceKey.localeCompare(b.deviceKey) || (a.modelId ?? "").localeCompare(b.modelId ?? ""))
 		});
 	}
 	return out;
 }
+function capacityOfRow(row) {
+	const { cell } = row;
+	return cell.phase === "pipeline" ? {
+		deviceKey: cell.deviceKey,
+		maxSessions: cell.maxSessions,
+		modelId: cell.modelId,
+		measuredAtMs: row.measuredAtMs,
+		phase: "pipeline"
+	} : {
+		deviceKey: cell.deviceKey,
+		maxSessions: sessionsOfDeviceCell(cell.kMax),
+		modelId: cell.modelId,
+		measuredAtMs: row.measuredAtMs,
+		phase: "device"
+	};
+}
+/** A pipeline cell beats a device cell; within a phase the newer one wins. */
+function betterCapacityCell(a, b) {
+	if (a.phase !== b.phase) return a.phase === "pipeline";
+	return a.measuredAtMs > b.measuredAtMs;
+}
 var BENCH_ROOT_STEP_ID = "object-detection";
 function slotAddons(schema, slot) {
 	return schema.slots.find((s) => s.id === slot)?.addons ?? [];
@@ -79025,6 +79117,44 @@ var NodeBenchmarkStore = class {
 		return true;
 	}
 };
+//#endregion
+//#region src/node-benchmark/auto-benchmark.ts
+/**
+* The automatic node benchmark: an enabled, available device that has no
+* measured result — a new node, or a device discovered later — is measured by
+* a `device`-phase run limited to it, so the automatic shares read a number
+* instead of the table without the operator starting anything.
+*
+*  - ONE auto run at a time, and never while any benchmark runs (the service
+*    allows one run cluster-wide);
+*  - a node that is busy is deferred: it carries more than
+*    {@link AUTO_BENCH_MAX_CAMERAS_ON_NODE} cameras (a benchmark moves them
+*    off and a camera with no other frame source would stay and spoil the
+*    numbers). The hub, which usually carries cameras, waits until it is quiet
+*    and the operator can still run it by hand;
+*  - a failed run is retried at most {@link AUTO_BENCH_MAX_ATTEMPTS} times,
+*    {@link AUTO_BENCH_RETRY_MS} apart. The ledger is persisted per (node,
+*    device) and restored at boot — a respawn right after a lost run would
+*    otherwise start the next one at once and hand the device a fresh budget;
+*    a device that gains a usable result is forgotten;
+*  - never for a device the operator disabled, one that is unplugged or
+*    excluded, or one that already has a usable result: a fresh cell for the
+*    model it runs NOW (the predicate the automatic shares read, so the two
+*    cannot disagree). A result for another model does not count — the device
+*    is measured again.
+*
+* The planner is pure; the scheduler injects every read and the start.
+*/
+/**
+* `nodeId\0deviceKey` of every device that has a usable result: a fresh
+* measurement for the model it runs now. The SAME predicate
+* ({@link capacityForModel}) the automatic shares size a node by.
+*/
+function benchmarkedDevices(runs, measured) {
+	const out = /* @__PURE__ */ new Set();
+	for (const run of runs) if (capacityForModel(measured.get(run.nodeId), run.deviceKey, run.modelId) !== void 0) out.add(benchKey(run.nodeId, run.deviceKey));
+	return out;
+}
 /** How often the scheduler looks (it is also nudged when a device set changes). */
 var AUTO_BENCH_TICK_MS = 6e4;
 /** Hard deadline of an auto run (a one-model device ramp takes well under this). */
@@ -79111,6 +79241,7 @@ var AutoBenchmarkScheduler = class {
 	attempts = /* @__PURE__ */ new Map();
 	/** The last decision logged per (node, device): a decision is logged when it CHANGES. */
 	lastLogged = /* @__PURE__ */ new Map();
+	restored = null;
 	inFlight = null;
 	timer = null;
 	constructor(deps) {
@@ -79140,6 +79271,27 @@ var AutoBenchmarkScheduler = class {
 		});
 		return this.inFlight;
 	}
+	/** Load the persisted ledger ONCE, before the first decision; a failed read is a fresh budget, said so. */
+	restore() {
+		this.restored ??= (async () => {
+			const persistence = this.deps.persistence;
+			if (persistence === void 0) return;
+			try {
+				for (const [key, attempt] of await persistence.load()) if (!this.attempts.has(key)) this.attempts.set(key, attempt);
+				this.deps.logger.info("auto benchmark: attempts restored", { meta: { devices: this.attempts.size } });
+			} catch (err) {
+				this.deps.logger.warn("auto benchmark: attempt ledger unreadable — starting with none", { meta: { error: errMsg$2(err) } });
+			}
+		})();
+		return this.restored;
+	}
+	async persist() {
+		try {
+			await this.deps.persistence?.save(this.attempts);
+		} catch (err) {
+			this.deps.logger.warn("auto benchmark: attempt ledger not saved — kept in memory", { meta: { error: errMsg$2(err) } });
+		}
+	}
 	note(decision, extra = "") {
 		const signature = `${decision.kind}:${"reason" in decision ? decision.reason : ""}${extra}`;
 		const key = benchKey(decision.nodeId, decision.deviceKey);
@@ -79156,14 +79308,20 @@ var AutoBenchmarkScheduler = class {
 	}
 	async run(trigger) {
 		const nowMs = this.deps.now?.() ?? Date.now();
+		await this.restore();
 		const plan = planAutoBenchmark({
 			nodes: await this.deps.readNodes(),
-			benchmarked: this.deps.benchmarked(),
+			benchmarked: await this.deps.benchmarked(),
 			runActive: this.deps.runActive(),
 			attempts: this.attempts,
 			nowMs
 		});
-		for (const decision of plan.decisions) if (decision.kind !== "start") this.note(decision);
+		let forgot = false;
+		for (const decision of plan.decisions) {
+			if (decision.kind !== "start") this.note(decision);
+			if (decision.kind === "skip" && decision.reason === "has-results") forgot = this.attempts.delete(benchKey(decision.nodeId, decision.deviceKey)) || forgot;
+		}
+		if (forgot) await this.persist();
 		if (plan.start === null) return;
 		const { nodeId, deviceKeys } = plan.start;
 		const models = await this.deps.pickModels(nodeId, deviceKeys);
@@ -79208,6 +79366,7 @@ var AutoBenchmarkScheduler = class {
 				lastAtMs: nowMs
 			});
 			this.lastLogged.delete(key);
+			await this.persist();
 			this.deps.logger.info("auto benchmark: started", {
 				tags: { nodeId },
 				meta: {
@@ -79221,6 +79380,29 @@ var AutoBenchmarkScheduler = class {
 	}
 };
 //#endregion
+//#region src/node-benchmark/auto-benchmark-attempts.ts
+function attemptsToBlob(attempts) {
+	const out = {};
+	for (const [key, attempt] of attempts) {
+		const sep = key.indexOf("\0");
+		if (sep < 0) continue;
+		const nodeId = key.slice(0, sep);
+		out[nodeId] = {
+			...out[nodeId],
+			[key.slice(sep + 1)]: attempt
+		};
+	}
+	return out;
+}
+function attemptsFromBlob(blob, keyOf) {
+	const out = /* @__PURE__ */ new Map();
+	for (const [nodeId, devices] of Object.entries(blob)) for (const [deviceKey, attempt] of Object.entries(devices)) out.set(keyOf(nodeId, deviceKey), {
+		count: attempt.count,
+		lastAtMs: attempt.lastAtMs
+	});
+	return out;
+}
+//#endregion
 //#region src/node-benchmark/node-benchmark-service.ts
 /**
 * The node benchmark lifecycle (D791) — the orchestrator owns it because it
@@ -89941,7 +90123,7 @@ var NodeStressController = class NodeStressController {
 	}
 	/** The configured mode, defaulting to {@link DEFAULT_NODE_STRESS_MODE}. */
 	mode() {
-		return parseNodeStressMode((this.deps.readGlobalSettings() ?? {}).nodeStressMode) ?? "observe";
+		return parseNodeStressMode((this.deps.readGlobalSettings() ?? {}).nodeStressMode) ?? "failover";
 	}
 	/**
 	* Record one camera metrics snapshot. Shares the `PipelineCameraMetricsSnapshot`
@@ -91595,6 +91777,10 @@ async function redispatchAffected(deps, cleared) {
 * schema is kept faithful to the on-disk shape rather than overly strict.
 */
 var NodeBindingsSchema = record(string(), record(string(), string()));
+var AutoBenchmarkAttemptsSchema = record(string(), record(string(), object({
+	count: number().int().min(0),
+	lastAtMs: number()
+})));
 var StoredAgentAddonConfigSchema = object({
 	enabled: boolean(),
 	modelId: string().optional(),
@@ -97097,7 +97283,7 @@ function buildGlobalSettingsSections(options) {
 					key: "nodeStressMode",
 					type: "select",
 					label: "Node stress handling",
-					description: "Per-NODE saturation verdict, computed from queue pressure, dropped-frame rate and the delivered-vs-configured fps of ACTIVE cameras only (an idle on-motion camera is never evidence). Observe = the verdict is logged when a node changes state, and nothing moves — this is how the thresholds get measured on your fleet. Failover = a saturated node additionally hands ONE camera at a time to an idle node, under a per-camera cooldown, a ban on returning it to the node it just left, and a cluster-wide cap of 4 moves an hour. A camera is never paused: running degraded beats running blind.",
+					description: "When a node is saturated (queue pressure, dropped frames and delivered-vs-configured fps of active cameras), Failover moves one of its cameras to an idle node. One camera at a time, a cooldown per camera, never back to the node it just left, at most 4 moves an hour across the cluster. A camera is never paused. Observe only logs the verdict; Off does nothing.",
 					options: [
 						{
 							value: "off",
@@ -97112,7 +97298,7 @@ function buildGlobalSettingsSections(options) {
 							label: "Failover (relocate a camera)"
 						}
 					],
-					default: "observe"
+					default: "failover"
 				},
 				{
 					key: "fpsLowWindowMs",
@@ -98050,6 +98236,7 @@ var AutoTuneService = class {
 		} catch (err) {
 			this.deps.logger.warn("load balancing: benchmark results unreadable — sized from hardware", { meta: { error: errMsg(err) } });
 		}
+		this.noteForeignCells(rec.recommendations, measured);
 		return computeCapacityShares({
 			recommendations: rec.recommendations,
 			source,
@@ -98059,6 +98246,28 @@ var AutoTuneService = class {
 			minSessionsPerDevice
 		});
 	}
+	/** Foreign-model cells already reported, so a recompute does not repeat the line. */
+	reportedForeign = /* @__PURE__ */ new Set();
+	/**
+	* A measured cell for a model the device does not run now is ignored by the
+	* shares (D913 follow-up); say so ONCE per (node, device, models), with the
+	* device key — silence would read as "measured and used".
+	*/
+	noteForeignCells(recommendations, measured) {
+		for (const cell of foreignCapacityCells(recommendations, measured)) {
+			const key = `${cell.nodeId}|${cell.deviceKey}|${cell.measuredModelId}|${cell.runsModelId ?? ""}`;
+			if (this.reportedForeign.has(key)) continue;
+			this.reportedForeign.add(key);
+			this.deps.logger.info("load balancing: benchmark cell for another model ignored", {
+				tags: { nodeId: cell.nodeId },
+				meta: {
+					deviceKey: cell.deviceKey,
+					measuredModelId: cell.measuredModelId,
+					runsModelId: cell.runsModelId
+				}
+			});
+		}
+	}
 	/**
 	* Gather every node and recommend. Nothing but hardware, catalog, the sizing
 	* and — for a plan — the custom pins goes in. `customPins` maps
@@ -100570,6 +100779,7 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
 	*/
 	startAutoBenchmark(store) {
 		this.autoBenchmark?.stop();
+		const attempts = this.state("autoBenchmarkAttempts", AutoBenchmarkAttemptsSchema, {});
 		this.autoBenchmark = new AutoBenchmarkScheduler({
 			logger: this.ctx.logger,
 			readNodes: async () => {
@@ -100583,7 +100793,19 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
 					devices: live[nodeId] ?? []
 				})));
 			},
-			benchmarked: () => new Set(store.listResults().map((r) => benchKey(r.nodeId, r.cell.deviceKey))),
+			benchmarked: async () => {
+				if (store.listResults().length === 0) return /* @__PURE__ */ new Set();
+				const hardware = await this.autoTune().suggestShares({
+					headroom: AUTO_SHARE_HEADROOM,
+					source: "hardware"
+				});
+				const measured = await this.readMeasuredCapacity(hardware.nodes.map((n) => n.nodeId));
+				return benchmarkedDevices(hardware.nodes.flatMap((n) => n.devices.map((d) => ({
+					nodeId: n.nodeId,
+					deviceKey: d.deviceKey,
+					modelId: d.modelId
+				}))), measured);
+			},
 			runActive: () => this.benchmarkService().isRunActive(),
 			pickModels: async (nodeId, deviceKeys) => {
 				const candidates = await this.getNodeBenchmarkCandidates({ nodeId });
@@ -100594,7 +100816,11 @@ var PipelineOrchestratorAddon = class PipelineOrchestratorAddon extends BaseAddo
 				}
 				return out;
 			},
-			start: (plan) => this.benchmarkService().start(plan)
+			start: (plan) => this.benchmarkService().start(plan),
+			persistence: {
+				load: async () => attemptsFromBlob(await attempts.get(), benchKey),
+				save: (map) => attempts.set(attemptsToBlob(map))
+			}
 		});
 		this.autoBenchmark.start();
 		this.autoBenchmark.tick("boot");