@camstack/system 1.2.455 → 1.2.457

package/dist/builtins/native-metrics/native-metrics.addon.js

@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
 	[Symbol.toStringTag]: { value: "Module" }
 });
 const require_chunk = require("../../chunk-Cek0wNdY.js");
-const require_dist = require("../../dist-DjQ8C-ku.js");
+const require_dist = require("../../dist-C89nqNnn.js");
 let node_fs_promises = require("node:fs/promises");
 let node_util = require("node:util");
 let node_os = require("node:os");
@@ -1025,16 +1025,107 @@ function mergeLoadSeries(input) {
 }
 //#endregion
 //#region src/builtins/native-metrics/node-metrics-history-recorder.ts
+/**
+* Node load history via device-history (D850, LTS P4) — hub only: the hub's
+* bus carries every node's snapshots, so the hub's writer is the cluster's.
+*
+* Per RUNNER, never per PID. A series was once scoped by process id, and a pid
+* changes on every runner restart: the hub held ~11.8k node series, most of
+* them one restart long, and no chart could follow a runner across a respawn.
+* The scope is now the fold's own function key (`load-series-fold.ts`): the
+* runner's addon id, or `__root__` / `__unattributed__`, so a runner keeps ONE
+* series for life and the cold tier folds exactly like the hot ring. The old
+* `process-*` keys are no longer written; their rows age out by retention.
+*
+* Node TOTALS ride the resources snapshot: CPU %, memory %, CPU temperature
+* and, when the node has one, GPU utilisation and temperature. A reading the
+* node did not take (`null`) is no point — never 0 (D393).
+*
+* The in-memory rings still serve the last hour for charts.
+*/
 var ADDON_ID = "metrics-native";
-var NODE_PROCESS_CPU_KEY = `addon:${ADDON_ID}/process-cpu-percent`;
-var NODE_PROCESS_RSS_KEY = `addon:${ADDON_ID}/process-rss-mib`;
-var NODE_CONTAINER_CURRENT_KEY = `addon:${ADDON_ID}/container-current-mib`;
+var MIB = 1048576;
+function keyOf(name) {
+	return `addon:${ADDON_ID}/${name}`;
+}
+/** Per-runner series (scope = the fold key). */
+var RUNNER_CPU_MAIN_KEY = keyOf("runner-cpu-main-percent");
+var RUNNER_CPU_GC_KEY = keyOf("runner-cpu-gc-percent");
+var RUNNER_CPU_LIFETIME_KEY = keyOf("runner-cpu-lifetime-percent");
+var RUNNER_RSS_KEY = keyOf("runner-rss-mib");
+var RUNNER_PROCESSES_KEY = keyOf("runner-processes");
+/** Node totals (scope ''). */
+var NODE_CPU_KEY = keyOf("node-cpu-percent");
+var NODE_MEMORY_KEY = keyOf("node-memory-percent");
+var NODE_CPU_TEMPERATURE_KEY = keyOf("node-cpu-temperature");
+var NODE_GPU_KEY = keyOf("node-gpu-percent");
+var NODE_GPU_TEMPERATURE_KEY = keyOf("node-gpu-temperature");
+var NODE_CONTAINER_CURRENT_KEY = keyOf("container-current-mib");
+keyOf("process-cpu-percent");
+keyOf("process-rss-mib");
+var DECLARATIONS = [
+	{
+		name: "runner-cpu-main-percent",
+		displayName: "Runner CPU % (main thread)",
+		unit: "%"
+	},
+	{
+		name: "runner-cpu-gc-percent",
+		displayName: "Runner CPU % (GC)",
+		unit: "%"
+	},
+	{
+		name: "runner-cpu-lifetime-percent",
+		displayName: "Runner CPU % (lifetime average)",
+		unit: "%"
+	},
+	{
+		name: "runner-rss-mib",
+		displayName: "Runner RSS (MiB)",
+		unit: "MiB"
+	},
+	{
+		name: "runner-processes",
+		displayName: "Runner processes",
+		unit: "count"
+	},
+	{
+		name: "node-cpu-percent",
+		displayName: "Node CPU %",
+		unit: "%"
+	},
+	{
+		name: "node-memory-percent",
+		displayName: "Node memory %",
+		unit: "%"
+	},
+	{
+		name: "node-cpu-temperature",
+		displayName: "Node CPU temperature",
+		unit: "°C"
+	},
+	{
+		name: "node-gpu-percent",
+		displayName: "Node GPU %",
+		unit: "%"
+	},
+	{
+		name: "node-gpu-temperature",
+		displayName: "Node GPU temperature",
+		unit: "°C"
+	},
+	{
+		name: "container-current-mib",
+		displayName: "Container memory current (MiB)",
+		unit: "MiB"
+	}
+];
 function nodeMetricsHistoryClient(api) {
 	return {
 		declareSeries: (input) => api.declareSeries.mutate(input),
 		record: (input) => api.record.mutate({ points: [...input.points] }),
 		queryNode: async (input) => {
-			return { series: (await api.queryNode.query(input)).series.map((s) => ({
+			return { series: (await api.queryNode.query({ ...input })).series.map((s) => ({
 				scope: s.scope,
 				points: s.points.map((p) => ({
 					t: p.t,
@@ -1050,78 +1141,112 @@ function refusalCounts(result) {
 	for (const r of result.refused) counts[r.reason] = (counts[r.reason] ?? 0) + r.count;
 	return counts;
 }
+/** The fold's function key of a process: the same rule as `foldSnapshotByFunction`. */
+function runnerScopeOf(process) {
+	if (process.addonId !== null) return process.addonId;
+	return process.classification === "root" ? require_dist.ROOT_BUCKET_KEY : require_dist.UNATTRIBUTED_BUCKET_KEY;
+}
+function round1(value) {
+	return Math.round(value * 10) / 10;
+}
 var NodeMetricsHistoryRecorder = class {
 	deps;
 	declared = false;
+	lastTotalsAt = /* @__PURE__ */ new Map();
+	/** A not-ready skip was logged and device-history has not been ready since. */
+	waitLogged = false;
 	constructor(deps) {
 		this.deps = deps;
 	}
 	async declare() {
 		if (this.declared) return;
-		const { deviceHistory } = this.deps;
-		await deviceHistory.declareSeries({
-			name: "process-cpu-percent",
+		for (const d of DECLARATIONS) await this.deps.deviceHistory.declareSeries({
+			name: d.name,
 			subjectKind: "node",
 			kind: "gauge",
-			displayName: "Process CPU %",
+			displayName: d.displayName,
 			ingest: "points",
-			unit: "%"
-		});
-		await deviceHistory.declareSeries({
-			name: "process-rss-mib",
-			subjectKind: "node",
-			kind: "gauge",
-			displayName: "Process RSS (MiB)",
-			ingest: "points",
-			unit: "MiB"
-		});
-		await deviceHistory.declareSeries({
-			name: "container-current-mib",
-			subjectKind: "node",
-			kind: "gauge",
-			displayName: "Container memory current (MiB)",
-			ingest: "points",
-			unit: "MiB"
+			unit: d.unit
 		});
 		this.declared = true;
 	}
+	/** One process snapshot: per-runner sums plus the container's memory. */
 	async recordSample(nodeId, atMs, processes, containerMemory) {
-		if (!await this.ensureDeclared()) return;
-		const points = [];
+		const subject = {
+			kind: "node",
+			nodeId
+		};
+		const point = (key, scope, value) => ({
+			key,
+			subject,
+			scope,
+			ts: atMs,
+			value
+		});
+		const byRunner = /* @__PURE__ */ new Map();
 		for (const p of processes) {
-			const scope = String(p.pid);
-			points.push({
-				key: NODE_PROCESS_CPU_KEY,
-				subject: {
-					kind: "node",
-					nodeId
-				},
-				scope,
-				ts: atMs,
-				value: p.cpuPercent
+			const scope = runnerScopeOf(p);
+			const t = byRunner.get(scope) ?? {
+				main: 0,
+				gc: 0,
+				splitKnown: true,
+				lifetime: 0,
+				rss: 0,
+				count: 0
+			};
+			const known = p.cpuMainPercent !== null && p.cpuGcPercent !== null;
+			byRunner.set(scope, {
+				main: t.main + (p.cpuMainPercent ?? 0),
+				gc: t.gc + (p.cpuGcPercent ?? 0),
+				splitKnown: t.splitKnown && known,
+				lifetime: t.lifetime + p.cpuPercent,
+				rss: t.rss + p.memoryRssBytes,
+				count: t.count + 1
 			});
+		}
+		const points = [];
+		for (const [scope, t] of byRunner) {
+			if (t.splitKnown) {
+				points.push(point(RUNNER_CPU_MAIN_KEY, scope, round1(t.main)));
+				points.push(point(RUNNER_CPU_GC_KEY, scope, round1(t.gc)));
+			}
+			points.push(point(RUNNER_CPU_LIFETIME_KEY, scope, round1(t.lifetime)));
+			points.push(point(RUNNER_RSS_KEY, scope, Math.round(t.rss / MIB)));
+			points.push(point(RUNNER_PROCESSES_KEY, scope, t.count));
+		}
+		if (containerMemory !== null) points.push(point(NODE_CONTAINER_CURRENT_KEY, "", Math.round(containerMemory.currentBytes / MIB)));
+		await this.write(nodeId, atMs, points);
+	}
+	/** One resources snapshot: the node's totals, at most every {@link NODE_TOTALS_MIN_INTERVAL_MS}. */
+	async recordTotals(nodeId, atMs, snapshot) {
+		const last = this.lastTotalsAt.get(nodeId);
+		if (last !== void 0 && atMs >= last && atMs - last < 15e3) return;
+		this.lastTotalsAt.set(nodeId, atMs);
+		const subject = {
+			kind: "node",
+			nodeId
+		};
+		const points = [];
+		const add = (key, value) => {
+			if (value === null || !Number.isFinite(value)) return;
 			points.push({
-				key: NODE_PROCESS_RSS_KEY,
-				subject: {
-					kind: "node",
-					nodeId
-				},
-				scope,
+				key,
+				subject,
+				scope: "",
 				ts: atMs,
-				value: Math.round(p.memoryRssBytes / 1048576)
+				value: round1(value)
 			});
-		}
-		if (containerMemory !== null) points.push({
-			key: NODE_CONTAINER_CURRENT_KEY,
-			subject: {
-				kind: "node",
-				nodeId
-			},
-			scope: "",
-			ts: atMs,
-			value: Math.round(containerMemory.currentBytes / 1048576)
-		});
+		};
+		add(NODE_CPU_KEY, snapshot.cpu.total);
+		add(NODE_MEMORY_KEY, snapshot.memory.percent);
+		add(NODE_CPU_TEMPERATURE_KEY, snapshot.cpuTemperature);
+		add(NODE_GPU_KEY, snapshot.gpu?.utilization ?? null);
+		add(NODE_GPU_TEMPERATURE_KEY, snapshot.gpu?.temperature ?? null);
+		await this.write(nodeId, atMs, points);
+	}
+	async write(nodeId, atMs, points) {
 		if (points.length === 0) return;
+		if (!await this.ensureDeclared()) return;
 		try {
 			const result = await this.deps.deviceHistory.record({ points });
 			if (result.refused.length > 0) {
@@ -1143,7 +1268,7 @@ var NodeMetricsHistoryRecorder = class {
 	async readContainerPoints(nodeId, sinceMs, maxPoints) {
 		if (!await this.ensureDeclared()) return [];
 		const result = await this.deps.deviceHistory.queryNode({
-			nodeId,
+			forNodeId: nodeId,
 			key: NODE_CONTAINER_CURRENT_KEY,
 			from: sinceMs + 1,
 			to: Date.now(),
@@ -1156,7 +1281,7 @@ var NodeMetricsHistoryRecorder = class {
 			out.push({
 				atMs: p.t,
 				source: "cgroup-v2",
-				currentBytes: p.value * 1048576,
+				currentBytes: p.value * MIB,
 				limitBytes: null,
 				anonBytes: null,
 				fileBytes: null,
@@ -1167,63 +1292,46 @@ var NodeMetricsHistoryRecorder = class {
 		}
 		return out.toSorted((a, b) => a.atMs - b.atMs);
 	}
+	/**
+	* The cold tier as process samples the fold reads like the hot ring's. A
+	* runner's sums come back as `processes` equal rows (sums preserved, count
+	* right); its CPU split is `null` where it was not recorded.
+	*/
 	async readProcessSamples(nodeId, sinceMs, maxPoints) {
 		if (!await this.ensureDeclared()) return [];
 		const to = Date.now();
-		const [cpuResult, rssResult] = await Promise.all([this.deps.deviceHistory.queryNode({
-			nodeId,
-			key: NODE_PROCESS_CPU_KEY,
+		const read = (key) => this.deps.deviceHistory.queryNode({
+			forNodeId: nodeId,
+			key,
 			from: sinceMs + 1,
 			to,
 			resolution: "raw",
 			maxPoints
-		}), this.deps.deviceHistory.queryNode({
-			nodeId,
-			key: NODE_PROCESS_RSS_KEY,
-			from: sinceMs + 1,
-			to,
-			resolution: "raw",
-			maxPoints
-		})]);
-		const byAt = /* @__PURE__ */ new Map();
-		const ingest = (series, field) => {
-			for (const s of series) {
-				const pid = Number(s.scope);
-				if (!Number.isFinite(pid)) continue;
-				for (const p of s.points) {
-					if (p.value === null || !Number.isFinite(p.value)) continue;
-					let perPid = byAt.get(p.t);
-					if (perPid === void 0) {
-						perPid = /* @__PURE__ */ new Map();
-						byAt.set(p.t, perPid);
-					}
-					const base = perPid.get(pid) ?? {
-						pid,
-						addonId: null,
-						classification: "managed",
-						cpuPercent: 0,
-						memoryRssBytes: 0,
-						cpuMainPercent: null,
-						cpuGcPercent: null
-					};
-					perPid.set(pid, field === "cpuPercent" ? {
-						...base,
-						cpuPercent: p.value
-					} : {
-						...base,
-						memoryRssBytes: p.value * 1048576
-					});
-				}
-			}
-		};
-		ingest(cpuResult.series, "cpuPercent");
-		ingest(rssResult.series, "memoryRssBytes");
-		return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, perPid]) => ({
-			atMs,
-			processes: [...perPid.values()]
-		}));
+		});
+		const [main, gc, lifetime, rss, count] = await Promise.all([
+			read(RUNNER_CPU_MAIN_KEY),
+			read(RUNNER_CPU_GC_KEY),
+			read(RUNNER_CPU_LIFETIME_KEY),
+			read(RUNNER_RSS_KEY),
+			read(RUNNER_PROCESSES_KEY)
+		]);
+		return assembleRunnerSamples({
+			main,
+			gc,
+			lifetime,
+			rss,
+			count
+		});
 	}
 	async ensureDeclared() {
+		if (this.deps.historyReady !== void 0 && !this.deps.historyReady()) {
+			if (!this.waitLogged) {
+				this.waitLogged = true;
+				this.deps.logger.info("node metrics history waits for device-history — samples until it is ready are not stored");
+			}
+			return false;
+		}
+		this.waitLogged = false;
 		try {
 			await this.declare();
 			return true;
@@ -1233,6 +1341,55 @@ var NodeMetricsHistoryRecorder = class {
 		}
 	}
 };
+/** `atMs -> scope -> value` of one key's answer, finite values only. */
+function indexAnswer(answer) {
+	const out = /* @__PURE__ */ new Map();
+	for (const s of answer.series) for (const p of s.points) {
+		if (p.value === null || !Number.isFinite(p.value)) continue;
+		const at = out.get(p.t) ?? /* @__PURE__ */ new Map();
+		at.set(s.scope, p.value);
+		out.set(p.t, at);
+	}
+	return out;
+}
+/**
+* Per-runner cold points back into process rows. A sample exists where the
+* runner's lifetime CPU and RSS were both recorded (always written together).
+*/
+function assembleRunnerSamples(answers) {
+	const main = indexAnswer(answers.main);
+	const gc = indexAnswer(answers.gc);
+	const rss = indexAnswer(answers.rss);
+	const count = indexAnswer(answers.count);
+	const samples = [];
+	for (const [atMs, lifetimeByScope] of indexAnswer(answers.lifetime)) {
+		const processes = [];
+		for (const [scope, lifetime] of lifetimeByScope) {
+			const rssMib = rss.get(atMs)?.get(scope);
+			if (rssMib === void 0) continue;
+			const n = Math.max(1, Math.round(count.get(atMs)?.get(scope) ?? 1));
+			const m = main.get(atMs)?.get(scope);
+			const g = gc.get(atMs)?.get(scope);
+			const split = m !== void 0 && g !== void 0;
+			const isRoot = scope === require_dist.ROOT_BUCKET_KEY;
+			const isUnattributed = scope === require_dist.UNATTRIBUTED_BUCKET_KEY;
+			for (let i = 0; i < n; i += 1) processes.push({
+				pid: 0,
+				addonId: isRoot || isUnattributed ? null : scope,
+				classification: isRoot ? "root" : isUnattributed ? "system" : "managed",
+				cpuPercent: lifetime / n,
+				memoryRssBytes: rssMib * MIB / n,
+				cpuMainPercent: split ? m / n : null,
+				cpuGcPercent: split ? g / n : null
+			});
+		}
+		if (processes.length > 0) samples.push({
+			atMs,
+			processes
+		});
+	}
+	return samples.toSorted((a, b) => a.atMs - b.atMs);
+}
 //#endregion
 //#region src/builtins/native-metrics/gpu-probe.ts
 /**
@@ -2196,6 +2353,8 @@ var ThreadCpuTracker = class {
 };
 //#endregion
 //#region src/builtins/native-metrics/native-metrics.addon.ts
+/** The hub's node id — `isHub` below is the same test; device-history lives there. */
+var HUB_NODE_ID = "hub";
 /**
 * Cadence for the per-node metric snapshot bus events. ~5 s matches
 * the existing UI polling cadence (ProcessesTab, ProcessResources)
@@ -2317,7 +2476,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
 	}
 	/** Is this the hub? The same test every other addon uses (`addon-ai`). */
 	get isHub() {
-		return (this.ctx.kernel.cluster?.broker?.nodeID ?? "hub") === "hub";
+		return (this.ctx.kernel.cluster?.broker?.nodeID ?? HUB_NODE_ID) === HUB_NODE_ID;
 	}
 	/** Process rows observed across the whole fleet, for the cost projection. */
 	observedFleetRows() {
@@ -2368,19 +2527,38 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
 		};
 		this.applyLoadSeriesConfig();
 		if (this.isHub) {
-			this.metricsHistory = new NodeMetricsHistoryRecorder({
+			const historyScope = {
+				type: "node",
+				nodeId: HUB_NODE_ID
+			};
+			const historyHandle = this.ctx.useCapability("device-history", historyScope);
+			const recorder = new NodeMetricsHistoryRecorder({
 				deviceHistory: nodeMetricsHistoryClient(this.ctx.api.deviceHistory),
-				logger: this.ctx.logger.child("NodeMetricsHistory")
-			});
-			this.metricsHistory.declare().catch((err) => {
-				this.ctx.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+				logger: this.ctx.logger.child("NodeMetricsHistory"),
+				historyReady: () => historyHandle.isReady
 			});
+			this.metricsHistory = recorder;
+			const declareNow = () => {
+				recorder.declare().catch((err) => {
+					this.ctx.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+				});
+			};
+			this.ctx.addDisposer(this.ctx.onCapabilityStateChange("device-history", historyScope, (state) => {
+				if (state === "ready") declareNow();
+			}));
+			if (historyHandle.isReady) declareNow();
 		}
 		this.memoryProbe = new ContainerMemoryProbe({ logger: this.ctx.logger.child("ContainerMemory") });
 		this.pidsProbe = new ContainerPidsProbe({ logger: this.ctx.logger.child("ContainerPids") });
 		if (this.snapshotTimer) clearInterval(this.snapshotTimer);
 		this.snapshotTimer = setInterval(() => this.runTick("resources snapshot", () => this.emitResourcesSnapshot()), METRICS_SNAPSHOT_INTERVAL_MS);
 		this.startProcessSnapshotTimer();
+		if (this.metricsHistory !== null) {
+			const history = this.metricsHistory;
+			this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: require_dist.EventCategory.MetricsNodeResourcesSnapshot }, (event) => {
+				history.recordTotals(event.data.nodeId, event.data.timestamp, event.data.snapshot);
+			}));
+		}
 		this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: require_dist.EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes, event.data.containerMemory)));
 		return [{
 			capability: require_dist.metricsProviderCapability,