@openclaw/diagnostics-prometheus 2026.9.1 → 2026.9.3

package/dist/index.js

@@ -1,7 +1,8 @@
 import { isInternalDiagnosticEventMetadata, redactSensitiveText } from "./api.js";
 import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
-import { normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
+import { isDiagnosticsEnabled, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
 import { asNonNegativeFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
+import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
 import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
 //#region extensions/diagnostics-prometheus/src/service.ts
 const DURATION_BUCKETS_SECONDS = [
@@ -70,7 +71,9 @@ function seconds(ms) {
 	return value === void 0 ? void 0 : value / 1e3;
 }
 function sortedLabels(labels) {
-	return Object.entries(labels).toSorted(([left], [right]) => left.localeCompare(right));
+	const entries = Object.entries(labels);
+	entries.sort(([left], [right]) => left.localeCompare(right));
+	return entries;
 }
 function metricKey(name, labels) {
 	return `${name}|${JSON.stringify(sortedLabels(labels))}`;
@@ -81,10 +84,13 @@ function escapeHelp(value) {
 function escapeLabelValue(value) {
 	return value.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, "\\\"");
 }
+function formatLabelEntry([key, value]) {
+	return `${key}="${escapeLabelValue(value)}"`;
+}
 function formatLabels(labels) {
 	const entries = sortedLabels(labels);
 	if (entries.length === 0) return "";
-	return `{${entries.map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(",")}}`;
+	return `{${entries.map(formatLabelEntry).join(",")}}`;
 }
 function formatPrometheusNumber(value) {
 	if (!Number.isFinite(value)) return "0";
@@ -95,9 +101,8 @@ function createPrometheusMetricStore() {
 	const gauges = /* @__PURE__ */ new Map();
 	const histograms = /* @__PURE__ */ new Map();
 	let droppedSeries = 0;
-	const canCreateSeries = (map, key, metricName) => {
+	const canCreateSeries = (map, key) => {
 		if (map.has(key)) return true;
-		if (metricName === DROPPED_SERIES_COUNTER_NAME) return true;
 		if (counters.size + gauges.size + histograms.size < MAX_PROMETHEUS_SERIES) return true;
 		droppedSeries += 1;
 		return false;
@@ -105,7 +110,7 @@ function createPrometheusMetricStore() {
 	const counter = (name, help, labels, amount = 1) => {
 		if (!Number.isFinite(amount) || amount <= 0) return;
 		const key = metricKey(name, labels);
-		if (!canCreateSeries(counters, key, name)) return;
+		if (!canCreateSeries(counters, key)) return;
 		const existing = counters.get(key);
 		if (existing) {
 			existing.value += amount;
@@ -120,7 +125,7 @@ function createPrometheusMetricStore() {
 	const gauge = (name, help, labels, value) => {
 		if (value === void 0 || !Number.isFinite(value)) return;
 		const key = metricKey(name, labels);
-		if (!canCreateSeries(gauges, key, name)) return;
+		if (!canCreateSeries(gauges, key)) return;
 		gauges.set(key, {
 			help,
 			labels,
@@ -130,7 +135,7 @@ function createPrometheusMetricStore() {
 	const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
 		if (value === void 0 || !Number.isFinite(value) || value < 0) return;
 		const key = metricKey(name, labels);
-		if (!canCreateSeries(histograms, key, name)) return;
+		if (!canCreateSeries(histograms, key)) return;
 		let sample = histograms.get(key);
 		if (!sample) {
 			sample = {
@@ -151,16 +156,16 @@ function createPrometheusMetricStore() {
 		}
 	};
 	const snapshot = () => {
-		const counterSnapshot = new Map(counters);
-		if (droppedSeries > 0) counterSnapshot.set(metricKey(DROPPED_SERIES_COUNTER_NAME, {}), {
+		const counterSnapshot = [...counters];
+		if (droppedSeries > 0) counterSnapshot.push([metricKey(DROPPED_SERIES_COUNTER_NAME, {}), {
 			help: "Prometheus metric series dropped because the exporter series cap was reached.",
 			labels: {},
 			value: droppedSeries
-		});
+		}]);
 		return {
 			counters: counterSnapshot,
-			gauges: new Map(gauges),
-			histograms: new Map(histograms)
+			gauges: [...gauges],
+			histograms: [...histograms]
 		};
 	};
 	const reset = () => {
@@ -194,36 +199,35 @@ function renderPrometheusMetrics(store) {
 		lines.push(`# HELP ${name} ${escapeHelp(help)}`);
 		lines.push(`# TYPE ${name} ${type}`);
 	};
-	const counterEntries = [...snapshot.counters.entries()].toSorted(([left], [right]) => left.localeCompare(right));
-	for (const [key, sample] of counterEntries) {
-		const name = key.split("|", 1)[0] ?? "";
-		emitHeader(name, "counter", sample.help);
-		lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
-	}
-	const gaugeEntries = [...snapshot.gauges.entries()].toSorted(([left], [right]) => left.localeCompare(right));
-	for (const [key, sample] of gaugeEntries) {
-		const name = key.split("|", 1)[0] ?? "";
-		emitHeader(name, "gauge", sample.help);
-		lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
+	for (const [type, entries] of [["counter", snapshot.counters], ["gauge", snapshot.gauges]]) {
+		entries.sort(([left], [right]) => left.localeCompare(right));
+		for (const [key, sample] of entries) {
+			const name = key.split("|", 1)[0] ?? "";
+			emitHeader(name, type, sample.help);
+			lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
+		}
 	}
-	const histogramEntries = [...snapshot.histograms.entries()].toSorted(([left], [right]) => left.localeCompare(right));
-	for (const [key, sample] of histogramEntries) {
+	snapshot.histograms.sort(([left], [right]) => left.localeCompare(right));
+	for (const [key, sample] of snapshot.histograms) {
 		const name = key.split("|", 1)[0] ?? "";
 		emitHeader(name, "histogram", sample.help);
+		const labels = formatLabels(sample.labels);
+		const bucketLabels = sortedLabels({
+			...sample.labels,
+			le: ""
+		});
+		const boundIndex = bucketLabels.findIndex(([labelKey]) => labelKey === "le");
+		const bucketFragments = bucketLabels.map(formatLabelEntry);
 		for (let index = 0; index < sample.buckets.length; index += 1) {
 			const bucket = sample.buckets[index];
 			if (bucket === void 0) continue;
-			lines.push(`${name}_bucket${formatLabels({
-				...sample.labels,
-				le: String(bucket)
-			})} ${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
+			bucketFragments[boundIndex] = `le="${String(bucket)}"`;
+			lines.push(`${name}_bucket{${bucketFragments.join(",")}} ${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
 		}
-		lines.push(`${name}_bucket${formatLabels({
-			...sample.labels,
-			le: "+Inf"
-		})} ${formatPrometheusNumber(sample.count)}`);
-		lines.push(`${name}_sum${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.sum)}`);
-		lines.push(`${name}_count${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.count)}`);
+		bucketFragments[boundIndex] = "le=\"+Inf\"";
+		lines.push(`${name}_bucket{${bucketFragments.join(",")}} ${formatPrometheusNumber(sample.count)}`);
+		lines.push(`${name}_sum${labels} ${formatPrometheusNumber(sample.sum)}`);
+		lines.push(`${name}_count${labels} ${formatPrometheusNumber(sample.count)}`);
 	}
 	lines.push("");
 	return lines.join("\n");
@@ -373,6 +377,30 @@ function recordModelUsage(store, evt) {
 function recordDiagnosticEvent(store, evt, metadata) {
 	if (!shouldRecordDiagnosticEvent(metadata)) return;
 	switch (evt.type) {
+		case "diagnostic.gc":
+			store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds for the hosting JavaScript isolate.", {}, seconds(evt.durationMs));
+			return;
+		case "gateway.event_loop.sample":
+			store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
+			store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
+			return;
+		case "gateway.rpc": {
+			const labels = { method: evt.method };
+			if (evt.phase === "received") {
+				store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
+				return;
+			}
+			store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
+				phase: evt.phase,
+				outcome: evt.outcome
+			});
+			if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) store.histogram("openclaw_gateway_rpc_first_response_seconds", "Elapsed time until the first Gateway RPC response is sent.", labels, seconds(evt.durationMs));
+			else if (evt.phase === "handler") {
+				store.histogram("openclaw_gateway_rpc_handler_seconds", "Gateway RPC handler duration until return or throw.", labels, seconds(evt.durationMs));
+				store.histogram("openclaw_gateway_rpc_admission_seconds", "Elapsed time from Gateway RPC receipt until handler invocation.", labels, seconds(evt.admissionMs));
+			} else if (evt.phase === "dispatch") store.histogram("openclaw_gateway_rpc_queue_wait_seconds", "Gateway operator request start queue wait.", labels, seconds(evt.queueWaitMs));
+			return;
+		}
 		case "model.usage":
 			recordModelUsage(store, evt);
 			return;
@@ -404,18 +432,16 @@ function recordDiagnosticEvent(store, evt, metadata) {
 			store.histogram("openclaw_harness_run_duration_seconds", "Agent harness run duration in seconds.", harnessLabels(evt), seconds(evt.durationMs));
 			store.counter("openclaw_harness_run_total", "Agent harness runs completed by outcome.", harnessLabels(evt));
 			return;
-		case "message.processed":
-			store.counter("openclaw_message_processed_total", "Inbound messages processed by outcome.", {
+		case "message.processed": {
+			const labels = {
 				channel: normalizeDiagnosticValue(evt.channel),
 				outcome: evt.outcome,
 				reason: normalizeDiagnosticValue(evt.reason, "none")
-			});
-			store.histogram("openclaw_message_processed_duration_seconds", "Inbound message processing duration in seconds.", {
-				channel: normalizeDiagnosticValue(evt.channel),
-				outcome: evt.outcome,
-				reason: normalizeDiagnosticValue(evt.reason, "none")
-			}, seconds(evt.durationMs));
+			};
+			store.counter("openclaw_message_processed_total", "Inbound messages processed by outcome.", labels);
+			store.histogram("openclaw_message_processed_duration_seconds", "Inbound message processing duration in seconds.", labels, seconds(evt.durationMs));
 			return;
+		}
 		case "webhook.received":
 			store.counter("openclaw_webhook_received_total", "Webhook requests received by channel and update type.", webhookLabels(evt));
 			return;
@@ -443,35 +469,29 @@ function recordDiagnosticEvent(store, evt, metadata) {
 				source: normalizeDiagnosticValue(evt.source)
 			});
 			return;
-		case "message.dispatch.completed":
-			store.counter("openclaw_message_dispatch_completed_total", "Inbound message dispatch attempts completed by outcome.", {
-				channel: normalizeDiagnosticValue(evt.channel),
-				outcome: evt.outcome,
-				reason: normalizeDiagnosticValue(evt.reason, "none"),
-				source: normalizeDiagnosticValue(evt.source)
-			});
-			store.histogram("openclaw_message_dispatch_duration_seconds", "Inbound message dispatch duration in seconds.", {
+		case "message.dispatch.completed": {
+			const labels = {
 				channel: normalizeDiagnosticValue(evt.channel),
 				outcome: evt.outcome,
 				reason: normalizeDiagnosticValue(evt.reason, "none"),
 				source: normalizeDiagnosticValue(evt.source)
-			}, seconds(evt.durationMs));
+			};
+			store.counter("openclaw_message_dispatch_completed_total", "Inbound message dispatch attempts completed by outcome.", labels);
+			store.histogram("openclaw_message_dispatch_duration_seconds", "Inbound message dispatch duration in seconds.", labels, seconds(evt.durationMs));
 			return;
+		}
 		case "message.delivery.completed":
-		case "message.delivery.error":
-			store.counter("openclaw_message_delivery_total", "Outbound message delivery attempts by outcome.", {
-				channel: normalizeDiagnosticValue(evt.channel),
-				delivery_kind: normalizeDiagnosticValue(evt.deliveryKind, "other"),
-				error_category: evt.type === "message.delivery.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
-				outcome: evt.type === "message.delivery.error" ? "error" : "completed"
-			});
-			store.histogram("openclaw_message_delivery_duration_seconds", "Outbound message delivery duration in seconds.", {
+		case "message.delivery.error": {
+			const labels = {
 				channel: normalizeDiagnosticValue(evt.channel),
 				delivery_kind: normalizeDiagnosticValue(evt.deliveryKind, "other"),
 				error_category: evt.type === "message.delivery.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
 				outcome: evt.type === "message.delivery.error" ? "error" : "completed"
-			}, seconds(evt.durationMs));
+			};
+			store.counter("openclaw_message_delivery_total", "Outbound message delivery attempts by outcome.", labels);
+			store.histogram("openclaw_message_delivery_duration_seconds", "Outbound message delivery duration in seconds.", labels, seconds(evt.durationMs));
 			return;
+		}
 		case "talk.event":
 			store.counter("openclaw_talk_event_total", "Talk events emitted by type.", talkLabels(evt));
 			store.histogram("openclaw_talk_event_duration_seconds", "Talk event duration in seconds when reported.", talkLabels(evt), seconds(evt.durationMs));
@@ -506,9 +526,11 @@ function recordDiagnosticEvent(store, evt, metadata) {
 			});
 			return;
 		case "diagnostic.memory.sample":
-			store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind: "rss" }, evt.memory.rssBytes);
-			store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind: "heap_total" }, evt.memory.heapTotalBytes);
-			store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind: "heap_used" }, evt.memory.heapUsedBytes);
+			for (const [kind, field] of [
+				["rss", "rssBytes"],
+				["heap_total", "heapTotalBytes"],
+				["heap_used", "heapUsedBytes"]
+			]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, evt.memory[field]);
 			store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(evt.memory.rssBytes), BYTE_BUCKETS);
 			return;
 		case "diagnostic.memory.pressure":
@@ -519,19 +541,26 @@ function recordDiagnosticEvent(store, evt, metadata) {
 			return;
 		case "diagnostic.liveness.warning":
 			store.counter("openclaw_liveness_warning_total", "Diagnostic liveness warning events.", livenessLabels(evt));
-			store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state: "active" }, asNonNegativeFiniteNumber(evt.active));
-			store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state: "waiting" }, asNonNegativeFiniteNumber(evt.waiting));
-			store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state: "queued" }, asNonNegativeFiniteNumber(evt.queued));
+			for (const state of [
+				"active",
+				"waiting",
+				"queued"
+			]) store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state }, asNonNegativeFiniteNumber(evt[state]));
 			store.histogram("openclaw_liveness_event_loop_delay_p99_seconds", "P99 event-loop delay reported by diagnostic liveness warnings in seconds.", livenessLabels(evt), seconds(evt.eventLoopDelayP99Ms));
 			store.histogram("openclaw_liveness_event_loop_delay_max_seconds", "Maximum event-loop delay reported by diagnostic liveness warnings in seconds.", livenessLabels(evt), seconds(evt.eventLoopDelayMaxMs));
 			store.histogram("openclaw_liveness_event_loop_utilization_ratio", "Event-loop utilization reported by diagnostic liveness warnings.", livenessLabels(evt), asNonNegativeFiniteNumber(evt.eventLoopUtilization), RATIO_BUCKETS);
-			store.histogram("openclaw_liveness_cpu_core_ratio", "CPU core ratio reported by diagnostic liveness warnings.", livenessLabels(evt), asNonNegativeFiniteNumber(evt.cpuCoreRatio), RATIO_BUCKETS);
+			store.histogram("openclaw_liveness_cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.", livenessLabels(evt), asNonNegativeFiniteNumber(evt.cpuCoreRatio), RATIO_BUCKETS);
 			return;
 		case "diagnostic.async_queue.dropped":
-			store.counter("openclaw_diagnostic_async_queue_dropped_total", "Async diagnostic queue drops by dropped event class.", { drop_class: "total" }, asNonNegativeFiniteNumber(evt.droppedEvents));
-			if (evt.droppedTrustedEvents !== void 0) store.counter("openclaw_diagnostic_async_queue_dropped_total", "Async diagnostic queue drops by dropped event class.", { drop_class: "trusted" }, asNonNegativeFiniteNumber(evt.droppedTrustedEvents));
-			if (evt.droppedUntrustedEvents !== void 0) store.counter("openclaw_diagnostic_async_queue_dropped_total", "Async diagnostic queue drops by dropped event class.", { drop_class: "untrusted" }, asNonNegativeFiniteNumber(evt.droppedUntrustedEvents));
-			if (evt.droppedPriorityEvents !== void 0) store.counter("openclaw_diagnostic_async_queue_dropped_total", "Async diagnostic queue drops by dropped event class.", { drop_class: "priority" }, asNonNegativeFiniteNumber(evt.droppedPriorityEvents));
+			for (const [dropClass, field] of [
+				["total", "droppedEvents"],
+				["trusted", "droppedTrustedEvents"],
+				["untrusted", "droppedUntrustedEvents"],
+				["priority", "droppedPriorityEvents"]
+			]) {
+				if (field !== "droppedEvents" && evt[field] === void 0) continue;
+				store.counter("openclaw_diagnostic_async_queue_dropped_total", "Async diagnostic queue drops by dropped event class.", { drop_class: dropClass }, asNonNegativeFiniteNumber(evt[field]));
+			}
 			store.gauge("openclaw_diagnostic_async_queue_length", "Latest async diagnostic queue length after a drop summary.", {}, asNonNegativeFiniteNumber(evt.queueLength));
 			break;
 		case "diagnostic.heartbeat": break;
@@ -548,8 +577,26 @@ function recordDiagnosticEvent(store, evt, metadata) {
 			store.histogram("openclaw_payload_large_bytes", "Oversized payload byte sizes by surface and action.", payloadLargeLabels(evt), asNonNegativeFiniteNumber(evt.bytes), BYTE_BUCKETS);
 	}
 }
+const METRICS_READ_SCOPE = "operator.read";
+const METRICS_READ_IMPLYING_SCOPES = [
+	METRICS_READ_SCOPE,
+	"operator.write",
+	"operator.admin"
+];
+function hasMetricsReadScope() {
+	const runtimeScopes = getPluginRuntimeGatewayRequestScope()?.client?.connect?.scopes;
+	const scopes = Array.isArray(runtimeScopes) ? runtimeScopes : [];
+	return METRICS_READ_IMPLYING_SCOPES.some((scope) => scopes.includes(scope));
+}
 function createMetricsHandler(store) {
 	return (req, res) => {
+		if (!hasMetricsReadScope()) {
+			res.statusCode = 403;
+			res.setHeader("Cache-Control", "no-store");
+			res.setHeader("Content-Type", "text/plain; charset=utf-8");
+			res.end(`missing scope: ${METRICS_READ_SCOPE}`);
+			return true;
+		}
 		if (req.method !== "GET" && req.method !== "HEAD") {
 			res.statusCode = 405;
 			res.setHeader("Allow", "GET, HEAD");
@@ -589,6 +636,11 @@ function createDiagnosticsPrometheusExporter() {
 					ctx.logger.error("diagnostics-prometheus: internal diagnostics capability unavailable");
 					return;
 				}
+				const identity = isDiagnosticsEnabled(ctx.config) ? ctx.internalDiagnostics?.getRuntimeIdentity?.() : void 0;
+				if (identity) store.gauge("openclaw_gateway_build_info", "Identity of the hosting process and its loaded build; not a health or exporter epoch.", {
+					process_instance_id: identity.processInstanceId,
+					...identity.buildId ? { build_id: identity.buildId } : {}
+				}, 1);
 				unsubscribe = subscribe((event, metadata) => {
 					try {
 						recordDiagnosticEvent(store, event, metadata);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openclaw/diagnostics-prometheus",
-  "version": "2026.9.1",
+  "version": "2026.9.3",
   "description": "OpenClaw diagnostics Prometheus exporter for runtime metrics.",
   "repository": {
     "type": "git",
@@ -18,10 +18,10 @@
       "minHostVersion": ">=2026.4.25"
     },
     "compat": {
-      "pluginApi": ">=2026.9.1"
+      "pluginApi": ">=2026.9.3"
     },
     "build": {
-      "openclawVersion": "2026.9.1"
+      "openclawVersion": "2026.9.3"
     },
     "release": {
       "publishToClawHub": true,
@@ -37,7 +37,7 @@
     "README.md"
   ],
   "peerDependencies": {
-    "openclaw": ">=2026.9.1"
+    "openclaw": ">=2026.9.3"
   },
   "peerDependenciesMeta": {
     "openclaw": {