@openclaw/diagnostics-prometheus 2026.9.5 → 2026.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/icon.png +0 -0
- package/dist/index.js +299 -236
- package/package.json +4 -4
package/assets/icon.png
CHANGED
|
Binary file
|
package/dist/index.js
CHANGED
|
@@ -4,72 +4,7 @@ import { isDiagnosticsEnabled, normalizeDiagnosticLane, normalizeDiagnosticValue
|
|
|
4
4
|
import { asNonNegativeFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
|
|
5
5
|
import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
|
|
6
6
|
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
7
|
-
//#region extensions/diagnostics-prometheus/src/
|
|
8
|
-
const DURATION_BUCKETS_SECONDS = [
|
|
9
|
-
.005,
|
|
10
|
-
.01,
|
|
11
|
-
.025,
|
|
12
|
-
.05,
|
|
13
|
-
.1,
|
|
14
|
-
.25,
|
|
15
|
-
.5,
|
|
16
|
-
1,
|
|
17
|
-
2.5,
|
|
18
|
-
5,
|
|
19
|
-
10,
|
|
20
|
-
30,
|
|
21
|
-
60,
|
|
22
|
-
120,
|
|
23
|
-
300,
|
|
24
|
-
600
|
|
25
|
-
];
|
|
26
|
-
const TOKEN_BUCKETS = [
|
|
27
|
-
1,
|
|
28
|
-
4,
|
|
29
|
-
16,
|
|
30
|
-
64,
|
|
31
|
-
256,
|
|
32
|
-
1024,
|
|
33
|
-
4096,
|
|
34
|
-
16384,
|
|
35
|
-
65536,
|
|
36
|
-
262144,
|
|
37
|
-
1048576
|
|
38
|
-
];
|
|
39
|
-
const BYTE_BUCKETS = [
|
|
40
|
-
1024,
|
|
41
|
-
4096,
|
|
42
|
-
16384,
|
|
43
|
-
65536,
|
|
44
|
-
262144,
|
|
45
|
-
1048576,
|
|
46
|
-
4194304,
|
|
47
|
-
16777216,
|
|
48
|
-
67108864,
|
|
49
|
-
268435456,
|
|
50
|
-
1073741824,
|
|
51
|
-
4294967296,
|
|
52
|
-
17179869184
|
|
53
|
-
];
|
|
54
|
-
const RATIO_BUCKETS = [
|
|
55
|
-
.01,
|
|
56
|
-
.05,
|
|
57
|
-
.1,
|
|
58
|
-
.25,
|
|
59
|
-
.5,
|
|
60
|
-
.75,
|
|
61
|
-
1,
|
|
62
|
-
2,
|
|
63
|
-
4,
|
|
64
|
-
8,
|
|
65
|
-
16
|
|
66
|
-
];
|
|
67
|
-
const MAX_PROMETHEUS_SERIES = 2048;
|
|
68
|
-
const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
|
|
69
|
-
function seconds(ms) {
|
|
70
|
-
const value = asNonNegativeFiniteNumber(ms);
|
|
71
|
-
return value === void 0 ? void 0 : value / 1e3;
|
|
72
|
-
}
|
|
7
|
+
//#region extensions/diagnostics-prometheus/src/prometheus-format.ts
|
|
73
8
|
function sortedLabels(labels) {
|
|
74
9
|
const entries = Object.entries(labels);
|
|
75
10
|
entries.sort(([left], [right]) => left.localeCompare(right));
|
|
@@ -96,6 +31,32 @@ function formatPrometheusNumber(value) {
|
|
|
96
31
|
if (!Number.isFinite(value)) return "0";
|
|
97
32
|
return Number.isInteger(value) ? String(value) : String(Number(value.toPrecision(12)));
|
|
98
33
|
}
|
|
34
|
+
function seconds(ms) {
|
|
35
|
+
const value = asNonNegativeFiniteNumber(ms);
|
|
36
|
+
return value === void 0 ? void 0 : value / 1e3;
|
|
37
|
+
}
|
|
38
|
+
//#endregion
|
|
39
|
+
//#region extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
|
|
40
|
+
const DURATION_BUCKETS_SECONDS = [
|
|
41
|
+
.005,
|
|
42
|
+
.01,
|
|
43
|
+
.025,
|
|
44
|
+
.05,
|
|
45
|
+
.1,
|
|
46
|
+
.25,
|
|
47
|
+
.5,
|
|
48
|
+
1,
|
|
49
|
+
2.5,
|
|
50
|
+
5,
|
|
51
|
+
10,
|
|
52
|
+
30,
|
|
53
|
+
60,
|
|
54
|
+
120,
|
|
55
|
+
300,
|
|
56
|
+
600
|
|
57
|
+
];
|
|
58
|
+
const MAX_PROMETHEUS_SERIES = 2048;
|
|
59
|
+
const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
|
|
99
60
|
function createPrometheusMetricStore() {
|
|
100
61
|
const counters = /* @__PURE__ */ new Map();
|
|
101
62
|
const gauges = /* @__PURE__ */ new Map();
|
|
@@ -132,6 +93,18 @@ function createPrometheusMetricStore() {
|
|
|
132
93
|
value
|
|
133
94
|
});
|
|
134
95
|
};
|
|
96
|
+
const counterValue = (name, help, labels, value) => {
|
|
97
|
+
if (!Number.isFinite(value) || value < 0) return;
|
|
98
|
+
const key = metricKey(name, labels);
|
|
99
|
+
if (canCreateSeries(counters, key)) counters.set(key, {
|
|
100
|
+
help,
|
|
101
|
+
labels,
|
|
102
|
+
value: Math.max(counters.get(key)?.value ?? 0, value)
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
const clearGauges = (name) => {
|
|
106
|
+
for (const key of gauges.keys()) if (key.startsWith(`${name}|`)) gauges.delete(key);
|
|
107
|
+
};
|
|
135
108
|
const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
|
|
136
109
|
if (value === void 0 || !Number.isFinite(value) || value < 0) return;
|
|
137
110
|
const key = metricKey(name, labels);
|
|
@@ -176,19 +149,136 @@ function createPrometheusMetricStore() {
|
|
|
176
149
|
};
|
|
177
150
|
return {
|
|
178
151
|
counter,
|
|
152
|
+
counterValue,
|
|
179
153
|
gauge,
|
|
154
|
+
clearGauges,
|
|
180
155
|
histogram,
|
|
181
156
|
reset,
|
|
182
157
|
snapshot
|
|
183
158
|
};
|
|
184
159
|
}
|
|
160
|
+
//#endregion
|
|
161
|
+
//#region extensions/diagnostics-prometheus/src/service-gateway-rpc.ts
|
|
162
|
+
const CATALOG_LIST_METHOD = "sessions.catalog.list";
|
|
163
|
+
const CATALOG_LIST_PHASE_PREFIX = `${CATALOG_LIST_METHOD}.`;
|
|
164
|
+
const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
|
|
165
|
+
"projection_initial",
|
|
166
|
+
"planning",
|
|
167
|
+
"provider",
|
|
168
|
+
"coalesced",
|
|
169
|
+
"projection_final",
|
|
170
|
+
"delivery"
|
|
171
|
+
]);
|
|
172
|
+
function recordGatewayRpcEvent(store, evt, metadata) {
|
|
173
|
+
switch (evt.type) {
|
|
174
|
+
case "diagnostic.phase.completed": {
|
|
175
|
+
if (!metadata.trusted || !evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
|
|
176
|
+
const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
|
|
177
|
+
const duration = seconds(evt.durationMs);
|
|
178
|
+
if (!CATALOG_LIST_PHASES.has(phase) || duration === void 0) return;
|
|
179
|
+
const labels = {
|
|
180
|
+
method: CATALOG_LIST_METHOD,
|
|
181
|
+
phase
|
|
182
|
+
};
|
|
183
|
+
store.histogram("openclaw_gateway_rpc_stage_seconds", "Elapsed time for completed Gateway RPC owner stages.", labels, duration);
|
|
184
|
+
const cpu = evt.details?.threadCpuMs;
|
|
185
|
+
if ((phase === "planning" || phase === "delivery") && typeof cpu === "number") store.histogram("openclaw_gateway_rpc_stage_thread_cpu_seconds", "Current-thread CPU for synchronous Gateway RPC owner stages.", labels, seconds(cpu));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
case "gateway.rpc": {
|
|
189
|
+
const labels = { method: evt.method };
|
|
190
|
+
if (evt.phase === "received") {
|
|
191
|
+
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
195
|
+
phase: evt.phase,
|
|
196
|
+
outcome: evt.outcome
|
|
197
|
+
});
|
|
198
|
+
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) store.histogram("openclaw_gateway_rpc_first_response_seconds", "Elapsed time until the first Gateway RPC response is sent.", labels, seconds(evt.durationMs));
|
|
199
|
+
else if (evt.phase === "handler") {
|
|
200
|
+
store.histogram("openclaw_gateway_rpc_handler_seconds", "Gateway RPC handler duration until return or throw.", labels, seconds(evt.durationMs));
|
|
201
|
+
store.histogram("openclaw_gateway_rpc_admission_seconds", "Elapsed time from Gateway RPC receipt until handler invocation.", labels, seconds(evt.admissionMs));
|
|
202
|
+
} else if (evt.phase === "dispatch") store.histogram("openclaw_gateway_rpc_queue_wait_seconds", "Gateway operator request start queue wait.", labels, seconds(evt.queueWaitMs));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
//#endregion
|
|
207
|
+
//#region extensions/diagnostics-prometheus/src/service-memory.ts
|
|
208
|
+
function recordMemorySample(store, memory, byteBuckets) {
|
|
209
|
+
for (const worker of memory.workerLifecycle ?? []) {
|
|
210
|
+
store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
|
|
211
|
+
for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
|
|
212
|
+
script: worker.script,
|
|
213
|
+
reason
|
|
214
|
+
}, count);
|
|
215
|
+
}
|
|
216
|
+
for (const [kind, field] of [
|
|
217
|
+
["rss", "rssBytes"],
|
|
218
|
+
["heap_total", "heapTotalBytes"],
|
|
219
|
+
["heap_used", "heapUsedBytes"],
|
|
220
|
+
["external", "externalBytes"],
|
|
221
|
+
["array_buffers", "arrayBuffersBytes"],
|
|
222
|
+
["worker_heap_total", "workerHeapTotalBytes"],
|
|
223
|
+
["worker_heap_used", "workerHeapUsedBytes"]
|
|
224
|
+
]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, asNonNegativeFiniteNumber(memory[field]));
|
|
225
|
+
for (const [name, field] of [["openclaw_worker_count", "workerCount"], ["openclaw_worker_heap_sampled_count", "workerHeapSampledCount"]]) store.gauge(name, "Worker isolate counts.", {}, asNonNegativeFiniteNumber(memory[field]));
|
|
226
|
+
const workerHeaps = /* @__PURE__ */ new Map();
|
|
227
|
+
for (const worker of memory.workerHeaps ?? []) {
|
|
228
|
+
const heapUsed = asNonNegativeFiniteNumber(worker.heapUsed);
|
|
229
|
+
if (heapUsed !== void 0) workerHeaps.set(worker.script, (workerHeaps.get(worker.script) ?? 0) + heapUsed);
|
|
230
|
+
}
|
|
231
|
+
store.clearGauges("openclaw_worker_heap_used_bytes");
|
|
232
|
+
for (const [script, heapUsed] of workerHeaps) store.gauge("openclaw_worker_heap_used_bytes", "Latest live Worker heap usage by bounded script basename.", { script }, heapUsed);
|
|
233
|
+
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
|
|
234
|
+
}
|
|
235
|
+
//#endregion
|
|
236
|
+
//#region extensions/diagnostics-prometheus/src/service.ts
|
|
237
|
+
const TOKEN_BUCKETS = [
|
|
238
|
+
1,
|
|
239
|
+
4,
|
|
240
|
+
16,
|
|
241
|
+
64,
|
|
242
|
+
256,
|
|
243
|
+
1024,
|
|
244
|
+
4096,
|
|
245
|
+
16384,
|
|
246
|
+
65536,
|
|
247
|
+
262144,
|
|
248
|
+
1048576
|
|
249
|
+
];
|
|
250
|
+
const BYTE_BUCKETS = [
|
|
251
|
+
1024,
|
|
252
|
+
4096,
|
|
253
|
+
16384,
|
|
254
|
+
65536,
|
|
255
|
+
262144,
|
|
256
|
+
1048576,
|
|
257
|
+
4194304,
|
|
258
|
+
16777216,
|
|
259
|
+
67108864,
|
|
260
|
+
268435456,
|
|
261
|
+
1073741824,
|
|
262
|
+
4294967296,
|
|
263
|
+
17179869184
|
|
264
|
+
];
|
|
265
|
+
const RATIO_BUCKETS = [
|
|
266
|
+
.01,
|
|
267
|
+
.05,
|
|
268
|
+
.1,
|
|
269
|
+
.25,
|
|
270
|
+
.5,
|
|
271
|
+
.75,
|
|
272
|
+
1,
|
|
273
|
+
2,
|
|
274
|
+
4,
|
|
275
|
+
8,
|
|
276
|
+
16
|
|
277
|
+
];
|
|
185
278
|
function safeErrorMessage(err) {
|
|
186
279
|
const message = err instanceof Error ? err.message ?? err.name : String(err);
|
|
187
280
|
return truncateUtf16Safe(redactSensitiveText(message).replaceAll("\0", " ").replace(/[\r\n\t\u2028\u2029]/gu, " "), 500);
|
|
188
281
|
}
|
|
189
|
-
function shouldRecordDiagnosticEvent(metadata) {
|
|
190
|
-
return metadata.trusted || isInternalDiagnosticEventMetadata(metadata);
|
|
191
|
-
}
|
|
192
282
|
function renderPrometheusMetrics(store) {
|
|
193
283
|
const snapshot = store.snapshot();
|
|
194
284
|
const lines = [];
|
|
@@ -232,118 +322,12 @@ function renderPrometheusMetrics(store) {
|
|
|
232
322
|
lines.push("");
|
|
233
323
|
return lines.join("\n");
|
|
234
324
|
}
|
|
235
|
-
function runLabels(evt) {
|
|
236
|
-
return {
|
|
237
|
-
...evt.blockedBy ? { blocked_by: normalizeDiagnosticValue(evt.blockedBy) } : {},
|
|
238
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
239
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
240
|
-
outcome: normalizeDiagnosticValue(evt.outcome, "unknown"),
|
|
241
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
242
|
-
trigger: normalizeDiagnosticValue(evt.trigger)
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
function modelCallLabels(evt) {
|
|
246
|
-
return {
|
|
247
|
-
api: normalizeDiagnosticValue(evt.api),
|
|
248
|
-
error_category: evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
249
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
250
|
-
observation_unit: evt.observationUnit === "turn" ? "turn" : "request",
|
|
251
|
-
outcome: evt.type === "model.call.error" ? "error" : "completed",
|
|
252
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
253
|
-
transport: normalizeDiagnosticValue(evt.transport)
|
|
254
|
-
};
|
|
255
|
-
}
|
|
256
|
-
function modelFailoverLabels(evt) {
|
|
257
|
-
return {
|
|
258
|
-
from_model: normalizeDiagnosticValue(evt.fromModel),
|
|
259
|
-
from_provider: normalizeDiagnosticValue(evt.fromProvider),
|
|
260
|
-
lane: normalizeDiagnosticLane(evt.lane),
|
|
261
|
-
reason: normalizeDiagnosticValue(evt.reason, "other"),
|
|
262
|
-
suspended: evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
263
|
-
to_model: normalizeDiagnosticValue(evt.toModel),
|
|
264
|
-
to_provider: normalizeDiagnosticValue(evt.toProvider)
|
|
265
|
-
};
|
|
266
|
-
}
|
|
267
|
-
function toolExecutionLabels(evt) {
|
|
268
|
-
return {
|
|
269
|
-
error_category: evt.type === "tool.execution.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
270
|
-
outcome: evt.type === "tool.execution.error" ? "error" : "completed",
|
|
271
|
-
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
272
|
-
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
273
|
-
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
274
|
-
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
function toolExecutionBlockedLabels(evt) {
|
|
278
|
-
return {
|
|
279
|
-
denied_reason: normalizeDiagnosticValue(evt.deniedReason, "other"),
|
|
280
|
-
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
281
|
-
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
282
|
-
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
283
|
-
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
284
|
-
};
|
|
285
|
-
}
|
|
286
|
-
function skillLabels(evt) {
|
|
287
|
-
return {
|
|
288
|
-
activation: normalizeDiagnosticValue(evt.activation, "unknown"),
|
|
289
|
-
agent: normalizeDiagnosticValue(evt.agentId),
|
|
290
|
-
skill: normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
291
|
-
source: normalizeDiagnosticValue(evt.skillSource)
|
|
292
|
-
};
|
|
293
|
-
}
|
|
294
|
-
function harnessLabels(evt) {
|
|
295
|
-
return {
|
|
296
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
297
|
-
error_category: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
298
|
-
harness: normalizeDiagnosticValue(evt.harnessId),
|
|
299
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
300
|
-
outcome: evt.type === "harness.run.error" ? "error" : normalizeDiagnosticValue(evt.outcome),
|
|
301
|
-
phase: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.phase) : "none",
|
|
302
|
-
plugin: normalizeDiagnosticValue(evt.pluginId),
|
|
303
|
-
provider: normalizeDiagnosticValue(evt.provider)
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
325
|
function webhookLabels(evt) {
|
|
307
326
|
return {
|
|
308
327
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
309
328
|
webhook: normalizeDiagnosticValue(evt.updateType)
|
|
310
329
|
};
|
|
311
330
|
}
|
|
312
|
-
function sessionStuckLabels(evt) {
|
|
313
|
-
return {
|
|
314
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
315
|
-
state: evt.state
|
|
316
|
-
};
|
|
317
|
-
}
|
|
318
|
-
function sessionRecoveryLabels(evt) {
|
|
319
|
-
return {
|
|
320
|
-
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
321
|
-
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
322
|
-
state: evt.state,
|
|
323
|
-
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
324
|
-
};
|
|
325
|
-
}
|
|
326
|
-
function livenessLabels(evt) {
|
|
327
|
-
return { reason: normalizeDiagnosticValue(evt.reasons.join(":"), "unknown") };
|
|
328
|
-
}
|
|
329
|
-
function payloadLargeLabels(evt) {
|
|
330
|
-
return {
|
|
331
|
-
action: evt.action,
|
|
332
|
-
channel: normalizeDiagnosticValue(evt.channel, "none"),
|
|
333
|
-
plugin: normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
334
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
335
|
-
surface: normalizeDiagnosticValue(evt.surface, "unknown")
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
function talkLabels(evt) {
|
|
339
|
-
return {
|
|
340
|
-
brain: normalizeDiagnosticValue(evt.brain),
|
|
341
|
-
event_type: normalizeDiagnosticValue(evt.talkEventType),
|
|
342
|
-
mode: normalizeDiagnosticValue(evt.mode),
|
|
343
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
344
|
-
transport: normalizeDiagnosticValue(evt.transport)
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
331
|
function recordModelUsage(store, evt) {
|
|
348
332
|
const labels = {
|
|
349
333
|
agent: normalizeDiagnosticValue(evt.agentId),
|
|
@@ -375,8 +359,12 @@ function recordModelUsage(store, evt) {
|
|
|
375
359
|
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
376
360
|
}
|
|
377
361
|
function recordDiagnosticEvent(store, evt, metadata) {
|
|
378
|
-
if (!
|
|
362
|
+
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
379
363
|
switch (evt.type) {
|
|
364
|
+
case "diagnostic.phase.completed":
|
|
365
|
+
case "gateway.rpc":
|
|
366
|
+
recordGatewayRpcEvent(store, evt, metadata);
|
|
367
|
+
return;
|
|
380
368
|
case "diagnostic.gc":
|
|
381
369
|
store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds for the hosting JavaScript isolate.", {}, seconds(evt.durationMs));
|
|
382
370
|
return;
|
|
@@ -384,54 +372,101 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
384
372
|
store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
|
|
385
373
|
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
386
374
|
return;
|
|
387
|
-
case "gateway.rpc": {
|
|
388
|
-
const labels = { method: evt.method };
|
|
389
|
-
if (evt.phase === "received") {
|
|
390
|
-
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
391
|
-
return;
|
|
392
|
-
}
|
|
393
|
-
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
394
|
-
phase: evt.phase,
|
|
395
|
-
outcome: evt.outcome
|
|
396
|
-
});
|
|
397
|
-
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) store.histogram("openclaw_gateway_rpc_first_response_seconds", "Elapsed time until the first Gateway RPC response is sent.", labels, seconds(evt.durationMs));
|
|
398
|
-
else if (evt.phase === "handler") {
|
|
399
|
-
store.histogram("openclaw_gateway_rpc_handler_seconds", "Gateway RPC handler duration until return or throw.", labels, seconds(evt.durationMs));
|
|
400
|
-
store.histogram("openclaw_gateway_rpc_admission_seconds", "Elapsed time from Gateway RPC receipt until handler invocation.", labels, seconds(evt.admissionMs));
|
|
401
|
-
} else if (evt.phase === "dispatch") store.histogram("openclaw_gateway_rpc_queue_wait_seconds", "Gateway operator request start queue wait.", labels, seconds(evt.queueWaitMs));
|
|
402
|
-
return;
|
|
403
|
-
}
|
|
404
375
|
case "model.usage":
|
|
405
376
|
recordModelUsage(store, evt);
|
|
406
377
|
return;
|
|
407
|
-
case "run.completed":
|
|
408
|
-
|
|
409
|
-
|
|
378
|
+
case "run.completed": {
|
|
379
|
+
const labels = {
|
|
380
|
+
...evt.blockedBy ? { blocked_by: normalizeDiagnosticValue(evt.blockedBy) } : {},
|
|
381
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
382
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
383
|
+
outcome: normalizeDiagnosticValue(evt.outcome, "unknown"),
|
|
384
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
385
|
+
trigger: normalizeDiagnosticValue(evt.trigger)
|
|
386
|
+
};
|
|
387
|
+
store.histogram("openclaw_run_duration_seconds", "Agent run duration in seconds.", labels, seconds(evt.durationMs));
|
|
388
|
+
store.counter("openclaw_run_completed_total", "Agent runs completed by outcome.", labels);
|
|
410
389
|
return;
|
|
390
|
+
}
|
|
411
391
|
case "model.call.completed":
|
|
412
|
-
case "model.call.error":
|
|
413
|
-
|
|
414
|
-
|
|
392
|
+
case "model.call.error": {
|
|
393
|
+
const labels = {
|
|
394
|
+
api: normalizeDiagnosticValue(evt.api),
|
|
395
|
+
error_category: evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
396
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
397
|
+
observation_unit: evt.observationUnit === "turn" ? "turn" : "request",
|
|
398
|
+
outcome: evt.type === "model.call.error" ? "error" : "completed",
|
|
399
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
400
|
+
transport: normalizeDiagnosticValue(evt.transport)
|
|
401
|
+
};
|
|
402
|
+
store.histogram("openclaw_model_call_duration_seconds", "Model request or synthetic agent-turn duration in seconds.", labels, seconds(evt.durationMs));
|
|
403
|
+
store.counter("openclaw_model_call_total", "Model requests or synthetic agent turns completed by outcome.", labels);
|
|
415
404
|
return;
|
|
416
|
-
|
|
417
|
-
|
|
405
|
+
}
|
|
406
|
+
case "model.failover": {
|
|
407
|
+
const labels = {
|
|
408
|
+
from_model: normalizeDiagnosticValue(evt.fromModel),
|
|
409
|
+
from_provider: normalizeDiagnosticValue(evt.fromProvider),
|
|
410
|
+
lane: normalizeDiagnosticLane(evt.lane),
|
|
411
|
+
reason: normalizeDiagnosticValue(evt.reason, "other"),
|
|
412
|
+
suspended: evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
413
|
+
to_model: normalizeDiagnosticValue(evt.toModel),
|
|
414
|
+
to_provider: normalizeDiagnosticValue(evt.toProvider)
|
|
415
|
+
};
|
|
416
|
+
store.counter("openclaw_model_failover_total", "Model failovers by source, destination, lane, and reason.", labels);
|
|
418
417
|
return;
|
|
418
|
+
}
|
|
419
419
|
case "tool.execution.completed":
|
|
420
|
-
case "tool.execution.error":
|
|
421
|
-
|
|
422
|
-
|
|
420
|
+
case "tool.execution.error": {
|
|
421
|
+
const labels = {
|
|
422
|
+
error_category: evt.type === "tool.execution.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
423
|
+
outcome: evt.type === "tool.execution.error" ? "error" : "completed",
|
|
424
|
+
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
425
|
+
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
426
|
+
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
427
|
+
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
428
|
+
};
|
|
429
|
+
store.histogram("openclaw_tool_execution_duration_seconds", "Tool execution duration in seconds.", labels, seconds(evt.durationMs));
|
|
430
|
+
store.counter("openclaw_tool_execution_total", "Tool executions completed by outcome.", labels);
|
|
423
431
|
return;
|
|
424
|
-
|
|
425
|
-
|
|
432
|
+
}
|
|
433
|
+
case "tool.execution.blocked": {
|
|
434
|
+
const labels = {
|
|
435
|
+
denied_reason: normalizeDiagnosticValue(evt.deniedReason, "other"),
|
|
436
|
+
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
437
|
+
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
438
|
+
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
439
|
+
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
440
|
+
};
|
|
441
|
+
store.counter("openclaw_tool_execution_blocked_total", "Tool executions blocked by policy or sandbox diagnostics.", labels);
|
|
426
442
|
return;
|
|
427
|
-
|
|
428
|
-
|
|
443
|
+
}
|
|
444
|
+
case "skill.used": {
|
|
445
|
+
const labels = {
|
|
446
|
+
activation: normalizeDiagnosticValue(evt.activation, "unknown"),
|
|
447
|
+
agent: normalizeDiagnosticValue(evt.agentId),
|
|
448
|
+
skill: normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
449
|
+
source: normalizeDiagnosticValue(evt.skillSource)
|
|
450
|
+
};
|
|
451
|
+
store.counter("openclaw_skill_used_total", "Skills used by agent runs.", labels);
|
|
429
452
|
return;
|
|
453
|
+
}
|
|
430
454
|
case "harness.run.completed":
|
|
431
|
-
case "harness.run.error":
|
|
432
|
-
|
|
433
|
-
|
|
455
|
+
case "harness.run.error": {
|
|
456
|
+
const labels = {
|
|
457
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
458
|
+
error_category: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
459
|
+
harness: normalizeDiagnosticValue(evt.harnessId),
|
|
460
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
461
|
+
outcome: evt.type === "harness.run.error" ? "error" : normalizeDiagnosticValue(evt.outcome),
|
|
462
|
+
phase: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.phase) : "none",
|
|
463
|
+
plugin: normalizeDiagnosticValue(evt.pluginId),
|
|
464
|
+
provider: normalizeDiagnosticValue(evt.provider)
|
|
465
|
+
};
|
|
466
|
+
store.histogram("openclaw_harness_run_duration_seconds", "Agent harness run duration in seconds.", labels, seconds(evt.durationMs));
|
|
467
|
+
store.counter("openclaw_harness_run_total", "Agent harness runs completed by outcome.", labels);
|
|
434
468
|
return;
|
|
469
|
+
}
|
|
435
470
|
case "message.processed": {
|
|
436
471
|
const labels = {
|
|
437
472
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
@@ -492,16 +527,31 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
492
527
|
store.histogram("openclaw_message_delivery_duration_seconds", "Outbound message delivery duration in seconds.", labels, seconds(evt.durationMs));
|
|
493
528
|
return;
|
|
494
529
|
}
|
|
495
|
-
case "talk.event":
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
530
|
+
case "talk.event": {
|
|
531
|
+
const labels = {
|
|
532
|
+
brain: normalizeDiagnosticValue(evt.brain),
|
|
533
|
+
event_type: normalizeDiagnosticValue(evt.talkEventType),
|
|
534
|
+
mode: normalizeDiagnosticValue(evt.mode),
|
|
535
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
536
|
+
transport: normalizeDiagnosticValue(evt.transport)
|
|
537
|
+
};
|
|
538
|
+
store.counter("openclaw_talk_event_total", "Talk events emitted by type.", labels);
|
|
539
|
+
store.histogram("openclaw_talk_event_duration_seconds", "Talk event duration in seconds when reported.", labels, seconds(evt.durationMs));
|
|
540
|
+
store.histogram("openclaw_talk_audio_bytes", "Talk audio frame byte lengths.", labels, asNonNegativeFiniteNumber(evt.byteLength), BYTE_BUCKETS);
|
|
499
541
|
return;
|
|
542
|
+
}
|
|
500
543
|
case "session.recovery.requested":
|
|
501
|
-
case "session.recovery.completed":
|
|
502
|
-
|
|
503
|
-
|
|
544
|
+
case "session.recovery.completed": {
|
|
545
|
+
const labels = {
|
|
546
|
+
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
547
|
+
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
548
|
+
state: evt.state,
|
|
549
|
+
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
550
|
+
};
|
|
551
|
+
store.counter("openclaw_session_recovery_total", "Session recovery observations by status and action.", labels);
|
|
552
|
+
store.histogram("openclaw_session_recovery_age_seconds", "Age of sessions selected for recovery in seconds.", labels, seconds(evt.ageMs));
|
|
504
553
|
return;
|
|
554
|
+
}
|
|
505
555
|
case "queue.lane.enqueue":
|
|
506
556
|
case "queue.lane.dequeue":
|
|
507
557
|
store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", { lane: normalizeDiagnosticLane(evt.lane) }, asNonNegativeFiniteNumber(evt.queueSize));
|
|
@@ -514,10 +564,15 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
514
564
|
});
|
|
515
565
|
if (evt.queueDepth !== void 0) store.gauge("openclaw_session_queue_depth", "Latest observed session queue depth.", { state: evt.state }, asNonNegativeFiniteNumber(evt.queueDepth));
|
|
516
566
|
return;
|
|
517
|
-
case "session.stuck":
|
|
518
|
-
|
|
519
|
-
|
|
567
|
+
case "session.stuck": {
|
|
568
|
+
const labels = {
|
|
569
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
570
|
+
state: evt.state
|
|
571
|
+
};
|
|
572
|
+
store.counter("openclaw_session_stuck_total", "Stale session bookkeeping observations with no active work.", labels);
|
|
573
|
+
store.histogram("openclaw_session_stuck_age_seconds", "Age of stale session bookkeeping observations in seconds.", labels, seconds(evt.ageMs));
|
|
520
574
|
return;
|
|
575
|
+
}
|
|
521
576
|
case "session.turn.created":
|
|
522
577
|
store.counter("openclaw_session_turn_created_total", "Agent session turns created.", {
|
|
523
578
|
agent: normalizeDiagnosticValue(evt.agentId),
|
|
@@ -525,13 +580,11 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
525
580
|
trigger: evt.trigger
|
|
526
581
|
});
|
|
527
582
|
return;
|
|
583
|
+
case "diagnostic.child_process.spawn":
|
|
584
|
+
store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", { family: normalizeDiagnosticValue(evt.family) }, asNonNegativeFiniteNumber(evt.count) ?? 0);
|
|
585
|
+
return;
|
|
528
586
|
case "diagnostic.memory.sample":
|
|
529
|
-
|
|
530
|
-
["rss", "rssBytes"],
|
|
531
|
-
["heap_total", "heapTotalBytes"],
|
|
532
|
-
["heap_used", "heapUsedBytes"]
|
|
533
|
-
]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, evt.memory[field]);
|
|
534
|
-
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(evt.memory.rssBytes), BYTE_BUCKETS);
|
|
587
|
+
recordMemorySample(store, evt.memory, BYTE_BUCKETS);
|
|
535
588
|
return;
|
|
536
589
|
case "diagnostic.memory.pressure":
|
|
537
590
|
store.counter("openclaw_memory_pressure_total", "Memory pressure events by level and reason.", {
|
|
@@ -539,18 +592,20 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
539
592
|
reason: evt.reason
|
|
540
593
|
});
|
|
541
594
|
return;
|
|
542
|
-
case "diagnostic.liveness.warning":
|
|
543
|
-
|
|
595
|
+
case "diagnostic.liveness.warning": {
|
|
596
|
+
const labels = { reason: normalizeDiagnosticValue(evt.reasons.join(":"), "unknown") };
|
|
597
|
+
store.counter("openclaw_liveness_warning_total", "Diagnostic liveness warning events.", labels);
|
|
544
598
|
for (const state of [
|
|
545
599
|
"active",
|
|
546
600
|
"waiting",
|
|
547
601
|
"queued"
|
|
548
602
|
]) store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state }, asNonNegativeFiniteNumber(evt[state]));
|
|
549
|
-
store.histogram("openclaw_liveness_event_loop_delay_p99_seconds", "P99 event-loop delay reported by diagnostic liveness warnings in seconds.",
|
|
550
|
-
store.histogram("openclaw_liveness_event_loop_delay_max_seconds", "Maximum event-loop delay reported by diagnostic liveness warnings in seconds.",
|
|
551
|
-
store.histogram("openclaw_liveness_event_loop_utilization_ratio", "Event-loop utilization reported by diagnostic liveness warnings.",
|
|
552
|
-
store.histogram("openclaw_liveness_cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.",
|
|
603
|
+
store.histogram("openclaw_liveness_event_loop_delay_p99_seconds", "P99 event-loop delay reported by diagnostic liveness warnings in seconds.", labels, seconds(evt.eventLoopDelayP99Ms));
|
|
604
|
+
store.histogram("openclaw_liveness_event_loop_delay_max_seconds", "Maximum event-loop delay reported by diagnostic liveness warnings in seconds.", labels, seconds(evt.eventLoopDelayMaxMs));
|
|
605
|
+
store.histogram("openclaw_liveness_event_loop_utilization_ratio", "Event-loop utilization reported by diagnostic liveness warnings.", labels, asNonNegativeFiniteNumber(evt.eventLoopUtilization), RATIO_BUCKETS);
|
|
606
|
+
store.histogram("openclaw_liveness_cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.", labels, asNonNegativeFiniteNumber(evt.cpuCoreRatio), RATIO_BUCKETS);
|
|
553
607
|
return;
|
|
608
|
+
}
|
|
554
609
|
case "diagnostic.async_queue.dropped":
|
|
555
610
|
for (const [dropClass, field] of [
|
|
556
611
|
["total", "droppedEvents"],
|
|
@@ -572,9 +627,17 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
572
627
|
status: evt.status
|
|
573
628
|
});
|
|
574
629
|
return;
|
|
575
|
-
case "payload.large":
|
|
576
|
-
|
|
577
|
-
|
|
630
|
+
case "payload.large": {
|
|
631
|
+
const labels = {
|
|
632
|
+
action: evt.action,
|
|
633
|
+
channel: normalizeDiagnosticValue(evt.channel, "none"),
|
|
634
|
+
plugin: normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
635
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
636
|
+
surface: normalizeDiagnosticValue(evt.surface, "unknown")
|
|
637
|
+
};
|
|
638
|
+
store.counter("openclaw_payload_large_total", "Oversized payload diagnostics by surface and action.", labels);
|
|
639
|
+
store.histogram("openclaw_payload_large_bytes", "Oversized payload byte sizes by surface and action.", labels, asNonNegativeFiniteNumber(evt.bytes), BYTE_BUCKETS);
|
|
640
|
+
}
|
|
578
641
|
}
|
|
579
642
|
}
|
|
580
643
|
const METRICS_READ_SCOPE = "operator.read";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-prometheus",
|
|
3
|
-
"version": "2026.9.
|
|
3
|
+
"version": "2026.9.7",
|
|
4
4
|
"description": "OpenClaw diagnostics Prometheus exporter for runtime metrics.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
"minHostVersion": ">=2026.4.25"
|
|
19
19
|
},
|
|
20
20
|
"compat": {
|
|
21
|
-
"pluginApi": ">=2026.9.
|
|
21
|
+
"pluginApi": ">=2026.9.7"
|
|
22
22
|
},
|
|
23
23
|
"build": {
|
|
24
|
-
"openclawVersion": "2026.9.
|
|
24
|
+
"openclawVersion": "2026.9.7"
|
|
25
25
|
},
|
|
26
26
|
"release": {
|
|
27
27
|
"publishToClawHub": true,
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"assets/activity.svg"
|
|
40
40
|
],
|
|
41
41
|
"peerDependencies": {
|
|
42
|
-
"openclaw": ">=2026.9.
|
|
42
|
+
"openclaw": ">=2026.9.7"
|
|
43
43
|
},
|
|
44
44
|
"peerDependenciesMeta": {
|
|
45
45
|
"openclaw": {
|