@openclaw/diagnostics-prometheus 2026.9.6 → 2026.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/icon.png +0 -0
- package/dist/index.js +227 -188
- package/package.json +4 -4
package/assets/icon.png
CHANGED
|
Binary file
|
package/dist/index.js
CHANGED
|
@@ -31,6 +31,10 @@ function formatPrometheusNumber(value) {
|
|
|
31
31
|
if (!Number.isFinite(value)) return "0";
|
|
32
32
|
return Number.isInteger(value) ? String(value) : String(Number(value.toPrecision(12)));
|
|
33
33
|
}
|
|
34
|
+
function seconds(ms) {
|
|
35
|
+
const value = asNonNegativeFiniteNumber(ms);
|
|
36
|
+
return value === void 0 ? void 0 : value / 1e3;
|
|
37
|
+
}
|
|
34
38
|
//#endregion
|
|
35
39
|
//#region extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
|
|
36
40
|
const DURATION_BUCKETS_SECONDS = [
|
|
@@ -89,6 +93,15 @@ function createPrometheusMetricStore() {
|
|
|
89
93
|
value
|
|
90
94
|
});
|
|
91
95
|
};
|
|
96
|
+
const counterValue = (name, help, labels, value) => {
|
|
97
|
+
if (!Number.isFinite(value) || value < 0) return;
|
|
98
|
+
const key = metricKey(name, labels);
|
|
99
|
+
if (canCreateSeries(counters, key)) counters.set(key, {
|
|
100
|
+
help,
|
|
101
|
+
labels,
|
|
102
|
+
value: Math.max(counters.get(key)?.value ?? 0, value)
|
|
103
|
+
});
|
|
104
|
+
};
|
|
92
105
|
const clearGauges = (name) => {
|
|
93
106
|
for (const key of gauges.keys()) if (key.startsWith(`${name}|`)) gauges.delete(key);
|
|
94
107
|
};
|
|
@@ -136,6 +149,7 @@ function createPrometheusMetricStore() {
|
|
|
136
149
|
};
|
|
137
150
|
return {
|
|
138
151
|
counter,
|
|
152
|
+
counterValue,
|
|
139
153
|
gauge,
|
|
140
154
|
clearGauges,
|
|
141
155
|
histogram,
|
|
@@ -144,6 +158,81 @@ function createPrometheusMetricStore() {
|
|
|
144
158
|
};
|
|
145
159
|
}
|
|
146
160
|
//#endregion
|
|
161
|
+
//#region extensions/diagnostics-prometheus/src/service-gateway-rpc.ts
|
|
162
|
+
const CATALOG_LIST_METHOD = "sessions.catalog.list";
|
|
163
|
+
const CATALOG_LIST_PHASE_PREFIX = `${CATALOG_LIST_METHOD}.`;
|
|
164
|
+
const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
|
|
165
|
+
"projection_initial",
|
|
166
|
+
"planning",
|
|
167
|
+
"provider",
|
|
168
|
+
"coalesced",
|
|
169
|
+
"projection_final",
|
|
170
|
+
"delivery"
|
|
171
|
+
]);
|
|
172
|
+
function recordGatewayRpcEvent(store, evt, metadata) {
|
|
173
|
+
switch (evt.type) {
|
|
174
|
+
case "diagnostic.phase.completed": {
|
|
175
|
+
if (!metadata.trusted || !evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
|
|
176
|
+
const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
|
|
177
|
+
const duration = seconds(evt.durationMs);
|
|
178
|
+
if (!CATALOG_LIST_PHASES.has(phase) || duration === void 0) return;
|
|
179
|
+
const labels = {
|
|
180
|
+
method: CATALOG_LIST_METHOD,
|
|
181
|
+
phase
|
|
182
|
+
};
|
|
183
|
+
store.histogram("openclaw_gateway_rpc_stage_seconds", "Elapsed time for completed Gateway RPC owner stages.", labels, duration);
|
|
184
|
+
const cpu = evt.details?.threadCpuMs;
|
|
185
|
+
if ((phase === "planning" || phase === "delivery") && typeof cpu === "number") store.histogram("openclaw_gateway_rpc_stage_thread_cpu_seconds", "Current-thread CPU for synchronous Gateway RPC owner stages.", labels, seconds(cpu));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
case "gateway.rpc": {
|
|
189
|
+
const labels = { method: evt.method };
|
|
190
|
+
if (evt.phase === "received") {
|
|
191
|
+
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
195
|
+
phase: evt.phase,
|
|
196
|
+
outcome: evt.outcome
|
|
197
|
+
});
|
|
198
|
+
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) store.histogram("openclaw_gateway_rpc_first_response_seconds", "Elapsed time until the first Gateway RPC response is sent.", labels, seconds(evt.durationMs));
|
|
199
|
+
else if (evt.phase === "handler") {
|
|
200
|
+
store.histogram("openclaw_gateway_rpc_handler_seconds", "Gateway RPC handler duration until return or throw.", labels, seconds(evt.durationMs));
|
|
201
|
+
store.histogram("openclaw_gateway_rpc_admission_seconds", "Elapsed time from Gateway RPC receipt until handler invocation.", labels, seconds(evt.admissionMs));
|
|
202
|
+
} else if (evt.phase === "dispatch") store.histogram("openclaw_gateway_rpc_queue_wait_seconds", "Gateway operator request start queue wait.", labels, seconds(evt.queueWaitMs));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
//#endregion
|
|
207
|
+
//#region extensions/diagnostics-prometheus/src/service-memory.ts
|
|
208
|
+
function recordMemorySample(store, memory, byteBuckets) {
|
|
209
|
+
for (const worker of memory.workerLifecycle ?? []) {
|
|
210
|
+
store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
|
|
211
|
+
for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
|
|
212
|
+
script: worker.script,
|
|
213
|
+
reason
|
|
214
|
+
}, count);
|
|
215
|
+
}
|
|
216
|
+
for (const [kind, field] of [
|
|
217
|
+
["rss", "rssBytes"],
|
|
218
|
+
["heap_total", "heapTotalBytes"],
|
|
219
|
+
["heap_used", "heapUsedBytes"],
|
|
220
|
+
["external", "externalBytes"],
|
|
221
|
+
["array_buffers", "arrayBuffersBytes"],
|
|
222
|
+
["worker_heap_total", "workerHeapTotalBytes"],
|
|
223
|
+
["worker_heap_used", "workerHeapUsedBytes"]
|
|
224
|
+
]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, asNonNegativeFiniteNumber(memory[field]));
|
|
225
|
+
for (const [name, field] of [["openclaw_worker_count", "workerCount"], ["openclaw_worker_heap_sampled_count", "workerHeapSampledCount"]]) store.gauge(name, "Worker isolate counts.", {}, asNonNegativeFiniteNumber(memory[field]));
|
|
226
|
+
const workerHeaps = /* @__PURE__ */ new Map();
|
|
227
|
+
for (const worker of memory.workerHeaps ?? []) {
|
|
228
|
+
const heapUsed = asNonNegativeFiniteNumber(worker.heapUsed);
|
|
229
|
+
if (heapUsed !== void 0) workerHeaps.set(worker.script, (workerHeaps.get(worker.script) ?? 0) + heapUsed);
|
|
230
|
+
}
|
|
231
|
+
store.clearGauges("openclaw_worker_heap_used_bytes");
|
|
232
|
+
for (const [script, heapUsed] of workerHeaps) store.gauge("openclaw_worker_heap_used_bytes", "Latest live Worker heap usage by bounded script basename.", { script }, heapUsed);
|
|
233
|
+
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
|
|
234
|
+
}
|
|
235
|
+
//#endregion
|
|
147
236
|
//#region extensions/diagnostics-prometheus/src/service.ts
|
|
148
237
|
const TOKEN_BUCKETS = [
|
|
149
238
|
1,
|
|
@@ -186,17 +275,10 @@ const RATIO_BUCKETS = [
|
|
|
186
275
|
8,
|
|
187
276
|
16
|
|
188
277
|
];
|
|
189
|
-
function seconds(ms) {
|
|
190
|
-
const value = asNonNegativeFiniteNumber(ms);
|
|
191
|
-
return value === void 0 ? void 0 : value / 1e3;
|
|
192
|
-
}
|
|
193
278
|
function safeErrorMessage(err) {
|
|
194
279
|
const message = err instanceof Error ? err.message ?? err.name : String(err);
|
|
195
280
|
return truncateUtf16Safe(redactSensitiveText(message).replaceAll("\0", " ").replace(/[\r\n\t\u2028\u2029]/gu, " "), 500);
|
|
196
281
|
}
|
|
197
|
-
function shouldRecordDiagnosticEvent(metadata) {
|
|
198
|
-
return metadata.trusted || isInternalDiagnosticEventMetadata(metadata);
|
|
199
|
-
}
|
|
200
282
|
function renderPrometheusMetrics(store) {
|
|
201
283
|
const snapshot = store.snapshot();
|
|
202
284
|
const lines = [];
|
|
@@ -240,118 +322,12 @@ function renderPrometheusMetrics(store) {
|
|
|
240
322
|
lines.push("");
|
|
241
323
|
return lines.join("\n");
|
|
242
324
|
}
|
|
243
|
-
function runLabels(evt) {
|
|
244
|
-
return {
|
|
245
|
-
...evt.blockedBy ? { blocked_by: normalizeDiagnosticValue(evt.blockedBy) } : {},
|
|
246
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
247
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
248
|
-
outcome: normalizeDiagnosticValue(evt.outcome, "unknown"),
|
|
249
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
250
|
-
trigger: normalizeDiagnosticValue(evt.trigger)
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
function modelCallLabels(evt) {
|
|
254
|
-
return {
|
|
255
|
-
api: normalizeDiagnosticValue(evt.api),
|
|
256
|
-
error_category: evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
257
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
258
|
-
observation_unit: evt.observationUnit === "turn" ? "turn" : "request",
|
|
259
|
-
outcome: evt.type === "model.call.error" ? "error" : "completed",
|
|
260
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
261
|
-
transport: normalizeDiagnosticValue(evt.transport)
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
|
-
function modelFailoverLabels(evt) {
|
|
265
|
-
return {
|
|
266
|
-
from_model: normalizeDiagnosticValue(evt.fromModel),
|
|
267
|
-
from_provider: normalizeDiagnosticValue(evt.fromProvider),
|
|
268
|
-
lane: normalizeDiagnosticLane(evt.lane),
|
|
269
|
-
reason: normalizeDiagnosticValue(evt.reason, "other"),
|
|
270
|
-
suspended: evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
271
|
-
to_model: normalizeDiagnosticValue(evt.toModel),
|
|
272
|
-
to_provider: normalizeDiagnosticValue(evt.toProvider)
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
function toolExecutionLabels(evt) {
|
|
276
|
-
return {
|
|
277
|
-
error_category: evt.type === "tool.execution.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
278
|
-
outcome: evt.type === "tool.execution.error" ? "error" : "completed",
|
|
279
|
-
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
280
|
-
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
281
|
-
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
282
|
-
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
283
|
-
};
|
|
284
|
-
}
|
|
285
|
-
function toolExecutionBlockedLabels(evt) {
|
|
286
|
-
return {
|
|
287
|
-
denied_reason: normalizeDiagnosticValue(evt.deniedReason, "other"),
|
|
288
|
-
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
289
|
-
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
290
|
-
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
291
|
-
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
292
|
-
};
|
|
293
|
-
}
|
|
294
|
-
function skillLabels(evt) {
|
|
295
|
-
return {
|
|
296
|
-
activation: normalizeDiagnosticValue(evt.activation, "unknown"),
|
|
297
|
-
agent: normalizeDiagnosticValue(evt.agentId),
|
|
298
|
-
skill: normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
299
|
-
source: normalizeDiagnosticValue(evt.skillSource)
|
|
300
|
-
};
|
|
301
|
-
}
|
|
302
|
-
function harnessLabels(evt) {
|
|
303
|
-
return {
|
|
304
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
305
|
-
error_category: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
306
|
-
harness: normalizeDiagnosticValue(evt.harnessId),
|
|
307
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
308
|
-
outcome: evt.type === "harness.run.error" ? "error" : normalizeDiagnosticValue(evt.outcome),
|
|
309
|
-
phase: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.phase) : "none",
|
|
310
|
-
plugin: normalizeDiagnosticValue(evt.pluginId),
|
|
311
|
-
provider: normalizeDiagnosticValue(evt.provider)
|
|
312
|
-
};
|
|
313
|
-
}
|
|
314
325
|
function webhookLabels(evt) {
|
|
315
326
|
return {
|
|
316
327
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
317
328
|
webhook: normalizeDiagnosticValue(evt.updateType)
|
|
318
329
|
};
|
|
319
330
|
}
|
|
320
|
-
function sessionStuckLabels(evt) {
|
|
321
|
-
return {
|
|
322
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
323
|
-
state: evt.state
|
|
324
|
-
};
|
|
325
|
-
}
|
|
326
|
-
function sessionRecoveryLabels(evt) {
|
|
327
|
-
return {
|
|
328
|
-
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
329
|
-
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
330
|
-
state: evt.state,
|
|
331
|
-
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
function livenessLabels(evt) {
|
|
335
|
-
return { reason: normalizeDiagnosticValue(evt.reasons.join(":"), "unknown") };
|
|
336
|
-
}
|
|
337
|
-
function payloadLargeLabels(evt) {
|
|
338
|
-
return {
|
|
339
|
-
action: evt.action,
|
|
340
|
-
channel: normalizeDiagnosticValue(evt.channel, "none"),
|
|
341
|
-
plugin: normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
342
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
343
|
-
surface: normalizeDiagnosticValue(evt.surface, "unknown")
|
|
344
|
-
};
|
|
345
|
-
}
|
|
346
|
-
function talkLabels(evt) {
|
|
347
|
-
return {
|
|
348
|
-
brain: normalizeDiagnosticValue(evt.brain),
|
|
349
|
-
event_type: normalizeDiagnosticValue(evt.talkEventType),
|
|
350
|
-
mode: normalizeDiagnosticValue(evt.mode),
|
|
351
|
-
provider: normalizeDiagnosticValue(evt.provider),
|
|
352
|
-
transport: normalizeDiagnosticValue(evt.transport)
|
|
353
|
-
};
|
|
354
|
-
}
|
|
355
331
|
function recordModelUsage(store, evt) {
|
|
356
332
|
const labels = {
|
|
357
333
|
agent: normalizeDiagnosticValue(evt.agentId),
|
|
@@ -383,8 +359,12 @@ function recordModelUsage(store, evt) {
|
|
|
383
359
|
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
384
360
|
}
|
|
385
361
|
function recordDiagnosticEvent(store, evt, metadata) {
|
|
386
|
-
if (!
|
|
362
|
+
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
387
363
|
switch (evt.type) {
|
|
364
|
+
case "diagnostic.phase.completed":
|
|
365
|
+
case "gateway.rpc":
|
|
366
|
+
recordGatewayRpcEvent(store, evt, metadata);
|
|
367
|
+
return;
|
|
388
368
|
case "diagnostic.gc":
|
|
389
369
|
store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds for the hosting JavaScript isolate.", {}, seconds(evt.durationMs));
|
|
390
370
|
return;
|
|
@@ -392,54 +372,101 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
392
372
|
store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
|
|
393
373
|
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
394
374
|
return;
|
|
395
|
-
case "gateway.rpc": {
|
|
396
|
-
const labels = { method: evt.method };
|
|
397
|
-
if (evt.phase === "received") {
|
|
398
|
-
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
399
|
-
return;
|
|
400
|
-
}
|
|
401
|
-
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
402
|
-
phase: evt.phase,
|
|
403
|
-
outcome: evt.outcome
|
|
404
|
-
});
|
|
405
|
-
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) store.histogram("openclaw_gateway_rpc_first_response_seconds", "Elapsed time until the first Gateway RPC response is sent.", labels, seconds(evt.durationMs));
|
|
406
|
-
else if (evt.phase === "handler") {
|
|
407
|
-
store.histogram("openclaw_gateway_rpc_handler_seconds", "Gateway RPC handler duration until return or throw.", labels, seconds(evt.durationMs));
|
|
408
|
-
store.histogram("openclaw_gateway_rpc_admission_seconds", "Elapsed time from Gateway RPC receipt until handler invocation.", labels, seconds(evt.admissionMs));
|
|
409
|
-
} else if (evt.phase === "dispatch") store.histogram("openclaw_gateway_rpc_queue_wait_seconds", "Gateway operator request start queue wait.", labels, seconds(evt.queueWaitMs));
|
|
410
|
-
return;
|
|
411
|
-
}
|
|
412
375
|
case "model.usage":
|
|
413
376
|
recordModelUsage(store, evt);
|
|
414
377
|
return;
|
|
415
|
-
case "run.completed":
|
|
416
|
-
|
|
417
|
-
|
|
378
|
+
case "run.completed": {
|
|
379
|
+
const labels = {
|
|
380
|
+
...evt.blockedBy ? { blocked_by: normalizeDiagnosticValue(evt.blockedBy) } : {},
|
|
381
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
382
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
383
|
+
outcome: normalizeDiagnosticValue(evt.outcome, "unknown"),
|
|
384
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
385
|
+
trigger: normalizeDiagnosticValue(evt.trigger)
|
|
386
|
+
};
|
|
387
|
+
store.histogram("openclaw_run_duration_seconds", "Agent run duration in seconds.", labels, seconds(evt.durationMs));
|
|
388
|
+
store.counter("openclaw_run_completed_total", "Agent runs completed by outcome.", labels);
|
|
418
389
|
return;
|
|
390
|
+
}
|
|
419
391
|
case "model.call.completed":
|
|
420
|
-
case "model.call.error":
|
|
421
|
-
|
|
422
|
-
|
|
392
|
+
case "model.call.error": {
|
|
393
|
+
const labels = {
|
|
394
|
+
api: normalizeDiagnosticValue(evt.api),
|
|
395
|
+
error_category: evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
396
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
397
|
+
observation_unit: evt.observationUnit === "turn" ? "turn" : "request",
|
|
398
|
+
outcome: evt.type === "model.call.error" ? "error" : "completed",
|
|
399
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
400
|
+
transport: normalizeDiagnosticValue(evt.transport)
|
|
401
|
+
};
|
|
402
|
+
store.histogram("openclaw_model_call_duration_seconds", "Model request or synthetic agent-turn duration in seconds.", labels, seconds(evt.durationMs));
|
|
403
|
+
store.counter("openclaw_model_call_total", "Model requests or synthetic agent turns completed by outcome.", labels);
|
|
423
404
|
return;
|
|
424
|
-
|
|
425
|
-
|
|
405
|
+
}
|
|
406
|
+
case "model.failover": {
|
|
407
|
+
const labels = {
|
|
408
|
+
from_model: normalizeDiagnosticValue(evt.fromModel),
|
|
409
|
+
from_provider: normalizeDiagnosticValue(evt.fromProvider),
|
|
410
|
+
lane: normalizeDiagnosticLane(evt.lane),
|
|
411
|
+
reason: normalizeDiagnosticValue(evt.reason, "other"),
|
|
412
|
+
suspended: evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
413
|
+
to_model: normalizeDiagnosticValue(evt.toModel),
|
|
414
|
+
to_provider: normalizeDiagnosticValue(evt.toProvider)
|
|
415
|
+
};
|
|
416
|
+
store.counter("openclaw_model_failover_total", "Model failovers by source, destination, lane, and reason.", labels);
|
|
426
417
|
return;
|
|
418
|
+
}
|
|
427
419
|
case "tool.execution.completed":
|
|
428
|
-
case "tool.execution.error":
|
|
429
|
-
|
|
430
|
-
|
|
420
|
+
case "tool.execution.error": {
|
|
421
|
+
const labels = {
|
|
422
|
+
error_category: evt.type === "tool.execution.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
423
|
+
outcome: evt.type === "tool.execution.error" ? "error" : "completed",
|
|
424
|
+
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
425
|
+
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
426
|
+
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
427
|
+
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
428
|
+
};
|
|
429
|
+
store.histogram("openclaw_tool_execution_duration_seconds", "Tool execution duration in seconds.", labels, seconds(evt.durationMs));
|
|
430
|
+
store.counter("openclaw_tool_execution_total", "Tool executions completed by outcome.", labels);
|
|
431
431
|
return;
|
|
432
|
-
|
|
433
|
-
|
|
432
|
+
}
|
|
433
|
+
case "tool.execution.blocked": {
|
|
434
|
+
const labels = {
|
|
435
|
+
denied_reason: normalizeDiagnosticValue(evt.deniedReason, "other"),
|
|
436
|
+
params_kind: normalizeDiagnosticValue(evt.paramsSummary?.kind),
|
|
437
|
+
tool: normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
438
|
+
tool_owner: normalizeDiagnosticValue(evt.toolOwner, "none"),
|
|
439
|
+
tool_source: normalizeDiagnosticValue(evt.toolSource, "core")
|
|
440
|
+
};
|
|
441
|
+
store.counter("openclaw_tool_execution_blocked_total", "Tool executions blocked by policy or sandbox diagnostics.", labels);
|
|
434
442
|
return;
|
|
435
|
-
|
|
436
|
-
|
|
443
|
+
}
|
|
444
|
+
case "skill.used": {
|
|
445
|
+
const labels = {
|
|
446
|
+
activation: normalizeDiagnosticValue(evt.activation, "unknown"),
|
|
447
|
+
agent: normalizeDiagnosticValue(evt.agentId),
|
|
448
|
+
skill: normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
449
|
+
source: normalizeDiagnosticValue(evt.skillSource)
|
|
450
|
+
};
|
|
451
|
+
store.counter("openclaw_skill_used_total", "Skills used by agent runs.", labels);
|
|
437
452
|
return;
|
|
453
|
+
}
|
|
438
454
|
case "harness.run.completed":
|
|
439
|
-
case "harness.run.error":
|
|
440
|
-
|
|
441
|
-
|
|
455
|
+
case "harness.run.error": {
|
|
456
|
+
const labels = {
|
|
457
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
458
|
+
error_category: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "none",
|
|
459
|
+
harness: normalizeDiagnosticValue(evt.harnessId),
|
|
460
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
461
|
+
outcome: evt.type === "harness.run.error" ? "error" : normalizeDiagnosticValue(evt.outcome),
|
|
462
|
+
phase: evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.phase) : "none",
|
|
463
|
+
plugin: normalizeDiagnosticValue(evt.pluginId),
|
|
464
|
+
provider: normalizeDiagnosticValue(evt.provider)
|
|
465
|
+
};
|
|
466
|
+
store.histogram("openclaw_harness_run_duration_seconds", "Agent harness run duration in seconds.", labels, seconds(evt.durationMs));
|
|
467
|
+
store.counter("openclaw_harness_run_total", "Agent harness runs completed by outcome.", labels);
|
|
442
468
|
return;
|
|
469
|
+
}
|
|
443
470
|
case "message.processed": {
|
|
444
471
|
const labels = {
|
|
445
472
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
@@ -500,16 +527,31 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
500
527
|
store.histogram("openclaw_message_delivery_duration_seconds", "Outbound message delivery duration in seconds.", labels, seconds(evt.durationMs));
|
|
501
528
|
return;
|
|
502
529
|
}
|
|
503
|
-
case "talk.event":
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
530
|
+
case "talk.event": {
|
|
531
|
+
const labels = {
|
|
532
|
+
brain: normalizeDiagnosticValue(evt.brain),
|
|
533
|
+
event_type: normalizeDiagnosticValue(evt.talkEventType),
|
|
534
|
+
mode: normalizeDiagnosticValue(evt.mode),
|
|
535
|
+
provider: normalizeDiagnosticValue(evt.provider),
|
|
536
|
+
transport: normalizeDiagnosticValue(evt.transport)
|
|
537
|
+
};
|
|
538
|
+
store.counter("openclaw_talk_event_total", "Talk events emitted by type.", labels);
|
|
539
|
+
store.histogram("openclaw_talk_event_duration_seconds", "Talk event duration in seconds when reported.", labels, seconds(evt.durationMs));
|
|
540
|
+
store.histogram("openclaw_talk_audio_bytes", "Talk audio frame byte lengths.", labels, asNonNegativeFiniteNumber(evt.byteLength), BYTE_BUCKETS);
|
|
507
541
|
return;
|
|
542
|
+
}
|
|
508
543
|
case "session.recovery.requested":
|
|
509
|
-
case "session.recovery.completed":
|
|
510
|
-
|
|
511
|
-
|
|
544
|
+
case "session.recovery.completed": {
|
|
545
|
+
const labels = {
|
|
546
|
+
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
547
|
+
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
548
|
+
state: evt.state,
|
|
549
|
+
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
550
|
+
};
|
|
551
|
+
store.counter("openclaw_session_recovery_total", "Session recovery observations by status and action.", labels);
|
|
552
|
+
store.histogram("openclaw_session_recovery_age_seconds", "Age of sessions selected for recovery in seconds.", labels, seconds(evt.ageMs));
|
|
512
553
|
return;
|
|
554
|
+
}
|
|
513
555
|
case "queue.lane.enqueue":
|
|
514
556
|
case "queue.lane.dequeue":
|
|
515
557
|
store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", { lane: normalizeDiagnosticLane(evt.lane) }, asNonNegativeFiniteNumber(evt.queueSize));
|
|
@@ -522,10 +564,15 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
522
564
|
});
|
|
523
565
|
if (evt.queueDepth !== void 0) store.gauge("openclaw_session_queue_depth", "Latest observed session queue depth.", { state: evt.state }, asNonNegativeFiniteNumber(evt.queueDepth));
|
|
524
566
|
return;
|
|
525
|
-
case "session.stuck":
|
|
526
|
-
|
|
527
|
-
|
|
567
|
+
case "session.stuck": {
|
|
568
|
+
const labels = {
|
|
569
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
570
|
+
state: evt.state
|
|
571
|
+
};
|
|
572
|
+
store.counter("openclaw_session_stuck_total", "Stale session bookkeeping observations with no active work.", labels);
|
|
573
|
+
store.histogram("openclaw_session_stuck_age_seconds", "Age of stale session bookkeeping observations in seconds.", labels, seconds(evt.ageMs));
|
|
528
574
|
return;
|
|
575
|
+
}
|
|
529
576
|
case "session.turn.created":
|
|
530
577
|
store.counter("openclaw_session_turn_created_total", "Agent session turns created.", {
|
|
531
578
|
agent: normalizeDiagnosticValue(evt.agentId),
|
|
@@ -536,45 +583,29 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
536
583
|
case "diagnostic.child_process.spawn":
|
|
537
584
|
store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", { family: normalizeDiagnosticValue(evt.family) }, asNonNegativeFiniteNumber(evt.count) ?? 0);
|
|
538
585
|
return;
|
|
539
|
-
case "diagnostic.memory.sample":
|
|
540
|
-
|
|
541
|
-
["rss", "rssBytes"],
|
|
542
|
-
["heap_total", "heapTotalBytes"],
|
|
543
|
-
["heap_used", "heapUsedBytes"],
|
|
544
|
-
["external", "externalBytes"],
|
|
545
|
-
["array_buffers", "arrayBuffersBytes"],
|
|
546
|
-
["worker_heap_total", "workerHeapTotalBytes"],
|
|
547
|
-
["worker_heap_used", "workerHeapUsedBytes"]
|
|
548
|
-
]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, asNonNegativeFiniteNumber(evt.memory[field]));
|
|
549
|
-
for (const [name, field] of [["openclaw_worker_count", "workerCount"], ["openclaw_worker_heap_sampled_count", "workerHeapSampledCount"]]) store.gauge(name, "Worker isolate counts.", {}, asNonNegativeFiniteNumber(evt.memory[field]));
|
|
550
|
-
const workerHeaps = /* @__PURE__ */ new Map();
|
|
551
|
-
for (const worker of evt.memory.workerHeaps ?? []) {
|
|
552
|
-
const heapUsed = asNonNegativeFiniteNumber(worker.heapUsed);
|
|
553
|
-
if (heapUsed !== void 0) workerHeaps.set(worker.script, (workerHeaps.get(worker.script) ?? 0) + heapUsed);
|
|
554
|
-
}
|
|
555
|
-
store.clearGauges("openclaw_worker_heap_used_bytes");
|
|
556
|
-
for (const [script, heapUsed] of workerHeaps) store.gauge("openclaw_worker_heap_used_bytes", "Latest live Worker heap usage by bounded script basename.", { script }, heapUsed);
|
|
557
|
-
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(evt.memory.rssBytes), BYTE_BUCKETS);
|
|
586
|
+
case "diagnostic.memory.sample":
|
|
587
|
+
recordMemorySample(store, evt.memory, BYTE_BUCKETS);
|
|
558
588
|
return;
|
|
559
|
-
}
|
|
560
589
|
case "diagnostic.memory.pressure":
|
|
561
590
|
store.counter("openclaw_memory_pressure_total", "Memory pressure events by level and reason.", {
|
|
562
591
|
level: evt.level,
|
|
563
592
|
reason: evt.reason
|
|
564
593
|
});
|
|
565
594
|
return;
|
|
566
|
-
case "diagnostic.liveness.warning":
|
|
567
|
-
|
|
595
|
+
case "diagnostic.liveness.warning": {
|
|
596
|
+
const labels = { reason: normalizeDiagnosticValue(evt.reasons.join(":"), "unknown") };
|
|
597
|
+
store.counter("openclaw_liveness_warning_total", "Diagnostic liveness warning events.", labels);
|
|
568
598
|
for (const state of [
|
|
569
599
|
"active",
|
|
570
600
|
"waiting",
|
|
571
601
|
"queued"
|
|
572
602
|
]) store.gauge("openclaw_liveness_sessions", "Latest session counts reported with diagnostic liveness warnings.", { state }, asNonNegativeFiniteNumber(evt[state]));
|
|
573
|
-
store.histogram("openclaw_liveness_event_loop_delay_p99_seconds", "P99 event-loop delay reported by diagnostic liveness warnings in seconds.",
|
|
574
|
-
store.histogram("openclaw_liveness_event_loop_delay_max_seconds", "Maximum event-loop delay reported by diagnostic liveness warnings in seconds.",
|
|
575
|
-
store.histogram("openclaw_liveness_event_loop_utilization_ratio", "Event-loop utilization reported by diagnostic liveness warnings.",
|
|
576
|
-
store.histogram("openclaw_liveness_cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.",
|
|
603
|
+
store.histogram("openclaw_liveness_event_loop_delay_p99_seconds", "P99 event-loop delay reported by diagnostic liveness warnings in seconds.", labels, seconds(evt.eventLoopDelayP99Ms));
|
|
604
|
+
store.histogram("openclaw_liveness_event_loop_delay_max_seconds", "Maximum event-loop delay reported by diagnostic liveness warnings in seconds.", labels, seconds(evt.eventLoopDelayMaxMs));
|
|
605
|
+
store.histogram("openclaw_liveness_event_loop_utilization_ratio", "Event-loop utilization reported by diagnostic liveness warnings.", labels, asNonNegativeFiniteNumber(evt.eventLoopUtilization), RATIO_BUCKETS);
|
|
606
|
+
store.histogram("openclaw_liveness_cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.", labels, asNonNegativeFiniteNumber(evt.cpuCoreRatio), RATIO_BUCKETS);
|
|
577
607
|
return;
|
|
608
|
+
}
|
|
578
609
|
case "diagnostic.async_queue.dropped":
|
|
579
610
|
for (const [dropClass, field] of [
|
|
580
611
|
["total", "droppedEvents"],
|
|
@@ -596,9 +627,17 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
596
627
|
status: evt.status
|
|
597
628
|
});
|
|
598
629
|
return;
|
|
599
|
-
case "payload.large":
|
|
600
|
-
|
|
601
|
-
|
|
630
|
+
case "payload.large": {
|
|
631
|
+
const labels = {
|
|
632
|
+
action: evt.action,
|
|
633
|
+
channel: normalizeDiagnosticValue(evt.channel, "none"),
|
|
634
|
+
plugin: normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
635
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
636
|
+
surface: normalizeDiagnosticValue(evt.surface, "unknown")
|
|
637
|
+
};
|
|
638
|
+
store.counter("openclaw_payload_large_total", "Oversized payload diagnostics by surface and action.", labels);
|
|
639
|
+
store.histogram("openclaw_payload_large_bytes", "Oversized payload byte sizes by surface and action.", labels, asNonNegativeFiniteNumber(evt.bytes), BYTE_BUCKETS);
|
|
640
|
+
}
|
|
602
641
|
}
|
|
603
642
|
}
|
|
604
643
|
const METRICS_READ_SCOPE = "operator.read";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-prometheus",
|
|
3
|
-
"version": "2026.9.
|
|
3
|
+
"version": "2026.9.7",
|
|
4
4
|
"description": "OpenClaw diagnostics Prometheus exporter for runtime metrics.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
"minHostVersion": ">=2026.4.25"
|
|
19
19
|
},
|
|
20
20
|
"compat": {
|
|
21
|
-
"pluginApi": ">=2026.9.
|
|
21
|
+
"pluginApi": ">=2026.9.7"
|
|
22
22
|
},
|
|
23
23
|
"build": {
|
|
24
|
-
"openclawVersion": "2026.9.
|
|
24
|
+
"openclawVersion": "2026.9.7"
|
|
25
25
|
},
|
|
26
26
|
"release": {
|
|
27
27
|
"publishToClawHub": true,
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"assets/activity.svg"
|
|
40
40
|
],
|
|
41
41
|
"peerDependencies": {
|
|
42
|
-
"openclaw": ">=2026.9.
|
|
42
|
+
"openclaw": ">=2026.9.7"
|
|
43
43
|
},
|
|
44
44
|
"peerDependenciesMeta": {
|
|
45
45
|
"openclaw": {
|