@openclaw/diagnostics-prometheus 2026.10.1-beta.2 → 2026.10.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +237 -144
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -16,11 +16,8 @@ function metricKey(name, labels) {
|
|
|
16
16
|
function escapeHelp(value) {
|
|
17
17
|
return value.replace(/\\/g, "\\\\").replace(/\n/g, "\\n");
|
|
18
18
|
}
|
|
19
|
-
function escapeLabelValue(value) {
|
|
20
|
-
return value.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, "\\\"");
|
|
21
|
-
}
|
|
22
19
|
function formatLabelEntry([key, value]) {
|
|
23
|
-
return `${key}="${
|
|
20
|
+
return `${key}="${escapeHelp(value).replace(/"/g, "\\\"")}"`;
|
|
24
21
|
}
|
|
25
22
|
function formatLabels(labels) {
|
|
26
23
|
const entries = sortedLabels(labels);
|
|
@@ -55,6 +52,12 @@ const DURATION_BUCKETS_SECONDS = [
|
|
|
55
52
|
300,
|
|
56
53
|
600
|
|
57
54
|
];
|
|
55
|
+
const AGENT_DURATION_BUCKETS_SECONDS = [
|
|
56
|
+
...DURATION_BUCKETS_SECONDS,
|
|
57
|
+
900,
|
|
58
|
+
1800,
|
|
59
|
+
3600
|
|
60
|
+
];
|
|
58
61
|
const MAX_PROMETHEUS_SERIES = 2048;
|
|
59
62
|
const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
|
|
60
63
|
function createPrometheusMetricStore() {
|
|
@@ -113,10 +116,14 @@ function createPrometheusMetricStore() {
|
|
|
113
116
|
if (!sample) {
|
|
114
117
|
sample = {
|
|
115
118
|
buckets,
|
|
119
|
+
bucketPrefixes: [...buckets, "+Inf"].map((le) => `${name}_bucket${formatLabels({
|
|
120
|
+
...labels,
|
|
121
|
+
le: String(le)
|
|
122
|
+
})} `),
|
|
116
123
|
counts: buckets.map(() => 0),
|
|
117
124
|
count: 0,
|
|
118
125
|
help,
|
|
119
|
-
labels,
|
|
126
|
+
labels: formatLabels(labels),
|
|
120
127
|
sum: 0
|
|
121
128
|
};
|
|
122
129
|
histograms.set(key, sample);
|
|
@@ -167,7 +174,91 @@ function recordChildProcessSpawn(store, evt) {
|
|
|
167
174
|
}, asNonNegativeFiniteNumber(evt.count) ?? 0);
|
|
168
175
|
}
|
|
169
176
|
//#endregion
|
|
170
|
-
//#region extensions/diagnostics-prometheus/src/service-
|
|
177
|
+
//#region extensions/diagnostics-prometheus/src/service-memory.ts
|
|
178
|
+
function recordMemorySample(store, memory, byteBuckets) {
|
|
179
|
+
for (const worker of memory.workerLifecycle ?? []) {
|
|
180
|
+
store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
|
|
181
|
+
for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
|
|
182
|
+
script: worker.script,
|
|
183
|
+
reason
|
|
184
|
+
}, count);
|
|
185
|
+
}
|
|
186
|
+
for (const [kind, field] of [
|
|
187
|
+
["rss", "rssBytes"],
|
|
188
|
+
["heap_total", "heapTotalBytes"],
|
|
189
|
+
["heap_used", "heapUsedBytes"],
|
|
190
|
+
["external", "externalBytes"],
|
|
191
|
+
["array_buffers", "arrayBuffersBytes"],
|
|
192
|
+
["worker_heap_total", "workerHeapTotalBytes"],
|
|
193
|
+
["worker_heap_used", "workerHeapUsedBytes"]
|
|
194
|
+
]) store.gauge("openclaw_memory_bytes", "Latest process memory usage by memory kind.", { kind }, asNonNegativeFiniteNumber(memory[field]));
|
|
195
|
+
for (const [name, field] of [["openclaw_worker_count", "workerCount"], ["openclaw_worker_heap_sampled_count", "workerHeapSampledCount"]]) store.gauge(name, "Worker isolate counts.", {}, asNonNegativeFiniteNumber(memory[field]));
|
|
196
|
+
store.clearGauges("openclaw_heap_space_bytes");
|
|
197
|
+
for (const space of memory.heapSpaces ?? []) for (const [stat, field] of [
|
|
198
|
+
["used", "space_used_size"],
|
|
199
|
+
["size", "space_size"],
|
|
200
|
+
["available", "space_available_size"],
|
|
201
|
+
["physical", "physical_space_size"]
|
|
202
|
+
]) store.gauge("openclaw_heap_space_bytes", "Latest main-isolate V8 heap space usage in bytes.", {
|
|
203
|
+
space: space.space_name,
|
|
204
|
+
stat
|
|
205
|
+
}, space[field]);
|
|
206
|
+
const workerHeaps = /* @__PURE__ */ new Map();
|
|
207
|
+
for (const worker of memory.workerHeaps ?? []) {
|
|
208
|
+
const heapUsed = asNonNegativeFiniteNumber(worker.heapUsed);
|
|
209
|
+
if (heapUsed !== void 0) workerHeaps.set(worker.script, (workerHeaps.get(worker.script) ?? 0) + heapUsed);
|
|
210
|
+
}
|
|
211
|
+
store.clearGauges("openclaw_worker_heap_used_bytes");
|
|
212
|
+
for (const [script, heapUsed] of workerHeaps) store.gauge("openclaw_worker_heap_used_bytes", "Latest live Worker heap usage by bounded script basename.", { script }, heapUsed);
|
|
213
|
+
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
|
|
214
|
+
}
|
|
215
|
+
//#endregion
|
|
216
|
+
//#region extensions/diagnostics-prometheus/src/service-model-usage.ts
|
|
217
|
+
const TOKEN_BUCKETS = [
|
|
218
|
+
1,
|
|
219
|
+
4,
|
|
220
|
+
16,
|
|
221
|
+
64,
|
|
222
|
+
256,
|
|
223
|
+
1024,
|
|
224
|
+
4096,
|
|
225
|
+
16384,
|
|
226
|
+
65536,
|
|
227
|
+
262144,
|
|
228
|
+
1048576
|
|
229
|
+
];
|
|
230
|
+
function recordModelUsage(store, evt) {
|
|
231
|
+
const labels = {
|
|
232
|
+
agent: normalizeDiagnosticValue(evt.agentId),
|
|
233
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
234
|
+
model: normalizeDiagnosticValue(evt.model),
|
|
235
|
+
provider: normalizeDiagnosticValue(evt.provider)
|
|
236
|
+
};
|
|
237
|
+
const usage = evt.usage;
|
|
238
|
+
const recordTokens = (tokenType, value) => {
|
|
239
|
+
const amount = asNonNegativeFiniteNumber(value);
|
|
240
|
+
if (amount === void 0 || amount === 0) return;
|
|
241
|
+
store.counter("openclaw_model_tokens_total", "Model tokens reported by diagnostic usage events.", {
|
|
242
|
+
...labels,
|
|
243
|
+
token_type: tokenType
|
|
244
|
+
}, amount);
|
|
245
|
+
if (tokenType === "input" || tokenType === "output") store.histogram("openclaw_gen_ai_client_token_usage", "GenAI token usage distribution for input and output tokens.", {
|
|
246
|
+
model: labels.model,
|
|
247
|
+
provider: labels.provider,
|
|
248
|
+
token_type: tokenType
|
|
249
|
+
}, amount, TOKEN_BUCKETS);
|
|
250
|
+
};
|
|
251
|
+
recordTokens("input", usage.input);
|
|
252
|
+
recordTokens("output", usage.output);
|
|
253
|
+
recordTokens("cache_read", usage.cacheRead);
|
|
254
|
+
recordTokens("cache_write", usage.cacheWrite);
|
|
255
|
+
recordTokens("prompt", usage.promptTokens);
|
|
256
|
+
recordTokens("total", usage.total);
|
|
257
|
+
store.counter("openclaw_model_cost_usd_total", "Estimated model cost in USD reported by diagnostic usage events.", labels, asNonNegativeFiniteNumber(evt.costUsd) ?? 0);
|
|
258
|
+
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
259
|
+
}
|
|
260
|
+
//#endregion
|
|
261
|
+
//#region extensions/diagnostics-prometheus/src/service-operation-timing.ts
|
|
171
262
|
const RESPONSE_BYTE_BUCKETS = Array.from({ length: 17 }, (_, index) => 1024 * 2 ** index);
|
|
172
263
|
const HEAP_DELTA_BYTE_BUCKETS = [
|
|
173
264
|
...RESPONSE_BYTE_BUCKETS.toReversed().map((bytes) => -bytes),
|
|
@@ -184,10 +275,35 @@ const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
|
|
|
184
275
|
"projection_final",
|
|
185
276
|
"delivery"
|
|
186
277
|
]);
|
|
187
|
-
|
|
278
|
+
const CHAT_SEND_PHASE = /^chat\.send\.(authority|admission|preparation|attachments|replyContext|authoring|persist|runAdmission|replyInitialization|snapshot|worktree|effects|response|dispatch)$/;
|
|
279
|
+
function recordOperationTimingEvent(store, evt, metadata) {
|
|
188
280
|
switch (evt.type) {
|
|
281
|
+
case "gateway.http.cancelled":
|
|
282
|
+
store.counter("openclaw_gateway_http_cancelled_total", "Gateway HTTP requests cancelled before completion.", { source: evt.source });
|
|
283
|
+
return;
|
|
284
|
+
case "diagnostic.gc":
|
|
285
|
+
store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds for the hosting JavaScript isolate.", {}, seconds(evt.durationMs));
|
|
286
|
+
return;
|
|
287
|
+
case "gateway.event_loop.sample":
|
|
288
|
+
store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
|
|
289
|
+
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
290
|
+
return;
|
|
189
291
|
case "diagnostic.phase.completed": {
|
|
190
|
-
if (!metadata.trusted
|
|
292
|
+
if (!metadata.trusted) return;
|
|
293
|
+
if (evt.name === "worktree.preparation") {
|
|
294
|
+
recordWorktreePreparation(store, evt);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const sendPhase = CHAT_SEND_PHASE.exec(evt.name)?.[1];
|
|
298
|
+
if (sendPhase) {
|
|
299
|
+
const stage = evt.details?.stage;
|
|
300
|
+
if (stage === "request" || stage === "startup") store.histogram("openclaw_chat_send_phase_seconds", "Elapsed chat.send owner phases before acknowledgement and during run startup.", {
|
|
301
|
+
phase: sendPhase,
|
|
302
|
+
stage
|
|
303
|
+
}, seconds(evt.durationMs));
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
if (!evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
|
|
191
307
|
const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
|
|
192
308
|
const duration = seconds(evt.durationMs);
|
|
193
309
|
if (!CATALOG_LIST_PHASES.has(phase) || duration === void 0) return;
|
|
@@ -224,60 +340,103 @@ function recordGatewayRpcEvent(store, evt, metadata) {
|
|
|
224
340
|
}
|
|
225
341
|
}
|
|
226
342
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
343
|
+
function recordWorktreePreparation(store, evt) {
|
|
344
|
+
const { kind, template, outcome } = evt.details ?? {};
|
|
345
|
+
if (kind !== "managed" && kind !== "sandbox" || ![
|
|
346
|
+
"warm",
|
|
347
|
+
"cold",
|
|
348
|
+
"unavailable",
|
|
349
|
+
"reused"
|
|
350
|
+
].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
|
|
351
|
+
for (const phase of [
|
|
352
|
+
"total",
|
|
353
|
+
"allocate",
|
|
354
|
+
"checkout",
|
|
355
|
+
"setup",
|
|
356
|
+
"templatePrepare",
|
|
357
|
+
"templateApply",
|
|
358
|
+
"snapshot",
|
|
359
|
+
"synchronizeCanonical",
|
|
360
|
+
"synchronizeProjection",
|
|
361
|
+
"containerStart",
|
|
362
|
+
"workspaceLayout"
|
|
363
|
+
]) {
|
|
364
|
+
const elapsed = phase === "total" ? evt.durationMs : evt.details?.[phase];
|
|
365
|
+
store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
|
|
366
|
+
kind,
|
|
367
|
+
template: String(template),
|
|
368
|
+
outcome,
|
|
369
|
+
phase
|
|
370
|
+
}, typeof elapsed === "number" ? seconds(elapsed) : void 0);
|
|
236
371
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
372
|
+
}
|
|
373
|
+
//#endregion
|
|
374
|
+
//#region extensions/diagnostics-prometheus/src/service-sessions.ts
|
|
375
|
+
function createGatewayWorkMetricsRecorder(store) {
|
|
376
|
+
return (snapshot) => {
|
|
377
|
+
if (!snapshot) {
|
|
378
|
+
store.clearGauges("openclaw_sessions_active");
|
|
379
|
+
store.clearGauges("openclaw_gateway_active_work");
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
|
|
383
|
+
for (const kind of [
|
|
384
|
+
"agentRuns",
|
|
385
|
+
"chatRuns",
|
|
386
|
+
"queuedTurns"
|
|
387
|
+
]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
function recordSessionDiagnosticEvent(store, evt) {
|
|
391
|
+
switch (evt.type) {
|
|
392
|
+
case "session.recovery.requested":
|
|
393
|
+
case "session.recovery.completed": {
|
|
394
|
+
const labels = {
|
|
395
|
+
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
396
|
+
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
397
|
+
state: evt.state,
|
|
398
|
+
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
399
|
+
};
|
|
400
|
+
store.counter("openclaw_session_recovery_total", "Session recovery observations by status and action.", labels);
|
|
401
|
+
store.histogram("openclaw_session_recovery_age_seconds", "Age of sessions selected for recovery in seconds.", labels, seconds(evt.ageMs));
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
case "session.state":
|
|
405
|
+
store.counter("openclaw_session_state_total", "Cumulative session state observations since exporter start; not current sessions.", {
|
|
406
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
407
|
+
state: evt.state
|
|
408
|
+
});
|
|
409
|
+
if (evt.queueDepth !== void 0) store.gauge("openclaw_session_queue_depth", "Latest observed session queue depth.", { state: evt.state }, asNonNegativeFiniteNumber(evt.queueDepth));
|
|
410
|
+
return;
|
|
411
|
+
case "session.stuck": {
|
|
412
|
+
const labels = {
|
|
413
|
+
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
414
|
+
state: evt.state
|
|
415
|
+
};
|
|
416
|
+
store.counter("openclaw_session_stuck_total", "Stale session bookkeeping observations with no active work.", labels);
|
|
417
|
+
store.histogram("openclaw_session_stuck_age_seconds", "Age of stale session bookkeeping observations in seconds.", labels, seconds(evt.ageMs));
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
case "session.turn.created": store.counter("openclaw_session_turn_created_total", "Agent session turns created.", {
|
|
421
|
+
agent: normalizeDiagnosticValue(evt.agentId),
|
|
422
|
+
channel: normalizeDiagnosticValue(evt.channel),
|
|
423
|
+
trigger: evt.trigger
|
|
424
|
+
});
|
|
261
425
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
426
|
+
}
|
|
427
|
+
//#endregion
|
|
428
|
+
//#region extensions/diagnostics-prometheus/src/service-worker.ts
|
|
429
|
+
function recordWorkerRequest(store, event) {
|
|
430
|
+
store.gauge("openclaw_worker_queue_depth", "Requests awaiting dispatch across worker owners of this kind.", { kind: event.kind }, event.queueDepth);
|
|
431
|
+
const labels = {
|
|
432
|
+
kind: event.kind,
|
|
433
|
+
request_class: event.requestClass
|
|
434
|
+
};
|
|
435
|
+
store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(event.queueWaitMs));
|
|
436
|
+
store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(event.durationMs));
|
|
265
437
|
}
|
|
266
438
|
//#endregion
|
|
267
439
|
//#region extensions/diagnostics-prometheus/src/service.ts
|
|
268
|
-
const TOKEN_BUCKETS = [
|
|
269
|
-
1,
|
|
270
|
-
4,
|
|
271
|
-
16,
|
|
272
|
-
64,
|
|
273
|
-
256,
|
|
274
|
-
1024,
|
|
275
|
-
4096,
|
|
276
|
-
16384,
|
|
277
|
-
65536,
|
|
278
|
-
262144,
|
|
279
|
-
1048576
|
|
280
|
-
];
|
|
281
440
|
const BYTE_BUCKETS = [
|
|
282
441
|
1024,
|
|
283
442
|
4096,
|
|
@@ -332,23 +491,10 @@ function renderPrometheusMetrics(store) {
|
|
|
332
491
|
for (const [key, sample] of snapshot.histograms) {
|
|
333
492
|
const name = key.split("|", 1)[0] ?? "";
|
|
334
493
|
emitHeader(name, "histogram", sample.help);
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
});
|
|
340
|
-
const boundIndex = bucketLabels.findIndex(([labelKey]) => labelKey === "le");
|
|
341
|
-
const bucketFragments = bucketLabels.map(formatLabelEntry);
|
|
342
|
-
for (let index = 0; index < sample.buckets.length; index += 1) {
|
|
343
|
-
const bucket = sample.buckets[index];
|
|
344
|
-
if (bucket === void 0) continue;
|
|
345
|
-
bucketFragments[boundIndex] = `le="${String(bucket)}"`;
|
|
346
|
-
lines.push(`${name}_bucket{${bucketFragments.join(",")}} ${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
|
|
347
|
-
}
|
|
348
|
-
bucketFragments[boundIndex] = "le=\"+Inf\"";
|
|
349
|
-
lines.push(`${name}_bucket{${bucketFragments.join(",")}} ${formatPrometheusNumber(sample.count)}`);
|
|
350
|
-
lines.push(`${name}_sum${labels} ${formatPrometheusNumber(sample.sum)}`);
|
|
351
|
-
lines.push(`${name}_count${labels} ${formatPrometheusNumber(sample.count)}`);
|
|
494
|
+
for (let index = 0; index < sample.buckets.length; index += 1) lines.push(`${sample.bucketPrefixes[index]}${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
|
|
495
|
+
lines.push(`${sample.bucketPrefixes[sample.buckets.length]}${formatPrometheusNumber(sample.count)}`);
|
|
496
|
+
lines.push(`${name}_sum${sample.labels} ${formatPrometheusNumber(sample.sum)}`);
|
|
497
|
+
lines.push(`${name}_count${sample.labels} ${formatPrometheusNumber(sample.count)}`);
|
|
352
498
|
}
|
|
353
499
|
lines.push("");
|
|
354
500
|
return lines.join("\n");
|
|
@@ -359,49 +505,18 @@ function webhookLabels(evt) {
|
|
|
359
505
|
webhook: normalizeDiagnosticValue(evt.updateType)
|
|
360
506
|
};
|
|
361
507
|
}
|
|
362
|
-
function recordModelUsage(store, evt) {
|
|
363
|
-
const labels = {
|
|
364
|
-
agent: normalizeDiagnosticValue(evt.agentId),
|
|
365
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
366
|
-
model: normalizeDiagnosticValue(evt.model),
|
|
367
|
-
provider: normalizeDiagnosticValue(evt.provider)
|
|
368
|
-
};
|
|
369
|
-
const usage = evt.usage;
|
|
370
|
-
const recordTokens = (tokenType, value) => {
|
|
371
|
-
const amount = asNonNegativeFiniteNumber(value);
|
|
372
|
-
if (amount === void 0 || amount === 0) return;
|
|
373
|
-
store.counter("openclaw_model_tokens_total", "Model tokens reported by diagnostic usage events.", {
|
|
374
|
-
...labels,
|
|
375
|
-
token_type: tokenType
|
|
376
|
-
}, amount);
|
|
377
|
-
if (tokenType === "input" || tokenType === "output") store.histogram("openclaw_gen_ai_client_token_usage", "GenAI token usage distribution for input and output tokens.", {
|
|
378
|
-
model: labels.model,
|
|
379
|
-
provider: labels.provider,
|
|
380
|
-
token_type: tokenType
|
|
381
|
-
}, amount, TOKEN_BUCKETS);
|
|
382
|
-
};
|
|
383
|
-
recordTokens("input", usage.input);
|
|
384
|
-
recordTokens("output", usage.output);
|
|
385
|
-
recordTokens("cache_read", usage.cacheRead);
|
|
386
|
-
recordTokens("cache_write", usage.cacheWrite);
|
|
387
|
-
recordTokens("prompt", usage.promptTokens);
|
|
388
|
-
recordTokens("total", usage.total);
|
|
389
|
-
store.counter("openclaw_model_cost_usd_total", "Estimated model cost in USD reported by diagnostic usage events.", labels, asNonNegativeFiniteNumber(evt.costUsd) ?? 0);
|
|
390
|
-
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
391
|
-
}
|
|
392
508
|
function recordDiagnosticEvent(store, evt, metadata) {
|
|
393
509
|
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
394
510
|
switch (evt.type) {
|
|
511
|
+
case "worker.request":
|
|
512
|
+
if (metadata.trusted) recordWorkerRequest(store, evt);
|
|
513
|
+
return;
|
|
395
514
|
case "diagnostic.phase.completed":
|
|
396
515
|
case "gateway.rpc":
|
|
397
|
-
|
|
398
|
-
return;
|
|
516
|
+
case "gateway.http.cancelled":
|
|
399
517
|
case "diagnostic.gc":
|
|
400
|
-
store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds for the hosting JavaScript isolate.", {}, seconds(evt.durationMs));
|
|
401
|
-
return;
|
|
402
518
|
case "gateway.event_loop.sample":
|
|
403
|
-
store
|
|
404
|
-
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
519
|
+
recordOperationTimingEvent(store, evt, metadata);
|
|
405
520
|
return;
|
|
406
521
|
case "model.usage":
|
|
407
522
|
recordModelUsage(store, evt);
|
|
@@ -415,7 +530,7 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
415
530
|
provider: normalizeDiagnosticValue(evt.provider),
|
|
416
531
|
trigger: normalizeDiagnosticValue(evt.trigger)
|
|
417
532
|
};
|
|
418
|
-
store.histogram("openclaw_run_duration_seconds", "Agent run duration in seconds.", labels, seconds(evt.durationMs));
|
|
533
|
+
store.histogram("openclaw_run_duration_seconds", "Agent run duration in seconds.", labels, seconds(evt.durationMs), AGENT_DURATION_BUCKETS_SECONDS);
|
|
419
534
|
store.counter("openclaw_run_completed_total", "Agent runs completed by outcome.", labels);
|
|
420
535
|
return;
|
|
421
536
|
}
|
|
@@ -430,7 +545,7 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
430
545
|
provider: normalizeDiagnosticValue(evt.provider),
|
|
431
546
|
transport: normalizeDiagnosticValue(evt.transport)
|
|
432
547
|
};
|
|
433
|
-
store.histogram("openclaw_model_call_duration_seconds", "Model request or synthetic agent-turn duration in seconds.", labels, seconds(evt.durationMs));
|
|
548
|
+
store.histogram("openclaw_model_call_duration_seconds", "Model request or synthetic agent-turn duration in seconds.", labels, seconds(evt.durationMs), AGENT_DURATION_BUCKETS_SECONDS);
|
|
434
549
|
store.counter("openclaw_model_call_total", "Model requests or synthetic agent turns completed by outcome.", labels);
|
|
435
550
|
return;
|
|
436
551
|
}
|
|
@@ -494,7 +609,7 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
494
609
|
plugin: normalizeDiagnosticValue(evt.pluginId),
|
|
495
610
|
provider: normalizeDiagnosticValue(evt.provider)
|
|
496
611
|
};
|
|
497
|
-
store.histogram("openclaw_harness_run_duration_seconds", "Agent harness run duration in seconds.", labels, seconds(evt.durationMs));
|
|
612
|
+
store.histogram("openclaw_harness_run_duration_seconds", "Agent harness run duration in seconds.", labels, seconds(evt.durationMs), AGENT_DURATION_BUCKETS_SECONDS);
|
|
498
613
|
store.counter("openclaw_harness_run_total", "Agent harness runs completed by outcome.", labels);
|
|
499
614
|
return;
|
|
500
615
|
}
|
|
@@ -543,7 +658,7 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
543
658
|
source: normalizeDiagnosticValue(evt.source)
|
|
544
659
|
};
|
|
545
660
|
store.counter("openclaw_message_dispatch_completed_total", "Inbound message dispatch attempts completed by outcome.", labels);
|
|
546
|
-
store.histogram("openclaw_message_dispatch_duration_seconds", "Inbound message dispatch duration in seconds.", labels, seconds(evt.durationMs));
|
|
661
|
+
store.histogram("openclaw_message_dispatch_duration_seconds", "Inbound message dispatch duration in seconds.", labels, seconds(evt.durationMs), AGENT_DURATION_BUCKETS_SECONDS);
|
|
547
662
|
return;
|
|
548
663
|
}
|
|
549
664
|
case "message.delivery.completed":
|
|
@@ -572,44 +687,18 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
572
687
|
return;
|
|
573
688
|
}
|
|
574
689
|
case "session.recovery.requested":
|
|
575
|
-
case "session.recovery.completed":
|
|
576
|
-
|
|
577
|
-
action: evt.type === "session.recovery.completed" ? normalizeDiagnosticValue(evt.action, "unknown") : evt.allowActiveAbort ? "abort" : "recover",
|
|
578
|
-
active_work_kind: normalizeDiagnosticValue(evt.activeWorkKind, "none"),
|
|
579
|
-
state: evt.state,
|
|
580
|
-
status: evt.type === "session.recovery.completed" ? evt.status : "requested"
|
|
581
|
-
};
|
|
582
|
-
store.counter("openclaw_session_recovery_total", "Session recovery observations by status and action.", labels);
|
|
583
|
-
store.histogram("openclaw_session_recovery_age_seconds", "Age of sessions selected for recovery in seconds.", labels, seconds(evt.ageMs));
|
|
690
|
+
case "session.recovery.completed":
|
|
691
|
+
recordSessionDiagnosticEvent(store, evt);
|
|
584
692
|
return;
|
|
585
|
-
}
|
|
586
693
|
case "queue.lane.enqueue":
|
|
587
694
|
case "queue.lane.dequeue":
|
|
588
695
|
store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", { lane: normalizeDiagnosticLane(evt.lane) }, asNonNegativeFiniteNumber(evt.queueSize));
|
|
589
696
|
if (evt.type === "queue.lane.dequeue") store.histogram("openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", { lane: normalizeDiagnosticLane(evt.lane) }, seconds(evt.waitMs));
|
|
590
697
|
return;
|
|
591
698
|
case "session.state":
|
|
592
|
-
|
|
593
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
594
|
-
state: evt.state
|
|
595
|
-
});
|
|
596
|
-
if (evt.queueDepth !== void 0) store.gauge("openclaw_session_queue_depth", "Latest observed session queue depth.", { state: evt.state }, asNonNegativeFiniteNumber(evt.queueDepth));
|
|
597
|
-
return;
|
|
598
|
-
case "session.stuck": {
|
|
599
|
-
const labels = {
|
|
600
|
-
reason: normalizeDiagnosticValue(evt.reason, "none"),
|
|
601
|
-
state: evt.state
|
|
602
|
-
};
|
|
603
|
-
store.counter("openclaw_session_stuck_total", "Stale session bookkeeping observations with no active work.", labels);
|
|
604
|
-
store.histogram("openclaw_session_stuck_age_seconds", "Age of stale session bookkeeping observations in seconds.", labels, seconds(evt.ageMs));
|
|
605
|
-
return;
|
|
606
|
-
}
|
|
699
|
+
case "session.stuck":
|
|
607
700
|
case "session.turn.created":
|
|
608
|
-
store
|
|
609
|
-
agent: normalizeDiagnosticValue(evt.agentId),
|
|
610
|
-
channel: normalizeDiagnosticValue(evt.channel),
|
|
611
|
-
trigger: evt.trigger
|
|
612
|
-
});
|
|
701
|
+
recordSessionDiagnosticEvent(store, evt);
|
|
613
702
|
return;
|
|
614
703
|
case "diagnostic.child_process.spawn":
|
|
615
704
|
recordChildProcessSpawn(store, evt);
|
|
@@ -713,6 +802,7 @@ function createMetricsHandler(store) {
|
|
|
713
802
|
function createDiagnosticsPrometheusExporter() {
|
|
714
803
|
const store = createPrometheusMetricStore();
|
|
715
804
|
let unsubscribe;
|
|
805
|
+
let unsubscribeWork;
|
|
716
806
|
let internalDiagnostics;
|
|
717
807
|
const reportExporterStatus = (update) => {
|
|
718
808
|
try {
|
|
@@ -741,6 +831,7 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
741
831
|
process_instance_id: identity.processInstanceId,
|
|
742
832
|
...identity.buildId ? { build_id: identity.buildId } : {}
|
|
743
833
|
}, 1);
|
|
834
|
+
unsubscribeWork = ctx.internalDiagnostics?.onGatewayWorkMetrics?.(createGatewayWorkMetricsRecorder(store));
|
|
744
835
|
unsubscribe = subscribe((event, metadata) => {
|
|
745
836
|
try {
|
|
746
837
|
recordDiagnosticEvent(store, event, metadata);
|
|
@@ -757,6 +848,8 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
757
848
|
});
|
|
758
849
|
},
|
|
759
850
|
stop() {
|
|
851
|
+
unsubscribeWork?.();
|
|
852
|
+
unsubscribeWork = void 0;
|
|
760
853
|
unsubscribe?.();
|
|
761
854
|
unsubscribe = void 0;
|
|
762
855
|
reportExporterStatus({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-prometheus",
|
|
3
|
-
"version": "2026.10.
|
|
3
|
+
"version": "2026.10.5-beta.1",
|
|
4
4
|
"description": "OpenClaw diagnostics Prometheus exporter for runtime metrics.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
"minHostVersion": ">=2026.4.25"
|
|
19
19
|
},
|
|
20
20
|
"compat": {
|
|
21
|
-
"pluginApi": ">=2026.10.
|
|
21
|
+
"pluginApi": ">=2026.10.5-beta.1"
|
|
22
22
|
},
|
|
23
23
|
"build": {
|
|
24
|
-
"openclawVersion": "2026.10.
|
|
24
|
+
"openclawVersion": "2026.10.5-beta.1"
|
|
25
25
|
},
|
|
26
26
|
"release": {
|
|
27
27
|
"publishToClawHub": true,
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"assets/activity.svg"
|
|
40
40
|
],
|
|
41
41
|
"peerDependencies": {
|
|
42
|
-
"openclaw": ">=2026.10.
|
|
42
|
+
"openclaw": ">=2026.10.5-beta.1"
|
|
43
43
|
},
|
|
44
44
|
"peerDependenciesMeta": {
|
|
45
45
|
"openclaw": {
|