@openclaw/diagnostics-prometheus 2026.10.10-beta.5 → 2026.11.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -4,7 +4,8 @@ import { isDiagnosticsEnabled, normalizeDiagnosticLane, normalizeDiagnosticValue
|
|
|
4
4
|
import { asNonNegativeFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
|
|
5
5
|
import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
|
|
6
6
|
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
7
|
-
|
|
7
|
+
import { expectDefined } from "openclaw/plugin-sdk/expect-runtime";
|
|
8
|
+
// extensions/diagnostics-prometheus/src/prometheus-format.ts
|
|
8
9
|
function sortedLabels(labels) {
|
|
9
10
|
const entries = Object.entries(labels);
|
|
10
11
|
entries.sort(([left], [right]) => left.localeCompare(right));
|
|
@@ -32,8 +33,166 @@ function seconds(ms) {
|
|
|
32
33
|
const value = asNonNegativeFiniteNumber(ms);
|
|
33
34
|
return value === void 0 ? void 0 : value / 1e3;
|
|
34
35
|
}
|
|
35
|
-
|
|
36
|
-
|
|
36
|
+
// extensions/diagnostics-prometheus/src/prometheus-family-budget.ts
|
|
37
|
+
const PROMETHEUS_FAMILY_RESERVES = {
|
|
38
|
+
openclaw_chat_history_requests_total: 4,
|
|
39
|
+
openclaw_chat_history_response_bytes_total: 4,
|
|
40
|
+
openclaw_chat_send_phase_seconds: 32,
|
|
41
|
+
openclaw_child_process_spawn_total: 4,
|
|
42
|
+
openclaw_code_mode_worker_heap_bytes: 128,
|
|
43
|
+
openclaw_code_mode_worker_memory_bytes: 128,
|
|
44
|
+
openclaw_control_ui_document_duration_seconds: 4,
|
|
45
|
+
openclaw_cron_list_requests_total: 4,
|
|
46
|
+
openclaw_diagnostic_async_queue_dropped_total: 4,
|
|
47
|
+
openclaw_diagnostic_async_queue_length: 4,
|
|
48
|
+
openclaw_diff_cache_misses_total: 4,
|
|
49
|
+
openclaw_diff_stage_seconds: 4,
|
|
50
|
+
openclaw_gateway_active_work: 4,
|
|
51
|
+
openclaw_gateway_build_info: 4,
|
|
52
|
+
openclaw_gateway_event_loop_delay_max_seconds: 4,
|
|
53
|
+
openclaw_gateway_event_loop_observed_seconds_total: 4,
|
|
54
|
+
openclaw_gateway_http_cancelled_total: 4,
|
|
55
|
+
openclaw_gateway_main_thread_stall_seconds: 4,
|
|
56
|
+
openclaw_gateway_main_thread_stalls_dropped_total: 4,
|
|
57
|
+
openclaw_gateway_rpc_admission_seconds: 256,
|
|
58
|
+
openclaw_gateway_rpc_client_requests_total: 512,
|
|
59
|
+
openclaw_gateway_rpc_first_response_seconds: 256,
|
|
60
|
+
openclaw_gateway_rpc_handler_heap_delta_bytes: 256,
|
|
61
|
+
openclaw_gateway_rpc_handler_heap_delta_exclusive_total: 256,
|
|
62
|
+
openclaw_gateway_rpc_handler_seconds: 256,
|
|
63
|
+
openclaw_gateway_rpc_outcomes_total: 256,
|
|
64
|
+
openclaw_gateway_rpc_queue_wait_seconds: 256,
|
|
65
|
+
openclaw_gateway_rpc_requests_total: 256,
|
|
66
|
+
openclaw_gateway_rpc_response_bytes: 256,
|
|
67
|
+
openclaw_gateway_rpc_stage_seconds: 256,
|
|
68
|
+
openclaw_gateway_rpc_stage_thread_cpu_seconds: 256,
|
|
69
|
+
openclaw_gateway_ws_event_bytes_total: 256,
|
|
70
|
+
openclaw_gateway_ws_event_frame_bytes: 2048,
|
|
71
|
+
openclaw_gateway_ws_event_wire_bytes_total: 256,
|
|
72
|
+
openclaw_gateway_ws_events_total: 256,
|
|
73
|
+
openclaw_gateway_ws_operator_connections: 4,
|
|
74
|
+
openclaw_gateway_ws_presence_bytes_total: 4,
|
|
75
|
+
openclaw_gateway_ws_presence_frames_total: 4,
|
|
76
|
+
openclaw_gateway_ws_session_event_bytes_total: 256,
|
|
77
|
+
openclaw_gateway_ws_session_event_frames_total: 256,
|
|
78
|
+
openclaw_gc_duration_seconds: 384,
|
|
79
|
+
openclaw_gen_ai_client_token_usage: 128,
|
|
80
|
+
openclaw_harness_run_duration_seconds: 128,
|
|
81
|
+
openclaw_harness_run_total: 128,
|
|
82
|
+
openclaw_heap_old_generation_after_gc_bytes: 128,
|
|
83
|
+
openclaw_heap_old_generation_after_gc_ratio: 128,
|
|
84
|
+
openclaw_heap_size_limit_bytes: 128,
|
|
85
|
+
openclaw_heap_space_bytes: 256,
|
|
86
|
+
openclaw_heap_used_after_major_gc_bytes: 64,
|
|
87
|
+
openclaw_heap_used_after_major_gc_timestamp_seconds: 64,
|
|
88
|
+
openclaw_heap_used_ratio: 128,
|
|
89
|
+
openclaw_liveness_cpu_core_ratio: 4,
|
|
90
|
+
openclaw_liveness_event_loop_delay_max_seconds: 4,
|
|
91
|
+
openclaw_liveness_event_loop_delay_p99_seconds: 4,
|
|
92
|
+
openclaw_liveness_event_loop_utilization_ratio: 4,
|
|
93
|
+
openclaw_liveness_sessions: 4,
|
|
94
|
+
openclaw_liveness_warning_total: 4,
|
|
95
|
+
openclaw_memory_bytes: 128,
|
|
96
|
+
openclaw_memory_pressure_total: 4,
|
|
97
|
+
openclaw_memory_rss_bytes: 4,
|
|
98
|
+
openclaw_message_delivery_duration_seconds: 4,
|
|
99
|
+
openclaw_message_delivery_started_total: 4,
|
|
100
|
+
openclaw_message_delivery_total: 4,
|
|
101
|
+
openclaw_message_dispatch_completed_total: 4,
|
|
102
|
+
openclaw_message_dispatch_duration_seconds: 4,
|
|
103
|
+
openclaw_message_dispatch_started_total: 4,
|
|
104
|
+
openclaw_message_processed_duration_seconds: 4,
|
|
105
|
+
openclaw_message_processed_total: 4,
|
|
106
|
+
openclaw_message_received_total: 4,
|
|
107
|
+
openclaw_model_call_duration_seconds: 128,
|
|
108
|
+
openclaw_model_call_total: 128,
|
|
109
|
+
openclaw_model_cost_usd_total: 128,
|
|
110
|
+
openclaw_model_failover_total: 128,
|
|
111
|
+
openclaw_model_tokens_total: 128,
|
|
112
|
+
openclaw_model_usage_duration_seconds: 128,
|
|
113
|
+
openclaw_models_list_requests_total: 4,
|
|
114
|
+
openclaw_payload_large_bytes: 4,
|
|
115
|
+
openclaw_payload_large_total: 4,
|
|
116
|
+
openclaw_process_anonymous_memory_bytes: 4,
|
|
117
|
+
openclaw_queue_lane_size: 4,
|
|
118
|
+
openclaw_queue_lane_wait_seconds: 4,
|
|
119
|
+
openclaw_run_completed_total: 128,
|
|
120
|
+
openclaw_run_duration_seconds: 128,
|
|
121
|
+
openclaw_secret_egress_requests_total: 4,
|
|
122
|
+
openclaw_session_event_rows_reset_total: 4,
|
|
123
|
+
openclaw_session_event_rows_retained_bytes: 4,
|
|
124
|
+
openclaw_session_event_rows_retained_entries: 4,
|
|
125
|
+
openclaw_session_queue_depth: 4,
|
|
126
|
+
openclaw_session_recovery_age_seconds: 4,
|
|
127
|
+
openclaw_session_recovery_total: 4,
|
|
128
|
+
openclaw_session_row_selection_resets_total: 4,
|
|
129
|
+
openclaw_session_row_partition_cache_total: 32,
|
|
130
|
+
openclaw_session_row_partition_revisions_total: 8,
|
|
131
|
+
openclaw_session_row_selection_rows_total: 64,
|
|
132
|
+
openclaw_session_row_selections_total: 64,
|
|
133
|
+
openclaw_session_state_total: 4,
|
|
134
|
+
openclaw_session_stuck_age_seconds: 4,
|
|
135
|
+
openclaw_session_stuck_total: 4,
|
|
136
|
+
openclaw_session_turn_created_total: 4,
|
|
137
|
+
openclaw_sessions_active: 4,
|
|
138
|
+
openclaw_sessions_list_base_misses_total: 4,
|
|
139
|
+
openclaw_sessions_list_baseline_retirements_total: 4,
|
|
140
|
+
openclaw_sessions_list_requests_total: 4,
|
|
141
|
+
openclaw_sessions_list_response_bytes_total: 4,
|
|
142
|
+
openclaw_sessions_list_retained_bytes: 4,
|
|
143
|
+
openclaw_sessions_list_retained_refs: 4,
|
|
144
|
+
openclaw_sessions_list_retained_views: 4,
|
|
145
|
+
openclaw_skill_used_total: 4,
|
|
146
|
+
openclaw_talk_audio_bytes: 4,
|
|
147
|
+
openclaw_talk_event_duration_seconds: 4,
|
|
148
|
+
openclaw_talk_event_total: 4,
|
|
149
|
+
openclaw_telemetry_exporter_total: 4,
|
|
150
|
+
openclaw_tool_catalog_tools_total: 128,
|
|
151
|
+
openclaw_tool_catalog_total: 128,
|
|
152
|
+
openclaw_tool_execution_blocked_total: 128,
|
|
153
|
+
openclaw_tool_execution_duration_seconds: 128,
|
|
154
|
+
openclaw_tool_execution_total: 128,
|
|
155
|
+
openclaw_webhook_duration_seconds: 4,
|
|
156
|
+
openclaw_webhook_error_total: 4,
|
|
157
|
+
openclaw_webhook_received_total: 4,
|
|
158
|
+
openclaw_worker_array_buffers_bytes: 128,
|
|
159
|
+
openclaw_worker_count: 128,
|
|
160
|
+
openclaw_worker_external_bytes: 128,
|
|
161
|
+
openclaw_worker_gc_collections_total: 128,
|
|
162
|
+
openclaw_worker_gc_duration_seconds_total: 128,
|
|
163
|
+
openclaw_worker_gc_reclaimed_bytes_total: 128,
|
|
164
|
+
openclaw_worker_heap_sampled_count: 128,
|
|
165
|
+
openclaw_worker_heap_space_used_bytes: 640,
|
|
166
|
+
openclaw_worker_heap_used_bytes: 128,
|
|
167
|
+
openclaw_worker_heap_used_high_water_bytes: 128,
|
|
168
|
+
openclaw_worker_message_payload_bytes_total: 512,
|
|
169
|
+
openclaw_worker_message_unmeasured_total: 64,
|
|
170
|
+
openclaw_worker_messages_total: 64,
|
|
171
|
+
openclaw_worker_queue_depth: 512,
|
|
172
|
+
openclaw_worker_queue_wait_seconds: 512,
|
|
173
|
+
openclaw_worker_request_phase_seconds: 1024,
|
|
174
|
+
openclaw_worker_request_seconds: 512,
|
|
175
|
+
openclaw_worker_retired_total: 64,
|
|
176
|
+
openclaw_worker_started_total: 64,
|
|
177
|
+
openclaw_worker_tasks_completed_total: 128,
|
|
178
|
+
openclaw_worktree_preparation_seconds: 32
|
|
179
|
+
};
|
|
180
|
+
const SHARED_PROMETHEUS_SERIES = 16384 - Object.values(PROMETHEUS_FAMILY_RESERVES).reduce((sum, n) => sum + n, 0);
|
|
181
|
+
// extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
|
|
182
|
+
function createSeries() {
|
|
183
|
+
return {
|
|
184
|
+
samples: [],
|
|
185
|
+
shapes: /* @__PURE__ */ new Map(),
|
|
186
|
+
sorted: true
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function sortedSamples(series) {
|
|
190
|
+
if (!series.sorted) {
|
|
191
|
+
series.samples.sort((left, right) => left.key.localeCompare(right.key));
|
|
192
|
+
series.sorted = true;
|
|
193
|
+
}
|
|
194
|
+
return series.samples;
|
|
195
|
+
}
|
|
37
196
|
const DURATION_BUCKETS_SECONDS = [
|
|
38
197
|
.005,
|
|
39
198
|
.01,
|
|
@@ -58,101 +217,270 @@ const AGENT_DURATION_BUCKETS_SECONDS = [
|
|
|
58
217
|
1800,
|
|
59
218
|
3600
|
|
60
219
|
];
|
|
61
|
-
const MAX_PROMETHEUS_SERIES = 2048;
|
|
62
220
|
const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const
|
|
221
|
+
const DROPPED_FAMILY_COUNTER_NAME = "openclaw_prometheus_family_series_dropped_total";
|
|
222
|
+
const warnedFamilies = /* @__PURE__ */ new Set();
|
|
223
|
+
function createPrometheusMetricStore(onSeriesDropped) {
|
|
224
|
+
const counters = createSeries();
|
|
225
|
+
const gauges = createSeries();
|
|
226
|
+
const histograms = createSeries();
|
|
67
227
|
let droppedSeries = 0;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
228
|
+
let sharedSeries = 0;
|
|
229
|
+
const familyCounts = /* @__PURE__ */ new Map();
|
|
230
|
+
const familyDrops = /* @__PURE__ */ new Map();
|
|
231
|
+
const headers = /* @__PURE__ */ new Map();
|
|
232
|
+
let prefixBuffer = Buffer.alloc(0);
|
|
233
|
+
let prefixOffset = 0;
|
|
234
|
+
const encodePrefix = (text) => {
|
|
235
|
+
const bytes = Buffer.byteLength(text);
|
|
236
|
+
if (bytes > 4096) {
|
|
237
|
+
const prefix = Buffer.allocUnsafeSlow(bytes);
|
|
238
|
+
prefix.write(text);
|
|
239
|
+
return prefix;
|
|
240
|
+
}
|
|
241
|
+
if (prefixOffset + bytes > prefixBuffer.length) {
|
|
242
|
+
prefixBuffer = Buffer.allocUnsafeSlow(4096);
|
|
243
|
+
prefixOffset = 0;
|
|
244
|
+
}
|
|
245
|
+
const start = prefixOffset;
|
|
246
|
+
prefixOffset += prefixBuffer.write(text, prefixOffset);
|
|
247
|
+
return prefixBuffer.subarray(start, prefixOffset);
|
|
73
248
|
};
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
249
|
+
let scratch = Buffer.alloc(0);
|
|
250
|
+
const series = (map, name, labels) => {
|
|
251
|
+
let labelCount = 0;
|
|
252
|
+
for (const key in labels) if (Object.hasOwn(labels, key)) labelCount++;
|
|
253
|
+
let shape;
|
|
254
|
+
shapes: for (const candidate of map.shapes.get(name) ?? []) {
|
|
255
|
+
if (candidate.keys.length !== labelCount) continue;
|
|
256
|
+
let node = candidate.root;
|
|
257
|
+
for (const key of candidate.keys) {
|
|
258
|
+
const value = labels[key];
|
|
259
|
+
if (value === void 0 || !Object.prototype.propertyIsEnumerable.call(labels, key)) continue shapes;
|
|
260
|
+
node = node?.values?.get(value);
|
|
261
|
+
}
|
|
262
|
+
shape = candidate;
|
|
263
|
+
if (node?.sample) return node;
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
const count = familyCounts.get(name) ?? 0;
|
|
267
|
+
const reserve = PROMETHEUS_FAMILY_RESERVES[name];
|
|
268
|
+
if (count >= Math.max(128, reserve) || count >= reserve && sharedSeries >= SHARED_PROMETHEUS_SERIES) {
|
|
269
|
+
droppedSeries++;
|
|
270
|
+
familyDrops.set(name, (familyDrops.get(name) ?? 0) + 1);
|
|
271
|
+
if (onSeriesDropped && !warnedFamilies.has(name)) {
|
|
272
|
+
warnedFamilies.add(name);
|
|
273
|
+
onSeriesDropped(name);
|
|
274
|
+
}
|
|
81
275
|
return;
|
|
82
276
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
value
|
|
97
|
-
|
|
277
|
+
familyCounts.set(name, count + 1);
|
|
278
|
+
if (count >= reserve) sharedSeries++;
|
|
279
|
+
if (!shape) {
|
|
280
|
+
shape = {
|
|
281
|
+
keys: Object.keys(labels).toSorted((left, right) => left.localeCompare(right)),
|
|
282
|
+
root: {}
|
|
283
|
+
};
|
|
284
|
+
const shapes = map.shapes.get(name);
|
|
285
|
+
if (shapes) shapes.push(shape);
|
|
286
|
+
else map.shapes.set(name, [shape]);
|
|
287
|
+
}
|
|
288
|
+
let node = shape.root;
|
|
289
|
+
for (const key of shape.keys) {
|
|
290
|
+
const value = expectDefined(labels[key], "Prometheus label");
|
|
291
|
+
const values = node.values ??= /* @__PURE__ */ new Map();
|
|
292
|
+
let child = values.get(value);
|
|
293
|
+
if (!child) {
|
|
294
|
+
child = {};
|
|
295
|
+
values.set(value, child);
|
|
296
|
+
}
|
|
297
|
+
node = child;
|
|
298
|
+
}
|
|
299
|
+
return node;
|
|
98
300
|
};
|
|
99
|
-
const
|
|
100
|
-
if (!Number.isFinite(value) || value < 0) return;
|
|
101
|
-
const
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
301
|
+
const scalar = (map, name, help, labels, value, mode) => {
|
|
302
|
+
if (value === void 0 || !Number.isFinite(value) || mode !== "set" && value < 0 || mode === "add" && value === 0) return;
|
|
303
|
+
const node = series(map, name, labels);
|
|
304
|
+
if (!node) return;
|
|
305
|
+
if (node.sample) {
|
|
306
|
+
const next = mode === "add" ? node.sample.value + value : mode === "max" ? Math.max(node.sample.value, value) : value;
|
|
307
|
+
if (next !== node.sample.value) {
|
|
308
|
+
node.sample.value = next;
|
|
309
|
+
node.sample.renderedDirty = true;
|
|
310
|
+
}
|
|
311
|
+
} else {
|
|
312
|
+
node.sample = {
|
|
313
|
+
help,
|
|
314
|
+
key: metricKey(name, labels),
|
|
315
|
+
name,
|
|
316
|
+
prefix: encodePrefix(`${name}${formatLabels(labels)} `),
|
|
317
|
+
value,
|
|
318
|
+
renderedDirty: true
|
|
319
|
+
};
|
|
320
|
+
map.samples.push(node.sample);
|
|
321
|
+
map.sorted = false;
|
|
322
|
+
}
|
|
107
323
|
};
|
|
324
|
+
const counter = (name, help, labels, amount = 1) => scalar(counters, name, help, labels, amount, "add");
|
|
325
|
+
const counterValue = (name, help, labels, value) => scalar(counters, name, help, labels, value, "max");
|
|
326
|
+
const gauge = (name, help, labels, value) => scalar(gauges, name, help, labels, value, "set");
|
|
108
327
|
const clearGauges = (name) => {
|
|
109
|
-
|
|
328
|
+
headers.delete(name);
|
|
329
|
+
const previousLength = gauges.samples.length;
|
|
330
|
+
gauges.shapes.delete(name);
|
|
331
|
+
gauges.samples = gauges.samples.filter((sample) => sample.name !== name);
|
|
332
|
+
const count = familyCounts.get(name) ?? 0;
|
|
333
|
+
const remaining = count - (previousLength - gauges.samples.length);
|
|
334
|
+
const reserve = PROMETHEUS_FAMILY_RESERVES[name];
|
|
335
|
+
sharedSeries -= Math.max(0, count - reserve) - Math.max(0, remaining - reserve);
|
|
336
|
+
familyCounts.set(name, remaining);
|
|
110
337
|
};
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
let sample = histograms.get(key);
|
|
338
|
+
const getHistogram = (name, help, labels, buckets = DURATION_BUCKETS_SECONDS) => {
|
|
339
|
+
const node = series(histograms, name, labels);
|
|
340
|
+
if (!node) return;
|
|
341
|
+
let sample = node.sample;
|
|
116
342
|
if (!sample) {
|
|
117
343
|
sample = {
|
|
118
344
|
buckets,
|
|
119
|
-
bucketPrefixes: [...buckets, "+Inf"].map((le) => `${name}_bucket${formatLabels({
|
|
345
|
+
bucketPrefixes: [...buckets, "+Inf"].map((le) => encodePrefix(`${name}_bucket${formatLabels({
|
|
120
346
|
...labels,
|
|
121
347
|
le: String(le)
|
|
122
|
-
})} `),
|
|
348
|
+
})} `)),
|
|
123
349
|
counts: buckets.map(() => 0),
|
|
124
350
|
count: 0,
|
|
125
351
|
help,
|
|
126
|
-
|
|
127
|
-
|
|
352
|
+
key: metricKey(name, labels),
|
|
353
|
+
name,
|
|
354
|
+
sumPrefix: encodePrefix(`${name}_sum${formatLabels(labels)} `),
|
|
355
|
+
countPrefix: encodePrefix(`${name}_count${formatLabels(labels)} `),
|
|
356
|
+
sum: 0,
|
|
357
|
+
renderedDirty: true
|
|
128
358
|
};
|
|
129
|
-
|
|
359
|
+
node.sample = sample;
|
|
360
|
+
histograms.samples.push(sample);
|
|
361
|
+
histograms.sorted = false;
|
|
130
362
|
}
|
|
363
|
+
return sample;
|
|
364
|
+
};
|
|
365
|
+
const reserveHistogram = (name, help, labels) => {
|
|
366
|
+
getHistogram(name, help, labels);
|
|
367
|
+
};
|
|
368
|
+
const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
|
|
369
|
+
if (value === void 0 || !Number.isFinite(value)) return;
|
|
370
|
+
const sample = getHistogram(name, help, labels, buckets);
|
|
371
|
+
if (!sample) return;
|
|
131
372
|
sample.count += 1;
|
|
132
373
|
sample.sum += value;
|
|
374
|
+
sample.renderedDirty = true;
|
|
133
375
|
for (let index = 0; index < sample.buckets.length; index += 1) {
|
|
134
376
|
const bucket = sample.buckets[index];
|
|
135
377
|
if (bucket !== void 0 && value <= bucket) sample.counts[index] = (sample.counts[index] ?? 0) + 1;
|
|
136
378
|
}
|
|
137
379
|
};
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
380
|
+
const render = () => {
|
|
381
|
+
let scratchOffset = 0;
|
|
382
|
+
let totalBytes = 0;
|
|
383
|
+
const chunks = [];
|
|
384
|
+
const append = (chunk) => {
|
|
385
|
+
chunks.push(chunk);
|
|
386
|
+
totalBytes += chunk.length;
|
|
387
|
+
};
|
|
388
|
+
const write = (prefix, value) => {
|
|
389
|
+
const number = formatPrometheusNumber(value);
|
|
390
|
+
const required = scratchOffset + prefix.length + number.length + 1;
|
|
391
|
+
if (required > scratch.length) {
|
|
392
|
+
const grown = Buffer.allocUnsafeSlow(Math.max(required, scratch.length * 2));
|
|
393
|
+
grown.set(scratch.subarray(0, scratchOffset));
|
|
394
|
+
scratch = grown;
|
|
395
|
+
}
|
|
396
|
+
scratch.set(prefix, scratchOffset);
|
|
397
|
+
scratchOffset += prefix.length;
|
|
398
|
+
scratchOffset += scratch.write(number, scratchOffset, "ascii");
|
|
399
|
+
scratch[scratchOffset++] = 10;
|
|
400
|
+
};
|
|
401
|
+
const finish = (previous) => {
|
|
402
|
+
const output = previous?.length === scratchOffset ? previous : new Uint8Array(scratchOffset);
|
|
403
|
+
scratch.copy(output, 0, 0, scratchOffset);
|
|
404
|
+
scratchOffset = 0;
|
|
405
|
+
return output;
|
|
406
|
+
};
|
|
407
|
+
const emitted = /* @__PURE__ */ new Set();
|
|
408
|
+
let previousName;
|
|
409
|
+
const header = (name, type, help) => {
|
|
410
|
+
if (previousName === name) return;
|
|
411
|
+
previousName = name;
|
|
412
|
+
if (!emitted.has(name)) {
|
|
413
|
+
emitted.add(name);
|
|
414
|
+
let cached = headers.get(name);
|
|
415
|
+
if (!cached || cached.type !== type || cached.help !== help) {
|
|
416
|
+
cached = {
|
|
417
|
+
type,
|
|
418
|
+
help,
|
|
419
|
+
bytes: encodePrefix(`# HELP ${name} ${escapeHelp(help)}\n# TYPE ${name} ${type}\n`)
|
|
420
|
+
};
|
|
421
|
+
headers.set(name, cached);
|
|
422
|
+
}
|
|
423
|
+
append(cached.bytes);
|
|
424
|
+
}
|
|
149
425
|
};
|
|
426
|
+
for (const [type, map] of [["counter", counters], ["gauge", gauges]]) {
|
|
427
|
+
for (const sample of sortedSamples(map)) {
|
|
428
|
+
header(sample.name, type, sample.help);
|
|
429
|
+
if (!sample.rendered || sample.renderedDirty) {
|
|
430
|
+
write(sample.prefix, sample.value);
|
|
431
|
+
sample.rendered = finish(sample.rendered);
|
|
432
|
+
sample.renderedDirty = false;
|
|
433
|
+
}
|
|
434
|
+
append(sample.rendered);
|
|
435
|
+
}
|
|
436
|
+
if (type === "counter" && droppedSeries > 0) {
|
|
437
|
+
header(DROPPED_SERIES_COUNTER_NAME, type, "New-series observations dropped because an exporter series budget was reached.");
|
|
438
|
+
write(Buffer.from(`${DROPPED_SERIES_COUNTER_NAME} `), droppedSeries);
|
|
439
|
+
append(finish());
|
|
440
|
+
header(DROPPED_FAMILY_COUNTER_NAME, type, "New-series observations dropped by bounded metric family.");
|
|
441
|
+
for (const [family, count] of familyDrops) write(Buffer.from(`${DROPPED_FAMILY_COUNTER_NAME}${formatLabels({ family })} `), count);
|
|
442
|
+
append(finish());
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
for (const sample of sortedSamples(histograms)) {
|
|
446
|
+
if (sample.count === 0) continue;
|
|
447
|
+
header(sample.name, "histogram", sample.help);
|
|
448
|
+
if (!sample.rendered || sample.renderedDirty) {
|
|
449
|
+
sample.bucketPrefixes.forEach((prefix, bucketIndex) => {
|
|
450
|
+
write(prefix, sample.counts[bucketIndex] ?? sample.count);
|
|
451
|
+
});
|
|
452
|
+
write(sample.sumPrefix, sample.sum);
|
|
453
|
+
write(sample.countPrefix, sample.count);
|
|
454
|
+
sample.rendered = finish(sample.rendered);
|
|
455
|
+
sample.renderedDirty = false;
|
|
456
|
+
}
|
|
457
|
+
append(sample.rendered);
|
|
458
|
+
}
|
|
459
|
+
const output = Buffer.from(new ArrayBuffer(totalBytes));
|
|
460
|
+
let offset = 0;
|
|
461
|
+
for (const chunk of chunks) {
|
|
462
|
+
output.set(chunk, offset);
|
|
463
|
+
offset += chunk.length;
|
|
464
|
+
}
|
|
465
|
+
return output;
|
|
150
466
|
};
|
|
151
467
|
const reset = () => {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
468
|
+
for (const map of [
|
|
469
|
+
counters,
|
|
470
|
+
gauges,
|
|
471
|
+
histograms
|
|
472
|
+
]) {
|
|
473
|
+
map.samples.length = 0;
|
|
474
|
+
map.shapes.clear();
|
|
475
|
+
}
|
|
155
476
|
droppedSeries = 0;
|
|
477
|
+
sharedSeries = 0;
|
|
478
|
+
familyCounts.clear();
|
|
479
|
+
familyDrops.clear();
|
|
480
|
+
headers.clear();
|
|
481
|
+
scratch = Buffer.alloc(0);
|
|
482
|
+
prefixBuffer = Buffer.alloc(0);
|
|
483
|
+
prefixOffset = 0;
|
|
156
484
|
};
|
|
157
485
|
return {
|
|
158
486
|
counter,
|
|
@@ -160,31 +488,140 @@ function createPrometheusMetricStore() {
|
|
|
160
488
|
gauge,
|
|
161
489
|
clearGauges,
|
|
162
490
|
histogram,
|
|
491
|
+
reserveHistogram,
|
|
163
492
|
reset,
|
|
164
|
-
|
|
493
|
+
render
|
|
165
494
|
};
|
|
166
495
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
496
|
+
// extensions/diagnostics-prometheus/src/service-gateway-ws.ts
|
|
497
|
+
function createGatewayWsMetricsRecorder(store) {
|
|
498
|
+
const labels = { event: "" };
|
|
499
|
+
const frameLabels = {
|
|
500
|
+
event: "",
|
|
501
|
+
le: ""
|
|
502
|
+
};
|
|
503
|
+
const presenceLabels = { mode: "" };
|
|
504
|
+
const sessionLabels = {
|
|
505
|
+
event: "",
|
|
506
|
+
reason: "",
|
|
507
|
+
delivery: ""
|
|
508
|
+
};
|
|
509
|
+
return (event, count, bytes, wireBytes, buckets, bucketLabels, sessionEvent, presenceMode) => {
|
|
510
|
+
labels.event = frameLabels.event = sessionLabels.event = event;
|
|
511
|
+
store.counter("openclaw_gateway_ws_events_total", "Outbound Gateway WebSocket event frames accepted by the sender per socket.", labels, count);
|
|
512
|
+
store.counter("openclaw_gateway_ws_event_bytes_total", "Encoded outbound Gateway WebSocket event bytes before framing or compression per socket.", labels, bytes);
|
|
513
|
+
if (wireBytes !== void 0) store.counter("openclaw_gateway_ws_event_wire_bytes_total", "Outbound Gateway WebSocket payload bytes after compression per socket, excluding framing.", labels, wireBytes);
|
|
514
|
+
if (count > 0) {
|
|
515
|
+
let cumulative = 0;
|
|
516
|
+
for (let i = 0; i < bucketLabels.length; i++) {
|
|
517
|
+
cumulative += buckets[i];
|
|
518
|
+
frameLabels.le = bucketLabels[i];
|
|
519
|
+
(cumulative === 0 ? store.counterValue : store.counter)("openclaw_gateway_ws_event_frame_bytes", "Cumulative counts of accepted raw Gateway event frames by byte-size upper bound.", frameLabels, cumulative);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (presenceMode) {
|
|
523
|
+
presenceLabels.mode = presenceMode;
|
|
524
|
+
store.counter("openclaw_gateway_ws_presence_frames_total", "Outbound presence frames accepted by the sender per socket, by delivery mode.", presenceLabels, count);
|
|
525
|
+
store.counter("openclaw_gateway_ws_presence_bytes_total", "Encoded outbound presence bytes before framing or compression per socket, by delivery mode.", presenceLabels, bytes);
|
|
526
|
+
}
|
|
527
|
+
if (sessionEvent) {
|
|
528
|
+
sessionLabels.reason = sessionEvent.reason;
|
|
529
|
+
sessionLabels.delivery = sessionEvent.delivery;
|
|
530
|
+
store.counter("openclaw_gateway_ws_session_event_frames_total", "Outbound session event frames by reason and exact message subscription per socket.", sessionLabels, count);
|
|
531
|
+
store.counter("openclaw_gateway_ws_session_event_bytes_total", "Encoded outbound session event bytes by reason and exact message subscription per socket.", sessionLabels, bytes);
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
function recordGatewayWsConnections(store, operators) {
|
|
536
|
+
store.gauge("openclaw_gateway_ws_operator_connections", "Connected authenticated Gateway operator WebSockets.", {}, operators);
|
|
537
|
+
}
|
|
538
|
+
// extensions/diagnostics-prometheus/src/service-memory.ts
|
|
539
|
+
const WORKER_MEMORY_METRICS = [
|
|
540
|
+
[
|
|
541
|
+
"heapUsed",
|
|
542
|
+
"openclaw_worker_heap_used_bytes",
|
|
543
|
+
"heap usage"
|
|
544
|
+
],
|
|
545
|
+
[
|
|
546
|
+
"external",
|
|
547
|
+
"openclaw_worker_external_bytes",
|
|
548
|
+
"external memory"
|
|
549
|
+
],
|
|
550
|
+
[
|
|
551
|
+
"arrayBuffers",
|
|
552
|
+
"openclaw_worker_array_buffers_bytes",
|
|
553
|
+
"ArrayBuffer memory (included in external)"
|
|
554
|
+
]
|
|
555
|
+
];
|
|
556
|
+
const HEAP_HEADROOM_METRICS = [
|
|
557
|
+
[
|
|
558
|
+
"limit",
|
|
559
|
+
"openclaw_heap_size_limit_bytes",
|
|
560
|
+
"Smallest live isolate V8 heap limit by thread group."
|
|
561
|
+
],
|
|
562
|
+
[
|
|
563
|
+
"usedRatio",
|
|
564
|
+
"openclaw_heap_used_ratio",
|
|
565
|
+
"Largest live isolate used/limit ratio by thread group."
|
|
566
|
+
],
|
|
567
|
+
[
|
|
568
|
+
"oldBytes",
|
|
569
|
+
"openclaw_heap_old_generation_after_gc_bytes",
|
|
570
|
+
"Largest live isolate old_space plus large_object_space usage at its last observed major GC."
|
|
571
|
+
],
|
|
572
|
+
[
|
|
573
|
+
"oldRatio",
|
|
574
|
+
"openclaw_heap_old_generation_after_gc_ratio",
|
|
575
|
+
"Largest live isolate post-major-GC old-generation/heap-limit ratio by thread group."
|
|
576
|
+
]
|
|
577
|
+
];
|
|
578
|
+
function collectHeapHeadroom(headroom, used, limit, oldBytes) {
|
|
579
|
+
if (limit === void 0 || limit <= 0) return;
|
|
580
|
+
headroom.limit = Math.min(headroom.limit ?? limit, limit);
|
|
581
|
+
headroom.usedRatio = Math.max(headroom.usedRatio ?? 0, used / limit);
|
|
582
|
+
if (oldBytes !== void 0) {
|
|
583
|
+
headroom.oldBytes = Math.max(headroom.oldBytes ?? 0, oldBytes);
|
|
584
|
+
headroom.oldRatio = Math.max(headroom.oldRatio ?? 0, oldBytes / limit);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
function recordHeapHeadroom(store, thread, headroom) {
|
|
588
|
+
for (const [field, name, description] of HEAP_HEADROOM_METRICS) store.gauge(name, description, { thread }, headroom[field]);
|
|
175
589
|
}
|
|
176
|
-
//#endregion
|
|
177
|
-
//#region extensions/diagnostics-prometheus/src/service-memory.ts
|
|
178
590
|
function recordMemorySample(store, memory, byteBuckets) {
|
|
591
|
+
for (const [, name] of HEAP_HEADROOM_METRICS) store.clearGauges(name);
|
|
592
|
+
const mainHeadroom = {};
|
|
593
|
+
collectHeapHeadroom(mainHeadroom, memory.heapUsedBytes, memory.heapSizeLimitBytes, memory.oldGenerationAfterGcBytes);
|
|
594
|
+
recordHeapHeadroom(store, "main", mainHeadroom);
|
|
595
|
+
store.clearGauges("openclaw_process_anonymous_memory_bytes");
|
|
596
|
+
for (const [kind, field] of [["rss", "anonymousRssBytes"], ["swap", "anonymousSwapBytes"]]) store.gauge("openclaw_process_anonymous_memory_bytes", "Latest Linux process anonymous RSS and swap; includes managed heaps and buffers, not allocator totals.", { kind }, asNonNegativeFiniteNumber(memory[field]));
|
|
179
597
|
for (const message of memory.workerMessages ?? []) {
|
|
180
598
|
const labels = {
|
|
181
599
|
kind: message.kind,
|
|
182
600
|
direction: message.direction
|
|
183
601
|
};
|
|
184
602
|
store.counterValue("openclaw_worker_messages_total", "Messages crossing the Gateway main-thread worker boundary.", labels, message.messages);
|
|
185
|
-
store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.",
|
|
603
|
+
for (const [requestClass, bytes] of Object.entries(message.payloadBytesByRequestClass ?? { [message.kind === "sqlite_request" || message.kind === "sqlite_reply" || message.kind === "authority" ? "other" : ""]: message.payloadBytes })) store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.", requestClass ? {
|
|
604
|
+
...labels,
|
|
605
|
+
request_class: requestClass
|
|
606
|
+
} : labels, bytes);
|
|
186
607
|
store.counterValue("openclaw_worker_message_unmeasured_total", "Worker messages without a known payload size; these do not contribute to payload bytes.", labels, message.unmeasuredMessages);
|
|
187
608
|
}
|
|
609
|
+
for (const selection of memory.sessionRowSelections?.selections ?? []) {
|
|
610
|
+
const labels = {
|
|
611
|
+
kind: selection.kind,
|
|
612
|
+
task: selection.task
|
|
613
|
+
};
|
|
614
|
+
store.counterValue("openclaw_session_row_selections_total", "Successful resident session-row selections by query shape and enclosing task.", labels, selection.calls);
|
|
615
|
+
store.counterValue("openclaw_session_row_selection_rows_total", "Resident session rows returned by selections, before federation and visibility filtering.", labels, selection.rows);
|
|
616
|
+
}
|
|
617
|
+
if (memory.sessionRowSelections) {
|
|
618
|
+
for (const partition of memory.sessionRowSelections.partitions) for (const outcome of ["hits", "misses"]) store.counterValue("openclaw_session_row_partition_cache_total", "Retained session-list partition lookups by outcome and enclosing selection task.", {
|
|
619
|
+
task: partition.task,
|
|
620
|
+
outcome: outcome === "hits" ? "hit" : "miss"
|
|
621
|
+
}, partition[outcome]);
|
|
622
|
+
for (const revision of memory.sessionRowSelections.partitionRevisions) store.counterValue("openclaw_session_row_partition_revisions_total", "Session-list partition revisions by synchronous internal change scope; excludes initial partitions.", { reason: revision.reason }, revision.changes);
|
|
623
|
+
store.counterValue("openclaw_session_row_selection_resets_total", "Broad session-row selection invalidations, including topology replacement and disposal.", {}, memory.sessionRowSelections.resets);
|
|
624
|
+
}
|
|
188
625
|
for (const worker of memory.workerLifecycle ?? []) {
|
|
189
626
|
store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
|
|
190
627
|
for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
|
|
@@ -192,6 +629,23 @@ function recordMemorySample(store, memory, byteBuckets) {
|
|
|
192
629
|
reason
|
|
193
630
|
}, count);
|
|
194
631
|
}
|
|
632
|
+
for (const worker of memory.workerTaskMetrics ?? []) {
|
|
633
|
+
const labels = {
|
|
634
|
+
worker_class: worker.workerClass,
|
|
635
|
+
script: worker.script
|
|
636
|
+
};
|
|
637
|
+
store.counterValue("openclaw_worker_tasks_completed_total", "Observed terminal task replies from pool workers, including handled failures.", labels, worker.tasksCompleted);
|
|
638
|
+
store.gauge("openclaw_worker_heap_used_high_water_bytes", "Largest observed pool worker heap usage, retained across worker retirement.", labels, worker.heapUsedHighWaterBytes);
|
|
639
|
+
for (const gc of worker.gc) {
|
|
640
|
+
const gcLabels = {
|
|
641
|
+
...labels,
|
|
642
|
+
kind: gc.kind
|
|
643
|
+
};
|
|
644
|
+
store.counterValue("openclaw_worker_gc_duration_seconds_total", "Observed pool worker garbage collection duration, retained across worker retirement.", gcLabels, gc.durationMs / 1e3);
|
|
645
|
+
store.counterValue("openclaw_worker_gc_collections_total", "Observed pool worker garbage collections, retained across worker retirement.", gcLabels, gc.count);
|
|
646
|
+
store.counterValue("openclaw_worker_gc_reclaimed_bytes_total", "Sum of positive heap usage decreases across observed pool worker garbage collections.", gcLabels, gc.reclaimedBytes);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
195
649
|
for (const [kind, field] of [
|
|
196
650
|
["rss", "rssBytes"],
|
|
197
651
|
["heap_total", "heapTotalBytes"],
|
|
@@ -217,41 +671,58 @@ function recordMemorySample(store, memory, byteBuckets) {
|
|
|
217
671
|
}, space[field]);
|
|
218
672
|
const workers = /* @__PURE__ */ new Map();
|
|
219
673
|
store.clearGauges("openclaw_code_mode_worker_heap_bytes");
|
|
674
|
+
store.clearGauges("openclaw_code_mode_worker_memory_bytes");
|
|
220
675
|
for (const worker of memory.workerHeaps ?? []) {
|
|
221
|
-
if (worker.workerSlot !== void 0)
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
676
|
+
if (worker.workerSlot !== void 0) {
|
|
677
|
+
for (const [kind, value] of [["used", worker.heapUsed], ["limit", worker.heapSizeLimitBytes]]) store.gauge("openclaw_code_mode_worker_heap_bytes", "Latest live Code Mode isolate heap usage and limit by reusable worker slot.", {
|
|
678
|
+
slot: String(worker.workerSlot),
|
|
679
|
+
kind
|
|
680
|
+
}, value);
|
|
681
|
+
for (const [kind, value] of [["external", worker.external], ["array_buffers", worker.arrayBuffers]]) store.gauge("openclaw_code_mode_worker_memory_bytes", "Latest live Code Mode isolate external and ArrayBuffer memory (included in external) by reusable worker slot.", {
|
|
682
|
+
slot: String(worker.workerSlot),
|
|
683
|
+
kind
|
|
684
|
+
}, value);
|
|
685
|
+
}
|
|
225
686
|
const totals = workers.get(worker.script) ?? {
|
|
226
687
|
count: 0,
|
|
227
|
-
sampled: 0
|
|
228
|
-
heapUsed: void 0
|
|
688
|
+
sampled: 0
|
|
229
689
|
};
|
|
230
690
|
totals.count++;
|
|
231
691
|
totals.sampled++;
|
|
232
|
-
|
|
233
|
-
|
|
692
|
+
collectHeapHeadroom(totals, worker.heapUsed, worker.heapSizeLimitBytes, worker.oldGenerationAfterGcBytes);
|
|
693
|
+
for (const [field] of WORKER_MEMORY_METRICS) {
|
|
694
|
+
const value = worker[field];
|
|
695
|
+
if (value !== void 0) totals[field] = (totals[field] ?? 0) + value;
|
|
696
|
+
}
|
|
697
|
+
for (const space of worker.heapSpaces ?? []) {
|
|
698
|
+
totals.heapSpaces ??= /* @__PURE__ */ new Map();
|
|
699
|
+
totals.heapSpaces.set(space.space_name, (totals.heapSpaces.get(space.space_name) ?? 0) + space.space_used_size);
|
|
700
|
+
}
|
|
234
701
|
workers.set(worker.script, totals);
|
|
235
702
|
}
|
|
236
703
|
for (const worker of memory.workerMemoryMissing ?? []) {
|
|
237
704
|
const totals = workers.get(worker.script) ?? {
|
|
238
705
|
count: 0,
|
|
239
|
-
sampled: 0
|
|
240
|
-
heapUsed: void 0
|
|
706
|
+
sampled: 0
|
|
241
707
|
};
|
|
242
708
|
totals.count++;
|
|
243
709
|
workers.set(worker.script, totals);
|
|
244
710
|
}
|
|
245
|
-
store.clearGauges(
|
|
711
|
+
for (const [, name] of WORKER_MEMORY_METRICS) store.clearGauges(name);
|
|
712
|
+
store.clearGauges("openclaw_worker_heap_space_used_bytes");
|
|
246
713
|
for (const [script, totals] of workers) {
|
|
714
|
+
recordHeapHeadroom(store, script, totals);
|
|
247
715
|
store.gauge("openclaw_worker_count", "Worker isolate counts.", { script }, totals.count);
|
|
248
716
|
store.gauge("openclaw_worker_heap_sampled_count", "Worker isolate counts.", { script }, totals.sampled);
|
|
249
|
-
store.gauge(
|
|
717
|
+
for (const [field, name, description] of WORKER_MEMORY_METRICS) store.gauge(name, `Latest live Worker ${description} by bounded script basename.`, { script }, totals[field]);
|
|
718
|
+
for (const [space, bytes] of totals.heapSpaces ?? []) store.gauge("openclaw_worker_heap_space_used_bytes", "Latest live Worker V8 heap space usage by bounded script basename.", {
|
|
719
|
+
script,
|
|
720
|
+
space
|
|
721
|
+
}, bytes);
|
|
250
722
|
}
|
|
251
723
|
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
|
|
252
724
|
}
|
|
253
|
-
|
|
254
|
-
//#region extensions/diagnostics-prometheus/src/service-model-usage.ts
|
|
725
|
+
// extensions/diagnostics-prometheus/src/service-model-usage.ts
|
|
255
726
|
const TOKEN_BUCKETS = [
|
|
256
727
|
1,
|
|
257
728
|
4,
|
|
@@ -295,8 +766,21 @@ function recordModelUsage(store, evt) {
|
|
|
295
766
|
store.counter("openclaw_model_cost_usd_total", "Estimated model cost in USD reported by diagnostic usage events.", labels, asNonNegativeFiniteNumber(evt.costUsd) ?? 0);
|
|
296
767
|
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
297
768
|
}
|
|
298
|
-
|
|
299
|
-
|
|
769
|
+
// extensions/diagnostics-prometheus/src/service-operation-timing.ts
|
|
770
|
+
const GC_DURATION_NAME = "openclaw_gc_duration_seconds";
|
|
771
|
+
const GC_DURATION_HELP = "Elapsed garbage collection duration in seconds by collection kind, callback flags, and isolate role.";
|
|
772
|
+
function reserveMainGcHistograms(store) {
|
|
773
|
+
for (const kind of [
|
|
774
|
+
"scavenge",
|
|
775
|
+
"mark_sweep_compact",
|
|
776
|
+
"incremental",
|
|
777
|
+
"weak_processing"
|
|
778
|
+
]) store.reserveHistogram(GC_DURATION_NAME, GC_DURATION_HELP, {
|
|
779
|
+
kind,
|
|
780
|
+
reason: "other",
|
|
781
|
+
thread: "main"
|
|
782
|
+
});
|
|
783
|
+
}
|
|
300
784
|
const RPC_DURATION_BUCKETS_SECONDS = [
|
|
301
785
|
.005,
|
|
302
786
|
.01,
|
|
@@ -341,6 +825,14 @@ const RPC_DURATION_BUCKETS_SECONDS = [
|
|
|
341
825
|
600
|
|
342
826
|
];
|
|
343
827
|
const RESPONSE_BYTE_BUCKETS = Array.from({ length: 17 }, (_, index) => 1024 * 2 ** index);
|
|
828
|
+
const MAIN_THREAD_STALL_BUCKETS_SECONDS = [
|
|
829
|
+
.25,
|
|
830
|
+
.5,
|
|
831
|
+
1,
|
|
832
|
+
2.5,
|
|
833
|
+
5,
|
|
834
|
+
10
|
|
835
|
+
];
|
|
344
836
|
const HEAP_DELTA_BYTE_BUCKETS = [
|
|
345
837
|
...RESPONSE_BYTE_BUCKETS.toReversed().map((bytes) => -bytes),
|
|
346
838
|
0,
|
|
@@ -356,9 +848,41 @@ const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
|
|
|
356
848
|
"projection_final",
|
|
357
849
|
"delivery"
|
|
358
850
|
]);
|
|
851
|
+
const CHAT_STARTUP_PHASE = /^chat\.startup\.(session_entry|pending_inputs|history_page|startup_projection|projection_final|session_info)$/;
|
|
852
|
+
const DIFF_CACHE_MISS_REASONS = /* @__PURE__ */ new Set([
|
|
853
|
+
"cold",
|
|
854
|
+
"evicted",
|
|
855
|
+
"pending",
|
|
856
|
+
"no-revision",
|
|
857
|
+
"no-revision-environment",
|
|
858
|
+
"no-revision-platform",
|
|
859
|
+
"no-revision-metadata",
|
|
860
|
+
"no-revision-git-installation",
|
|
861
|
+
"no-revision-config",
|
|
862
|
+
"no-revision-reftable",
|
|
863
|
+
"no-revision-alternates",
|
|
864
|
+
"no-revision-symlink",
|
|
865
|
+
"no-revision-nested-repository",
|
|
866
|
+
"no-revision-status",
|
|
867
|
+
"no-revision-unmerged",
|
|
868
|
+
"no-revision-submodule",
|
|
869
|
+
"no-revision-permission",
|
|
870
|
+
"no-revision-missing",
|
|
871
|
+
"no-revision-io",
|
|
872
|
+
"oversized",
|
|
873
|
+
"revision-changed-head",
|
|
874
|
+
"revision-changed-index",
|
|
875
|
+
"revision-changed-untracked-set",
|
|
876
|
+
"revision-changed-worktree-mtime",
|
|
877
|
+
"revision-changed-metadata"
|
|
878
|
+
]);
|
|
359
879
|
const CHAT_SEND_PHASE = /^chat\.send\.(authority|admission|preparation|attachments|replyContext|authoring|persist|runAdmission|replyInitialization|snapshot|worktree|effects|response|dispatch)$/;
|
|
880
|
+
const CONTROL_UI_DOCUMENT_STAGE = /^control-ui\.document\.(auth_grants|identity_avatar|plugin_catalog|git_branch|preferences|render|total)$/;
|
|
360
881
|
function recordOperationTimingEvent(store, evt, metadata) {
|
|
361
882
|
switch (evt.type) {
|
|
883
|
+
case "sessions.list.baseline.retired":
|
|
884
|
+
if (metadata.trusted) store.counter("openclaw_sessions_list_baseline_retirements_total", "Session list baseline retirement events by cause, not rows or views retired.", { cause: evt.cause });
|
|
885
|
+
return;
|
|
362
886
|
case "secret.egress":
|
|
363
887
|
store.counter("openclaw_secret_egress_requests_total", "Secret egress proxy requests by outcome and credential substitution.", {
|
|
364
888
|
outcome: evt.outcome,
|
|
@@ -378,20 +902,87 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
378
902
|
store.counter("openclaw_gateway_http_cancelled_total", "Gateway HTTP requests cancelled before completion.", { source: evt.source });
|
|
379
903
|
return;
|
|
380
904
|
case "diagnostic.gc":
|
|
381
|
-
|
|
905
|
+
if (evt.kind === "mark_sweep_compact") {
|
|
906
|
+
const labels = { thread: evt.thread ?? "main" };
|
|
907
|
+
store.gauge("openclaw_heap_used_after_major_gc_bytes", "Native used heap after the latest received major collection by isolate role.", labels, evt.heapUsedAfterMajorGcBytes);
|
|
908
|
+
store.gauge("openclaw_heap_used_after_major_gc_timestamp_seconds", "Unix observation time of the latest received native major-GC heap sample.", labels, evt.heapUsedAfterMajorGcTimestampSeconds);
|
|
909
|
+
}
|
|
910
|
+
store.histogram(GC_DURATION_NAME, GC_DURATION_HELP, {
|
|
382
911
|
kind: evt.kind ?? "unknown",
|
|
912
|
+
reason: evt.reason ?? "other",
|
|
383
913
|
thread: evt.thread ?? "main",
|
|
384
914
|
...evt.workerSlot === void 0 ? {} : { slot: String(evt.workerSlot) }
|
|
385
915
|
}, seconds(evt.durationMs));
|
|
386
916
|
return;
|
|
917
|
+
case "gateway.main_thread.stall":
|
|
918
|
+
for (const stall of evt.stalls) store.histogram("openclaw_gateway_main_thread_stall_seconds", "Elapsed main-thread tasks lasting at least 250 milliseconds, by work scope.", { task: normalizeDiagnosticValue(stall.task) }, seconds(stall.durationMs), MAIN_THREAD_STALL_BUCKETS_SECONDS);
|
|
919
|
+
store.counter("openclaw_gateway_main_thread_stalls_dropped_total", "Main-thread stall records dropped because the attribution buffer was full.", {}, evt.dropped);
|
|
920
|
+
return;
|
|
387
921
|
case "gateway.event_loop.sample":
|
|
388
922
|
store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
|
|
389
923
|
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
390
924
|
return;
|
|
391
925
|
case "diagnostic.phase.completed": {
|
|
392
926
|
if (!metadata.trusted) return;
|
|
927
|
+
const documentStage = CONTROL_UI_DOCUMENT_STAGE.exec(evt.name)?.[1];
|
|
928
|
+
if (documentStage) {
|
|
929
|
+
store.histogram("openclaw_control_ui_document_duration_seconds", "Elapsed Control UI HTML document handler stages in seconds; stages can overlap.", { stage: documentStage }, seconds(evt.durationMs));
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
if (evt.name === "sessions.diff" || evt.name === "checkout.diff") {
|
|
933
|
+
const outcome = evt.details?.outcome;
|
|
934
|
+
if (outcome !== "cache-hit" && outcome !== "cache-miss") return;
|
|
935
|
+
const reason = evt.details?.missReason;
|
|
936
|
+
if (outcome === "cache-miss" && typeof reason === "string" && DIFF_CACHE_MISS_REASONS.has(reason)) store.counter("openclaw_diff_cache_misses_total", "Local diff cache misses by bounded reason; pending reads join an existing load.", {
|
|
937
|
+
method: evt.name,
|
|
938
|
+
reason
|
|
939
|
+
});
|
|
940
|
+
for (const stage of [
|
|
941
|
+
"admission",
|
|
942
|
+
"checkout-resolve",
|
|
943
|
+
"cache-validate",
|
|
944
|
+
"worker-queue",
|
|
945
|
+
"git-wall",
|
|
946
|
+
"encode"
|
|
947
|
+
]) {
|
|
948
|
+
const elapsed = evt.details?.[stage];
|
|
949
|
+
store.histogram("openclaw_diff_stage_seconds", "Elapsed local diff stages; cache validation includes its worker queue and Git wall time.", {
|
|
950
|
+
method: evt.name,
|
|
951
|
+
stage,
|
|
952
|
+
outcome
|
|
953
|
+
}, typeof elapsed === "number" ? seconds(elapsed) : void 0, RPC_DURATION_BUCKETS_SECONDS);
|
|
954
|
+
}
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
393
957
|
if (evt.name === "worktree.preparation") {
|
|
394
|
-
|
|
958
|
+
const { kind, template, outcome } = evt.details ?? {};
|
|
959
|
+
if (kind !== "managed" && kind !== "sandbox" || ![
|
|
960
|
+
"warm",
|
|
961
|
+
"cold",
|
|
962
|
+
"unavailable",
|
|
963
|
+
"reused"
|
|
964
|
+
].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
|
|
965
|
+
for (const preparationPhase of [
|
|
966
|
+
"total",
|
|
967
|
+
"allocate",
|
|
968
|
+
"checkout",
|
|
969
|
+
"setup",
|
|
970
|
+
"templatePrepare",
|
|
971
|
+
"templateApply",
|
|
972
|
+
"snapshot",
|
|
973
|
+
"synchronizeCanonical",
|
|
974
|
+
"synchronizeProjection",
|
|
975
|
+
"containerStart",
|
|
976
|
+
"workspaceLayout"
|
|
977
|
+
]) {
|
|
978
|
+
const elapsed = preparationPhase === "total" ? evt.durationMs : evt.details?.[preparationPhase];
|
|
979
|
+
store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
|
|
980
|
+
kind,
|
|
981
|
+
template: String(template),
|
|
982
|
+
outcome,
|
|
983
|
+
phase: preparationPhase
|
|
984
|
+
}, typeof elapsed === "number" ? seconds(elapsed) : void 0);
|
|
985
|
+
}
|
|
395
986
|
return;
|
|
396
987
|
}
|
|
397
988
|
const sendPhase = CHAT_SEND_PHASE.exec(evt.name)?.[1];
|
|
@@ -403,6 +994,14 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
403
994
|
}, seconds(evt.durationMs));
|
|
404
995
|
return;
|
|
405
996
|
}
|
|
997
|
+
const startupPhase = CHAT_STARTUP_PHASE.exec(evt.name)?.[1];
|
|
998
|
+
if (startupPhase) {
|
|
999
|
+
store.histogram("openclaw_gateway_rpc_stage_seconds", "Elapsed time for completed Gateway RPC owner stages.", {
|
|
1000
|
+
method: "chat.startup",
|
|
1001
|
+
phase: startupPhase
|
|
1002
|
+
}, seconds(evt.durationMs));
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
406
1005
|
if (!evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
|
|
407
1006
|
const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
|
|
408
1007
|
const duration = seconds(evt.durationMs);
|
|
@@ -420,6 +1019,10 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
420
1019
|
const labels = { method: evt.method };
|
|
421
1020
|
if (evt.phase === "received") {
|
|
422
1021
|
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
1022
|
+
store.counter("openclaw_gateway_rpc_client_requests_total", "Gateway RPC requests by bounded handshake client kind.", {
|
|
1023
|
+
method: evt.method,
|
|
1024
|
+
client_kind: evt.clientKind ?? "other"
|
|
1025
|
+
});
|
|
423
1026
|
if (evt.method === "models.list" && evt.modelsList) {
|
|
424
1027
|
const catalog = evt.modelsList;
|
|
425
1028
|
store.counter("openclaw_models_list_requests_total", "Model catalog requests by client kind and bounded request shape.", {
|
|
@@ -431,10 +1034,54 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
431
1034
|
prepared_only: String(catalog.preparedOnly)
|
|
432
1035
|
});
|
|
433
1036
|
}
|
|
1037
|
+
if (evt.method === "sessions.list" && evt.sessionsList) {
|
|
1038
|
+
const sessions = evt.sessionsList;
|
|
1039
|
+
store.counter("openclaw_sessions_list_requests_total", "Session list requests by client kind and bounded request shape.", {
|
|
1040
|
+
client_kind: sessions.clientKind,
|
|
1041
|
+
source: sessions.source,
|
|
1042
|
+
caller_source: sessions.callerSource,
|
|
1043
|
+
row_mode: sessions.rowMode,
|
|
1044
|
+
filter: sessions.filter
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
if (evt.method === "cron.list" && evt.cronList) {
|
|
1048
|
+
const cron = evt.cronList;
|
|
1049
|
+
store.counter("openclaw_cron_list_requests_total", "Cron list requests by client kind and bounded request shape.", {
|
|
1050
|
+
client_kind: cron.clientKind,
|
|
1051
|
+
source: cron.source,
|
|
1052
|
+
scope: cron.scope,
|
|
1053
|
+
compact: String(cron.compact),
|
|
1054
|
+
page: cron.page
|
|
1055
|
+
});
|
|
1056
|
+
}
|
|
434
1057
|
return;
|
|
435
1058
|
}
|
|
436
1059
|
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) {
|
|
1060
|
+
if (evt.outcome === "ok" && evt.firstResponse !== false && evt.sessionsList?.baseMissReason) store.counter("openclaw_sessions_list_base_misses_total", "Successful full sidebar responses with a supplied base, by miss reason.", { reason: evt.sessionsList.baseMissReason });
|
|
437
1061
|
store.histogram("openclaw_gateway_rpc_response_bytes", "Encoded Gateway RPC response frame size in bytes, including later frames.", labels, evt.responseBytes, RESPONSE_BYTE_BUCKETS);
|
|
1062
|
+
if (evt.responseBytes !== void 0 && evt.sessionsList) {
|
|
1063
|
+
const sessions = evt.sessionsList;
|
|
1064
|
+
store.counter("openclaw_sessions_list_response_bytes_total", "Encoded successful session list response bytes by request shape and row outcome.", {
|
|
1065
|
+
client_kind: sessions.clientKind,
|
|
1066
|
+
source: sessions.source,
|
|
1067
|
+
caller_source: sessions.callerSource,
|
|
1068
|
+
row_mode: sessions.rowMode,
|
|
1069
|
+
filter: sessions.filter,
|
|
1070
|
+
rows_outcome: sessions.rowsOutcome
|
|
1071
|
+
}, evt.responseBytes);
|
|
1072
|
+
}
|
|
1073
|
+
if (evt.chatHistory) {
|
|
1074
|
+
const history = evt.chatHistory;
|
|
1075
|
+
const historyLabels = {
|
|
1076
|
+
client_kind: history.clientKind,
|
|
1077
|
+
source: history.source,
|
|
1078
|
+
shape: history.shape,
|
|
1079
|
+
limit_bucket: history.limitBucket,
|
|
1080
|
+
outcome: history.outcome
|
|
1081
|
+
};
|
|
1082
|
+
if (evt.firstResponse !== false) store.counter("openclaw_chat_history_requests_total", "Successful chat history requests by request shape and response outcome.", historyLabels);
|
|
1083
|
+
if (evt.responseBytes !== void 0) store.counter("openclaw_chat_history_response_bytes_total", "Encoded successful chat history response bytes by request shape and response outcome.", historyLabels, evt.responseBytes);
|
|
1084
|
+
}
|
|
438
1085
|
if (evt.firstResponse === false) return;
|
|
439
1086
|
}
|
|
440
1087
|
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
@@ -451,53 +1098,22 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
451
1098
|
}
|
|
452
1099
|
}
|
|
453
1100
|
}
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
if (
|
|
457
|
-
"
|
|
458
|
-
|
|
459
|
-
"unavailable",
|
|
460
|
-
"reused"
|
|
461
|
-
].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
|
|
462
|
-
for (const phase of [
|
|
463
|
-
"total",
|
|
464
|
-
"allocate",
|
|
465
|
-
"checkout",
|
|
466
|
-
"setup",
|
|
467
|
-
"templatePrepare",
|
|
468
|
-
"templateApply",
|
|
469
|
-
"snapshot",
|
|
470
|
-
"synchronizeCanonical",
|
|
471
|
-
"synchronizeProjection",
|
|
472
|
-
"containerStart",
|
|
473
|
-
"workspaceLayout"
|
|
474
|
-
]) {
|
|
475
|
-
const elapsed = phase === "total" ? evt.durationMs : evt.details?.[phase];
|
|
476
|
-
store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
|
|
477
|
-
kind,
|
|
478
|
-
template: String(template),
|
|
479
|
-
outcome,
|
|
480
|
-
phase
|
|
481
|
-
}, typeof elapsed === "number" ? seconds(elapsed) : void 0);
|
|
1101
|
+
// extensions/diagnostics-prometheus/src/service-retention.ts
|
|
1102
|
+
function recordSessionRetention(store, evt) {
|
|
1103
|
+
if (evt.type === "sessions.event.rows.reset") {
|
|
1104
|
+
store.counter("openclaw_session_event_rows_reset_total", "Retained session-event wire baselines retired by cause.", { cause: evt.cause }, evt.count);
|
|
1105
|
+
return;
|
|
482
1106
|
}
|
|
1107
|
+
if (evt.type === "sessions.list.retention") {
|
|
1108
|
+
store.gauge("openclaw_sessions_list_retained_views", "Retained session-list views across all publications.", {}, evt.views);
|
|
1109
|
+
store.gauge("openclaw_sessions_list_retained_refs", "Row references and selection entries retained by session-list views.", {}, evt.refs);
|
|
1110
|
+
store.gauge("openclaw_sessions_list_retained_bytes", "Estimated memory retained by session-list views.", {}, evt.estimatedBytes);
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
store.gauge("openclaw_session_event_rows_retained_entries", "Session-event field-hash vectors retained across all connections.", {}, evt.entries);
|
|
1114
|
+
store.gauge("openclaw_session_event_rows_retained_bytes", "Estimated memory retained by session-event field-hash vectors.", {}, evt.estimatedBytes);
|
|
483
1115
|
}
|
|
484
|
-
|
|
485
|
-
//#region extensions/diagnostics-prometheus/src/service-sessions.ts
|
|
486
|
-
function createGatewayWorkMetricsRecorder(store) {
|
|
487
|
-
return (snapshot) => {
|
|
488
|
-
if (!snapshot) {
|
|
489
|
-
store.clearGauges("openclaw_sessions_active");
|
|
490
|
-
store.clearGauges("openclaw_gateway_active_work");
|
|
491
|
-
return;
|
|
492
|
-
}
|
|
493
|
-
for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
|
|
494
|
-
for (const kind of [
|
|
495
|
-
"agentRuns",
|
|
496
|
-
"chatRuns",
|
|
497
|
-
"queuedTurns"
|
|
498
|
-
]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
|
|
499
|
-
};
|
|
500
|
-
}
|
|
1116
|
+
// extensions/diagnostics-prometheus/src/service-sessions.ts
|
|
501
1117
|
function recordSessionDiagnosticEvent(store, evt) {
|
|
502
1118
|
switch (evt.type) {
|
|
503
1119
|
case "session.recovery.requested":
|
|
@@ -535,65 +1151,65 @@ function recordSessionDiagnosticEvent(store, evt) {
|
|
|
535
1151
|
});
|
|
536
1152
|
}
|
|
537
1153
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
1154
|
+
// extensions/diagnostics-prometheus/src/service-worker.ts
|
|
1155
|
+
function createWorkerRequestMetricsRecorder(store) {
|
|
1156
|
+
return (kind, requestClass, requestPhase, queueDepth, queueWaitMs, durationMs, timing) => {
|
|
1157
|
+
store.gauge("openclaw_worker_queue_depth", "Requests awaiting dispatch across worker owners of this kind.", { kind }, queueDepth);
|
|
1158
|
+
const labels = {
|
|
1159
|
+
kind,
|
|
1160
|
+
request_class: requestClass
|
|
1161
|
+
};
|
|
1162
|
+
store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(queueWaitMs));
|
|
1163
|
+
store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(durationMs));
|
|
1164
|
+
if (requestPhase !== "completed" || !timing) return;
|
|
1165
|
+
for (const [phase, start, end] of [
|
|
1166
|
+
[
|
|
1167
|
+
"dispatch",
|
|
1168
|
+
"dispatched",
|
|
1169
|
+
"posted"
|
|
1170
|
+
],
|
|
1171
|
+
[
|
|
1172
|
+
"delivery",
|
|
1173
|
+
"posted",
|
|
1174
|
+
"received"
|
|
1175
|
+
],
|
|
1176
|
+
[
|
|
1177
|
+
"input",
|
|
1178
|
+
"received",
|
|
1179
|
+
"inputReady"
|
|
1180
|
+
],
|
|
1181
|
+
[
|
|
1182
|
+
"prepare",
|
|
1183
|
+
"inputReady",
|
|
1184
|
+
"executeStarted"
|
|
1185
|
+
],
|
|
1186
|
+
[
|
|
1187
|
+
"execute",
|
|
1188
|
+
"executeStarted",
|
|
1189
|
+
"executeFinished"
|
|
1190
|
+
],
|
|
1191
|
+
[
|
|
1192
|
+
"result",
|
|
1193
|
+
"executeFinished",
|
|
1194
|
+
"replied"
|
|
1195
|
+
],
|
|
1196
|
+
[
|
|
1197
|
+
"reply",
|
|
1198
|
+
"replied",
|
|
1199
|
+
"replyReceived"
|
|
1200
|
+
]
|
|
1201
|
+
]) {
|
|
1202
|
+
const started = timing[start];
|
|
1203
|
+
const finished = timing[end];
|
|
1204
|
+
if (started === void 0 || finished === void 0) continue;
|
|
1205
|
+
store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
|
|
1206
|
+
...labels,
|
|
1207
|
+
phase
|
|
1208
|
+
}, seconds(finished - started));
|
|
1209
|
+
}
|
|
545
1210
|
};
|
|
546
|
-
store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(event.queueWaitMs));
|
|
547
|
-
store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(event.durationMs));
|
|
548
|
-
if (event.phase !== "completed" || !event.timing) return;
|
|
549
|
-
for (const [phase, start, end] of [
|
|
550
|
-
[
|
|
551
|
-
"dispatch",
|
|
552
|
-
"dispatched",
|
|
553
|
-
"posted"
|
|
554
|
-
],
|
|
555
|
-
[
|
|
556
|
-
"delivery",
|
|
557
|
-
"posted",
|
|
558
|
-
"received"
|
|
559
|
-
],
|
|
560
|
-
[
|
|
561
|
-
"input",
|
|
562
|
-
"received",
|
|
563
|
-
"inputReady"
|
|
564
|
-
],
|
|
565
|
-
[
|
|
566
|
-
"prepare",
|
|
567
|
-
"inputReady",
|
|
568
|
-
"executeStarted"
|
|
569
|
-
],
|
|
570
|
-
[
|
|
571
|
-
"execute",
|
|
572
|
-
"executeStarted",
|
|
573
|
-
"executeFinished"
|
|
574
|
-
],
|
|
575
|
-
[
|
|
576
|
-
"result",
|
|
577
|
-
"executeFinished",
|
|
578
|
-
"replied"
|
|
579
|
-
],
|
|
580
|
-
[
|
|
581
|
-
"reply",
|
|
582
|
-
"replied",
|
|
583
|
-
"replyReceived"
|
|
584
|
-
]
|
|
585
|
-
]) {
|
|
586
|
-
const started = event.timing[start];
|
|
587
|
-
const finished = event.timing[end];
|
|
588
|
-
if (started === void 0 || finished === void 0) continue;
|
|
589
|
-
store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
|
|
590
|
-
...labels,
|
|
591
|
-
phase
|
|
592
|
-
}, seconds(finished - started));
|
|
593
|
-
}
|
|
594
1211
|
}
|
|
595
|
-
|
|
596
|
-
//#region extensions/diagnostics-prometheus/src/service.ts
|
|
1212
|
+
// extensions/diagnostics-prometheus/src/service.ts
|
|
597
1213
|
const BYTE_BUCKETS = [
|
|
598
1214
|
1024,
|
|
599
1215
|
4096,
|
|
@@ -626,36 +1242,6 @@ function safeErrorMessage(err) {
|
|
|
626
1242
|
const message = err instanceof Error ? err.message ?? err.name : String(err);
|
|
627
1243
|
return truncateUtf16Safe(redactSensitiveText(message).replaceAll("\0", " ").replace(/[\r\n\t\u2028\u2029]/gu, " "), 500);
|
|
628
1244
|
}
|
|
629
|
-
function renderPrometheusMetrics(store) {
|
|
630
|
-
const snapshot = store.snapshot();
|
|
631
|
-
const lines = [];
|
|
632
|
-
const emitted = /* @__PURE__ */ new Set();
|
|
633
|
-
const emitHeader = (name, type, help) => {
|
|
634
|
-
if (emitted.has(name)) return;
|
|
635
|
-
emitted.add(name);
|
|
636
|
-
lines.push(`# HELP ${name} ${escapeHelp(help)}`);
|
|
637
|
-
lines.push(`# TYPE ${name} ${type}`);
|
|
638
|
-
};
|
|
639
|
-
for (const [type, entries] of [["counter", snapshot.counters], ["gauge", snapshot.gauges]]) {
|
|
640
|
-
entries.sort(([left], [right]) => left.localeCompare(right));
|
|
641
|
-
for (const [key, sample] of entries) {
|
|
642
|
-
const name = key.split("|", 1)[0] ?? "";
|
|
643
|
-
emitHeader(name, type, sample.help);
|
|
644
|
-
lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
|
|
645
|
-
}
|
|
646
|
-
}
|
|
647
|
-
snapshot.histograms.sort(([left], [right]) => left.localeCompare(right));
|
|
648
|
-
for (const [key, sample] of snapshot.histograms) {
|
|
649
|
-
const name = key.split("|", 1)[0] ?? "";
|
|
650
|
-
emitHeader(name, "histogram", sample.help);
|
|
651
|
-
for (let index = 0; index < sample.buckets.length; index += 1) lines.push(`${sample.bucketPrefixes[index]}${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
|
|
652
|
-
lines.push(`${sample.bucketPrefixes[sample.buckets.length]}${formatPrometheusNumber(sample.count)}`);
|
|
653
|
-
lines.push(`${name}_sum${sample.labels} ${formatPrometheusNumber(sample.sum)}`);
|
|
654
|
-
lines.push(`${name}_count${sample.labels} ${formatPrometheusNumber(sample.count)}`);
|
|
655
|
-
}
|
|
656
|
-
lines.push("");
|
|
657
|
-
return lines.join("\n");
|
|
658
|
-
}
|
|
659
1245
|
function webhookLabels(evt) {
|
|
660
1246
|
return {
|
|
661
1247
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
@@ -665,16 +1251,23 @@ function webhookLabels(evt) {
|
|
|
665
1251
|
function recordDiagnosticEvent(store, evt, metadata) {
|
|
666
1252
|
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
667
1253
|
switch (evt.type) {
|
|
668
|
-
case "
|
|
669
|
-
if (metadata.trusted)
|
|
1254
|
+
case "gateway.ws.connections":
|
|
1255
|
+
if (metadata.trusted) recordGatewayWsConnections(store, evt.operators);
|
|
1256
|
+
return;
|
|
1257
|
+
case "sessions.list.retention":
|
|
1258
|
+
case "sessions.event.retention":
|
|
1259
|
+
case "sessions.event.rows.reset":
|
|
1260
|
+
if (metadata.trusted) recordSessionRetention(store, evt);
|
|
670
1261
|
return;
|
|
671
1262
|
case "diagnostic.phase.completed":
|
|
672
1263
|
case "secret.egress":
|
|
673
1264
|
case "tool.catalog":
|
|
674
1265
|
case "gateway.rpc":
|
|
1266
|
+
case "sessions.list.baseline.retired":
|
|
675
1267
|
case "gateway.http.cancelled":
|
|
676
1268
|
case "diagnostic.gc":
|
|
677
1269
|
case "gateway.event_loop.sample":
|
|
1270
|
+
case "gateway.main_thread.stall":
|
|
678
1271
|
recordOperationTimingEvent(store, evt, metadata);
|
|
679
1272
|
return;
|
|
680
1273
|
case "model.usage":
|
|
@@ -850,17 +1443,22 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
850
1443
|
recordSessionDiagnosticEvent(store, evt);
|
|
851
1444
|
return;
|
|
852
1445
|
case "queue.lane.enqueue":
|
|
853
|
-
case "queue.lane.dequeue":
|
|
854
|
-
|
|
855
|
-
|
|
1446
|
+
case "queue.lane.dequeue": {
|
|
1447
|
+
const labels = { lane: normalizeDiagnosticLane(evt.lane) };
|
|
1448
|
+
store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", labels, asNonNegativeFiniteNumber(evt.queueSize));
|
|
1449
|
+
if (evt.type === "queue.lane.dequeue") store.histogram("openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", labels, seconds(evt.waitMs));
|
|
856
1450
|
return;
|
|
1451
|
+
}
|
|
857
1452
|
case "session.state":
|
|
858
1453
|
case "session.stuck":
|
|
859
1454
|
case "session.turn.created":
|
|
860
1455
|
recordSessionDiagnosticEvent(store, evt);
|
|
861
1456
|
return;
|
|
862
1457
|
case "diagnostic.child_process.spawn":
|
|
863
|
-
|
|
1458
|
+
store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", {
|
|
1459
|
+
family: normalizeDiagnosticValue(evt.family),
|
|
1460
|
+
operation: normalizeDiagnosticValue(evt.operation ?? (evt.family === "git" ? "unknown" : "none"))
|
|
1461
|
+
}, asNonNegativeFiniteNumber(evt.count) ?? 0);
|
|
864
1462
|
return;
|
|
865
1463
|
case "diagnostic.memory.sample":
|
|
866
1464
|
recordMemorySample(store, evt.memory, BYTE_BUCKETS);
|
|
@@ -945,23 +1543,24 @@ function createMetricsHandler(store) {
|
|
|
945
1543
|
res.end("Method Not Allowed");
|
|
946
1544
|
return true;
|
|
947
1545
|
}
|
|
948
|
-
const body =
|
|
1546
|
+
const body = store.render();
|
|
949
1547
|
res.statusCode = 200;
|
|
950
1548
|
res.setHeader("Cache-Control", "no-store");
|
|
951
1549
|
res.setHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
952
|
-
res.setHeader("Content-Length", String(
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
}
|
|
957
|
-
res.end(body);
|
|
1550
|
+
res.setHeader("Content-Length", String(body.length));
|
|
1551
|
+
res.flushHeaders();
|
|
1552
|
+
res.end(req.method === "HEAD" ? void 0 : body, (error) => {
|
|
1553
|
+
if (!error) body.buffer.transfer(0);
|
|
1554
|
+
});
|
|
958
1555
|
return true;
|
|
959
1556
|
};
|
|
960
1557
|
}
|
|
961
1558
|
function createDiagnosticsPrometheusExporter() {
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
1559
|
+
let logger;
|
|
1560
|
+
const store = createPrometheusMetricStore((family) => {
|
|
1561
|
+
logger?.warn(`diagnostics-prometheus: series budget reached for ${family}; new label sets are being dropped. See openclaw_prometheus_family_series_dropped_total.`);
|
|
1562
|
+
});
|
|
1563
|
+
let subscriptions = [];
|
|
965
1564
|
let internalDiagnostics;
|
|
966
1565
|
const reportExporterStatus = (update) => {
|
|
967
1566
|
try {
|
|
@@ -976,7 +1575,6 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
976
1575
|
};
|
|
977
1576
|
return {
|
|
978
1577
|
handler: createMetricsHandler(store),
|
|
979
|
-
render: () => renderPrometheusMetrics(store),
|
|
980
1578
|
service: {
|
|
981
1579
|
id: "diagnostics-prometheus",
|
|
982
1580
|
start(ctx) {
|
|
@@ -985,19 +1583,44 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
985
1583
|
ctx.logger.error("diagnostics-prometheus: internal diagnostics capability unavailable");
|
|
986
1584
|
return;
|
|
987
1585
|
}
|
|
1586
|
+
logger = ctx.logger;
|
|
1587
|
+
reserveMainGcHistograms(store);
|
|
988
1588
|
const identity = isDiagnosticsEnabled(ctx.config) ? ctx.internalDiagnostics?.getRuntimeIdentity?.() : void 0;
|
|
989
1589
|
if (identity) store.gauge("openclaw_gateway_build_info", "Identity of the hosting process and its loaded build; not a health or exporter epoch.", {
|
|
990
1590
|
process_instance_id: identity.processInstanceId,
|
|
991
1591
|
...identity.buildId ? { build_id: identity.buildId } : {}
|
|
992
1592
|
}, 1);
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1593
|
+
subscriptions = [
|
|
1594
|
+
ctx.internalDiagnostics?.onGatewayWorkMetrics?.((snapshot) => {
|
|
1595
|
+
if (!snapshot) {
|
|
1596
|
+
store.clearGauges("openclaw_sessions_active");
|
|
1597
|
+
store.clearGauges("openclaw_gateway_active_work");
|
|
1598
|
+
return;
|
|
1599
|
+
}
|
|
1600
|
+
for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
|
|
1601
|
+
for (const kind of [
|
|
1602
|
+
"agentRuns",
|
|
1603
|
+
"chatRuns",
|
|
1604
|
+
"queuedTurns"
|
|
1605
|
+
]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
|
|
1606
|
+
}),
|
|
1607
|
+
ctx.internalDiagnostics?.onGatewayWsMetrics?.(createGatewayWsMetricsRecorder(store)),
|
|
1608
|
+
ctx.internalDiagnostics?.onWorkerRequestMetrics?.(createWorkerRequestMetricsRecorder(store)),
|
|
1609
|
+
subscribe((event, metadata) => {
|
|
1610
|
+
try {
|
|
1611
|
+
recordDiagnosticEvent(store, event, metadata);
|
|
1612
|
+
} catch (err) {
|
|
1613
|
+
ctx.logger.error(`diagnostics-prometheus: event handler failed (${event.type}): ${safeErrorMessage(err)}`);
|
|
1614
|
+
}
|
|
1615
|
+
}, {
|
|
1616
|
+
exclude: [
|
|
1617
|
+
"log.record",
|
|
1618
|
+
"worker.request",
|
|
1619
|
+
"gateway.ws.event"
|
|
1620
|
+
],
|
|
1621
|
+
includePromptStats: false
|
|
1622
|
+
}, { includePrivateData: false })
|
|
1623
|
+
];
|
|
1001
1624
|
internalDiagnostics = ctx.internalDiagnostics;
|
|
1002
1625
|
reportExporterStatus({
|
|
1003
1626
|
signal: "metrics",
|
|
@@ -1007,10 +1630,8 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
1007
1630
|
});
|
|
1008
1631
|
},
|
|
1009
1632
|
stop() {
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
unsubscribe?.();
|
|
1013
|
-
unsubscribe = void 0;
|
|
1633
|
+
for (const unsubscribe of subscriptions) unsubscribe?.();
|
|
1634
|
+
subscriptions = [];
|
|
1014
1635
|
reportExporterStatus({
|
|
1015
1636
|
signal: "metrics",
|
|
1016
1637
|
transport: "prometheus-scrape",
|
|
@@ -1018,12 +1639,12 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
1018
1639
|
});
|
|
1019
1640
|
internalDiagnostics = void 0;
|
|
1020
1641
|
store.reset();
|
|
1642
|
+
logger = void 0;
|
|
1021
1643
|
}
|
|
1022
1644
|
}
|
|
1023
1645
|
};
|
|
1024
1646
|
}
|
|
1025
|
-
|
|
1026
|
-
//#region extensions/diagnostics-prometheus/index.ts
|
|
1647
|
+
// extensions/diagnostics-prometheus/index.ts
|
|
1027
1648
|
const exporter = createDiagnosticsPrometheusExporter();
|
|
1028
1649
|
var diagnostics_prometheus_default = definePluginEntry({
|
|
1029
1650
|
id: "diagnostics-prometheus",
|
|
@@ -1040,5 +1661,4 @@ var diagnostics_prometheus_default = definePluginEntry({
|
|
|
1040
1661
|
});
|
|
1041
1662
|
}
|
|
1042
1663
|
});
|
|
1043
|
-
//#endregion
|
|
1044
1664
|
export { diagnostics_prometheus_default as default };
|