@openclaw/diagnostics-prometheus 2026.10.10 → 2026.11.5-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import { isDiagnosticsEnabled, normalizeDiagnosticLane, normalizeDiagnosticValue
|
|
|
4
4
|
import { asNonNegativeFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
|
|
5
5
|
import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
|
|
6
6
|
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
7
|
+
import { expectDefined } from "openclaw/plugin-sdk/expect-runtime";
|
|
7
8
|
// extensions/diagnostics-prometheus/src/prometheus-format.ts
|
|
8
9
|
function sortedLabels(labels) {
|
|
9
10
|
const entries = Object.entries(labels);
|
|
@@ -32,7 +33,166 @@ function seconds(ms) {
|
|
|
32
33
|
const value = asNonNegativeFiniteNumber(ms);
|
|
33
34
|
return value === void 0 ? void 0 : value / 1e3;
|
|
34
35
|
}
|
|
36
|
+
// extensions/diagnostics-prometheus/src/prometheus-family-budget.ts
|
|
37
|
+
const PROMETHEUS_FAMILY_RESERVES = {
|
|
38
|
+
openclaw_chat_history_requests_total: 4,
|
|
39
|
+
openclaw_chat_history_response_bytes_total: 4,
|
|
40
|
+
openclaw_chat_send_phase_seconds: 32,
|
|
41
|
+
openclaw_child_process_spawn_total: 4,
|
|
42
|
+
openclaw_code_mode_worker_heap_bytes: 128,
|
|
43
|
+
openclaw_code_mode_worker_memory_bytes: 128,
|
|
44
|
+
openclaw_control_ui_document_duration_seconds: 4,
|
|
45
|
+
openclaw_cron_list_requests_total: 4,
|
|
46
|
+
openclaw_diagnostic_async_queue_dropped_total: 4,
|
|
47
|
+
openclaw_diagnostic_async_queue_length: 4,
|
|
48
|
+
openclaw_diff_cache_misses_total: 4,
|
|
49
|
+
openclaw_diff_stage_seconds: 4,
|
|
50
|
+
openclaw_gateway_active_work: 4,
|
|
51
|
+
openclaw_gateway_build_info: 4,
|
|
52
|
+
openclaw_gateway_event_loop_delay_max_seconds: 4,
|
|
53
|
+
openclaw_gateway_event_loop_observed_seconds_total: 4,
|
|
54
|
+
openclaw_gateway_http_cancelled_total: 4,
|
|
55
|
+
openclaw_gateway_main_thread_stall_seconds: 4,
|
|
56
|
+
openclaw_gateway_main_thread_stalls_dropped_total: 4,
|
|
57
|
+
openclaw_gateway_rpc_admission_seconds: 256,
|
|
58
|
+
openclaw_gateway_rpc_client_requests_total: 512,
|
|
59
|
+
openclaw_gateway_rpc_first_response_seconds: 256,
|
|
60
|
+
openclaw_gateway_rpc_handler_heap_delta_bytes: 256,
|
|
61
|
+
openclaw_gateway_rpc_handler_heap_delta_exclusive_total: 256,
|
|
62
|
+
openclaw_gateway_rpc_handler_seconds: 256,
|
|
63
|
+
openclaw_gateway_rpc_outcomes_total: 256,
|
|
64
|
+
openclaw_gateway_rpc_queue_wait_seconds: 256,
|
|
65
|
+
openclaw_gateway_rpc_requests_total: 256,
|
|
66
|
+
openclaw_gateway_rpc_response_bytes: 256,
|
|
67
|
+
openclaw_gateway_rpc_stage_seconds: 256,
|
|
68
|
+
openclaw_gateway_rpc_stage_thread_cpu_seconds: 256,
|
|
69
|
+
openclaw_gateway_ws_event_bytes_total: 256,
|
|
70
|
+
openclaw_gateway_ws_event_frame_bytes: 2048,
|
|
71
|
+
openclaw_gateway_ws_event_wire_bytes_total: 256,
|
|
72
|
+
openclaw_gateway_ws_events_total: 256,
|
|
73
|
+
openclaw_gateway_ws_operator_connections: 4,
|
|
74
|
+
openclaw_gateway_ws_presence_bytes_total: 4,
|
|
75
|
+
openclaw_gateway_ws_presence_frames_total: 4,
|
|
76
|
+
openclaw_gateway_ws_session_event_bytes_total: 256,
|
|
77
|
+
openclaw_gateway_ws_session_event_frames_total: 256,
|
|
78
|
+
openclaw_gc_duration_seconds: 384,
|
|
79
|
+
openclaw_gen_ai_client_token_usage: 128,
|
|
80
|
+
openclaw_harness_run_duration_seconds: 128,
|
|
81
|
+
openclaw_harness_run_total: 128,
|
|
82
|
+
openclaw_heap_old_generation_after_gc_bytes: 128,
|
|
83
|
+
openclaw_heap_old_generation_after_gc_ratio: 128,
|
|
84
|
+
openclaw_heap_size_limit_bytes: 128,
|
|
85
|
+
openclaw_heap_space_bytes: 256,
|
|
86
|
+
openclaw_heap_used_after_major_gc_bytes: 64,
|
|
87
|
+
openclaw_heap_used_after_major_gc_timestamp_seconds: 64,
|
|
88
|
+
openclaw_heap_used_ratio: 128,
|
|
89
|
+
openclaw_liveness_cpu_core_ratio: 4,
|
|
90
|
+
openclaw_liveness_event_loop_delay_max_seconds: 4,
|
|
91
|
+
openclaw_liveness_event_loop_delay_p99_seconds: 4,
|
|
92
|
+
openclaw_liveness_event_loop_utilization_ratio: 4,
|
|
93
|
+
openclaw_liveness_sessions: 4,
|
|
94
|
+
openclaw_liveness_warning_total: 4,
|
|
95
|
+
openclaw_memory_bytes: 128,
|
|
96
|
+
openclaw_memory_pressure_total: 4,
|
|
97
|
+
openclaw_memory_rss_bytes: 4,
|
|
98
|
+
openclaw_message_delivery_duration_seconds: 4,
|
|
99
|
+
openclaw_message_delivery_started_total: 4,
|
|
100
|
+
openclaw_message_delivery_total: 4,
|
|
101
|
+
openclaw_message_dispatch_completed_total: 4,
|
|
102
|
+
openclaw_message_dispatch_duration_seconds: 4,
|
|
103
|
+
openclaw_message_dispatch_started_total: 4,
|
|
104
|
+
openclaw_message_processed_duration_seconds: 4,
|
|
105
|
+
openclaw_message_processed_total: 4,
|
|
106
|
+
openclaw_message_received_total: 4,
|
|
107
|
+
openclaw_model_call_duration_seconds: 128,
|
|
108
|
+
openclaw_model_call_total: 128,
|
|
109
|
+
openclaw_model_cost_usd_total: 128,
|
|
110
|
+
openclaw_model_failover_total: 128,
|
|
111
|
+
openclaw_model_tokens_total: 128,
|
|
112
|
+
openclaw_model_usage_duration_seconds: 128,
|
|
113
|
+
openclaw_models_list_requests_total: 4,
|
|
114
|
+
openclaw_payload_large_bytes: 4,
|
|
115
|
+
openclaw_payload_large_total: 4,
|
|
116
|
+
openclaw_process_anonymous_memory_bytes: 4,
|
|
117
|
+
openclaw_queue_lane_size: 4,
|
|
118
|
+
openclaw_queue_lane_wait_seconds: 4,
|
|
119
|
+
openclaw_run_completed_total: 128,
|
|
120
|
+
openclaw_run_duration_seconds: 128,
|
|
121
|
+
openclaw_secret_egress_requests_total: 4,
|
|
122
|
+
openclaw_session_event_rows_reset_total: 4,
|
|
123
|
+
openclaw_session_event_rows_retained_bytes: 4,
|
|
124
|
+
openclaw_session_event_rows_retained_entries: 4,
|
|
125
|
+
openclaw_session_queue_depth: 4,
|
|
126
|
+
openclaw_session_recovery_age_seconds: 4,
|
|
127
|
+
openclaw_session_recovery_total: 4,
|
|
128
|
+
openclaw_session_row_selection_resets_total: 4,
|
|
129
|
+
openclaw_session_row_partition_cache_total: 32,
|
|
130
|
+
openclaw_session_row_partition_revisions_total: 8,
|
|
131
|
+
openclaw_session_row_selection_rows_total: 64,
|
|
132
|
+
openclaw_session_row_selections_total: 64,
|
|
133
|
+
openclaw_session_state_total: 4,
|
|
134
|
+
openclaw_session_stuck_age_seconds: 4,
|
|
135
|
+
openclaw_session_stuck_total: 4,
|
|
136
|
+
openclaw_session_turn_created_total: 4,
|
|
137
|
+
openclaw_sessions_active: 4,
|
|
138
|
+
openclaw_sessions_list_base_misses_total: 4,
|
|
139
|
+
openclaw_sessions_list_baseline_retirements_total: 4,
|
|
140
|
+
openclaw_sessions_list_requests_total: 4,
|
|
141
|
+
openclaw_sessions_list_response_bytes_total: 4,
|
|
142
|
+
openclaw_sessions_list_retained_bytes: 4,
|
|
143
|
+
openclaw_sessions_list_retained_refs: 4,
|
|
144
|
+
openclaw_sessions_list_retained_views: 4,
|
|
145
|
+
openclaw_skill_used_total: 4,
|
|
146
|
+
openclaw_talk_audio_bytes: 4,
|
|
147
|
+
openclaw_talk_event_duration_seconds: 4,
|
|
148
|
+
openclaw_talk_event_total: 4,
|
|
149
|
+
openclaw_telemetry_exporter_total: 4,
|
|
150
|
+
openclaw_tool_catalog_tools_total: 128,
|
|
151
|
+
openclaw_tool_catalog_total: 128,
|
|
152
|
+
openclaw_tool_execution_blocked_total: 128,
|
|
153
|
+
openclaw_tool_execution_duration_seconds: 128,
|
|
154
|
+
openclaw_tool_execution_total: 128,
|
|
155
|
+
openclaw_webhook_duration_seconds: 4,
|
|
156
|
+
openclaw_webhook_error_total: 4,
|
|
157
|
+
openclaw_webhook_received_total: 4,
|
|
158
|
+
openclaw_worker_array_buffers_bytes: 128,
|
|
159
|
+
openclaw_worker_count: 128,
|
|
160
|
+
openclaw_worker_external_bytes: 128,
|
|
161
|
+
openclaw_worker_gc_collections_total: 128,
|
|
162
|
+
openclaw_worker_gc_duration_seconds_total: 128,
|
|
163
|
+
openclaw_worker_gc_reclaimed_bytes_total: 128,
|
|
164
|
+
openclaw_worker_heap_sampled_count: 128,
|
|
165
|
+
openclaw_worker_heap_space_used_bytes: 640,
|
|
166
|
+
openclaw_worker_heap_used_bytes: 128,
|
|
167
|
+
openclaw_worker_heap_used_high_water_bytes: 128,
|
|
168
|
+
openclaw_worker_message_payload_bytes_total: 512,
|
|
169
|
+
openclaw_worker_message_unmeasured_total: 64,
|
|
170
|
+
openclaw_worker_messages_total: 64,
|
|
171
|
+
openclaw_worker_queue_depth: 512,
|
|
172
|
+
openclaw_worker_queue_wait_seconds: 512,
|
|
173
|
+
openclaw_worker_request_phase_seconds: 1024,
|
|
174
|
+
openclaw_worker_request_seconds: 512,
|
|
175
|
+
openclaw_worker_retired_total: 64,
|
|
176
|
+
openclaw_worker_started_total: 64,
|
|
177
|
+
openclaw_worker_tasks_completed_total: 128,
|
|
178
|
+
openclaw_worktree_preparation_seconds: 32
|
|
179
|
+
};
|
|
180
|
+
const SHARED_PROMETHEUS_SERIES = 16384 - Object.values(PROMETHEUS_FAMILY_RESERVES).reduce((sum, n) => sum + n, 0);
|
|
35
181
|
// extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
|
|
182
|
+
function createSeries() {
|
|
183
|
+
return {
|
|
184
|
+
samples: [],
|
|
185
|
+
shapes: /* @__PURE__ */ new Map(),
|
|
186
|
+
sorted: true
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function sortedSamples(series) {
|
|
190
|
+
if (!series.sorted) {
|
|
191
|
+
series.samples.sort((left, right) => left.key.localeCompare(right.key));
|
|
192
|
+
series.sorted = true;
|
|
193
|
+
}
|
|
194
|
+
return series.samples;
|
|
195
|
+
}
|
|
36
196
|
const DURATION_BUCKETS_SECONDS = [
|
|
37
197
|
.005,
|
|
38
198
|
.01,
|
|
@@ -57,101 +217,270 @@ const AGENT_DURATION_BUCKETS_SECONDS = [
|
|
|
57
217
|
1800,
|
|
58
218
|
3600
|
|
59
219
|
];
|
|
60
|
-
const MAX_PROMETHEUS_SERIES = 2048;
|
|
61
220
|
const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const
|
|
221
|
+
const DROPPED_FAMILY_COUNTER_NAME = "openclaw_prometheus_family_series_dropped_total";
|
|
222
|
+
const warnedFamilies = /* @__PURE__ */ new Set();
|
|
223
|
+
function createPrometheusMetricStore(onSeriesDropped) {
|
|
224
|
+
const counters = createSeries();
|
|
225
|
+
const gauges = createSeries();
|
|
226
|
+
const histograms = createSeries();
|
|
66
227
|
let droppedSeries = 0;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
228
|
+
let sharedSeries = 0;
|
|
229
|
+
const familyCounts = /* @__PURE__ */ new Map();
|
|
230
|
+
const familyDrops = /* @__PURE__ */ new Map();
|
|
231
|
+
const headers = /* @__PURE__ */ new Map();
|
|
232
|
+
let prefixBuffer = Buffer.alloc(0);
|
|
233
|
+
let prefixOffset = 0;
|
|
234
|
+
const encodePrefix = (text) => {
|
|
235
|
+
const bytes = Buffer.byteLength(text);
|
|
236
|
+
if (bytes > 4096) {
|
|
237
|
+
const prefix = Buffer.allocUnsafeSlow(bytes);
|
|
238
|
+
prefix.write(text);
|
|
239
|
+
return prefix;
|
|
240
|
+
}
|
|
241
|
+
if (prefixOffset + bytes > prefixBuffer.length) {
|
|
242
|
+
prefixBuffer = Buffer.allocUnsafeSlow(4096);
|
|
243
|
+
prefixOffset = 0;
|
|
244
|
+
}
|
|
245
|
+
const start = prefixOffset;
|
|
246
|
+
prefixOffset += prefixBuffer.write(text, prefixOffset);
|
|
247
|
+
return prefixBuffer.subarray(start, prefixOffset);
|
|
72
248
|
};
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
if (
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
249
|
+
let scratch = Buffer.alloc(0);
|
|
250
|
+
const series = (map, name, labels) => {
|
|
251
|
+
let labelCount = 0;
|
|
252
|
+
for (const key in labels) if (Object.hasOwn(labels, key)) labelCount++;
|
|
253
|
+
let shape;
|
|
254
|
+
shapes: for (const candidate of map.shapes.get(name) ?? []) {
|
|
255
|
+
if (candidate.keys.length !== labelCount) continue;
|
|
256
|
+
let node = candidate.root;
|
|
257
|
+
for (const key of candidate.keys) {
|
|
258
|
+
const value = labels[key];
|
|
259
|
+
if (value === void 0 || !Object.prototype.propertyIsEnumerable.call(labels, key)) continue shapes;
|
|
260
|
+
node = node?.values?.get(value);
|
|
261
|
+
}
|
|
262
|
+
shape = candidate;
|
|
263
|
+
if (node?.sample) return node;
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
const count = familyCounts.get(name) ?? 0;
|
|
267
|
+
const reserve = PROMETHEUS_FAMILY_RESERVES[name];
|
|
268
|
+
if (count >= Math.max(128, reserve) || count >= reserve && sharedSeries >= SHARED_PROMETHEUS_SERIES) {
|
|
269
|
+
droppedSeries++;
|
|
270
|
+
familyDrops.set(name, (familyDrops.get(name) ?? 0) + 1);
|
|
271
|
+
if (onSeriesDropped && !warnedFamilies.has(name)) {
|
|
272
|
+
warnedFamilies.add(name);
|
|
273
|
+
onSeriesDropped(name);
|
|
274
|
+
}
|
|
80
275
|
return;
|
|
81
276
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
value
|
|
96
|
-
|
|
277
|
+
familyCounts.set(name, count + 1);
|
|
278
|
+
if (count >= reserve) sharedSeries++;
|
|
279
|
+
if (!shape) {
|
|
280
|
+
shape = {
|
|
281
|
+
keys: Object.keys(labels).toSorted((left, right) => left.localeCompare(right)),
|
|
282
|
+
root: {}
|
|
283
|
+
};
|
|
284
|
+
const shapes = map.shapes.get(name);
|
|
285
|
+
if (shapes) shapes.push(shape);
|
|
286
|
+
else map.shapes.set(name, [shape]);
|
|
287
|
+
}
|
|
288
|
+
let node = shape.root;
|
|
289
|
+
for (const key of shape.keys) {
|
|
290
|
+
const value = expectDefined(labels[key], "Prometheus label");
|
|
291
|
+
const values = node.values ??= /* @__PURE__ */ new Map();
|
|
292
|
+
let child = values.get(value);
|
|
293
|
+
if (!child) {
|
|
294
|
+
child = {};
|
|
295
|
+
values.set(value, child);
|
|
296
|
+
}
|
|
297
|
+
node = child;
|
|
298
|
+
}
|
|
299
|
+
return node;
|
|
97
300
|
};
|
|
98
|
-
const
|
|
99
|
-
if (!Number.isFinite(value) || value < 0) return;
|
|
100
|
-
const
|
|
101
|
-
if (
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
301
|
+
const scalar = (map, name, help, labels, value, mode) => {
|
|
302
|
+
if (value === void 0 || !Number.isFinite(value) || mode !== "set" && value < 0 || mode === "add" && value === 0) return;
|
|
303
|
+
const node = series(map, name, labels);
|
|
304
|
+
if (!node) return;
|
|
305
|
+
if (node.sample) {
|
|
306
|
+
const next = mode === "add" ? node.sample.value + value : mode === "max" ? Math.max(node.sample.value, value) : value;
|
|
307
|
+
if (next !== node.sample.value) {
|
|
308
|
+
node.sample.value = next;
|
|
309
|
+
node.sample.renderedDirty = true;
|
|
310
|
+
}
|
|
311
|
+
} else {
|
|
312
|
+
node.sample = {
|
|
313
|
+
help,
|
|
314
|
+
key: metricKey(name, labels),
|
|
315
|
+
name,
|
|
316
|
+
prefix: encodePrefix(`${name}${formatLabels(labels)} `),
|
|
317
|
+
value,
|
|
318
|
+
renderedDirty: true
|
|
319
|
+
};
|
|
320
|
+
map.samples.push(node.sample);
|
|
321
|
+
map.sorted = false;
|
|
322
|
+
}
|
|
106
323
|
};
|
|
324
|
+
const counter = (name, help, labels, amount = 1) => scalar(counters, name, help, labels, amount, "add");
|
|
325
|
+
const counterValue = (name, help, labels, value) => scalar(counters, name, help, labels, value, "max");
|
|
326
|
+
const gauge = (name, help, labels, value) => scalar(gauges, name, help, labels, value, "set");
|
|
107
327
|
const clearGauges = (name) => {
|
|
108
|
-
|
|
328
|
+
headers.delete(name);
|
|
329
|
+
const previousLength = gauges.samples.length;
|
|
330
|
+
gauges.shapes.delete(name);
|
|
331
|
+
gauges.samples = gauges.samples.filter((sample) => sample.name !== name);
|
|
332
|
+
const count = familyCounts.get(name) ?? 0;
|
|
333
|
+
const remaining = count - (previousLength - gauges.samples.length);
|
|
334
|
+
const reserve = PROMETHEUS_FAMILY_RESERVES[name];
|
|
335
|
+
sharedSeries -= Math.max(0, count - reserve) - Math.max(0, remaining - reserve);
|
|
336
|
+
familyCounts.set(name, remaining);
|
|
109
337
|
};
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
let sample = histograms.get(key);
|
|
338
|
+
const getHistogram = (name, help, labels, buckets = DURATION_BUCKETS_SECONDS) => {
|
|
339
|
+
const node = series(histograms, name, labels);
|
|
340
|
+
if (!node) return;
|
|
341
|
+
let sample = node.sample;
|
|
115
342
|
if (!sample) {
|
|
116
343
|
sample = {
|
|
117
344
|
buckets,
|
|
118
|
-
bucketPrefixes: [...buckets, "+Inf"].map((le) => `${name}_bucket${formatLabels({
|
|
345
|
+
bucketPrefixes: [...buckets, "+Inf"].map((le) => encodePrefix(`${name}_bucket${formatLabels({
|
|
119
346
|
...labels,
|
|
120
347
|
le: String(le)
|
|
121
|
-
})} `),
|
|
348
|
+
})} `)),
|
|
122
349
|
counts: buckets.map(() => 0),
|
|
123
350
|
count: 0,
|
|
124
351
|
help,
|
|
125
|
-
|
|
126
|
-
|
|
352
|
+
key: metricKey(name, labels),
|
|
353
|
+
name,
|
|
354
|
+
sumPrefix: encodePrefix(`${name}_sum${formatLabels(labels)} `),
|
|
355
|
+
countPrefix: encodePrefix(`${name}_count${formatLabels(labels)} `),
|
|
356
|
+
sum: 0,
|
|
357
|
+
renderedDirty: true
|
|
127
358
|
};
|
|
128
|
-
|
|
359
|
+
node.sample = sample;
|
|
360
|
+
histograms.samples.push(sample);
|
|
361
|
+
histograms.sorted = false;
|
|
129
362
|
}
|
|
363
|
+
return sample;
|
|
364
|
+
};
|
|
365
|
+
const reserveHistogram = (name, help, labels) => {
|
|
366
|
+
getHistogram(name, help, labels);
|
|
367
|
+
};
|
|
368
|
+
const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
|
|
369
|
+
if (value === void 0 || !Number.isFinite(value)) return;
|
|
370
|
+
const sample = getHistogram(name, help, labels, buckets);
|
|
371
|
+
if (!sample) return;
|
|
130
372
|
sample.count += 1;
|
|
131
373
|
sample.sum += value;
|
|
374
|
+
sample.renderedDirty = true;
|
|
132
375
|
for (let index = 0; index < sample.buckets.length; index += 1) {
|
|
133
376
|
const bucket = sample.buckets[index];
|
|
134
377
|
if (bucket !== void 0 && value <= bucket) sample.counts[index] = (sample.counts[index] ?? 0) + 1;
|
|
135
378
|
}
|
|
136
379
|
};
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
380
|
+
const render = () => {
|
|
381
|
+
let scratchOffset = 0;
|
|
382
|
+
let totalBytes = 0;
|
|
383
|
+
const chunks = [];
|
|
384
|
+
const append = (chunk) => {
|
|
385
|
+
chunks.push(chunk);
|
|
386
|
+
totalBytes += chunk.length;
|
|
387
|
+
};
|
|
388
|
+
const write = (prefix, value) => {
|
|
389
|
+
const number = formatPrometheusNumber(value);
|
|
390
|
+
const required = scratchOffset + prefix.length + number.length + 1;
|
|
391
|
+
if (required > scratch.length) {
|
|
392
|
+
const grown = Buffer.allocUnsafeSlow(Math.max(required, scratch.length * 2));
|
|
393
|
+
grown.set(scratch.subarray(0, scratchOffset));
|
|
394
|
+
scratch = grown;
|
|
395
|
+
}
|
|
396
|
+
scratch.set(prefix, scratchOffset);
|
|
397
|
+
scratchOffset += prefix.length;
|
|
398
|
+
scratchOffset += scratch.write(number, scratchOffset, "ascii");
|
|
399
|
+
scratch[scratchOffset++] = 10;
|
|
400
|
+
};
|
|
401
|
+
const finish = (previous) => {
|
|
402
|
+
const output = previous?.length === scratchOffset ? previous : new Uint8Array(scratchOffset);
|
|
403
|
+
scratch.copy(output, 0, 0, scratchOffset);
|
|
404
|
+
scratchOffset = 0;
|
|
405
|
+
return output;
|
|
406
|
+
};
|
|
407
|
+
const emitted = /* @__PURE__ */ new Set();
|
|
408
|
+
let previousName;
|
|
409
|
+
const header = (name, type, help) => {
|
|
410
|
+
if (previousName === name) return;
|
|
411
|
+
previousName = name;
|
|
412
|
+
if (!emitted.has(name)) {
|
|
413
|
+
emitted.add(name);
|
|
414
|
+
let cached = headers.get(name);
|
|
415
|
+
if (!cached || cached.type !== type || cached.help !== help) {
|
|
416
|
+
cached = {
|
|
417
|
+
type,
|
|
418
|
+
help,
|
|
419
|
+
bytes: encodePrefix(`# HELP ${name} ${escapeHelp(help)}\n# TYPE ${name} ${type}\n`)
|
|
420
|
+
};
|
|
421
|
+
headers.set(name, cached);
|
|
422
|
+
}
|
|
423
|
+
append(cached.bytes);
|
|
424
|
+
}
|
|
148
425
|
};
|
|
426
|
+
for (const [type, map] of [["counter", counters], ["gauge", gauges]]) {
|
|
427
|
+
for (const sample of sortedSamples(map)) {
|
|
428
|
+
header(sample.name, type, sample.help);
|
|
429
|
+
if (!sample.rendered || sample.renderedDirty) {
|
|
430
|
+
write(sample.prefix, sample.value);
|
|
431
|
+
sample.rendered = finish(sample.rendered);
|
|
432
|
+
sample.renderedDirty = false;
|
|
433
|
+
}
|
|
434
|
+
append(sample.rendered);
|
|
435
|
+
}
|
|
436
|
+
if (type === "counter" && droppedSeries > 0) {
|
|
437
|
+
header(DROPPED_SERIES_COUNTER_NAME, type, "New-series observations dropped because an exporter series budget was reached.");
|
|
438
|
+
write(Buffer.from(`${DROPPED_SERIES_COUNTER_NAME} `), droppedSeries);
|
|
439
|
+
append(finish());
|
|
440
|
+
header(DROPPED_FAMILY_COUNTER_NAME, type, "New-series observations dropped by bounded metric family.");
|
|
441
|
+
for (const [family, count] of familyDrops) write(Buffer.from(`${DROPPED_FAMILY_COUNTER_NAME}${formatLabels({ family })} `), count);
|
|
442
|
+
append(finish());
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
for (const sample of sortedSamples(histograms)) {
|
|
446
|
+
if (sample.count === 0) continue;
|
|
447
|
+
header(sample.name, "histogram", sample.help);
|
|
448
|
+
if (!sample.rendered || sample.renderedDirty) {
|
|
449
|
+
sample.bucketPrefixes.forEach((prefix, bucketIndex) => {
|
|
450
|
+
write(prefix, sample.counts[bucketIndex] ?? sample.count);
|
|
451
|
+
});
|
|
452
|
+
write(sample.sumPrefix, sample.sum);
|
|
453
|
+
write(sample.countPrefix, sample.count);
|
|
454
|
+
sample.rendered = finish(sample.rendered);
|
|
455
|
+
sample.renderedDirty = false;
|
|
456
|
+
}
|
|
457
|
+
append(sample.rendered);
|
|
458
|
+
}
|
|
459
|
+
const output = Buffer.from(new ArrayBuffer(totalBytes));
|
|
460
|
+
let offset = 0;
|
|
461
|
+
for (const chunk of chunks) {
|
|
462
|
+
output.set(chunk, offset);
|
|
463
|
+
offset += chunk.length;
|
|
464
|
+
}
|
|
465
|
+
return output;
|
|
149
466
|
};
|
|
150
467
|
const reset = () => {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
468
|
+
for (const map of [
|
|
469
|
+
counters,
|
|
470
|
+
gauges,
|
|
471
|
+
histograms
|
|
472
|
+
]) {
|
|
473
|
+
map.samples.length = 0;
|
|
474
|
+
map.shapes.clear();
|
|
475
|
+
}
|
|
154
476
|
droppedSeries = 0;
|
|
477
|
+
sharedSeries = 0;
|
|
478
|
+
familyCounts.clear();
|
|
479
|
+
familyDrops.clear();
|
|
480
|
+
headers.clear();
|
|
481
|
+
scratch = Buffer.alloc(0);
|
|
482
|
+
prefixBuffer = Buffer.alloc(0);
|
|
483
|
+
prefixOffset = 0;
|
|
155
484
|
};
|
|
156
485
|
return {
|
|
157
486
|
counter,
|
|
@@ -159,29 +488,140 @@ function createPrometheusMetricStore() {
|
|
|
159
488
|
gauge,
|
|
160
489
|
clearGauges,
|
|
161
490
|
histogram,
|
|
491
|
+
reserveHistogram,
|
|
162
492
|
reset,
|
|
163
|
-
|
|
493
|
+
render
|
|
164
494
|
};
|
|
165
495
|
}
|
|
166
|
-
// extensions/diagnostics-prometheus/src/service-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
}
|
|
496
|
+
// extensions/diagnostics-prometheus/src/service-gateway-ws.ts
|
|
497
|
+
function createGatewayWsMetricsRecorder(store) {
|
|
498
|
+
const labels = { event: "" };
|
|
499
|
+
const frameLabels = {
|
|
500
|
+
event: "",
|
|
501
|
+
le: ""
|
|
502
|
+
};
|
|
503
|
+
const presenceLabels = { mode: "" };
|
|
504
|
+
const sessionLabels = {
|
|
505
|
+
event: "",
|
|
506
|
+
reason: "",
|
|
507
|
+
delivery: ""
|
|
508
|
+
};
|
|
509
|
+
return (event, count, bytes, wireBytes, buckets, bucketLabels, sessionEvent, presenceMode) => {
|
|
510
|
+
labels.event = frameLabels.event = sessionLabels.event = event;
|
|
511
|
+
store.counter("openclaw_gateway_ws_events_total", "Outbound Gateway WebSocket event frames accepted by the sender per socket.", labels, count);
|
|
512
|
+
store.counter("openclaw_gateway_ws_event_bytes_total", "Encoded outbound Gateway WebSocket event bytes before framing or compression per socket.", labels, bytes);
|
|
513
|
+
if (wireBytes !== void 0) store.counter("openclaw_gateway_ws_event_wire_bytes_total", "Outbound Gateway WebSocket payload bytes after compression per socket, excluding framing.", labels, wireBytes);
|
|
514
|
+
if (count > 0) {
|
|
515
|
+
let cumulative = 0;
|
|
516
|
+
for (let i = 0; i < bucketLabels.length; i++) {
|
|
517
|
+
cumulative += buckets[i];
|
|
518
|
+
frameLabels.le = bucketLabels[i];
|
|
519
|
+
(cumulative === 0 ? store.counterValue : store.counter)("openclaw_gateway_ws_event_frame_bytes", "Cumulative counts of accepted raw Gateway event frames by byte-size upper bound.", frameLabels, cumulative);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (presenceMode) {
|
|
523
|
+
presenceLabels.mode = presenceMode;
|
|
524
|
+
store.counter("openclaw_gateway_ws_presence_frames_total", "Outbound presence frames accepted by the sender per socket, by delivery mode.", presenceLabels, count);
|
|
525
|
+
store.counter("openclaw_gateway_ws_presence_bytes_total", "Encoded outbound presence bytes before framing or compression per socket, by delivery mode.", presenceLabels, bytes);
|
|
526
|
+
}
|
|
527
|
+
if (sessionEvent) {
|
|
528
|
+
sessionLabels.reason = sessionEvent.reason;
|
|
529
|
+
sessionLabels.delivery = sessionEvent.delivery;
|
|
530
|
+
store.counter("openclaw_gateway_ws_session_event_frames_total", "Outbound session event frames by reason and exact message subscription per socket.", sessionLabels, count);
|
|
531
|
+
store.counter("openclaw_gateway_ws_session_event_bytes_total", "Encoded outbound session event bytes by reason and exact message subscription per socket.", sessionLabels, bytes);
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
function recordGatewayWsConnections(store, operators) {
|
|
536
|
+
store.gauge("openclaw_gateway_ws_operator_connections", "Connected authenticated Gateway operator WebSockets.", {}, operators);
|
|
173
537
|
}
|
|
174
538
|
// extensions/diagnostics-prometheus/src/service-memory.ts
|
|
539
|
+
const WORKER_MEMORY_METRICS = [
|
|
540
|
+
[
|
|
541
|
+
"heapUsed",
|
|
542
|
+
"openclaw_worker_heap_used_bytes",
|
|
543
|
+
"heap usage"
|
|
544
|
+
],
|
|
545
|
+
[
|
|
546
|
+
"external",
|
|
547
|
+
"openclaw_worker_external_bytes",
|
|
548
|
+
"external memory"
|
|
549
|
+
],
|
|
550
|
+
[
|
|
551
|
+
"arrayBuffers",
|
|
552
|
+
"openclaw_worker_array_buffers_bytes",
|
|
553
|
+
"ArrayBuffer memory (included in external)"
|
|
554
|
+
]
|
|
555
|
+
];
|
|
556
|
+
const HEAP_HEADROOM_METRICS = [
|
|
557
|
+
[
|
|
558
|
+
"limit",
|
|
559
|
+
"openclaw_heap_size_limit_bytes",
|
|
560
|
+
"Smallest live isolate V8 heap limit by thread group."
|
|
561
|
+
],
|
|
562
|
+
[
|
|
563
|
+
"usedRatio",
|
|
564
|
+
"openclaw_heap_used_ratio",
|
|
565
|
+
"Largest live isolate used/limit ratio by thread group."
|
|
566
|
+
],
|
|
567
|
+
[
|
|
568
|
+
"oldBytes",
|
|
569
|
+
"openclaw_heap_old_generation_after_gc_bytes",
|
|
570
|
+
"Largest live isolate old_space plus large_object_space usage at its last observed major GC."
|
|
571
|
+
],
|
|
572
|
+
[
|
|
573
|
+
"oldRatio",
|
|
574
|
+
"openclaw_heap_old_generation_after_gc_ratio",
|
|
575
|
+
"Largest live isolate post-major-GC old-generation/heap-limit ratio by thread group."
|
|
576
|
+
]
|
|
577
|
+
];
|
|
578
|
+
function collectHeapHeadroom(headroom, used, limit, oldBytes) {
|
|
579
|
+
if (limit === void 0 || limit <= 0) return;
|
|
580
|
+
headroom.limit = Math.min(headroom.limit ?? limit, limit);
|
|
581
|
+
headroom.usedRatio = Math.max(headroom.usedRatio ?? 0, used / limit);
|
|
582
|
+
if (oldBytes !== void 0) {
|
|
583
|
+
headroom.oldBytes = Math.max(headroom.oldBytes ?? 0, oldBytes);
|
|
584
|
+
headroom.oldRatio = Math.max(headroom.oldRatio ?? 0, oldBytes / limit);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
function recordHeapHeadroom(store, thread, headroom) {
|
|
588
|
+
for (const [field, name, description] of HEAP_HEADROOM_METRICS) store.gauge(name, description, { thread }, headroom[field]);
|
|
589
|
+
}
|
|
175
590
|
function recordMemorySample(store, memory, byteBuckets) {
|
|
591
|
+
for (const [, name] of HEAP_HEADROOM_METRICS) store.clearGauges(name);
|
|
592
|
+
const mainHeadroom = {};
|
|
593
|
+
collectHeapHeadroom(mainHeadroom, memory.heapUsedBytes, memory.heapSizeLimitBytes, memory.oldGenerationAfterGcBytes);
|
|
594
|
+
recordHeapHeadroom(store, "main", mainHeadroom);
|
|
595
|
+
store.clearGauges("openclaw_process_anonymous_memory_bytes");
|
|
596
|
+
for (const [kind, field] of [["rss", "anonymousRssBytes"], ["swap", "anonymousSwapBytes"]]) store.gauge("openclaw_process_anonymous_memory_bytes", "Latest Linux process anonymous RSS and swap; includes managed heaps and buffers, not allocator totals.", { kind }, asNonNegativeFiniteNumber(memory[field]));
|
|
176
597
|
for (const message of memory.workerMessages ?? []) {
|
|
177
598
|
const labels = {
|
|
178
599
|
kind: message.kind,
|
|
179
600
|
direction: message.direction
|
|
180
601
|
};
|
|
181
602
|
store.counterValue("openclaw_worker_messages_total", "Messages crossing the Gateway main-thread worker boundary.", labels, message.messages);
|
|
182
|
-
store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.",
|
|
603
|
+
for (const [requestClass, bytes] of Object.entries(message.payloadBytesByRequestClass ?? { [message.kind === "sqlite_request" || message.kind === "sqlite_reply" || message.kind === "authority" ? "other" : ""]: message.payloadBytes })) store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.", requestClass ? {
|
|
604
|
+
...labels,
|
|
605
|
+
request_class: requestClass
|
|
606
|
+
} : labels, bytes);
|
|
183
607
|
store.counterValue("openclaw_worker_message_unmeasured_total", "Worker messages without a known payload size; these do not contribute to payload bytes.", labels, message.unmeasuredMessages);
|
|
184
608
|
}
|
|
609
|
+
for (const selection of memory.sessionRowSelections?.selections ?? []) {
|
|
610
|
+
const labels = {
|
|
611
|
+
kind: selection.kind,
|
|
612
|
+
task: selection.task
|
|
613
|
+
};
|
|
614
|
+
store.counterValue("openclaw_session_row_selections_total", "Successful resident session-row selections by query shape and enclosing task.", labels, selection.calls);
|
|
615
|
+
store.counterValue("openclaw_session_row_selection_rows_total", "Resident session rows returned by selections, before federation and visibility filtering.", labels, selection.rows);
|
|
616
|
+
}
|
|
617
|
+
if (memory.sessionRowSelections) {
|
|
618
|
+
for (const partition of memory.sessionRowSelections.partitions) for (const outcome of ["hits", "misses"]) store.counterValue("openclaw_session_row_partition_cache_total", "Retained session-list partition lookups by outcome and enclosing selection task.", {
|
|
619
|
+
task: partition.task,
|
|
620
|
+
outcome: outcome === "hits" ? "hit" : "miss"
|
|
621
|
+
}, partition[outcome]);
|
|
622
|
+
for (const revision of memory.sessionRowSelections.partitionRevisions) store.counterValue("openclaw_session_row_partition_revisions_total", "Session-list partition revisions by synchronous internal change scope; excludes initial partitions.", { reason: revision.reason }, revision.changes);
|
|
623
|
+
store.counterValue("openclaw_session_row_selection_resets_total", "Broad session-row selection invalidations, including topology replacement and disposal.", {}, memory.sessionRowSelections.resets);
|
|
624
|
+
}
|
|
185
625
|
for (const worker of memory.workerLifecycle ?? []) {
|
|
186
626
|
store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
|
|
187
627
|
for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
|
|
@@ -189,6 +629,23 @@ function recordMemorySample(store, memory, byteBuckets) {
|
|
|
189
629
|
reason
|
|
190
630
|
}, count);
|
|
191
631
|
}
|
|
632
|
+
for (const worker of memory.workerTaskMetrics ?? []) {
|
|
633
|
+
const labels = {
|
|
634
|
+
worker_class: worker.workerClass,
|
|
635
|
+
script: worker.script
|
|
636
|
+
};
|
|
637
|
+
store.counterValue("openclaw_worker_tasks_completed_total", "Observed terminal task replies from pool workers, including handled failures.", labels, worker.tasksCompleted);
|
|
638
|
+
store.gauge("openclaw_worker_heap_used_high_water_bytes", "Largest observed pool worker heap usage, retained across worker retirement.", labels, worker.heapUsedHighWaterBytes);
|
|
639
|
+
for (const gc of worker.gc) {
|
|
640
|
+
const gcLabels = {
|
|
641
|
+
...labels,
|
|
642
|
+
kind: gc.kind
|
|
643
|
+
};
|
|
644
|
+
store.counterValue("openclaw_worker_gc_duration_seconds_total", "Observed pool worker garbage collection duration, retained across worker retirement.", gcLabels, gc.durationMs / 1e3);
|
|
645
|
+
store.counterValue("openclaw_worker_gc_collections_total", "Observed pool worker garbage collections, retained across worker retirement.", gcLabels, gc.count);
|
|
646
|
+
store.counterValue("openclaw_worker_gc_reclaimed_bytes_total", "Sum of positive heap usage decreases across observed pool worker garbage collections.", gcLabels, gc.reclaimedBytes);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
192
649
|
for (const [kind, field] of [
|
|
193
650
|
["rss", "rssBytes"],
|
|
194
651
|
["heap_total", "heapTotalBytes"],
|
|
@@ -214,36 +671,54 @@ function recordMemorySample(store, memory, byteBuckets) {
|
|
|
214
671
|
}, space[field]);
|
|
215
672
|
const workers = /* @__PURE__ */ new Map();
|
|
216
673
|
store.clearGauges("openclaw_code_mode_worker_heap_bytes");
|
|
674
|
+
store.clearGauges("openclaw_code_mode_worker_memory_bytes");
|
|
217
675
|
for (const worker of memory.workerHeaps ?? []) {
|
|
218
|
-
if (worker.workerSlot !== void 0)
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
676
|
+
if (worker.workerSlot !== void 0) {
|
|
677
|
+
for (const [kind, value] of [["used", worker.heapUsed], ["limit", worker.heapSizeLimitBytes]]) store.gauge("openclaw_code_mode_worker_heap_bytes", "Latest live Code Mode isolate heap usage and limit by reusable worker slot.", {
|
|
678
|
+
slot: String(worker.workerSlot),
|
|
679
|
+
kind
|
|
680
|
+
}, value);
|
|
681
|
+
for (const [kind, value] of [["external", worker.external], ["array_buffers", worker.arrayBuffers]]) store.gauge("openclaw_code_mode_worker_memory_bytes", "Latest live Code Mode isolate external and ArrayBuffer memory (included in external) by reusable worker slot.", {
|
|
682
|
+
slot: String(worker.workerSlot),
|
|
683
|
+
kind
|
|
684
|
+
}, value);
|
|
685
|
+
}
|
|
222
686
|
const totals = workers.get(worker.script) ?? {
|
|
223
687
|
count: 0,
|
|
224
|
-
sampled: 0
|
|
225
|
-
heapUsed: void 0
|
|
688
|
+
sampled: 0
|
|
226
689
|
};
|
|
227
690
|
totals.count++;
|
|
228
691
|
totals.sampled++;
|
|
229
|
-
|
|
230
|
-
|
|
692
|
+
collectHeapHeadroom(totals, worker.heapUsed, worker.heapSizeLimitBytes, worker.oldGenerationAfterGcBytes);
|
|
693
|
+
for (const [field] of WORKER_MEMORY_METRICS) {
|
|
694
|
+
const value = worker[field];
|
|
695
|
+
if (value !== void 0) totals[field] = (totals[field] ?? 0) + value;
|
|
696
|
+
}
|
|
697
|
+
for (const space of worker.heapSpaces ?? []) {
|
|
698
|
+
totals.heapSpaces ??= /* @__PURE__ */ new Map();
|
|
699
|
+
totals.heapSpaces.set(space.space_name, (totals.heapSpaces.get(space.space_name) ?? 0) + space.space_used_size);
|
|
700
|
+
}
|
|
231
701
|
workers.set(worker.script, totals);
|
|
232
702
|
}
|
|
233
703
|
for (const worker of memory.workerMemoryMissing ?? []) {
|
|
234
704
|
const totals = workers.get(worker.script) ?? {
|
|
235
705
|
count: 0,
|
|
236
|
-
sampled: 0
|
|
237
|
-
heapUsed: void 0
|
|
706
|
+
sampled: 0
|
|
238
707
|
};
|
|
239
708
|
totals.count++;
|
|
240
709
|
workers.set(worker.script, totals);
|
|
241
710
|
}
|
|
242
|
-
store.clearGauges(
|
|
711
|
+
for (const [, name] of WORKER_MEMORY_METRICS) store.clearGauges(name);
|
|
712
|
+
store.clearGauges("openclaw_worker_heap_space_used_bytes");
|
|
243
713
|
for (const [script, totals] of workers) {
|
|
714
|
+
recordHeapHeadroom(store, script, totals);
|
|
244
715
|
store.gauge("openclaw_worker_count", "Worker isolate counts.", { script }, totals.count);
|
|
245
716
|
store.gauge("openclaw_worker_heap_sampled_count", "Worker isolate counts.", { script }, totals.sampled);
|
|
246
|
-
store.gauge(
|
|
717
|
+
for (const [field, name, description] of WORKER_MEMORY_METRICS) store.gauge(name, `Latest live Worker ${description} by bounded script basename.`, { script }, totals[field]);
|
|
718
|
+
for (const [space, bytes] of totals.heapSpaces ?? []) store.gauge("openclaw_worker_heap_space_used_bytes", "Latest live Worker V8 heap space usage by bounded script basename.", {
|
|
719
|
+
script,
|
|
720
|
+
space
|
|
721
|
+
}, bytes);
|
|
247
722
|
}
|
|
248
723
|
store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
|
|
249
724
|
}
|
|
@@ -292,6 +767,20 @@ function recordModelUsage(store, evt) {
|
|
|
292
767
|
store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
|
|
293
768
|
}
|
|
294
769
|
// extensions/diagnostics-prometheus/src/service-operation-timing.ts
|
|
770
|
+
const GC_DURATION_NAME = "openclaw_gc_duration_seconds";
|
|
771
|
+
const GC_DURATION_HELP = "Elapsed garbage collection duration in seconds by collection kind, callback flags, and isolate role.";
|
|
772
|
+
function reserveMainGcHistograms(store) {
|
|
773
|
+
for (const kind of [
|
|
774
|
+
"scavenge",
|
|
775
|
+
"mark_sweep_compact",
|
|
776
|
+
"incremental",
|
|
777
|
+
"weak_processing"
|
|
778
|
+
]) store.reserveHistogram(GC_DURATION_NAME, GC_DURATION_HELP, {
|
|
779
|
+
kind,
|
|
780
|
+
reason: "other",
|
|
781
|
+
thread: "main"
|
|
782
|
+
});
|
|
783
|
+
}
|
|
295
784
|
const RPC_DURATION_BUCKETS_SECONDS = [
|
|
296
785
|
.005,
|
|
297
786
|
.01,
|
|
@@ -336,6 +825,14 @@ const RPC_DURATION_BUCKETS_SECONDS = [
|
|
|
336
825
|
600
|
|
337
826
|
];
|
|
338
827
|
const RESPONSE_BYTE_BUCKETS = Array.from({ length: 17 }, (_, index) => 1024 * 2 ** index);
|
|
828
|
+
const MAIN_THREAD_STALL_BUCKETS_SECONDS = [
|
|
829
|
+
.25,
|
|
830
|
+
.5,
|
|
831
|
+
1,
|
|
832
|
+
2.5,
|
|
833
|
+
5,
|
|
834
|
+
10
|
|
835
|
+
];
|
|
339
836
|
const HEAP_DELTA_BYTE_BUCKETS = [
|
|
340
837
|
...RESPONSE_BYTE_BUCKETS.toReversed().map((bytes) => -bytes),
|
|
341
838
|
0,
|
|
@@ -351,9 +848,41 @@ const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
|
|
|
351
848
|
"projection_final",
|
|
352
849
|
"delivery"
|
|
353
850
|
]);
|
|
851
|
+
const CHAT_STARTUP_PHASE = /^chat\.startup\.(session_entry|pending_inputs|history_page|startup_projection|projection_final|session_info)$/;
|
|
852
|
+
const DIFF_CACHE_MISS_REASONS = /* @__PURE__ */ new Set([
|
|
853
|
+
"cold",
|
|
854
|
+
"evicted",
|
|
855
|
+
"pending",
|
|
856
|
+
"no-revision",
|
|
857
|
+
"no-revision-environment",
|
|
858
|
+
"no-revision-platform",
|
|
859
|
+
"no-revision-metadata",
|
|
860
|
+
"no-revision-git-installation",
|
|
861
|
+
"no-revision-config",
|
|
862
|
+
"no-revision-reftable",
|
|
863
|
+
"no-revision-alternates",
|
|
864
|
+
"no-revision-symlink",
|
|
865
|
+
"no-revision-nested-repository",
|
|
866
|
+
"no-revision-status",
|
|
867
|
+
"no-revision-unmerged",
|
|
868
|
+
"no-revision-submodule",
|
|
869
|
+
"no-revision-permission",
|
|
870
|
+
"no-revision-missing",
|
|
871
|
+
"no-revision-io",
|
|
872
|
+
"oversized",
|
|
873
|
+
"revision-changed-head",
|
|
874
|
+
"revision-changed-index",
|
|
875
|
+
"revision-changed-untracked-set",
|
|
876
|
+
"revision-changed-worktree-mtime",
|
|
877
|
+
"revision-changed-metadata"
|
|
878
|
+
]);
|
|
354
879
|
const CHAT_SEND_PHASE = /^chat\.send\.(authority|admission|preparation|attachments|replyContext|authoring|persist|runAdmission|replyInitialization|snapshot|worktree|effects|response|dispatch)$/;
|
|
880
|
+
const CONTROL_UI_DOCUMENT_STAGE = /^control-ui\.document\.(auth_grants|identity_avatar|plugin_catalog|git_branch|preferences|render|total)$/;
|
|
355
881
|
function recordOperationTimingEvent(store, evt, metadata) {
|
|
356
882
|
switch (evt.type) {
|
|
883
|
+
case "sessions.list.baseline.retired":
|
|
884
|
+
if (metadata.trusted) store.counter("openclaw_sessions_list_baseline_retirements_total", "Session list baseline retirement events by cause, not rows or views retired.", { cause: evt.cause });
|
|
885
|
+
return;
|
|
357
886
|
case "secret.egress":
|
|
358
887
|
store.counter("openclaw_secret_egress_requests_total", "Secret egress proxy requests by outcome and credential substitution.", {
|
|
359
888
|
outcome: evt.outcome,
|
|
@@ -373,20 +902,87 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
373
902
|
store.counter("openclaw_gateway_http_cancelled_total", "Gateway HTTP requests cancelled before completion.", { source: evt.source });
|
|
374
903
|
return;
|
|
375
904
|
case "diagnostic.gc":
|
|
376
|
-
|
|
905
|
+
if (evt.kind === "mark_sweep_compact") {
|
|
906
|
+
const labels = { thread: evt.thread ?? "main" };
|
|
907
|
+
store.gauge("openclaw_heap_used_after_major_gc_bytes", "Native used heap after the latest received major collection by isolate role.", labels, evt.heapUsedAfterMajorGcBytes);
|
|
908
|
+
store.gauge("openclaw_heap_used_after_major_gc_timestamp_seconds", "Unix observation time of the latest received native major-GC heap sample.", labels, evt.heapUsedAfterMajorGcTimestampSeconds);
|
|
909
|
+
}
|
|
910
|
+
store.histogram(GC_DURATION_NAME, GC_DURATION_HELP, {
|
|
377
911
|
kind: evt.kind ?? "unknown",
|
|
912
|
+
reason: evt.reason ?? "other",
|
|
378
913
|
thread: evt.thread ?? "main",
|
|
379
914
|
...evt.workerSlot === void 0 ? {} : { slot: String(evt.workerSlot) }
|
|
380
915
|
}, seconds(evt.durationMs));
|
|
381
916
|
return;
|
|
917
|
+
case "gateway.main_thread.stall":
|
|
918
|
+
for (const stall of evt.stalls) store.histogram("openclaw_gateway_main_thread_stall_seconds", "Elapsed main-thread tasks lasting at least 250 milliseconds, by work scope.", { task: normalizeDiagnosticValue(stall.task) }, seconds(stall.durationMs), MAIN_THREAD_STALL_BUCKETS_SECONDS);
|
|
919
|
+
store.counter("openclaw_gateway_main_thread_stalls_dropped_total", "Main-thread stall records dropped because the attribution buffer was full.", {}, evt.dropped);
|
|
920
|
+
return;
|
|
382
921
|
case "gateway.event_loop.sample":
|
|
383
922
|
store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
|
|
384
923
|
store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
|
|
385
924
|
return;
|
|
386
925
|
case "diagnostic.phase.completed": {
|
|
387
926
|
if (!metadata.trusted) return;
|
|
927
|
+
const documentStage = CONTROL_UI_DOCUMENT_STAGE.exec(evt.name)?.[1];
|
|
928
|
+
if (documentStage) {
|
|
929
|
+
store.histogram("openclaw_control_ui_document_duration_seconds", "Elapsed Control UI HTML document handler stages in seconds; stages can overlap.", { stage: documentStage }, seconds(evt.durationMs));
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
if (evt.name === "sessions.diff" || evt.name === "checkout.diff") {
|
|
933
|
+
const outcome = evt.details?.outcome;
|
|
934
|
+
if (outcome !== "cache-hit" && outcome !== "cache-miss") return;
|
|
935
|
+
const reason = evt.details?.missReason;
|
|
936
|
+
if (outcome === "cache-miss" && typeof reason === "string" && DIFF_CACHE_MISS_REASONS.has(reason)) store.counter("openclaw_diff_cache_misses_total", "Local diff cache misses by bounded reason; pending reads join an existing load.", {
|
|
937
|
+
method: evt.name,
|
|
938
|
+
reason
|
|
939
|
+
});
|
|
940
|
+
for (const stage of [
|
|
941
|
+
"admission",
|
|
942
|
+
"checkout-resolve",
|
|
943
|
+
"cache-validate",
|
|
944
|
+
"worker-queue",
|
|
945
|
+
"git-wall",
|
|
946
|
+
"encode"
|
|
947
|
+
]) {
|
|
948
|
+
const elapsed = evt.details?.[stage];
|
|
949
|
+
store.histogram("openclaw_diff_stage_seconds", "Elapsed local diff stages; cache validation includes its worker queue and Git wall time.", {
|
|
950
|
+
method: evt.name,
|
|
951
|
+
stage,
|
|
952
|
+
outcome
|
|
953
|
+
}, typeof elapsed === "number" ? seconds(elapsed) : void 0, RPC_DURATION_BUCKETS_SECONDS);
|
|
954
|
+
}
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
388
957
|
if (evt.name === "worktree.preparation") {
|
|
389
|
-
|
|
958
|
+
const { kind, template, outcome } = evt.details ?? {};
|
|
959
|
+
if (kind !== "managed" && kind !== "sandbox" || ![
|
|
960
|
+
"warm",
|
|
961
|
+
"cold",
|
|
962
|
+
"unavailable",
|
|
963
|
+
"reused"
|
|
964
|
+
].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
|
|
965
|
+
for (const preparationPhase of [
|
|
966
|
+
"total",
|
|
967
|
+
"allocate",
|
|
968
|
+
"checkout",
|
|
969
|
+
"setup",
|
|
970
|
+
"templatePrepare",
|
|
971
|
+
"templateApply",
|
|
972
|
+
"snapshot",
|
|
973
|
+
"synchronizeCanonical",
|
|
974
|
+
"synchronizeProjection",
|
|
975
|
+
"containerStart",
|
|
976
|
+
"workspaceLayout"
|
|
977
|
+
]) {
|
|
978
|
+
const elapsed = preparationPhase === "total" ? evt.durationMs : evt.details?.[preparationPhase];
|
|
979
|
+
store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
|
|
980
|
+
kind,
|
|
981
|
+
template: String(template),
|
|
982
|
+
outcome,
|
|
983
|
+
phase: preparationPhase
|
|
984
|
+
}, typeof elapsed === "number" ? seconds(elapsed) : void 0);
|
|
985
|
+
}
|
|
390
986
|
return;
|
|
391
987
|
}
|
|
392
988
|
const sendPhase = CHAT_SEND_PHASE.exec(evt.name)?.[1];
|
|
@@ -398,6 +994,14 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
398
994
|
}, seconds(evt.durationMs));
|
|
399
995
|
return;
|
|
400
996
|
}
|
|
997
|
+
const startupPhase = CHAT_STARTUP_PHASE.exec(evt.name)?.[1];
|
|
998
|
+
if (startupPhase) {
|
|
999
|
+
store.histogram("openclaw_gateway_rpc_stage_seconds", "Elapsed time for completed Gateway RPC owner stages.", {
|
|
1000
|
+
method: "chat.startup",
|
|
1001
|
+
phase: startupPhase
|
|
1002
|
+
}, seconds(evt.durationMs));
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
401
1005
|
if (!evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
|
|
402
1006
|
const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
|
|
403
1007
|
const duration = seconds(evt.durationMs);
|
|
@@ -415,6 +1019,10 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
415
1019
|
const labels = { method: evt.method };
|
|
416
1020
|
if (evt.phase === "received") {
|
|
417
1021
|
store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
|
|
1022
|
+
store.counter("openclaw_gateway_rpc_client_requests_total", "Gateway RPC requests by bounded handshake client kind.", {
|
|
1023
|
+
method: evt.method,
|
|
1024
|
+
client_kind: evt.clientKind ?? "other"
|
|
1025
|
+
});
|
|
418
1026
|
if (evt.method === "models.list" && evt.modelsList) {
|
|
419
1027
|
const catalog = evt.modelsList;
|
|
420
1028
|
store.counter("openclaw_models_list_requests_total", "Model catalog requests by client kind and bounded request shape.", {
|
|
@@ -426,10 +1034,54 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
426
1034
|
prepared_only: String(catalog.preparedOnly)
|
|
427
1035
|
});
|
|
428
1036
|
}
|
|
1037
|
+
if (evt.method === "sessions.list" && evt.sessionsList) {
|
|
1038
|
+
const sessions = evt.sessionsList;
|
|
1039
|
+
store.counter("openclaw_sessions_list_requests_total", "Session list requests by client kind and bounded request shape.", {
|
|
1040
|
+
client_kind: sessions.clientKind,
|
|
1041
|
+
source: sessions.source,
|
|
1042
|
+
caller_source: sessions.callerSource,
|
|
1043
|
+
row_mode: sessions.rowMode,
|
|
1044
|
+
filter: sessions.filter
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
if (evt.method === "cron.list" && evt.cronList) {
|
|
1048
|
+
const cron = evt.cronList;
|
|
1049
|
+
store.counter("openclaw_cron_list_requests_total", "Cron list requests by client kind and bounded request shape.", {
|
|
1050
|
+
client_kind: cron.clientKind,
|
|
1051
|
+
source: cron.source,
|
|
1052
|
+
scope: cron.scope,
|
|
1053
|
+
compact: String(cron.compact),
|
|
1054
|
+
page: cron.page
|
|
1055
|
+
});
|
|
1056
|
+
}
|
|
429
1057
|
return;
|
|
430
1058
|
}
|
|
431
1059
|
if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) {
|
|
1060
|
+
if (evt.outcome === "ok" && evt.firstResponse !== false && evt.sessionsList?.baseMissReason) store.counter("openclaw_sessions_list_base_misses_total", "Successful full sidebar responses with a supplied base, by miss reason.", { reason: evt.sessionsList.baseMissReason });
|
|
432
1061
|
store.histogram("openclaw_gateway_rpc_response_bytes", "Encoded Gateway RPC response frame size in bytes, including later frames.", labels, evt.responseBytes, RESPONSE_BYTE_BUCKETS);
|
|
1062
|
+
if (evt.responseBytes !== void 0 && evt.sessionsList) {
|
|
1063
|
+
const sessions = evt.sessionsList;
|
|
1064
|
+
store.counter("openclaw_sessions_list_response_bytes_total", "Encoded successful session list response bytes by request shape and row outcome.", {
|
|
1065
|
+
client_kind: sessions.clientKind,
|
|
1066
|
+
source: sessions.source,
|
|
1067
|
+
caller_source: sessions.callerSource,
|
|
1068
|
+
row_mode: sessions.rowMode,
|
|
1069
|
+
filter: sessions.filter,
|
|
1070
|
+
rows_outcome: sessions.rowsOutcome
|
|
1071
|
+
}, evt.responseBytes);
|
|
1072
|
+
}
|
|
1073
|
+
if (evt.chatHistory) {
|
|
1074
|
+
const history = evt.chatHistory;
|
|
1075
|
+
const historyLabels = {
|
|
1076
|
+
client_kind: history.clientKind,
|
|
1077
|
+
source: history.source,
|
|
1078
|
+
shape: history.shape,
|
|
1079
|
+
limit_bucket: history.limitBucket,
|
|
1080
|
+
outcome: history.outcome
|
|
1081
|
+
};
|
|
1082
|
+
if (evt.firstResponse !== false) store.counter("openclaw_chat_history_requests_total", "Successful chat history requests by request shape and response outcome.", historyLabels);
|
|
1083
|
+
if (evt.responseBytes !== void 0) store.counter("openclaw_chat_history_response_bytes_total", "Encoded successful chat history response bytes by request shape and response outcome.", historyLabels, evt.responseBytes);
|
|
1084
|
+
}
|
|
433
1085
|
if (evt.firstResponse === false) return;
|
|
434
1086
|
}
|
|
435
1087
|
store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
|
|
@@ -446,52 +1098,22 @@ function recordOperationTimingEvent(store, evt, metadata) {
|
|
|
446
1098
|
}
|
|
447
1099
|
}
|
|
448
1100
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
if (
|
|
452
|
-
"
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
"
|
|
459
|
-
|
|
460
|
-
"checkout",
|
|
461
|
-
"setup",
|
|
462
|
-
"templatePrepare",
|
|
463
|
-
"templateApply",
|
|
464
|
-
"snapshot",
|
|
465
|
-
"synchronizeCanonical",
|
|
466
|
-
"synchronizeProjection",
|
|
467
|
-
"containerStart",
|
|
468
|
-
"workspaceLayout"
|
|
469
|
-
]) {
|
|
470
|
-
const elapsed = phase === "total" ? evt.durationMs : evt.details?.[phase];
|
|
471
|
-
store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
|
|
472
|
-
kind,
|
|
473
|
-
template: String(template),
|
|
474
|
-
outcome,
|
|
475
|
-
phase
|
|
476
|
-
}, typeof elapsed === "number" ? seconds(elapsed) : void 0);
|
|
1101
|
+
// extensions/diagnostics-prometheus/src/service-retention.ts
|
|
1102
|
+
function recordSessionRetention(store, evt) {
|
|
1103
|
+
if (evt.type === "sessions.event.rows.reset") {
|
|
1104
|
+
store.counter("openclaw_session_event_rows_reset_total", "Retained session-event wire baselines retired by cause.", { cause: evt.cause }, evt.count);
|
|
1105
|
+
return;
|
|
1106
|
+
}
|
|
1107
|
+
if (evt.type === "sessions.list.retention") {
|
|
1108
|
+
store.gauge("openclaw_sessions_list_retained_views", "Retained session-list views across all publications.", {}, evt.views);
|
|
1109
|
+
store.gauge("openclaw_sessions_list_retained_refs", "Row references and selection entries retained by session-list views.", {}, evt.refs);
|
|
1110
|
+
store.gauge("openclaw_sessions_list_retained_bytes", "Estimated memory retained by session-list views.", {}, evt.estimatedBytes);
|
|
1111
|
+
return;
|
|
477
1112
|
}
|
|
1113
|
+
store.gauge("openclaw_session_event_rows_retained_entries", "Session-event field-hash vectors retained across all connections.", {}, evt.entries);
|
|
1114
|
+
store.gauge("openclaw_session_event_rows_retained_bytes", "Estimated memory retained by session-event field-hash vectors.", {}, evt.estimatedBytes);
|
|
478
1115
|
}
|
|
479
1116
|
// extensions/diagnostics-prometheus/src/service-sessions.ts
|
|
480
|
-
function createGatewayWorkMetricsRecorder(store) {
|
|
481
|
-
return (snapshot) => {
|
|
482
|
-
if (!snapshot) {
|
|
483
|
-
store.clearGauges("openclaw_sessions_active");
|
|
484
|
-
store.clearGauges("openclaw_gateway_active_work");
|
|
485
|
-
return;
|
|
486
|
-
}
|
|
487
|
-
for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
|
|
488
|
-
for (const kind of [
|
|
489
|
-
"agentRuns",
|
|
490
|
-
"chatRuns",
|
|
491
|
-
"queuedTurns"
|
|
492
|
-
]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
|
|
493
|
-
};
|
|
494
|
-
}
|
|
495
1117
|
function recordSessionDiagnosticEvent(store, evt) {
|
|
496
1118
|
switch (evt.type) {
|
|
497
1119
|
case "session.recovery.requested":
|
|
@@ -530,60 +1152,62 @@ function recordSessionDiagnosticEvent(store, evt) {
|
|
|
530
1152
|
}
|
|
531
1153
|
}
|
|
532
1154
|
// extensions/diagnostics-prometheus/src/service-worker.ts
|
|
533
|
-
function
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
1155
|
+
function createWorkerRequestMetricsRecorder(store) {
|
|
1156
|
+
return (kind, requestClass, requestPhase, queueDepth, queueWaitMs, durationMs, timing) => {
|
|
1157
|
+
store.gauge("openclaw_worker_queue_depth", "Requests awaiting dispatch across worker owners of this kind.", { kind }, queueDepth);
|
|
1158
|
+
const labels = {
|
|
1159
|
+
kind,
|
|
1160
|
+
request_class: requestClass
|
|
1161
|
+
};
|
|
1162
|
+
store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(queueWaitMs));
|
|
1163
|
+
store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(durationMs));
|
|
1164
|
+
if (requestPhase !== "completed" || !timing) return;
|
|
1165
|
+
for (const [phase, start, end] of [
|
|
1166
|
+
[
|
|
1167
|
+
"dispatch",
|
|
1168
|
+
"dispatched",
|
|
1169
|
+
"posted"
|
|
1170
|
+
],
|
|
1171
|
+
[
|
|
1172
|
+
"delivery",
|
|
1173
|
+
"posted",
|
|
1174
|
+
"received"
|
|
1175
|
+
],
|
|
1176
|
+
[
|
|
1177
|
+
"input",
|
|
1178
|
+
"received",
|
|
1179
|
+
"inputReady"
|
|
1180
|
+
],
|
|
1181
|
+
[
|
|
1182
|
+
"prepare",
|
|
1183
|
+
"inputReady",
|
|
1184
|
+
"executeStarted"
|
|
1185
|
+
],
|
|
1186
|
+
[
|
|
1187
|
+
"execute",
|
|
1188
|
+
"executeStarted",
|
|
1189
|
+
"executeFinished"
|
|
1190
|
+
],
|
|
1191
|
+
[
|
|
1192
|
+
"result",
|
|
1193
|
+
"executeFinished",
|
|
1194
|
+
"replied"
|
|
1195
|
+
],
|
|
1196
|
+
[
|
|
1197
|
+
"reply",
|
|
1198
|
+
"replied",
|
|
1199
|
+
"replyReceived"
|
|
1200
|
+
]
|
|
1201
|
+
]) {
|
|
1202
|
+
const started = timing[start];
|
|
1203
|
+
const finished = timing[end];
|
|
1204
|
+
if (started === void 0 || finished === void 0) continue;
|
|
1205
|
+
store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
|
|
1206
|
+
...labels,
|
|
1207
|
+
phase
|
|
1208
|
+
}, seconds(finished - started));
|
|
1209
|
+
}
|
|
538
1210
|
};
|
|
539
|
-
store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(event.queueWaitMs));
|
|
540
|
-
store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(event.durationMs));
|
|
541
|
-
if (event.phase !== "completed" || !event.timing) return;
|
|
542
|
-
for (const [phase, start, end] of [
|
|
543
|
-
[
|
|
544
|
-
"dispatch",
|
|
545
|
-
"dispatched",
|
|
546
|
-
"posted"
|
|
547
|
-
],
|
|
548
|
-
[
|
|
549
|
-
"delivery",
|
|
550
|
-
"posted",
|
|
551
|
-
"received"
|
|
552
|
-
],
|
|
553
|
-
[
|
|
554
|
-
"input",
|
|
555
|
-
"received",
|
|
556
|
-
"inputReady"
|
|
557
|
-
],
|
|
558
|
-
[
|
|
559
|
-
"prepare",
|
|
560
|
-
"inputReady",
|
|
561
|
-
"executeStarted"
|
|
562
|
-
],
|
|
563
|
-
[
|
|
564
|
-
"execute",
|
|
565
|
-
"executeStarted",
|
|
566
|
-
"executeFinished"
|
|
567
|
-
],
|
|
568
|
-
[
|
|
569
|
-
"result",
|
|
570
|
-
"executeFinished",
|
|
571
|
-
"replied"
|
|
572
|
-
],
|
|
573
|
-
[
|
|
574
|
-
"reply",
|
|
575
|
-
"replied",
|
|
576
|
-
"replyReceived"
|
|
577
|
-
]
|
|
578
|
-
]) {
|
|
579
|
-
const started = event.timing[start];
|
|
580
|
-
const finished = event.timing[end];
|
|
581
|
-
if (started === void 0 || finished === void 0) continue;
|
|
582
|
-
store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
|
|
583
|
-
...labels,
|
|
584
|
-
phase
|
|
585
|
-
}, seconds(finished - started));
|
|
586
|
-
}
|
|
587
1211
|
}
|
|
588
1212
|
// extensions/diagnostics-prometheus/src/service.ts
|
|
589
1213
|
const BYTE_BUCKETS = [
|
|
@@ -618,36 +1242,6 @@ function safeErrorMessage(err) {
|
|
|
618
1242
|
const message = err instanceof Error ? err.message ?? err.name : String(err);
|
|
619
1243
|
return truncateUtf16Safe(redactSensitiveText(message).replaceAll("\0", " ").replace(/[\r\n\t\u2028\u2029]/gu, " "), 500);
|
|
620
1244
|
}
|
|
621
|
-
function renderPrometheusMetrics(store) {
|
|
622
|
-
const snapshot = store.snapshot();
|
|
623
|
-
const lines = [];
|
|
624
|
-
const emitted = /* @__PURE__ */ new Set();
|
|
625
|
-
const emitHeader = (name, type, help) => {
|
|
626
|
-
if (emitted.has(name)) return;
|
|
627
|
-
emitted.add(name);
|
|
628
|
-
lines.push(`# HELP ${name} ${escapeHelp(help)}`);
|
|
629
|
-
lines.push(`# TYPE ${name} ${type}`);
|
|
630
|
-
};
|
|
631
|
-
for (const [type, entries] of [["counter", snapshot.counters], ["gauge", snapshot.gauges]]) {
|
|
632
|
-
entries.sort(([left], [right]) => left.localeCompare(right));
|
|
633
|
-
for (const [key, sample] of entries) {
|
|
634
|
-
const name = key.split("|", 1)[0] ?? "";
|
|
635
|
-
emitHeader(name, type, sample.help);
|
|
636
|
-
lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
snapshot.histograms.sort(([left], [right]) => left.localeCompare(right));
|
|
640
|
-
for (const [key, sample] of snapshot.histograms) {
|
|
641
|
-
const name = key.split("|", 1)[0] ?? "";
|
|
642
|
-
emitHeader(name, "histogram", sample.help);
|
|
643
|
-
for (let index = 0; index < sample.buckets.length; index += 1) lines.push(`${sample.bucketPrefixes[index]}${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
|
|
644
|
-
lines.push(`${sample.bucketPrefixes[sample.buckets.length]}${formatPrometheusNumber(sample.count)}`);
|
|
645
|
-
lines.push(`${name}_sum${sample.labels} ${formatPrometheusNumber(sample.sum)}`);
|
|
646
|
-
lines.push(`${name}_count${sample.labels} ${formatPrometheusNumber(sample.count)}`);
|
|
647
|
-
}
|
|
648
|
-
lines.push("");
|
|
649
|
-
return lines.join("\n");
|
|
650
|
-
}
|
|
651
1245
|
function webhookLabels(evt) {
|
|
652
1246
|
return {
|
|
653
1247
|
channel: normalizeDiagnosticValue(evt.channel),
|
|
@@ -657,16 +1251,23 @@ function webhookLabels(evt) {
|
|
|
657
1251
|
function recordDiagnosticEvent(store, evt, metadata) {
|
|
658
1252
|
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
659
1253
|
switch (evt.type) {
|
|
660
|
-
case "
|
|
661
|
-
if (metadata.trusted)
|
|
1254
|
+
case "gateway.ws.connections":
|
|
1255
|
+
if (metadata.trusted) recordGatewayWsConnections(store, evt.operators);
|
|
1256
|
+
return;
|
|
1257
|
+
case "sessions.list.retention":
|
|
1258
|
+
case "sessions.event.retention":
|
|
1259
|
+
case "sessions.event.rows.reset":
|
|
1260
|
+
if (metadata.trusted) recordSessionRetention(store, evt);
|
|
662
1261
|
return;
|
|
663
1262
|
case "diagnostic.phase.completed":
|
|
664
1263
|
case "secret.egress":
|
|
665
1264
|
case "tool.catalog":
|
|
666
1265
|
case "gateway.rpc":
|
|
1266
|
+
case "sessions.list.baseline.retired":
|
|
667
1267
|
case "gateway.http.cancelled":
|
|
668
1268
|
case "diagnostic.gc":
|
|
669
1269
|
case "gateway.event_loop.sample":
|
|
1270
|
+
case "gateway.main_thread.stall":
|
|
670
1271
|
recordOperationTimingEvent(store, evt, metadata);
|
|
671
1272
|
return;
|
|
672
1273
|
case "model.usage":
|
|
@@ -842,17 +1443,22 @@ function recordDiagnosticEvent(store, evt, metadata) {
|
|
|
842
1443
|
recordSessionDiagnosticEvent(store, evt);
|
|
843
1444
|
return;
|
|
844
1445
|
case "queue.lane.enqueue":
|
|
845
|
-
case "queue.lane.dequeue":
|
|
846
|
-
|
|
847
|
-
|
|
1446
|
+
case "queue.lane.dequeue": {
|
|
1447
|
+
const labels = { lane: normalizeDiagnosticLane(evt.lane) };
|
|
1448
|
+
store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", labels, asNonNegativeFiniteNumber(evt.queueSize));
|
|
1449
|
+
if (evt.type === "queue.lane.dequeue") store.histogram("openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", labels, seconds(evt.waitMs));
|
|
848
1450
|
return;
|
|
1451
|
+
}
|
|
849
1452
|
case "session.state":
|
|
850
1453
|
case "session.stuck":
|
|
851
1454
|
case "session.turn.created":
|
|
852
1455
|
recordSessionDiagnosticEvent(store, evt);
|
|
853
1456
|
return;
|
|
854
1457
|
case "diagnostic.child_process.spawn":
|
|
855
|
-
|
|
1458
|
+
store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", {
|
|
1459
|
+
family: normalizeDiagnosticValue(evt.family),
|
|
1460
|
+
operation: normalizeDiagnosticValue(evt.operation ?? (evt.family === "git" ? "unknown" : "none"))
|
|
1461
|
+
}, asNonNegativeFiniteNumber(evt.count) ?? 0);
|
|
856
1462
|
return;
|
|
857
1463
|
case "diagnostic.memory.sample":
|
|
858
1464
|
recordMemorySample(store, evt.memory, BYTE_BUCKETS);
|
|
@@ -937,23 +1543,24 @@ function createMetricsHandler(store) {
|
|
|
937
1543
|
res.end("Method Not Allowed");
|
|
938
1544
|
return true;
|
|
939
1545
|
}
|
|
940
|
-
const body =
|
|
1546
|
+
const body = store.render();
|
|
941
1547
|
res.statusCode = 200;
|
|
942
1548
|
res.setHeader("Cache-Control", "no-store");
|
|
943
1549
|
res.setHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
944
|
-
res.setHeader("Content-Length", String(
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
}
|
|
949
|
-
res.end(body);
|
|
1550
|
+
res.setHeader("Content-Length", String(body.length));
|
|
1551
|
+
res.flushHeaders();
|
|
1552
|
+
res.end(req.method === "HEAD" ? void 0 : body, (error) => {
|
|
1553
|
+
if (!error) body.buffer.transfer(0);
|
|
1554
|
+
});
|
|
950
1555
|
return true;
|
|
951
1556
|
};
|
|
952
1557
|
}
|
|
953
1558
|
function createDiagnosticsPrometheusExporter() {
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
1559
|
+
let logger;
|
|
1560
|
+
const store = createPrometheusMetricStore((family) => {
|
|
1561
|
+
logger?.warn(`diagnostics-prometheus: series budget reached for ${family}; new label sets are being dropped. See openclaw_prometheus_family_series_dropped_total.`);
|
|
1562
|
+
});
|
|
1563
|
+
let subscriptions = [];
|
|
957
1564
|
let internalDiagnostics;
|
|
958
1565
|
const reportExporterStatus = (update) => {
|
|
959
1566
|
try {
|
|
@@ -968,7 +1575,6 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
968
1575
|
};
|
|
969
1576
|
return {
|
|
970
1577
|
handler: createMetricsHandler(store),
|
|
971
|
-
render: () => renderPrometheusMetrics(store),
|
|
972
1578
|
service: {
|
|
973
1579
|
id: "diagnostics-prometheus",
|
|
974
1580
|
start(ctx) {
|
|
@@ -977,19 +1583,44 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
977
1583
|
ctx.logger.error("diagnostics-prometheus: internal diagnostics capability unavailable");
|
|
978
1584
|
return;
|
|
979
1585
|
}
|
|
1586
|
+
logger = ctx.logger;
|
|
1587
|
+
reserveMainGcHistograms(store);
|
|
980
1588
|
const identity = isDiagnosticsEnabled(ctx.config) ? ctx.internalDiagnostics?.getRuntimeIdentity?.() : void 0;
|
|
981
1589
|
if (identity) store.gauge("openclaw_gateway_build_info", "Identity of the hosting process and its loaded build; not a health or exporter epoch.", {
|
|
982
1590
|
process_instance_id: identity.processInstanceId,
|
|
983
1591
|
...identity.buildId ? { build_id: identity.buildId } : {}
|
|
984
1592
|
}, 1);
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1593
|
+
subscriptions = [
|
|
1594
|
+
ctx.internalDiagnostics?.onGatewayWorkMetrics?.((snapshot) => {
|
|
1595
|
+
if (!snapshot) {
|
|
1596
|
+
store.clearGauges("openclaw_sessions_active");
|
|
1597
|
+
store.clearGauges("openclaw_gateway_active_work");
|
|
1598
|
+
return;
|
|
1599
|
+
}
|
|
1600
|
+
for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
|
|
1601
|
+
for (const kind of [
|
|
1602
|
+
"agentRuns",
|
|
1603
|
+
"chatRuns",
|
|
1604
|
+
"queuedTurns"
|
|
1605
|
+
]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
|
|
1606
|
+
}),
|
|
1607
|
+
ctx.internalDiagnostics?.onGatewayWsMetrics?.(createGatewayWsMetricsRecorder(store)),
|
|
1608
|
+
ctx.internalDiagnostics?.onWorkerRequestMetrics?.(createWorkerRequestMetricsRecorder(store)),
|
|
1609
|
+
subscribe((event, metadata) => {
|
|
1610
|
+
try {
|
|
1611
|
+
recordDiagnosticEvent(store, event, metadata);
|
|
1612
|
+
} catch (err) {
|
|
1613
|
+
ctx.logger.error(`diagnostics-prometheus: event handler failed (${event.type}): ${safeErrorMessage(err)}`);
|
|
1614
|
+
}
|
|
1615
|
+
}, {
|
|
1616
|
+
exclude: [
|
|
1617
|
+
"log.record",
|
|
1618
|
+
"worker.request",
|
|
1619
|
+
"gateway.ws.event"
|
|
1620
|
+
],
|
|
1621
|
+
includePromptStats: false
|
|
1622
|
+
}, { includePrivateData: false })
|
|
1623
|
+
];
|
|
993
1624
|
internalDiagnostics = ctx.internalDiagnostics;
|
|
994
1625
|
reportExporterStatus({
|
|
995
1626
|
signal: "metrics",
|
|
@@ -999,10 +1630,8 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
999
1630
|
});
|
|
1000
1631
|
},
|
|
1001
1632
|
stop() {
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
unsubscribe?.();
|
|
1005
|
-
unsubscribe = void 0;
|
|
1633
|
+
for (const unsubscribe of subscriptions) unsubscribe?.();
|
|
1634
|
+
subscriptions = [];
|
|
1006
1635
|
reportExporterStatus({
|
|
1007
1636
|
signal: "metrics",
|
|
1008
1637
|
transport: "prometheus-scrape",
|
|
@@ -1010,6 +1639,7 @@ function createDiagnosticsPrometheusExporter() {
|
|
|
1010
1639
|
});
|
|
1011
1640
|
internalDiagnostics = void 0;
|
|
1012
1641
|
store.reset();
|
|
1642
|
+
logger = void 0;
|
|
1013
1643
|
}
|
|
1014
1644
|
}
|
|
1015
1645
|
};
|