@openclaw/diagnostics-prometheus 2026.10.10-beta.5 → 2026.11.5-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -4,7 +4,8 @@ import { isDiagnosticsEnabled, normalizeDiagnosticLane, normalizeDiagnosticValue
4
4
  import { asNonNegativeFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
5
5
  import { getPluginRuntimeGatewayRequestScope } from "openclaw/plugin-sdk/plugin-runtime";
6
6
  import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
7
- //#region extensions/diagnostics-prometheus/src/prometheus-format.ts
7
+ import { expectDefined } from "openclaw/plugin-sdk/expect-runtime";
8
+ // extensions/diagnostics-prometheus/src/prometheus-format.ts
8
9
  function sortedLabels(labels) {
9
10
  const entries = Object.entries(labels);
10
11
  entries.sort(([left], [right]) => left.localeCompare(right));
@@ -32,8 +33,166 @@ function seconds(ms) {
32
33
  const value = asNonNegativeFiniteNumber(ms);
33
34
  return value === void 0 ? void 0 : value / 1e3;
34
35
  }
35
- //#endregion
36
- //#region extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
36
+ // extensions/diagnostics-prometheus/src/prometheus-family-budget.ts
37
+ const PROMETHEUS_FAMILY_RESERVES = {
38
+ openclaw_chat_history_requests_total: 4,
39
+ openclaw_chat_history_response_bytes_total: 4,
40
+ openclaw_chat_send_phase_seconds: 32,
41
+ openclaw_child_process_spawn_total: 4,
42
+ openclaw_code_mode_worker_heap_bytes: 128,
43
+ openclaw_code_mode_worker_memory_bytes: 128,
44
+ openclaw_control_ui_document_duration_seconds: 4,
45
+ openclaw_cron_list_requests_total: 4,
46
+ openclaw_diagnostic_async_queue_dropped_total: 4,
47
+ openclaw_diagnostic_async_queue_length: 4,
48
+ openclaw_diff_cache_misses_total: 4,
49
+ openclaw_diff_stage_seconds: 4,
50
+ openclaw_gateway_active_work: 4,
51
+ openclaw_gateway_build_info: 4,
52
+ openclaw_gateway_event_loop_delay_max_seconds: 4,
53
+ openclaw_gateway_event_loop_observed_seconds_total: 4,
54
+ openclaw_gateway_http_cancelled_total: 4,
55
+ openclaw_gateway_main_thread_stall_seconds: 4,
56
+ openclaw_gateway_main_thread_stalls_dropped_total: 4,
57
+ openclaw_gateway_rpc_admission_seconds: 256,
58
+ openclaw_gateway_rpc_client_requests_total: 512,
59
+ openclaw_gateway_rpc_first_response_seconds: 256,
60
+ openclaw_gateway_rpc_handler_heap_delta_bytes: 256,
61
+ openclaw_gateway_rpc_handler_heap_delta_exclusive_total: 256,
62
+ openclaw_gateway_rpc_handler_seconds: 256,
63
+ openclaw_gateway_rpc_outcomes_total: 256,
64
+ openclaw_gateway_rpc_queue_wait_seconds: 256,
65
+ openclaw_gateway_rpc_requests_total: 256,
66
+ openclaw_gateway_rpc_response_bytes: 256,
67
+ openclaw_gateway_rpc_stage_seconds: 256,
68
+ openclaw_gateway_rpc_stage_thread_cpu_seconds: 256,
69
+ openclaw_gateway_ws_event_bytes_total: 256,
70
+ openclaw_gateway_ws_event_frame_bytes: 2048,
71
+ openclaw_gateway_ws_event_wire_bytes_total: 256,
72
+ openclaw_gateway_ws_events_total: 256,
73
+ openclaw_gateway_ws_operator_connections: 4,
74
+ openclaw_gateway_ws_presence_bytes_total: 4,
75
+ openclaw_gateway_ws_presence_frames_total: 4,
76
+ openclaw_gateway_ws_session_event_bytes_total: 256,
77
+ openclaw_gateway_ws_session_event_frames_total: 256,
78
+ openclaw_gc_duration_seconds: 384,
79
+ openclaw_gen_ai_client_token_usage: 128,
80
+ openclaw_harness_run_duration_seconds: 128,
81
+ openclaw_harness_run_total: 128,
82
+ openclaw_heap_old_generation_after_gc_bytes: 128,
83
+ openclaw_heap_old_generation_after_gc_ratio: 128,
84
+ openclaw_heap_size_limit_bytes: 128,
85
+ openclaw_heap_space_bytes: 256,
86
+ openclaw_heap_used_after_major_gc_bytes: 64,
87
+ openclaw_heap_used_after_major_gc_timestamp_seconds: 64,
88
+ openclaw_heap_used_ratio: 128,
89
+ openclaw_liveness_cpu_core_ratio: 4,
90
+ openclaw_liveness_event_loop_delay_max_seconds: 4,
91
+ openclaw_liveness_event_loop_delay_p99_seconds: 4,
92
+ openclaw_liveness_event_loop_utilization_ratio: 4,
93
+ openclaw_liveness_sessions: 4,
94
+ openclaw_liveness_warning_total: 4,
95
+ openclaw_memory_bytes: 128,
96
+ openclaw_memory_pressure_total: 4,
97
+ openclaw_memory_rss_bytes: 4,
98
+ openclaw_message_delivery_duration_seconds: 4,
99
+ openclaw_message_delivery_started_total: 4,
100
+ openclaw_message_delivery_total: 4,
101
+ openclaw_message_dispatch_completed_total: 4,
102
+ openclaw_message_dispatch_duration_seconds: 4,
103
+ openclaw_message_dispatch_started_total: 4,
104
+ openclaw_message_processed_duration_seconds: 4,
105
+ openclaw_message_processed_total: 4,
106
+ openclaw_message_received_total: 4,
107
+ openclaw_model_call_duration_seconds: 128,
108
+ openclaw_model_call_total: 128,
109
+ openclaw_model_cost_usd_total: 128,
110
+ openclaw_model_failover_total: 128,
111
+ openclaw_model_tokens_total: 128,
112
+ openclaw_model_usage_duration_seconds: 128,
113
+ openclaw_models_list_requests_total: 4,
114
+ openclaw_payload_large_bytes: 4,
115
+ openclaw_payload_large_total: 4,
116
+ openclaw_process_anonymous_memory_bytes: 4,
117
+ openclaw_queue_lane_size: 4,
118
+ openclaw_queue_lane_wait_seconds: 4,
119
+ openclaw_run_completed_total: 128,
120
+ openclaw_run_duration_seconds: 128,
121
+ openclaw_secret_egress_requests_total: 4,
122
+ openclaw_session_event_rows_reset_total: 4,
123
+ openclaw_session_event_rows_retained_bytes: 4,
124
+ openclaw_session_event_rows_retained_entries: 4,
125
+ openclaw_session_queue_depth: 4,
126
+ openclaw_session_recovery_age_seconds: 4,
127
+ openclaw_session_recovery_total: 4,
128
+ openclaw_session_row_selection_resets_total: 4,
129
+ openclaw_session_row_partition_cache_total: 32,
130
+ openclaw_session_row_partition_revisions_total: 8,
131
+ openclaw_session_row_selection_rows_total: 64,
132
+ openclaw_session_row_selections_total: 64,
133
+ openclaw_session_state_total: 4,
134
+ openclaw_session_stuck_age_seconds: 4,
135
+ openclaw_session_stuck_total: 4,
136
+ openclaw_session_turn_created_total: 4,
137
+ openclaw_sessions_active: 4,
138
+ openclaw_sessions_list_base_misses_total: 4,
139
+ openclaw_sessions_list_baseline_retirements_total: 4,
140
+ openclaw_sessions_list_requests_total: 4,
141
+ openclaw_sessions_list_response_bytes_total: 4,
142
+ openclaw_sessions_list_retained_bytes: 4,
143
+ openclaw_sessions_list_retained_refs: 4,
144
+ openclaw_sessions_list_retained_views: 4,
145
+ openclaw_skill_used_total: 4,
146
+ openclaw_talk_audio_bytes: 4,
147
+ openclaw_talk_event_duration_seconds: 4,
148
+ openclaw_talk_event_total: 4,
149
+ openclaw_telemetry_exporter_total: 4,
150
+ openclaw_tool_catalog_tools_total: 128,
151
+ openclaw_tool_catalog_total: 128,
152
+ openclaw_tool_execution_blocked_total: 128,
153
+ openclaw_tool_execution_duration_seconds: 128,
154
+ openclaw_tool_execution_total: 128,
155
+ openclaw_webhook_duration_seconds: 4,
156
+ openclaw_webhook_error_total: 4,
157
+ openclaw_webhook_received_total: 4,
158
+ openclaw_worker_array_buffers_bytes: 128,
159
+ openclaw_worker_count: 128,
160
+ openclaw_worker_external_bytes: 128,
161
+ openclaw_worker_gc_collections_total: 128,
162
+ openclaw_worker_gc_duration_seconds_total: 128,
163
+ openclaw_worker_gc_reclaimed_bytes_total: 128,
164
+ openclaw_worker_heap_sampled_count: 128,
165
+ openclaw_worker_heap_space_used_bytes: 640,
166
+ openclaw_worker_heap_used_bytes: 128,
167
+ openclaw_worker_heap_used_high_water_bytes: 128,
168
+ openclaw_worker_message_payload_bytes_total: 512,
169
+ openclaw_worker_message_unmeasured_total: 64,
170
+ openclaw_worker_messages_total: 64,
171
+ openclaw_worker_queue_depth: 512,
172
+ openclaw_worker_queue_wait_seconds: 512,
173
+ openclaw_worker_request_phase_seconds: 1024,
174
+ openclaw_worker_request_seconds: 512,
175
+ openclaw_worker_retired_total: 64,
176
+ openclaw_worker_started_total: 64,
177
+ openclaw_worker_tasks_completed_total: 128,
178
+ openclaw_worktree_preparation_seconds: 32
179
+ };
180
+ const SHARED_PROMETHEUS_SERIES = 16384 - Object.values(PROMETHEUS_FAMILY_RESERVES).reduce((sum, n) => sum + n, 0);
181
+ // extensions/diagnostics-prometheus/src/prometheus-metric-store.ts
182
+ function createSeries() {
183
+ return {
184
+ samples: [],
185
+ shapes: /* @__PURE__ */ new Map(),
186
+ sorted: true
187
+ };
188
+ }
189
+ function sortedSamples(series) {
190
+ if (!series.sorted) {
191
+ series.samples.sort((left, right) => left.key.localeCompare(right.key));
192
+ series.sorted = true;
193
+ }
194
+ return series.samples;
195
+ }
37
196
  const DURATION_BUCKETS_SECONDS = [
38
197
  .005,
39
198
  .01,
@@ -58,101 +217,270 @@ const AGENT_DURATION_BUCKETS_SECONDS = [
58
217
  1800,
59
218
  3600
60
219
  ];
61
- const MAX_PROMETHEUS_SERIES = 2048;
62
220
  const DROPPED_SERIES_COUNTER_NAME = "openclaw_prometheus_series_dropped_total";
63
- function createPrometheusMetricStore() {
64
- const counters = /* @__PURE__ */ new Map();
65
- const gauges = /* @__PURE__ */ new Map();
66
- const histograms = /* @__PURE__ */ new Map();
221
+ const DROPPED_FAMILY_COUNTER_NAME = "openclaw_prometheus_family_series_dropped_total";
222
+ const warnedFamilies = /* @__PURE__ */ new Set();
223
+ function createPrometheusMetricStore(onSeriesDropped) {
224
+ const counters = createSeries();
225
+ const gauges = createSeries();
226
+ const histograms = createSeries();
67
227
  let droppedSeries = 0;
68
- const canCreateSeries = (map, key) => {
69
- if (map.has(key)) return true;
70
- if (counters.size + gauges.size + histograms.size < MAX_PROMETHEUS_SERIES) return true;
71
- droppedSeries += 1;
72
- return false;
228
+ let sharedSeries = 0;
229
+ const familyCounts = /* @__PURE__ */ new Map();
230
+ const familyDrops = /* @__PURE__ */ new Map();
231
+ const headers = /* @__PURE__ */ new Map();
232
+ let prefixBuffer = Buffer.alloc(0);
233
+ let prefixOffset = 0;
234
+ const encodePrefix = (text) => {
235
+ const bytes = Buffer.byteLength(text);
236
+ if (bytes > 4096) {
237
+ const prefix = Buffer.allocUnsafeSlow(bytes);
238
+ prefix.write(text);
239
+ return prefix;
240
+ }
241
+ if (prefixOffset + bytes > prefixBuffer.length) {
242
+ prefixBuffer = Buffer.allocUnsafeSlow(4096);
243
+ prefixOffset = 0;
244
+ }
245
+ const start = prefixOffset;
246
+ prefixOffset += prefixBuffer.write(text, prefixOffset);
247
+ return prefixBuffer.subarray(start, prefixOffset);
73
248
  };
74
- const counter = (name, help, labels, amount = 1) => {
75
- if (!Number.isFinite(amount) || amount <= 0) return;
76
- const key = metricKey(name, labels);
77
- if (!canCreateSeries(counters, key)) return;
78
- const existing = counters.get(key);
79
- if (existing) {
80
- existing.value += amount;
249
+ let scratch = Buffer.alloc(0);
250
+ const series = (map, name, labels) => {
251
+ let labelCount = 0;
252
+ for (const key in labels) if (Object.hasOwn(labels, key)) labelCount++;
253
+ let shape;
254
+ shapes: for (const candidate of map.shapes.get(name) ?? []) {
255
+ if (candidate.keys.length !== labelCount) continue;
256
+ let node = candidate.root;
257
+ for (const key of candidate.keys) {
258
+ const value = labels[key];
259
+ if (value === void 0 || !Object.prototype.propertyIsEnumerable.call(labels, key)) continue shapes;
260
+ node = node?.values?.get(value);
261
+ }
262
+ shape = candidate;
263
+ if (node?.sample) return node;
264
+ break;
265
+ }
266
+ const count = familyCounts.get(name) ?? 0;
267
+ const reserve = PROMETHEUS_FAMILY_RESERVES[name];
268
+ if (count >= Math.max(128, reserve) || count >= reserve && sharedSeries >= SHARED_PROMETHEUS_SERIES) {
269
+ droppedSeries++;
270
+ familyDrops.set(name, (familyDrops.get(name) ?? 0) + 1);
271
+ if (onSeriesDropped && !warnedFamilies.has(name)) {
272
+ warnedFamilies.add(name);
273
+ onSeriesDropped(name);
274
+ }
81
275
  return;
82
276
  }
83
- counters.set(key, {
84
- help,
85
- labels,
86
- value: amount
87
- });
88
- };
89
- const gauge = (name, help, labels, value) => {
90
- if (value === void 0 || !Number.isFinite(value)) return;
91
- const key = metricKey(name, labels);
92
- if (!canCreateSeries(gauges, key)) return;
93
- gauges.set(key, {
94
- help,
95
- labels,
96
- value
97
- });
277
+ familyCounts.set(name, count + 1);
278
+ if (count >= reserve) sharedSeries++;
279
+ if (!shape) {
280
+ shape = {
281
+ keys: Object.keys(labels).toSorted((left, right) => left.localeCompare(right)),
282
+ root: {}
283
+ };
284
+ const shapes = map.shapes.get(name);
285
+ if (shapes) shapes.push(shape);
286
+ else map.shapes.set(name, [shape]);
287
+ }
288
+ let node = shape.root;
289
+ for (const key of shape.keys) {
290
+ const value = expectDefined(labels[key], "Prometheus label");
291
+ const values = node.values ??= /* @__PURE__ */ new Map();
292
+ let child = values.get(value);
293
+ if (!child) {
294
+ child = {};
295
+ values.set(value, child);
296
+ }
297
+ node = child;
298
+ }
299
+ return node;
98
300
  };
99
- const counterValue = (name, help, labels, value) => {
100
- if (!Number.isFinite(value) || value < 0) return;
101
- const key = metricKey(name, labels);
102
- if (canCreateSeries(counters, key)) counters.set(key, {
103
- help,
104
- labels,
105
- value: Math.max(counters.get(key)?.value ?? 0, value)
106
- });
301
+ const scalar = (map, name, help, labels, value, mode) => {
302
+ if (value === void 0 || !Number.isFinite(value) || mode !== "set" && value < 0 || mode === "add" && value === 0) return;
303
+ const node = series(map, name, labels);
304
+ if (!node) return;
305
+ if (node.sample) {
306
+ const next = mode === "add" ? node.sample.value + value : mode === "max" ? Math.max(node.sample.value, value) : value;
307
+ if (next !== node.sample.value) {
308
+ node.sample.value = next;
309
+ node.sample.renderedDirty = true;
310
+ }
311
+ } else {
312
+ node.sample = {
313
+ help,
314
+ key: metricKey(name, labels),
315
+ name,
316
+ prefix: encodePrefix(`${name}${formatLabels(labels)} `),
317
+ value,
318
+ renderedDirty: true
319
+ };
320
+ map.samples.push(node.sample);
321
+ map.sorted = false;
322
+ }
107
323
  };
324
+ const counter = (name, help, labels, amount = 1) => scalar(counters, name, help, labels, amount, "add");
325
+ const counterValue = (name, help, labels, value) => scalar(counters, name, help, labels, value, "max");
326
+ const gauge = (name, help, labels, value) => scalar(gauges, name, help, labels, value, "set");
108
327
  const clearGauges = (name) => {
109
- for (const key of gauges.keys()) if (key.startsWith(`${name}|`)) gauges.delete(key);
328
+ headers.delete(name);
329
+ const previousLength = gauges.samples.length;
330
+ gauges.shapes.delete(name);
331
+ gauges.samples = gauges.samples.filter((sample) => sample.name !== name);
332
+ const count = familyCounts.get(name) ?? 0;
333
+ const remaining = count - (previousLength - gauges.samples.length);
334
+ const reserve = PROMETHEUS_FAMILY_RESERVES[name];
335
+ sharedSeries -= Math.max(0, count - reserve) - Math.max(0, remaining - reserve);
336
+ familyCounts.set(name, remaining);
110
337
  };
111
- const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
112
- if (value === void 0 || !Number.isFinite(value)) return;
113
- const key = metricKey(name, labels);
114
- if (!canCreateSeries(histograms, key)) return;
115
- let sample = histograms.get(key);
338
+ const getHistogram = (name, help, labels, buckets = DURATION_BUCKETS_SECONDS) => {
339
+ const node = series(histograms, name, labels);
340
+ if (!node) return;
341
+ let sample = node.sample;
116
342
  if (!sample) {
117
343
  sample = {
118
344
  buckets,
119
- bucketPrefixes: [...buckets, "+Inf"].map((le) => `${name}_bucket${formatLabels({
345
+ bucketPrefixes: [...buckets, "+Inf"].map((le) => encodePrefix(`${name}_bucket${formatLabels({
120
346
  ...labels,
121
347
  le: String(le)
122
- })} `),
348
+ })} `)),
123
349
  counts: buckets.map(() => 0),
124
350
  count: 0,
125
351
  help,
126
- labels: formatLabels(labels),
127
- sum: 0
352
+ key: metricKey(name, labels),
353
+ name,
354
+ sumPrefix: encodePrefix(`${name}_sum${formatLabels(labels)} `),
355
+ countPrefix: encodePrefix(`${name}_count${formatLabels(labels)} `),
356
+ sum: 0,
357
+ renderedDirty: true
128
358
  };
129
- histograms.set(key, sample);
359
+ node.sample = sample;
360
+ histograms.samples.push(sample);
361
+ histograms.sorted = false;
130
362
  }
363
+ return sample;
364
+ };
365
+ const reserveHistogram = (name, help, labels) => {
366
+ getHistogram(name, help, labels);
367
+ };
368
+ const histogram = (name, help, labels, value, buckets = DURATION_BUCKETS_SECONDS) => {
369
+ if (value === void 0 || !Number.isFinite(value)) return;
370
+ const sample = getHistogram(name, help, labels, buckets);
371
+ if (!sample) return;
131
372
  sample.count += 1;
132
373
  sample.sum += value;
374
+ sample.renderedDirty = true;
133
375
  for (let index = 0; index < sample.buckets.length; index += 1) {
134
376
  const bucket = sample.buckets[index];
135
377
  if (bucket !== void 0 && value <= bucket) sample.counts[index] = (sample.counts[index] ?? 0) + 1;
136
378
  }
137
379
  };
138
- const snapshot = () => {
139
- const counterSnapshot = [...counters];
140
- if (droppedSeries > 0) counterSnapshot.push([metricKey(DROPPED_SERIES_COUNTER_NAME, {}), {
141
- help: "Prometheus metric series dropped because the exporter series cap was reached.",
142
- labels: {},
143
- value: droppedSeries
144
- }]);
145
- return {
146
- counters: counterSnapshot,
147
- gauges: [...gauges],
148
- histograms: [...histograms]
380
+ const render = () => {
381
+ let scratchOffset = 0;
382
+ let totalBytes = 0;
383
+ const chunks = [];
384
+ const append = (chunk) => {
385
+ chunks.push(chunk);
386
+ totalBytes += chunk.length;
387
+ };
388
+ const write = (prefix, value) => {
389
+ const number = formatPrometheusNumber(value);
390
+ const required = scratchOffset + prefix.length + number.length + 1;
391
+ if (required > scratch.length) {
392
+ const grown = Buffer.allocUnsafeSlow(Math.max(required, scratch.length * 2));
393
+ grown.set(scratch.subarray(0, scratchOffset));
394
+ scratch = grown;
395
+ }
396
+ scratch.set(prefix, scratchOffset);
397
+ scratchOffset += prefix.length;
398
+ scratchOffset += scratch.write(number, scratchOffset, "ascii");
399
+ scratch[scratchOffset++] = 10;
400
+ };
401
+ const finish = (previous) => {
402
+ const output = previous?.length === scratchOffset ? previous : new Uint8Array(scratchOffset);
403
+ scratch.copy(output, 0, 0, scratchOffset);
404
+ scratchOffset = 0;
405
+ return output;
406
+ };
407
+ const emitted = /* @__PURE__ */ new Set();
408
+ let previousName;
409
+ const header = (name, type, help) => {
410
+ if (previousName === name) return;
411
+ previousName = name;
412
+ if (!emitted.has(name)) {
413
+ emitted.add(name);
414
+ let cached = headers.get(name);
415
+ if (!cached || cached.type !== type || cached.help !== help) {
416
+ cached = {
417
+ type,
418
+ help,
419
+ bytes: encodePrefix(`# HELP ${name} ${escapeHelp(help)}\n# TYPE ${name} ${type}\n`)
420
+ };
421
+ headers.set(name, cached);
422
+ }
423
+ append(cached.bytes);
424
+ }
149
425
  };
426
+ for (const [type, map] of [["counter", counters], ["gauge", gauges]]) {
427
+ for (const sample of sortedSamples(map)) {
428
+ header(sample.name, type, sample.help);
429
+ if (!sample.rendered || sample.renderedDirty) {
430
+ write(sample.prefix, sample.value);
431
+ sample.rendered = finish(sample.rendered);
432
+ sample.renderedDirty = false;
433
+ }
434
+ append(sample.rendered);
435
+ }
436
+ if (type === "counter" && droppedSeries > 0) {
437
+ header(DROPPED_SERIES_COUNTER_NAME, type, "New-series observations dropped because an exporter series budget was reached.");
438
+ write(Buffer.from(`${DROPPED_SERIES_COUNTER_NAME} `), droppedSeries);
439
+ append(finish());
440
+ header(DROPPED_FAMILY_COUNTER_NAME, type, "New-series observations dropped by bounded metric family.");
441
+ for (const [family, count] of familyDrops) write(Buffer.from(`${DROPPED_FAMILY_COUNTER_NAME}${formatLabels({ family })} `), count);
442
+ append(finish());
443
+ }
444
+ }
445
+ for (const sample of sortedSamples(histograms)) {
446
+ if (sample.count === 0) continue;
447
+ header(sample.name, "histogram", sample.help);
448
+ if (!sample.rendered || sample.renderedDirty) {
449
+ sample.bucketPrefixes.forEach((prefix, bucketIndex) => {
450
+ write(prefix, sample.counts[bucketIndex] ?? sample.count);
451
+ });
452
+ write(sample.sumPrefix, sample.sum);
453
+ write(sample.countPrefix, sample.count);
454
+ sample.rendered = finish(sample.rendered);
455
+ sample.renderedDirty = false;
456
+ }
457
+ append(sample.rendered);
458
+ }
459
+ const output = Buffer.from(new ArrayBuffer(totalBytes));
460
+ let offset = 0;
461
+ for (const chunk of chunks) {
462
+ output.set(chunk, offset);
463
+ offset += chunk.length;
464
+ }
465
+ return output;
150
466
  };
151
467
  const reset = () => {
152
- counters.clear();
153
- gauges.clear();
154
- histograms.clear();
468
+ for (const map of [
469
+ counters,
470
+ gauges,
471
+ histograms
472
+ ]) {
473
+ map.samples.length = 0;
474
+ map.shapes.clear();
475
+ }
155
476
  droppedSeries = 0;
477
+ sharedSeries = 0;
478
+ familyCounts.clear();
479
+ familyDrops.clear();
480
+ headers.clear();
481
+ scratch = Buffer.alloc(0);
482
+ prefixBuffer = Buffer.alloc(0);
483
+ prefixOffset = 0;
156
484
  };
157
485
  return {
158
486
  counter,
@@ -160,31 +488,140 @@ function createPrometheusMetricStore() {
160
488
  gauge,
161
489
  clearGauges,
162
490
  histogram,
491
+ reserveHistogram,
163
492
  reset,
164
- snapshot
493
+ render
165
494
  };
166
495
  }
167
- //#endregion
168
- //#region extensions/diagnostics-prometheus/src/service-child-process.ts
169
- /** Git owners and SQLite session transports carry bounded operation labels. */
170
- function recordChildProcessSpawn(store, evt) {
171
- store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", {
172
- family: normalizeDiagnosticValue(evt.family),
173
- operation: normalizeDiagnosticValue(evt.operation ?? (evt.family === "git" ? "unknown" : "none"))
174
- }, asNonNegativeFiniteNumber(evt.count) ?? 0);
496
+ // extensions/diagnostics-prometheus/src/service-gateway-ws.ts
497
+ function createGatewayWsMetricsRecorder(store) {
498
+ const labels = { event: "" };
499
+ const frameLabels = {
500
+ event: "",
501
+ le: ""
502
+ };
503
+ const presenceLabels = { mode: "" };
504
+ const sessionLabels = {
505
+ event: "",
506
+ reason: "",
507
+ delivery: ""
508
+ };
509
+ return (event, count, bytes, wireBytes, buckets, bucketLabels, sessionEvent, presenceMode) => {
510
+ labels.event = frameLabels.event = sessionLabels.event = event;
511
+ store.counter("openclaw_gateway_ws_events_total", "Outbound Gateway WebSocket event frames accepted by the sender per socket.", labels, count);
512
+ store.counter("openclaw_gateway_ws_event_bytes_total", "Encoded outbound Gateway WebSocket event bytes before framing or compression per socket.", labels, bytes);
513
+ if (wireBytes !== void 0) store.counter("openclaw_gateway_ws_event_wire_bytes_total", "Outbound Gateway WebSocket payload bytes after compression per socket, excluding framing.", labels, wireBytes);
514
+ if (count > 0) {
515
+ let cumulative = 0;
516
+ for (let i = 0; i < bucketLabels.length; i++) {
517
+ cumulative += buckets[i];
518
+ frameLabels.le = bucketLabels[i];
519
+ (cumulative === 0 ? store.counterValue : store.counter)("openclaw_gateway_ws_event_frame_bytes", "Cumulative counts of accepted raw Gateway event frames by byte-size upper bound.", frameLabels, cumulative);
520
+ }
521
+ }
522
+ if (presenceMode) {
523
+ presenceLabels.mode = presenceMode;
524
+ store.counter("openclaw_gateway_ws_presence_frames_total", "Outbound presence frames accepted by the sender per socket, by delivery mode.", presenceLabels, count);
525
+ store.counter("openclaw_gateway_ws_presence_bytes_total", "Encoded outbound presence bytes before framing or compression per socket, by delivery mode.", presenceLabels, bytes);
526
+ }
527
+ if (sessionEvent) {
528
+ sessionLabels.reason = sessionEvent.reason;
529
+ sessionLabels.delivery = sessionEvent.delivery;
530
+ store.counter("openclaw_gateway_ws_session_event_frames_total", "Outbound session event frames by reason and exact message subscription per socket.", sessionLabels, count);
531
+ store.counter("openclaw_gateway_ws_session_event_bytes_total", "Encoded outbound session event bytes by reason and exact message subscription per socket.", sessionLabels, bytes);
532
+ }
533
+ };
534
+ }
535
+ function recordGatewayWsConnections(store, operators) {
536
+ store.gauge("openclaw_gateway_ws_operator_connections", "Connected authenticated Gateway operator WebSockets.", {}, operators);
537
+ }
538
+ // extensions/diagnostics-prometheus/src/service-memory.ts
539
+ const WORKER_MEMORY_METRICS = [
540
+ [
541
+ "heapUsed",
542
+ "openclaw_worker_heap_used_bytes",
543
+ "heap usage"
544
+ ],
545
+ [
546
+ "external",
547
+ "openclaw_worker_external_bytes",
548
+ "external memory"
549
+ ],
550
+ [
551
+ "arrayBuffers",
552
+ "openclaw_worker_array_buffers_bytes",
553
+ "ArrayBuffer memory (included in external)"
554
+ ]
555
+ ];
556
+ const HEAP_HEADROOM_METRICS = [
557
+ [
558
+ "limit",
559
+ "openclaw_heap_size_limit_bytes",
560
+ "Smallest live isolate V8 heap limit by thread group."
561
+ ],
562
+ [
563
+ "usedRatio",
564
+ "openclaw_heap_used_ratio",
565
+ "Largest live isolate used/limit ratio by thread group."
566
+ ],
567
+ [
568
+ "oldBytes",
569
+ "openclaw_heap_old_generation_after_gc_bytes",
570
+ "Largest live isolate old_space plus large_object_space usage at its last observed major GC."
571
+ ],
572
+ [
573
+ "oldRatio",
574
+ "openclaw_heap_old_generation_after_gc_ratio",
575
+ "Largest live isolate post-major-GC old-generation/heap-limit ratio by thread group."
576
+ ]
577
+ ];
578
+ function collectHeapHeadroom(headroom, used, limit, oldBytes) {
579
+ if (limit === void 0 || limit <= 0) return;
580
+ headroom.limit = Math.min(headroom.limit ?? limit, limit);
581
+ headroom.usedRatio = Math.max(headroom.usedRatio ?? 0, used / limit);
582
+ if (oldBytes !== void 0) {
583
+ headroom.oldBytes = Math.max(headroom.oldBytes ?? 0, oldBytes);
584
+ headroom.oldRatio = Math.max(headroom.oldRatio ?? 0, oldBytes / limit);
585
+ }
586
+ }
587
+ function recordHeapHeadroom(store, thread, headroom) {
588
+ for (const [field, name, description] of HEAP_HEADROOM_METRICS) store.gauge(name, description, { thread }, headroom[field]);
175
589
  }
176
- //#endregion
177
- //#region extensions/diagnostics-prometheus/src/service-memory.ts
178
590
  function recordMemorySample(store, memory, byteBuckets) {
591
+ for (const [, name] of HEAP_HEADROOM_METRICS) store.clearGauges(name);
592
+ const mainHeadroom = {};
593
+ collectHeapHeadroom(mainHeadroom, memory.heapUsedBytes, memory.heapSizeLimitBytes, memory.oldGenerationAfterGcBytes);
594
+ recordHeapHeadroom(store, "main", mainHeadroom);
595
+ store.clearGauges("openclaw_process_anonymous_memory_bytes");
596
+ for (const [kind, field] of [["rss", "anonymousRssBytes"], ["swap", "anonymousSwapBytes"]]) store.gauge("openclaw_process_anonymous_memory_bytes", "Latest Linux process anonymous RSS and swap; includes managed heaps and buffers, not allocator totals.", { kind }, asNonNegativeFiniteNumber(memory[field]));
179
597
  for (const message of memory.workerMessages ?? []) {
180
598
  const labels = {
181
599
  kind: message.kind,
182
600
  direction: message.direction
183
601
  };
184
602
  store.counterValue("openclaw_worker_messages_total", "Messages crossing the Gateway main-thread worker boundary.", labels, message.messages);
185
- store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.", labels, message.payloadBytes);
603
+ for (const [requestClass, bytes] of Object.entries(message.payloadBytesByRequestClass ?? { [message.kind === "sqlite_request" || message.kind === "sqlite_reply" || message.kind === "authority" ? "other" : ""]: message.payloadBytes })) store.counterValue("openclaw_worker_message_payload_bytes_total", "Known worker payload bytes, excluding transport framing and unmeasured object graphs.", requestClass ? {
604
+ ...labels,
605
+ request_class: requestClass
606
+ } : labels, bytes);
186
607
  store.counterValue("openclaw_worker_message_unmeasured_total", "Worker messages without a known payload size; these do not contribute to payload bytes.", labels, message.unmeasuredMessages);
187
608
  }
609
+ for (const selection of memory.sessionRowSelections?.selections ?? []) {
610
+ const labels = {
611
+ kind: selection.kind,
612
+ task: selection.task
613
+ };
614
+ store.counterValue("openclaw_session_row_selections_total", "Successful resident session-row selections by query shape and enclosing task.", labels, selection.calls);
615
+ store.counterValue("openclaw_session_row_selection_rows_total", "Resident session rows returned by selections, before federation and visibility filtering.", labels, selection.rows);
616
+ }
617
+ if (memory.sessionRowSelections) {
618
+ for (const partition of memory.sessionRowSelections.partitions) for (const outcome of ["hits", "misses"]) store.counterValue("openclaw_session_row_partition_cache_total", "Retained session-list partition lookups by outcome and enclosing selection task.", {
619
+ task: partition.task,
620
+ outcome: outcome === "hits" ? "hit" : "miss"
621
+ }, partition[outcome]);
622
+ for (const revision of memory.sessionRowSelections.partitionRevisions) store.counterValue("openclaw_session_row_partition_revisions_total", "Session-list partition revisions by synchronous internal change scope; excludes initial partitions.", { reason: revision.reason }, revision.changes);
623
+ store.counterValue("openclaw_session_row_selection_resets_total", "Broad session-row selection invalidations, including topology replacement and disposal.", {}, memory.sessionRowSelections.resets);
624
+ }
188
625
  for (const worker of memory.workerLifecycle ?? []) {
189
626
  store.counterValue("openclaw_worker_started_total", "Worker isolates created by bounded script basename.", { script: worker.script }, worker.started);
190
627
  for (const { reason, count } of worker.retired) store.counterValue("openclaw_worker_retired_total", "Worker isolates confirmed exited by owner retirement reason.", {
@@ -192,6 +629,23 @@ function recordMemorySample(store, memory, byteBuckets) {
192
629
  reason
193
630
  }, count);
194
631
  }
632
+ for (const worker of memory.workerTaskMetrics ?? []) {
633
+ const labels = {
634
+ worker_class: worker.workerClass,
635
+ script: worker.script
636
+ };
637
+ store.counterValue("openclaw_worker_tasks_completed_total", "Observed terminal task replies from pool workers, including handled failures.", labels, worker.tasksCompleted);
638
+ store.gauge("openclaw_worker_heap_used_high_water_bytes", "Largest observed pool worker heap usage, retained across worker retirement.", labels, worker.heapUsedHighWaterBytes);
639
+ for (const gc of worker.gc) {
640
+ const gcLabels = {
641
+ ...labels,
642
+ kind: gc.kind
643
+ };
644
+ store.counterValue("openclaw_worker_gc_duration_seconds_total", "Observed pool worker garbage collection duration, retained across worker retirement.", gcLabels, gc.durationMs / 1e3);
645
+ store.counterValue("openclaw_worker_gc_collections_total", "Observed pool worker garbage collections, retained across worker retirement.", gcLabels, gc.count);
646
+ store.counterValue("openclaw_worker_gc_reclaimed_bytes_total", "Sum of positive heap usage decreases across observed pool worker garbage collections.", gcLabels, gc.reclaimedBytes);
647
+ }
648
+ }
195
649
  for (const [kind, field] of [
196
650
  ["rss", "rssBytes"],
197
651
  ["heap_total", "heapTotalBytes"],
@@ -217,41 +671,58 @@ function recordMemorySample(store, memory, byteBuckets) {
217
671
  }, space[field]);
218
672
  const workers = /* @__PURE__ */ new Map();
219
673
  store.clearGauges("openclaw_code_mode_worker_heap_bytes");
674
+ store.clearGauges("openclaw_code_mode_worker_memory_bytes");
220
675
  for (const worker of memory.workerHeaps ?? []) {
221
- if (worker.workerSlot !== void 0) for (const [kind, value] of [["used", worker.heapUsed], ["limit", worker.heapSizeLimitBytes]]) store.gauge("openclaw_code_mode_worker_heap_bytes", "Latest live Code Mode isolate heap usage and limit by reusable worker slot.", {
222
- slot: String(worker.workerSlot),
223
- kind
224
- }, value);
676
+ if (worker.workerSlot !== void 0) {
677
+ for (const [kind, value] of [["used", worker.heapUsed], ["limit", worker.heapSizeLimitBytes]]) store.gauge("openclaw_code_mode_worker_heap_bytes", "Latest live Code Mode isolate heap usage and limit by reusable worker slot.", {
678
+ slot: String(worker.workerSlot),
679
+ kind
680
+ }, value);
681
+ for (const [kind, value] of [["external", worker.external], ["array_buffers", worker.arrayBuffers]]) store.gauge("openclaw_code_mode_worker_memory_bytes", "Latest live Code Mode isolate external and ArrayBuffer memory (included in external) by reusable worker slot.", {
682
+ slot: String(worker.workerSlot),
683
+ kind
684
+ }, value);
685
+ }
225
686
  const totals = workers.get(worker.script) ?? {
226
687
  count: 0,
227
- sampled: 0,
228
- heapUsed: void 0
688
+ sampled: 0
229
689
  };
230
690
  totals.count++;
231
691
  totals.sampled++;
232
- const heapUsed = asNonNegativeFiniteNumber(worker.heapUsed);
233
- if (heapUsed !== void 0) totals.heapUsed = (totals.heapUsed ?? 0) + heapUsed;
692
+ collectHeapHeadroom(totals, worker.heapUsed, worker.heapSizeLimitBytes, worker.oldGenerationAfterGcBytes);
693
+ for (const [field] of WORKER_MEMORY_METRICS) {
694
+ const value = worker[field];
695
+ if (value !== void 0) totals[field] = (totals[field] ?? 0) + value;
696
+ }
697
+ for (const space of worker.heapSpaces ?? []) {
698
+ totals.heapSpaces ??= /* @__PURE__ */ new Map();
699
+ totals.heapSpaces.set(space.space_name, (totals.heapSpaces.get(space.space_name) ?? 0) + space.space_used_size);
700
+ }
234
701
  workers.set(worker.script, totals);
235
702
  }
236
703
  for (const worker of memory.workerMemoryMissing ?? []) {
237
704
  const totals = workers.get(worker.script) ?? {
238
705
  count: 0,
239
- sampled: 0,
240
- heapUsed: void 0
706
+ sampled: 0
241
707
  };
242
708
  totals.count++;
243
709
  workers.set(worker.script, totals);
244
710
  }
245
- store.clearGauges("openclaw_worker_heap_used_bytes");
711
+ for (const [, name] of WORKER_MEMORY_METRICS) store.clearGauges(name);
712
+ store.clearGauges("openclaw_worker_heap_space_used_bytes");
246
713
  for (const [script, totals] of workers) {
714
+ recordHeapHeadroom(store, script, totals);
247
715
  store.gauge("openclaw_worker_count", "Worker isolate counts.", { script }, totals.count);
248
716
  store.gauge("openclaw_worker_heap_sampled_count", "Worker isolate counts.", { script }, totals.sampled);
249
- store.gauge("openclaw_worker_heap_used_bytes", "Latest live Worker heap usage by bounded script basename.", { script }, totals.heapUsed);
717
+ for (const [field, name, description] of WORKER_MEMORY_METRICS) store.gauge(name, `Latest live Worker ${description} by bounded script basename.`, { script }, totals[field]);
718
+ for (const [space, bytes] of totals.heapSpaces ?? []) store.gauge("openclaw_worker_heap_space_used_bytes", "Latest live Worker V8 heap space usage by bounded script basename.", {
719
+ script,
720
+ space
721
+ }, bytes);
250
722
  }
251
723
  store.histogram("openclaw_memory_rss_bytes", "RSS memory sample distribution in bytes.", {}, asNonNegativeFiniteNumber(memory.rssBytes), byteBuckets);
252
724
  }
253
- //#endregion
254
- //#region extensions/diagnostics-prometheus/src/service-model-usage.ts
725
+ // extensions/diagnostics-prometheus/src/service-model-usage.ts
255
726
  const TOKEN_BUCKETS = [
256
727
  1,
257
728
  4,
@@ -295,8 +766,21 @@ function recordModelUsage(store, evt) {
295
766
  store.counter("openclaw_model_cost_usd_total", "Estimated model cost in USD reported by diagnostic usage events.", labels, asNonNegativeFiniteNumber(evt.costUsd) ?? 0);
296
767
  store.histogram("openclaw_model_usage_duration_seconds", "Model usage event duration in seconds.", labels, seconds(evt.durationMs));
297
768
  }
298
- //#endregion
299
- //#region extensions/diagnostics-prometheus/src/service-operation-timing.ts
769
+ // extensions/diagnostics-prometheus/src/service-operation-timing.ts
770
+ const GC_DURATION_NAME = "openclaw_gc_duration_seconds";
771
+ const GC_DURATION_HELP = "Elapsed garbage collection duration in seconds by collection kind, callback flags, and isolate role.";
772
+ function reserveMainGcHistograms(store) {
773
+ for (const kind of [
774
+ "scavenge",
775
+ "mark_sweep_compact",
776
+ "incremental",
777
+ "weak_processing"
778
+ ]) store.reserveHistogram(GC_DURATION_NAME, GC_DURATION_HELP, {
779
+ kind,
780
+ reason: "other",
781
+ thread: "main"
782
+ });
783
+ }
300
784
  const RPC_DURATION_BUCKETS_SECONDS = [
301
785
  .005,
302
786
  .01,
@@ -341,6 +825,14 @@ const RPC_DURATION_BUCKETS_SECONDS = [
341
825
  600
342
826
  ];
343
827
  const RESPONSE_BYTE_BUCKETS = Array.from({ length: 17 }, (_, index) => 1024 * 2 ** index);
828
+ const MAIN_THREAD_STALL_BUCKETS_SECONDS = [
829
+ .25,
830
+ .5,
831
+ 1,
832
+ 2.5,
833
+ 5,
834
+ 10
835
+ ];
344
836
  const HEAP_DELTA_BYTE_BUCKETS = [
345
837
  ...RESPONSE_BYTE_BUCKETS.toReversed().map((bytes) => -bytes),
346
838
  0,
@@ -356,9 +848,41 @@ const CATALOG_LIST_PHASES = /* @__PURE__ */ new Set([
356
848
  "projection_final",
357
849
  "delivery"
358
850
  ]);
851
+ const CHAT_STARTUP_PHASE = /^chat\.startup\.(session_entry|pending_inputs|history_page|startup_projection|projection_final|session_info)$/;
852
+ const DIFF_CACHE_MISS_REASONS = /* @__PURE__ */ new Set([
853
+ "cold",
854
+ "evicted",
855
+ "pending",
856
+ "no-revision",
857
+ "no-revision-environment",
858
+ "no-revision-platform",
859
+ "no-revision-metadata",
860
+ "no-revision-git-installation",
861
+ "no-revision-config",
862
+ "no-revision-reftable",
863
+ "no-revision-alternates",
864
+ "no-revision-symlink",
865
+ "no-revision-nested-repository",
866
+ "no-revision-status",
867
+ "no-revision-unmerged",
868
+ "no-revision-submodule",
869
+ "no-revision-permission",
870
+ "no-revision-missing",
871
+ "no-revision-io",
872
+ "oversized",
873
+ "revision-changed-head",
874
+ "revision-changed-index",
875
+ "revision-changed-untracked-set",
876
+ "revision-changed-worktree-mtime",
877
+ "revision-changed-metadata"
878
+ ]);
359
879
  const CHAT_SEND_PHASE = /^chat\.send\.(authority|admission|preparation|attachments|replyContext|authoring|persist|runAdmission|replyInitialization|snapshot|worktree|effects|response|dispatch)$/;
880
+ const CONTROL_UI_DOCUMENT_STAGE = /^control-ui\.document\.(auth_grants|identity_avatar|plugin_catalog|git_branch|preferences|render|total)$/;
360
881
  function recordOperationTimingEvent(store, evt, metadata) {
361
882
  switch (evt.type) {
883
+ case "sessions.list.baseline.retired":
884
+ if (metadata.trusted) store.counter("openclaw_sessions_list_baseline_retirements_total", "Session list baseline retirement events by cause, not rows or views retired.", { cause: evt.cause });
885
+ return;
362
886
  case "secret.egress":
363
887
  store.counter("openclaw_secret_egress_requests_total", "Secret egress proxy requests by outcome and credential substitution.", {
364
888
  outcome: evt.outcome,
@@ -378,20 +902,87 @@ function recordOperationTimingEvent(store, evt, metadata) {
378
902
  store.counter("openclaw_gateway_http_cancelled_total", "Gateway HTTP requests cancelled before completion.", { source: evt.source });
379
903
  return;
380
904
  case "diagnostic.gc":
381
- store.histogram("openclaw_gc_duration_seconds", "Elapsed garbage collection duration in seconds by collection kind and isolate role.", {
905
+ if (evt.kind === "mark_sweep_compact") {
906
+ const labels = { thread: evt.thread ?? "main" };
907
+ store.gauge("openclaw_heap_used_after_major_gc_bytes", "Native used heap after the latest received major collection by isolate role.", labels, evt.heapUsedAfterMajorGcBytes);
908
+ store.gauge("openclaw_heap_used_after_major_gc_timestamp_seconds", "Unix observation time of the latest received native major-GC heap sample.", labels, evt.heapUsedAfterMajorGcTimestampSeconds);
909
+ }
910
+ store.histogram(GC_DURATION_NAME, GC_DURATION_HELP, {
382
911
  kind: evt.kind ?? "unknown",
912
+ reason: evt.reason ?? "other",
383
913
  thread: evt.thread ?? "main",
384
914
  ...evt.workerSlot === void 0 ? {} : { slot: String(evt.workerSlot) }
385
915
  }, seconds(evt.durationMs));
386
916
  return;
917
+ case "gateway.main_thread.stall":
918
+ for (const stall of evt.stalls) store.histogram("openclaw_gateway_main_thread_stall_seconds", "Elapsed main-thread tasks lasting at least 250 milliseconds, by work scope.", { task: normalizeDiagnosticValue(stall.task) }, seconds(stall.durationMs), MAIN_THREAD_STALL_BUCKETS_SECONDS);
919
+ store.counter("openclaw_gateway_main_thread_stalls_dropped_total", "Main-thread stall records dropped because the attribution buffer was full.", {}, evt.dropped);
920
+ return;
387
921
  case "gateway.event_loop.sample":
388
922
  store.histogram("openclaw_gateway_event_loop_delay_max_seconds", "Maximum event-loop delay per completed Gateway observation window in seconds.", {}, seconds(evt.delayMaxMs));
389
923
  store.counter("openclaw_gateway_event_loop_observed_seconds_total", "Elapsed seconds covered by completed Gateway event-loop observation windows.", {}, evt.intervalMs / 1e3);
390
924
  return;
391
925
  case "diagnostic.phase.completed": {
392
926
  if (!metadata.trusted) return;
927
+ const documentStage = CONTROL_UI_DOCUMENT_STAGE.exec(evt.name)?.[1];
928
+ if (documentStage) {
929
+ store.histogram("openclaw_control_ui_document_duration_seconds", "Elapsed Control UI HTML document handler stages in seconds; stages can overlap.", { stage: documentStage }, seconds(evt.durationMs));
930
+ return;
931
+ }
932
+ if (evt.name === "sessions.diff" || evt.name === "checkout.diff") {
933
+ const outcome = evt.details?.outcome;
934
+ if (outcome !== "cache-hit" && outcome !== "cache-miss") return;
935
+ const reason = evt.details?.missReason;
936
+ if (outcome === "cache-miss" && typeof reason === "string" && DIFF_CACHE_MISS_REASONS.has(reason)) store.counter("openclaw_diff_cache_misses_total", "Local diff cache misses by bounded reason; pending reads join an existing load.", {
937
+ method: evt.name,
938
+ reason
939
+ });
940
+ for (const stage of [
941
+ "admission",
942
+ "checkout-resolve",
943
+ "cache-validate",
944
+ "worker-queue",
945
+ "git-wall",
946
+ "encode"
947
+ ]) {
948
+ const elapsed = evt.details?.[stage];
949
+ store.histogram("openclaw_diff_stage_seconds", "Elapsed local diff stages; cache validation includes its worker queue and Git wall time.", {
950
+ method: evt.name,
951
+ stage,
952
+ outcome
953
+ }, typeof elapsed === "number" ? seconds(elapsed) : void 0, RPC_DURATION_BUCKETS_SECONDS);
954
+ }
955
+ return;
956
+ }
393
957
  if (evt.name === "worktree.preparation") {
394
- recordWorktreePreparation(store, evt);
958
+ const { kind, template, outcome } = evt.details ?? {};
959
+ if (kind !== "managed" && kind !== "sandbox" || ![
960
+ "warm",
961
+ "cold",
962
+ "unavailable",
963
+ "reused"
964
+ ].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
965
+ for (const preparationPhase of [
966
+ "total",
967
+ "allocate",
968
+ "checkout",
969
+ "setup",
970
+ "templatePrepare",
971
+ "templateApply",
972
+ "snapshot",
973
+ "synchronizeCanonical",
974
+ "synchronizeProjection",
975
+ "containerStart",
976
+ "workspaceLayout"
977
+ ]) {
978
+ const elapsed = preparationPhase === "total" ? evt.durationMs : evt.details?.[preparationPhase];
979
+ store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
980
+ kind,
981
+ template: String(template),
982
+ outcome,
983
+ phase: preparationPhase
984
+ }, typeof elapsed === "number" ? seconds(elapsed) : void 0);
985
+ }
395
986
  return;
396
987
  }
397
988
  const sendPhase = CHAT_SEND_PHASE.exec(evt.name)?.[1];
@@ -403,6 +994,14 @@ function recordOperationTimingEvent(store, evt, metadata) {
403
994
  }, seconds(evt.durationMs));
404
995
  return;
405
996
  }
997
+ const startupPhase = CHAT_STARTUP_PHASE.exec(evt.name)?.[1];
998
+ if (startupPhase) {
999
+ store.histogram("openclaw_gateway_rpc_stage_seconds", "Elapsed time for completed Gateway RPC owner stages.", {
1000
+ method: "chat.startup",
1001
+ phase: startupPhase
1002
+ }, seconds(evt.durationMs));
1003
+ return;
1004
+ }
406
1005
  if (!evt.name.startsWith(CATALOG_LIST_PHASE_PREFIX)) return;
407
1006
  const phase = evt.name.slice(CATALOG_LIST_PHASE_PREFIX.length);
408
1007
  const duration = seconds(evt.durationMs);
@@ -420,6 +1019,10 @@ function recordOperationTimingEvent(store, evt, metadata) {
420
1019
  const labels = { method: evt.method };
421
1020
  if (evt.phase === "received") {
422
1021
  store.counter("openclaw_gateway_rpc_requests_total", "Authenticated Gateway WebSocket requests received.", labels);
1022
+ store.counter("openclaw_gateway_rpc_client_requests_total", "Gateway RPC requests by bounded handshake client kind.", {
1023
+ method: evt.method,
1024
+ client_kind: evt.clientKind ?? "other"
1025
+ });
423
1026
  if (evt.method === "models.list" && evt.modelsList) {
424
1027
  const catalog = evt.modelsList;
425
1028
  store.counter("openclaw_models_list_requests_total", "Model catalog requests by client kind and bounded request shape.", {
@@ -431,10 +1034,54 @@ function recordOperationTimingEvent(store, evt, metadata) {
431
1034
  prepared_only: String(catalog.preparedOnly)
432
1035
  });
433
1036
  }
1037
+ if (evt.method === "sessions.list" && evt.sessionsList) {
1038
+ const sessions = evt.sessionsList;
1039
+ store.counter("openclaw_sessions_list_requests_total", "Session list requests by client kind and bounded request shape.", {
1040
+ client_kind: sessions.clientKind,
1041
+ source: sessions.source,
1042
+ caller_source: sessions.callerSource,
1043
+ row_mode: sessions.rowMode,
1044
+ filter: sessions.filter
1045
+ });
1046
+ }
1047
+ if (evt.method === "cron.list" && evt.cronList) {
1048
+ const cron = evt.cronList;
1049
+ store.counter("openclaw_cron_list_requests_total", "Cron list requests by client kind and bounded request shape.", {
1050
+ client_kind: cron.clientKind,
1051
+ source: cron.source,
1052
+ scope: cron.scope,
1053
+ compact: String(cron.compact),
1054
+ page: cron.page
1055
+ });
1056
+ }
434
1057
  return;
435
1058
  }
436
1059
  if (evt.phase === "response" && (evt.outcome === "ok" || evt.outcome === "error")) {
1060
+ if (evt.outcome === "ok" && evt.firstResponse !== false && evt.sessionsList?.baseMissReason) store.counter("openclaw_sessions_list_base_misses_total", "Successful full sidebar responses with a supplied base, by miss reason.", { reason: evt.sessionsList.baseMissReason });
437
1061
  store.histogram("openclaw_gateway_rpc_response_bytes", "Encoded Gateway RPC response frame size in bytes, including later frames.", labels, evt.responseBytes, RESPONSE_BYTE_BUCKETS);
1062
+ if (evt.responseBytes !== void 0 && evt.sessionsList) {
1063
+ const sessions = evt.sessionsList;
1064
+ store.counter("openclaw_sessions_list_response_bytes_total", "Encoded successful session list response bytes by request shape and row outcome.", {
1065
+ client_kind: sessions.clientKind,
1066
+ source: sessions.source,
1067
+ caller_source: sessions.callerSource,
1068
+ row_mode: sessions.rowMode,
1069
+ filter: sessions.filter,
1070
+ rows_outcome: sessions.rowsOutcome
1071
+ }, evt.responseBytes);
1072
+ }
1073
+ if (evt.chatHistory) {
1074
+ const history = evt.chatHistory;
1075
+ const historyLabels = {
1076
+ client_kind: history.clientKind,
1077
+ source: history.source,
1078
+ shape: history.shape,
1079
+ limit_bucket: history.limitBucket,
1080
+ outcome: history.outcome
1081
+ };
1082
+ if (evt.firstResponse !== false) store.counter("openclaw_chat_history_requests_total", "Successful chat history requests by request shape and response outcome.", historyLabels);
1083
+ if (evt.responseBytes !== void 0) store.counter("openclaw_chat_history_response_bytes_total", "Encoded successful chat history response bytes by request shape and response outcome.", historyLabels, evt.responseBytes);
1084
+ }
438
1085
  if (evt.firstResponse === false) return;
439
1086
  }
440
1087
  store.counter("openclaw_gateway_rpc_outcomes_total", "Gateway RPC observations by phase and outcome.", {
@@ -451,53 +1098,22 @@ function recordOperationTimingEvent(store, evt, metadata) {
451
1098
  }
452
1099
  }
453
1100
  }
454
- function recordWorktreePreparation(store, evt) {
455
- const { kind, template, outcome } = evt.details ?? {};
456
- if (kind !== "managed" && kind !== "sandbox" || ![
457
- "warm",
458
- "cold",
459
- "unavailable",
460
- "reused"
461
- ].includes(String(template)) || outcome !== "returned" && outcome !== "threw") return;
462
- for (const phase of [
463
- "total",
464
- "allocate",
465
- "checkout",
466
- "setup",
467
- "templatePrepare",
468
- "templateApply",
469
- "snapshot",
470
- "synchronizeCanonical",
471
- "synchronizeProjection",
472
- "containerStart",
473
- "workspaceLayout"
474
- ]) {
475
- const elapsed = phase === "total" ? evt.durationMs : evt.details?.[phase];
476
- store.histogram("openclaw_worktree_preparation_seconds", "Elapsed managed worktree preparation time; nested phases are inclusive.", {
477
- kind,
478
- template: String(template),
479
- outcome,
480
- phase
481
- }, typeof elapsed === "number" ? seconds(elapsed) : void 0);
1101
+ // extensions/diagnostics-prometheus/src/service-retention.ts
1102
+ function recordSessionRetention(store, evt) {
1103
+ if (evt.type === "sessions.event.rows.reset") {
1104
+ store.counter("openclaw_session_event_rows_reset_total", "Retained session-event wire baselines retired by cause.", { cause: evt.cause }, evt.count);
1105
+ return;
482
1106
  }
1107
+ if (evt.type === "sessions.list.retention") {
1108
+ store.gauge("openclaw_sessions_list_retained_views", "Retained session-list views across all publications.", {}, evt.views);
1109
+ store.gauge("openclaw_sessions_list_retained_refs", "Row references and selection entries retained by session-list views.", {}, evt.refs);
1110
+ store.gauge("openclaw_sessions_list_retained_bytes", "Estimated memory retained by session-list views.", {}, evt.estimatedBytes);
1111
+ return;
1112
+ }
1113
+ store.gauge("openclaw_session_event_rows_retained_entries", "Session-event field-hash vectors retained across all connections.", {}, evt.entries);
1114
+ store.gauge("openclaw_session_event_rows_retained_bytes", "Estimated memory retained by session-event field-hash vectors.", {}, evt.estimatedBytes);
483
1115
  }
484
- //#endregion
485
- //#region extensions/diagnostics-prometheus/src/service-sessions.ts
486
- function createGatewayWorkMetricsRecorder(store) {
487
- return (snapshot) => {
488
- if (!snapshot) {
489
- store.clearGauges("openclaw_sessions_active");
490
- store.clearGauges("openclaw_gateway_active_work");
491
- return;
492
- }
493
- for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
494
- for (const kind of [
495
- "agentRuns",
496
- "chatRuns",
497
- "queuedTurns"
498
- ]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
499
- };
500
- }
1116
+ // extensions/diagnostics-prometheus/src/service-sessions.ts
501
1117
  function recordSessionDiagnosticEvent(store, evt) {
502
1118
  switch (evt.type) {
503
1119
  case "session.recovery.requested":
@@ -535,65 +1151,65 @@ function recordSessionDiagnosticEvent(store, evt) {
535
1151
  });
536
1152
  }
537
1153
  }
538
- //#endregion
539
- //#region extensions/diagnostics-prometheus/src/service-worker.ts
540
- function recordWorkerRequest(store, event) {
541
- store.gauge("openclaw_worker_queue_depth", "Requests awaiting dispatch across worker owners of this kind.", { kind: event.kind }, event.queueDepth);
542
- const labels = {
543
- kind: event.kind,
544
- request_class: event.requestClass
1154
+ // extensions/diagnostics-prometheus/src/service-worker.ts
1155
+ function createWorkerRequestMetricsRecorder(store) {
1156
+ return (kind, requestClass, requestPhase, queueDepth, queueWaitMs, durationMs, timing) => {
1157
+ store.gauge("openclaw_worker_queue_depth", "Requests awaiting dispatch across worker owners of this kind.", { kind }, queueDepth);
1158
+ const labels = {
1159
+ kind,
1160
+ request_class: requestClass
1161
+ };
1162
+ store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(queueWaitMs));
1163
+ store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(durationMs));
1164
+ if (requestPhase !== "completed" || !timing) return;
1165
+ for (const [phase, start, end] of [
1166
+ [
1167
+ "dispatch",
1168
+ "dispatched",
1169
+ "posted"
1170
+ ],
1171
+ [
1172
+ "delivery",
1173
+ "posted",
1174
+ "received"
1175
+ ],
1176
+ [
1177
+ "input",
1178
+ "received",
1179
+ "inputReady"
1180
+ ],
1181
+ [
1182
+ "prepare",
1183
+ "inputReady",
1184
+ "executeStarted"
1185
+ ],
1186
+ [
1187
+ "execute",
1188
+ "executeStarted",
1189
+ "executeFinished"
1190
+ ],
1191
+ [
1192
+ "result",
1193
+ "executeFinished",
1194
+ "replied"
1195
+ ],
1196
+ [
1197
+ "reply",
1198
+ "replied",
1199
+ "replyReceived"
1200
+ ]
1201
+ ]) {
1202
+ const started = timing[start];
1203
+ const finished = timing[end];
1204
+ if (started === void 0 || finished === void 0) continue;
1205
+ store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
1206
+ ...labels,
1207
+ phase
1208
+ }, seconds(finished - started));
1209
+ }
545
1210
  };
546
- store.histogram("openclaw_worker_queue_wait_seconds", "Elapsed time from worker request enqueue to dispatch.", labels, seconds(event.queueWaitMs));
547
- store.histogram("openclaw_worker_request_seconds", "Elapsed time from worker dispatch to reply or failure, including preparation and transport.", labels, seconds(event.durationMs));
548
- if (event.phase !== "completed" || !event.timing) return;
549
- for (const [phase, start, end] of [
550
- [
551
- "dispatch",
552
- "dispatched",
553
- "posted"
554
- ],
555
- [
556
- "delivery",
557
- "posted",
558
- "received"
559
- ],
560
- [
561
- "input",
562
- "received",
563
- "inputReady"
564
- ],
565
- [
566
- "prepare",
567
- "inputReady",
568
- "executeStarted"
569
- ],
570
- [
571
- "execute",
572
- "executeStarted",
573
- "executeFinished"
574
- ],
575
- [
576
- "result",
577
- "executeFinished",
578
- "replied"
579
- ],
580
- [
581
- "reply",
582
- "replied",
583
- "replyReceived"
584
- ]
585
- ]) {
586
- const started = event.timing[start];
587
- const finished = event.timing[end];
588
- if (started === void 0 || finished === void 0) continue;
589
- store.histogram("openclaw_worker_request_phase_seconds", "Elapsed time within an observed SQLite worker request phase, including transport.", {
590
- ...labels,
591
- phase
592
- }, seconds(finished - started));
593
- }
594
1211
  }
595
- //#endregion
596
- //#region extensions/diagnostics-prometheus/src/service.ts
1212
+ // extensions/diagnostics-prometheus/src/service.ts
597
1213
  const BYTE_BUCKETS = [
598
1214
  1024,
599
1215
  4096,
@@ -626,36 +1242,6 @@ function safeErrorMessage(err) {
626
1242
  const message = err instanceof Error ? err.message ?? err.name : String(err);
627
1243
  return truncateUtf16Safe(redactSensitiveText(message).replaceAll("\0", " ").replace(/[\r\n\t\u2028\u2029]/gu, " "), 500);
628
1244
  }
629
- function renderPrometheusMetrics(store) {
630
- const snapshot = store.snapshot();
631
- const lines = [];
632
- const emitted = /* @__PURE__ */ new Set();
633
- const emitHeader = (name, type, help) => {
634
- if (emitted.has(name)) return;
635
- emitted.add(name);
636
- lines.push(`# HELP ${name} ${escapeHelp(help)}`);
637
- lines.push(`# TYPE ${name} ${type}`);
638
- };
639
- for (const [type, entries] of [["counter", snapshot.counters], ["gauge", snapshot.gauges]]) {
640
- entries.sort(([left], [right]) => left.localeCompare(right));
641
- for (const [key, sample] of entries) {
642
- const name = key.split("|", 1)[0] ?? "";
643
- emitHeader(name, type, sample.help);
644
- lines.push(`${name}${formatLabels(sample.labels)} ${formatPrometheusNumber(sample.value)}`);
645
- }
646
- }
647
- snapshot.histograms.sort(([left], [right]) => left.localeCompare(right));
648
- for (const [key, sample] of snapshot.histograms) {
649
- const name = key.split("|", 1)[0] ?? "";
650
- emitHeader(name, "histogram", sample.help);
651
- for (let index = 0; index < sample.buckets.length; index += 1) lines.push(`${sample.bucketPrefixes[index]}${formatPrometheusNumber(sample.counts[index] ?? 0)}`);
652
- lines.push(`${sample.bucketPrefixes[sample.buckets.length]}${formatPrometheusNumber(sample.count)}`);
653
- lines.push(`${name}_sum${sample.labels} ${formatPrometheusNumber(sample.sum)}`);
654
- lines.push(`${name}_count${sample.labels} ${formatPrometheusNumber(sample.count)}`);
655
- }
656
- lines.push("");
657
- return lines.join("\n");
658
- }
659
1245
  function webhookLabels(evt) {
660
1246
  return {
661
1247
  channel: normalizeDiagnosticValue(evt.channel),
@@ -665,16 +1251,23 @@ function webhookLabels(evt) {
665
1251
  function recordDiagnosticEvent(store, evt, metadata) {
666
1252
  if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
667
1253
  switch (evt.type) {
668
- case "worker.request":
669
- if (metadata.trusted) recordWorkerRequest(store, evt);
1254
+ case "gateway.ws.connections":
1255
+ if (metadata.trusted) recordGatewayWsConnections(store, evt.operators);
1256
+ return;
1257
+ case "sessions.list.retention":
1258
+ case "sessions.event.retention":
1259
+ case "sessions.event.rows.reset":
1260
+ if (metadata.trusted) recordSessionRetention(store, evt);
670
1261
  return;
671
1262
  case "diagnostic.phase.completed":
672
1263
  case "secret.egress":
673
1264
  case "tool.catalog":
674
1265
  case "gateway.rpc":
1266
+ case "sessions.list.baseline.retired":
675
1267
  case "gateway.http.cancelled":
676
1268
  case "diagnostic.gc":
677
1269
  case "gateway.event_loop.sample":
1270
+ case "gateway.main_thread.stall":
678
1271
  recordOperationTimingEvent(store, evt, metadata);
679
1272
  return;
680
1273
  case "model.usage":
@@ -850,17 +1443,22 @@ function recordDiagnosticEvent(store, evt, metadata) {
850
1443
  recordSessionDiagnosticEvent(store, evt);
851
1444
  return;
852
1445
  case "queue.lane.enqueue":
853
- case "queue.lane.dequeue":
854
- store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", { lane: normalizeDiagnosticLane(evt.lane) }, asNonNegativeFiniteNumber(evt.queueSize));
855
- if (evt.type === "queue.lane.dequeue") store.histogram("openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", { lane: normalizeDiagnosticLane(evt.lane) }, seconds(evt.waitMs));
1446
+ case "queue.lane.dequeue": {
1447
+ const labels = { lane: normalizeDiagnosticLane(evt.lane) };
1448
+ store.gauge("openclaw_queue_lane_size", "Current diagnostic queue lane size.", labels, asNonNegativeFiniteNumber(evt.queueSize));
1449
+ if (evt.type === "queue.lane.dequeue") store.histogram("openclaw_queue_lane_wait_seconds", "Queue lane wait time in seconds.", labels, seconds(evt.waitMs));
856
1450
  return;
1451
+ }
857
1452
  case "session.state":
858
1453
  case "session.stuck":
859
1454
  case "session.turn.created":
860
1455
  recordSessionDiagnosticEvent(store, evt);
861
1456
  return;
862
1457
  case "diagnostic.child_process.spawn":
863
- recordChildProcessSpawn(store, evt);
1458
+ store.counter("openclaw_child_process_spawn_total", "Successful child launches through the shared spawn and exec owners.", {
1459
+ family: normalizeDiagnosticValue(evt.family),
1460
+ operation: normalizeDiagnosticValue(evt.operation ?? (evt.family === "git" ? "unknown" : "none"))
1461
+ }, asNonNegativeFiniteNumber(evt.count) ?? 0);
864
1462
  return;
865
1463
  case "diagnostic.memory.sample":
866
1464
  recordMemorySample(store, evt.memory, BYTE_BUCKETS);
@@ -945,23 +1543,24 @@ function createMetricsHandler(store) {
945
1543
  res.end("Method Not Allowed");
946
1544
  return true;
947
1545
  }
948
- const body = renderPrometheusMetrics(store);
1546
+ const body = store.render();
949
1547
  res.statusCode = 200;
950
1548
  res.setHeader("Cache-Control", "no-store");
951
1549
  res.setHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
952
- res.setHeader("Content-Length", String(Buffer.byteLength(body)));
953
- if (req.method === "HEAD") {
954
- res.end();
955
- return true;
956
- }
957
- res.end(body);
1550
+ res.setHeader("Content-Length", String(body.length));
1551
+ res.flushHeaders();
1552
+ res.end(req.method === "HEAD" ? void 0 : body, (error) => {
1553
+ if (!error) body.buffer.transfer(0);
1554
+ });
958
1555
  return true;
959
1556
  };
960
1557
  }
961
1558
  function createDiagnosticsPrometheusExporter() {
962
- const store = createPrometheusMetricStore();
963
- let unsubscribe;
964
- let unsubscribeWork;
1559
+ let logger;
1560
+ const store = createPrometheusMetricStore((family) => {
1561
+ logger?.warn(`diagnostics-prometheus: series budget reached for ${family}; new label sets are being dropped. See openclaw_prometheus_family_series_dropped_total.`);
1562
+ });
1563
+ let subscriptions = [];
965
1564
  let internalDiagnostics;
966
1565
  const reportExporterStatus = (update) => {
967
1566
  try {
@@ -976,7 +1575,6 @@ function createDiagnosticsPrometheusExporter() {
976
1575
  };
977
1576
  return {
978
1577
  handler: createMetricsHandler(store),
979
- render: () => renderPrometheusMetrics(store),
980
1578
  service: {
981
1579
  id: "diagnostics-prometheus",
982
1580
  start(ctx) {
@@ -985,19 +1583,44 @@ function createDiagnosticsPrometheusExporter() {
985
1583
  ctx.logger.error("diagnostics-prometheus: internal diagnostics capability unavailable");
986
1584
  return;
987
1585
  }
1586
+ logger = ctx.logger;
1587
+ reserveMainGcHistograms(store);
988
1588
  const identity = isDiagnosticsEnabled(ctx.config) ? ctx.internalDiagnostics?.getRuntimeIdentity?.() : void 0;
989
1589
  if (identity) store.gauge("openclaw_gateway_build_info", "Identity of the hosting process and its loaded build; not a health or exporter epoch.", {
990
1590
  process_instance_id: identity.processInstanceId,
991
1591
  ...identity.buildId ? { build_id: identity.buildId } : {}
992
1592
  }, 1);
993
- unsubscribeWork = ctx.internalDiagnostics?.onGatewayWorkMetrics?.(createGatewayWorkMetricsRecorder(store));
994
- unsubscribe = subscribe((event, metadata) => {
995
- try {
996
- recordDiagnosticEvent(store, event, metadata);
997
- } catch (err) {
998
- ctx.logger.error(`diagnostics-prometheus: event handler failed (${event.type}): ${safeErrorMessage(err)}`);
999
- }
1000
- }, { exclude: ["log.record"] }, { includePrivateData: false });
1593
+ subscriptions = [
1594
+ ctx.internalDiagnostics?.onGatewayWorkMetrics?.((snapshot) => {
1595
+ if (!snapshot) {
1596
+ store.clearGauges("openclaw_sessions_active");
1597
+ store.clearGauges("openclaw_gateway_active_work");
1598
+ return;
1599
+ }
1600
+ for (const state of ["running", "queued"]) store.gauge("openclaw_sessions_active", "Current active sessions in the Gateway session roster by run state.", { state }, snapshot.sessions[state]);
1601
+ for (const kind of [
1602
+ "agentRuns",
1603
+ "chatRuns",
1604
+ "queuedTurns"
1605
+ ]) store.gauge("openclaw_gateway_active_work", "Current Gateway work by owner; categories may overlap.", { kind }, snapshot.work[kind]);
1606
+ }),
1607
+ ctx.internalDiagnostics?.onGatewayWsMetrics?.(createGatewayWsMetricsRecorder(store)),
1608
+ ctx.internalDiagnostics?.onWorkerRequestMetrics?.(createWorkerRequestMetricsRecorder(store)),
1609
+ subscribe((event, metadata) => {
1610
+ try {
1611
+ recordDiagnosticEvent(store, event, metadata);
1612
+ } catch (err) {
1613
+ ctx.logger.error(`diagnostics-prometheus: event handler failed (${event.type}): ${safeErrorMessage(err)}`);
1614
+ }
1615
+ }, {
1616
+ exclude: [
1617
+ "log.record",
1618
+ "worker.request",
1619
+ "gateway.ws.event"
1620
+ ],
1621
+ includePromptStats: false
1622
+ }, { includePrivateData: false })
1623
+ ];
1001
1624
  internalDiagnostics = ctx.internalDiagnostics;
1002
1625
  reportExporterStatus({
1003
1626
  signal: "metrics",
@@ -1007,10 +1630,8 @@ function createDiagnosticsPrometheusExporter() {
1007
1630
  });
1008
1631
  },
1009
1632
  stop() {
1010
- unsubscribeWork?.();
1011
- unsubscribeWork = void 0;
1012
- unsubscribe?.();
1013
- unsubscribe = void 0;
1633
+ for (const unsubscribe of subscriptions) unsubscribe?.();
1634
+ subscriptions = [];
1014
1635
  reportExporterStatus({
1015
1636
  signal: "metrics",
1016
1637
  transport: "prometheus-scrape",
@@ -1018,12 +1639,12 @@ function createDiagnosticsPrometheusExporter() {
1018
1639
  });
1019
1640
  internalDiagnostics = void 0;
1020
1641
  store.reset();
1642
+ logger = void 0;
1021
1643
  }
1022
1644
  }
1023
1645
  };
1024
1646
  }
1025
- //#endregion
1026
- //#region extensions/diagnostics-prometheus/index.ts
1647
+ // extensions/diagnostics-prometheus/index.ts
1027
1648
  const exporter = createDiagnosticsPrometheusExporter();
1028
1649
  var diagnostics_prometheus_default = definePluginEntry({
1029
1650
  id: "diagnostics-prometheus",
@@ -1040,5 +1661,4 @@ var diagnostics_prometheus_default = definePluginEntry({
1040
1661
  });
1041
1662
  }
1042
1663
  });
1043
- //#endregion
1044
1664
  export { diagnostics_prometheus_default as default };