@openclaw/diagnostics-otel 2026.7.1-beta.6 → 2026.7.2-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2773 @@
1
+ import { isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, redactSensitiveText } from "./api.js";
2
+ import { SpanKind, SpanStatusCode, TraceFlags, context, metrics, trace } from "@opentelemetry/api";
3
+ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
4
+ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
5
+ import { resourceFromAttributes } from "@opentelemetry/resources";
6
+ import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
7
+ import { NodeSDK } from "@opentelemetry/sdk-node";
8
+ import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
9
+ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
10
+ import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
11
+ import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
12
+ import { waitForDiagnosticEventsDrained } from "openclaw/plugin-sdk/diagnostic-runtime";
13
+ import { readFileSync } from "node:fs";
14
+ import nodePath from "node:path";
15
+ import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime";
16
+ import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
17
+ import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
18
+ import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_TOOL_DEFINITIONS, GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL } from "@opentelemetry/semantic-conventions/incubating";
19
+ import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
20
+ const DROPPED_OTEL_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
21
+ "openclaw.callId",
22
+ "openclaw.call_id",
23
+ "openclaw.chatId",
24
+ "openclaw.chat_id",
25
+ "openclaw.messageId",
26
+ "openclaw.message_id",
27
+ "openclaw.parentSpanId",
28
+ "openclaw.parent_span_id",
29
+ "openclaw.runId",
30
+ "openclaw.run_id",
31
+ "openclaw.sessionId",
32
+ "openclaw.session_id",
33
+ "openclaw.sessionKey",
34
+ "openclaw.session_key",
35
+ "openclaw.spanId",
36
+ "openclaw.span_id",
37
+ "openclaw.toolCallId",
38
+ "openclaw.tool_call_id",
39
+ "openclaw.traceId",
40
+ "openclaw.trace_id"
41
+ ]);
42
+ const LOW_CARDINALITY_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u;
43
+ const SECURITY_TARGET_NAME_VALUE_RE = /^[A-Za-z0-9@/_.:-]{1,256}$/u;
44
+ const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024;
45
+ const MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS = 4 * 1024;
46
+ const OTEL_LOG_RAW_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,64}$/u;
47
+ const OTEL_LOG_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,96}$/u;
48
+ const BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
49
+ "__proto__",
50
+ "prototype",
51
+ "constructor"
52
+ ]);
53
+ const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
54
+ const OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT";
55
+ const OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT";
56
+ const OTEL_EXPORTER_OTLP_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CERTIFICATE";
57
+ const OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE";
58
+ const OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV = "OTEL_EXPORTER_OTLP_CLIENT_KEY";
59
+ const OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN";
60
+ const GEN_AI_LATEST_EXPERIMENTAL_OPT_IN = "gen_ai_latest_experimental";
61
+ const GEN_AI_TOKEN_USAGE_BUCKETS = [
62
+ 1,
63
+ 4,
64
+ 16,
65
+ 64,
66
+ 256,
67
+ 1024,
68
+ 4096,
69
+ 16384,
70
+ 65536,
71
+ 262144,
72
+ 1048576,
73
+ 4194304,
74
+ 16777216,
75
+ 67108864
76
+ ];
77
+ const GEN_AI_OPERATION_DURATION_BUCKETS = [
78
+ .01,
79
+ .02,
80
+ .04,
81
+ .08,
82
+ .16,
83
+ .32,
84
+ .64,
85
+ 1.28,
86
+ 2.56,
87
+ 5.12,
88
+ 10.24,
89
+ 20.48,
90
+ 40.96,
91
+ 81.92
92
+ ];
93
+ const OTEL_DEFAULT_HISTOGRAM_BUCKETS = [
94
+ 0,
95
+ 5,
96
+ 10,
97
+ 25,
98
+ 50,
99
+ 75,
100
+ 100,
101
+ 250,
102
+ 500,
103
+ 750,
104
+ 1e3,
105
+ 2500,
106
+ 5e3,
107
+ 7500,
108
+ 1e4
109
+ ];
110
+ const AGENT_DURATION_MS_BUCKETS = [
111
+ ...OTEL_DEFAULT_HISTOGRAM_BUCKETS,
112
+ 15e3,
113
+ 2e4,
114
+ 3e4,
115
+ 45e3,
116
+ 6e4,
117
+ 12e4,
118
+ 18e4,
119
+ 24e4,
120
+ 3e5,
121
+ 6e5,
122
+ 9e5,
123
+ 18e5,
124
+ 36e5
125
+ ];
126
+ const CONTEXT_TOKENS_BUCKETS = [
127
+ ...OTEL_DEFAULT_HISTOGRAM_BUCKETS,
128
+ 16e3,
129
+ 32e3,
130
+ 64e3,
131
+ 128e3,
132
+ 2e5,
133
+ 4e5,
134
+ 1e6,
135
+ 2e6
136
+ ];
137
+ const MAX_RETAINED_TRUSTED_SPAN_CONTEXTS = 1024;
138
+ const RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS = 5e3;
139
+ //#endregion
140
+ //#region extensions/diagnostics-otel/src/service-content-normalization.ts
141
+ const MAX_OTEL_CONTENT_ATTRIBUTE_CHARS = 128 * 1024;
142
+ const MAX_OTEL_ERROR_MESSAGE_CHARS = 4 * 1024;
143
+ const PRELOADED_OTEL_SDK_ENV = "OPENCLAW_OTEL_PRELOADED";
144
+ const NO_CONTENT_CAPTURE = {
145
+ inputMessages: false,
146
+ outputMessages: false,
147
+ toolInputs: false,
148
+ toolOutputs: false,
149
+ systemPrompt: false,
150
+ toolDefinitions: false,
151
+ logBodies: false
152
+ };
153
+ function clampOtelLogText(value, maxChars) {
154
+ return value.length > maxChars ? `${truncateUtf16Safe(value, maxChars)}...(truncated)` : value;
155
+ }
156
+ function normalizeOtelLogString(value, maxChars) {
157
+ return clampOtelLogText(redactSensitiveText(value), maxChars);
158
+ }
159
+ function normalizeOtelErrorMessage(value) {
160
+ if (!value) return;
161
+ return normalizeOtelLogString(value.trim(), MAX_OTEL_ERROR_MESSAGE_CHARS) || void 0;
162
+ }
163
+ function resolveContentCapturePolicy(value) {
164
+ if (value === true) return {
165
+ inputMessages: true,
166
+ outputMessages: true,
167
+ toolInputs: true,
168
+ toolOutputs: true,
169
+ systemPrompt: false,
170
+ toolDefinitions: true,
171
+ logBodies: true
172
+ };
173
+ if (!value || typeof value !== "object" || Array.isArray(value)) return NO_CONTENT_CAPTURE;
174
+ const config = value;
175
+ if (config.enabled !== true) return NO_CONTENT_CAPTURE;
176
+ return {
177
+ inputMessages: config.inputMessages === true,
178
+ outputMessages: config.outputMessages === true,
179
+ toolInputs: config.toolInputs === true,
180
+ toolOutputs: config.toolOutputs === true,
181
+ systemPrompt: config.systemPrompt === true,
182
+ toolDefinitions: config.toolDefinitions === true,
183
+ logBodies: false
184
+ };
185
+ }
186
+ function hasPreloadedOtelSdk() {
187
+ return process.env[PRELOADED_OTEL_SDK_ENV] === "1";
188
+ }
189
+ function normalizeOtelContentValue(value) {
190
+ if (typeof value === "string") return normalizeOtelLogString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
191
+ if (Array.isArray(value)) {
192
+ const items = [];
193
+ for (const item of value.slice(0, 200)) if (typeof item === "string") items.push(item);
194
+ if (items.length > 0) return normalizeOtelLogString(items.join("\n"), MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
195
+ }
196
+ const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
197
+ if (json) return json;
198
+ }
199
+ const TRUNCATED_JSON_TEXT_SUFFIX = "...(truncated)";
200
+ const JSON_TRUNCATION_STRING_BUDGETS = [
201
+ 8192,
202
+ 4096,
203
+ 2048,
204
+ 1024,
205
+ 512,
206
+ 256,
207
+ 128,
208
+ 64,
209
+ 32
210
+ ];
211
+ const JSON_TRUNCATION_ARRAY_ITEM_BUDGETS = [
212
+ 200,
213
+ 100,
214
+ 50,
215
+ 25,
216
+ 10,
217
+ 5,
218
+ 1
219
+ ];
220
+ const JSON_TRUNCATION_MAX_OBJECT_FIELDS = 64;
221
+ const JSON_TRUNCATION_MAX_DEPTH = 8;
222
+ function safeJsonString(value, maxChars) {
223
+ if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
224
+ const exact = stringifyJsonForOtelAttribute(value);
225
+ if (exact && exact.length <= maxChars) return exact;
226
+ for (const maxArrayItems of JSON_TRUNCATION_ARRAY_ITEM_BUDGETS) for (const maxStringChars of JSON_TRUNCATION_STRING_BUDGETS) {
227
+ const json = stringifyJsonForOtelAttribute(truncateJsonValueForOtelAttribute(value, {
228
+ maxArrayItems,
229
+ maxDepth: JSON_TRUNCATION_MAX_DEPTH,
230
+ maxObjectFields: JSON_TRUNCATION_MAX_OBJECT_FIELDS,
231
+ maxStringChars,
232
+ seen: /* @__PURE__ */ new WeakSet()
233
+ }));
234
+ if (json && json.length <= maxChars) return json;
235
+ }
236
+ const summary = stringifyJsonForOtelAttribute({
237
+ truncated: true,
238
+ reason: exact ? "max_attribute_size" : "unserializable_value",
239
+ type: describeJsonValue(value)
240
+ });
241
+ return summary && summary.length <= maxChars ? summary : void 0;
242
+ }
243
+ function stringifyJsonForOtelAttribute(value) {
244
+ try {
245
+ const json = JSON.stringify(value);
246
+ if (!json) return;
247
+ return redactSensitiveText(json);
248
+ } catch {
249
+ return;
250
+ }
251
+ }
252
+ function truncateJsonValueForOtelAttribute(value, options) {
253
+ if (typeof value === "string") return truncateJsonTextForOtelAttribute(value, options.maxStringChars);
254
+ if (typeof value === "number" || typeof value === "boolean" || value === null) return value;
255
+ if (typeof value === "bigint") return truncateJsonTextForOtelAttribute(String(value), options.maxStringChars);
256
+ if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
257
+ if (options.maxDepth <= 0) return {
258
+ truncated: true,
259
+ reason: "max_depth"
260
+ };
261
+ if (Array.isArray(value)) return truncateJsonArrayForOtelAttribute(value, options);
262
+ if (typeof value === "object") return truncateJsonObjectForOtelAttribute(value, options);
263
+ }
264
+ function truncateJsonArrayForOtelAttribute(value, options) {
265
+ if (options.seen.has(value)) return [{
266
+ truncated: true,
267
+ reason: "circular_reference"
268
+ }];
269
+ options.seen.add(value);
270
+ const nextOptions = {
271
+ ...options,
272
+ maxDepth: options.maxDepth - 1
273
+ };
274
+ const items = value.slice(0, options.maxArrayItems).map((item) => truncateJsonValueForOtelAttribute(item, nextOptions));
275
+ if (value.length > items.length) items.push({
276
+ truncated: true,
277
+ omittedItems: value.length - items.length
278
+ });
279
+ options.seen.delete(value);
280
+ return items;
281
+ }
282
+ function truncateJsonObjectForOtelAttribute(value, options) {
283
+ if (options.seen.has(value)) return {
284
+ truncated: true,
285
+ reason: "circular_reference"
286
+ };
287
+ options.seen.add(value);
288
+ const nextOptions = {
289
+ ...options,
290
+ maxDepth: options.maxDepth - 1
291
+ };
292
+ const result = {};
293
+ const entries = Object.entries(value).filter(([, field]) => field !== void 0 && typeof field !== "function" && typeof field !== "symbol");
294
+ for (const [key, field] of entries.slice(0, options.maxObjectFields)) result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
295
+ if (entries.length > options.maxObjectFields) {
296
+ result.truncated = true;
297
+ result.omittedFields = entries.length - options.maxObjectFields;
298
+ }
299
+ options.seen.delete(value);
300
+ return result;
301
+ }
302
+ function truncateJsonTextForOtelAttribute(value, maxChars) {
303
+ const redacted = redactSensitiveText(value);
304
+ if (redacted.length <= maxChars) return redacted;
305
+ const suffixBudget = Math.min(14, maxChars);
306
+ return `${truncateUtf16Safe(redacted, Math.max(0, maxChars - suffixBudget))}${TRUNCATED_JSON_TEXT_SUFFIX.slice(14 - suffixBudget)}`;
307
+ }
308
+ function describeJsonValue(value) {
309
+ if (Array.isArray(value)) return "array";
310
+ if (value === null) return "null";
311
+ return typeof value;
312
+ }
313
+ //#endregion
314
+ //#region extensions/diagnostics-otel/src/service-attributes.ts
315
+ function redactOtelAttributes(attributes) {
316
+ const redactedAttributes = {};
317
+ for (const [key, value] of Object.entries(attributes)) {
318
+ if (DROPPED_OTEL_ATTRIBUTE_KEYS.has(key)) continue;
319
+ redactedAttributes[key] = typeof value === "string" ? redactSensitiveText(value) : value;
320
+ }
321
+ return redactedAttributes;
322
+ }
323
+ function lowCardinalityAttr(value, fallback = "unknown") {
324
+ if (!value) return fallback;
325
+ const redacted = redactSensitiveText(value.trim());
326
+ const redactedLower = redacted.toLowerCase();
327
+ if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
328
+ return LOW_CARDINALITY_VALUE_RE.test(redacted) ? redacted : fallback;
329
+ }
330
+ function securityTargetNameAttr(value, fallback = "unknown") {
331
+ if (!value) return fallback;
332
+ const redacted = redactSensitiveText(value.trim());
333
+ const redactedLower = redacted.toLowerCase();
334
+ if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
335
+ return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : fallback;
336
+ }
337
+ function lowCardinalityQueueLaneAttr(value, fallback = "unknown") {
338
+ if (!value) return fallback;
339
+ const redacted = redactSensitiveText(value.trim());
340
+ if (redacted.toLowerCase().startsWith("agent:")) return fallback;
341
+ const scopedLaneIndex = redacted.indexOf(":");
342
+ const lane = scopedLaneIndex >= 0 ? redacted.slice(0, scopedLaneIndex) : redacted;
343
+ return LOW_CARDINALITY_VALUE_RE.test(lane) ? lane : fallback;
344
+ }
345
+ function shouldCaptureOtelLogBody(policy) {
346
+ return policy.logBodies;
347
+ }
348
+ function otelLogTimestampIso(timestamp) {
349
+ if (timestamp instanceof Date) return timestamp.toISOString();
350
+ if (typeof timestamp === "number" && Number.isFinite(timestamp)) return new Date(timestamp).toISOString();
351
+ if (Array.isArray(timestamp)) {
352
+ const [seconds, nanoseconds] = timestamp;
353
+ if (Number.isFinite(seconds) && Number.isFinite(nanoseconds)) return new Date(seconds * 1e3 + Math.trunc(nanoseconds / 1e6)).toISOString();
354
+ }
355
+ return (/* @__PURE__ */ new Date()).toISOString();
356
+ }
357
+ function writeStdoutDiagnosticLogRecord(params) {
358
+ const { logRecord, serviceName, traceContext } = params;
359
+ const line = {
360
+ ts: otelLogTimestampIso(logRecord.timestamp),
361
+ signal: "openclaw.diagnostic.log",
362
+ "service.name": serviceName,
363
+ severityText: logRecord.severityText,
364
+ severityNumber: logRecord.severityNumber,
365
+ body: logRecord.body,
366
+ attributes: logRecord.attributes ?? {},
367
+ ...traceContext?.traceId ? { trace_id: traceContext.traceId } : {},
368
+ ...traceContext?.spanId ? { span_id: traceContext.spanId } : {},
369
+ ...traceContext?.traceFlags ? { trace_flags: traceContext.traceFlags } : {}
370
+ };
371
+ process.stdout.write(`${JSON.stringify(line)}\n`);
372
+ }
373
+ function assignOtelLogAttribute(attributes, key, value) {
374
+ if (Object.keys(attributes).length >= 64) return;
375
+ if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) return;
376
+ if (redactSensitiveText(key) !== key) return;
377
+ if (!OTEL_LOG_ATTRIBUTE_KEY_RE.test(key)) return;
378
+ if (typeof value === "string") {
379
+ attributes[key] = normalizeOtelLogString(value, MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS);
380
+ return;
381
+ }
382
+ if (typeof value === "number" && Number.isFinite(value)) {
383
+ attributes[key] = value;
384
+ return;
385
+ }
386
+ if (typeof value === "boolean") attributes[key] = value;
387
+ }
388
+ function assignOtelLogEventAttributes(attributes, eventAttributes) {
389
+ if (!eventAttributes) return;
390
+ for (const [rawKey, value] of Object.entries(eventAttributes)) {
391
+ if (Object.keys(attributes).length >= 64) break;
392
+ const key = rawKey.trim();
393
+ if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
394
+ if (redactSensitiveText(key) !== key) continue;
395
+ if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
396
+ assignOtelLogAttribute(attributes, `openclaw.${key}`, value);
397
+ }
398
+ }
399
+ function assignOtelSecurityEventAttributes(attributes, eventAttributes) {
400
+ if (!eventAttributes) return;
401
+ for (const [rawKey, value] of Object.entries(eventAttributes)) {
402
+ if (Object.keys(attributes).length >= 64) break;
403
+ const key = rawKey.trim();
404
+ if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
405
+ if (redactSensitiveText(key) !== key) continue;
406
+ if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
407
+ assignOtelLogAttribute(attributes, `openclaw.security.attribute.${key}`, typeof value === "string" ? lowCardinalityAttr(value) : value);
408
+ }
409
+ }
410
+ function securitySeverityText(severity) {
411
+ switch (severity) {
412
+ case "critical": return "FATAL";
413
+ case "high": return "ERROR";
414
+ case "medium": return "WARN";
415
+ case "info":
416
+ case "low": return "INFO";
417
+ }
418
+ return severity;
419
+ }
420
+ function assignOtelSecurityAttributes(attributes, evt) {
421
+ assignOtelLogAttribute(attributes, "openclaw.security.event_id", evt.eventId);
422
+ assignOtelLogAttribute(attributes, "openclaw.security.category", evt.category);
423
+ assignOtelLogAttribute(attributes, "openclaw.security.action", lowCardinalityAttr(evt.action));
424
+ assignOtelLogAttribute(attributes, "openclaw.security.outcome", evt.outcome);
425
+ assignOtelLogAttribute(attributes, "openclaw.security.severity", evt.severity);
426
+ if (evt.reason) assignOtelLogAttribute(attributes, "openclaw.security.reason", lowCardinalityAttr(evt.reason));
427
+ if (evt.actor) {
428
+ assignOtelLogAttribute(attributes, "openclaw.security.actor.kind", evt.actor.kind);
429
+ if (evt.actor.idHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.id_hash", lowCardinalityAttr(evt.actor.idHash));
430
+ if (evt.actor.deviceIdHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.device_id_hash", lowCardinalityAttr(evt.actor.deviceIdHash));
431
+ if (evt.actor.channel) assignOtelLogAttribute(attributes, "openclaw.security.actor.channel", lowCardinalityAttr(evt.actor.channel));
432
+ if (evt.actor.role) assignOtelLogAttribute(attributes, "openclaw.security.actor.role", lowCardinalityAttr(evt.actor.role));
433
+ if (evt.actor.scopes?.length) assignOtelLogAttribute(attributes, "openclaw.security.actor.scopes", evt.actor.scopes.map((scope) => lowCardinalityAttr(scope)).join(","));
434
+ }
435
+ if (evt.target) {
436
+ assignOtelLogAttribute(attributes, "openclaw.security.target.kind", evt.target.kind);
437
+ if (evt.target.idHash) assignOtelLogAttribute(attributes, "openclaw.security.target.id_hash", lowCardinalityAttr(evt.target.idHash));
438
+ if (evt.target.name) assignOtelLogAttribute(attributes, "openclaw.security.target.name", securityTargetNameAttr(evt.target.name));
439
+ if (evt.target.owner) assignOtelLogAttribute(attributes, "openclaw.security.target.owner", lowCardinalityAttr(evt.target.owner));
440
+ }
441
+ if (evt.policy) {
442
+ if (evt.policy.id) assignOtelLogAttribute(attributes, "openclaw.security.policy.id", lowCardinalityAttr(evt.policy.id));
443
+ if (evt.policy.decision) assignOtelLogAttribute(attributes, "openclaw.security.policy.decision", evt.policy.decision);
444
+ if (evt.policy.reason) assignOtelLogAttribute(attributes, "openclaw.security.policy.reason", lowCardinalityAttr(evt.policy.reason));
445
+ }
446
+ if (evt.control) {
447
+ if (evt.control.id) assignOtelLogAttribute(attributes, "openclaw.security.control.id", lowCardinalityAttr(evt.control.id));
448
+ if (evt.control.family) assignOtelLogAttribute(attributes, "openclaw.security.control.family", evt.control.family);
449
+ }
450
+ assignOtelSecurityEventAttributes(attributes, evt.attributes);
451
+ }
452
+ //#endregion
453
+ //#region extensions/diagnostics-otel/src/service-exporter.ts
454
+ function normalizeEndpoint(endpoint) {
455
+ const trimmed = endpoint?.trim();
456
+ return trimmed ? trimmed.replace(/\/+$/, "") : void 0;
457
+ }
458
+ function resolveOtelUrl(endpoint, path) {
459
+ if (!endpoint) return;
460
+ const endpointWithoutQueryOrFragment = endpoint.split(/[?#]/, 1)[0] ?? endpoint;
461
+ if (/\/v1\/(?:traces|metrics|logs)$/i.test(endpointWithoutQueryOrFragment)) return endpoint;
462
+ if (/[?#]/u.test(endpoint)) try {
463
+ const url = new URL(endpoint);
464
+ url.pathname = `${url.pathname.replace(/\/+$/u, "")}/${path}`;
465
+ return url.toString();
466
+ } catch {}
467
+ return `${endpoint}/${path}`;
468
+ }
469
+ function resolveSignalOtelUrl(params) {
470
+ return resolveOtelUrl(normalizeEndpoint(params.signalEndpoint ?? params.signalEnvEndpoint) ?? params.endpoint, params.path);
471
+ }
472
+ function readOtelEnvFile(params) {
473
+ const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`;
474
+ const filePath = normalizeOtelEnvValue(process.env[signalEnvName]) ?? normalizeOtelEnvValue(process.env[params.sharedEnvName]);
475
+ if (!filePath) return;
476
+ try {
477
+ return readFileSync(nodePath.resolve(process.cwd(), filePath));
478
+ } catch {
479
+ params.logger.warn(`diagnostics-otel: ${params.warning}`);
480
+ return;
481
+ }
482
+ }
483
+ function normalizeOtelEnvValue(value) {
484
+ const trimmed = value?.trim();
485
+ return trimmed ? trimmed : void 0;
486
+ }
487
+ function resolveOtelHttpAgentOptions(params) {
488
+ const { url, signalIdentifier, logger } = params;
489
+ if (!url) return;
490
+ const ca = readOtelEnvFile({
491
+ signalIdentifier,
492
+ signalSuffix: "CERTIFICATE",
493
+ sharedEnvName: OTEL_EXPORTER_OTLP_CERTIFICATE_ENV,
494
+ logger,
495
+ warning: "failed to read root certificate file"
496
+ });
497
+ const cert = readOtelEnvFile({
498
+ signalIdentifier,
499
+ signalSuffix: "CLIENT_CERTIFICATE",
500
+ sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV,
501
+ logger,
502
+ warning: "failed to read client certificate chain file"
503
+ });
504
+ const key = readOtelEnvFile({
505
+ signalIdentifier,
506
+ signalSuffix: "CLIENT_KEY",
507
+ sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV,
508
+ logger,
509
+ warning: "failed to read client certificate private key file"
510
+ });
511
+ const agentOptions = {
512
+ keepAlive: true,
513
+ ...ca !== void 0 ? { ca } : {},
514
+ ...cert !== void 0 ? { cert } : {},
515
+ ...key !== void 0 ? { key } : {}
516
+ };
517
+ try {
518
+ const agent = createNodeProxyAgent({
519
+ mode: "env",
520
+ targetUrl: url,
521
+ agentOptions
522
+ });
523
+ return agent ? () => agent : void 0;
524
+ } catch {
525
+ logger.warn(`diagnostics-otel: env proxy agent unavailable for OTLP ${signalIdentifier.toLowerCase()} exporter; falling back to default Node agent`);
526
+ return;
527
+ }
528
+ }
529
+ function resolveSampleRate(value) {
530
+ if (typeof value !== "number" || !Number.isFinite(value)) return;
531
+ if (value < 0 || value > 1) return;
532
+ return value;
533
+ }
534
+ function formatError(err) {
535
+ if (err instanceof Error) return err.stack ?? err.message;
536
+ if (typeof err === "string") return err;
537
+ try {
538
+ return JSON.stringify(err);
539
+ } catch {
540
+ return String(err);
541
+ }
542
+ }
543
+ function errorCategory(err) {
544
+ try {
545
+ if (err instanceof Error && typeof err.name === "string" && err.name.trim()) return lowCardinalityAttr(err.name, "Error");
546
+ return lowCardinalityAttr(typeof err, "unknown");
547
+ } catch {
548
+ return "unknown";
549
+ }
550
+ }
551
+ function collectNestedErrorCandidates(err) {
552
+ const queue = [err];
553
+ const seen = /* @__PURE__ */ new Set();
554
+ const candidates = [];
555
+ while (queue.length > 0) {
556
+ const current = queue.shift();
557
+ if (current == null || seen.has(current)) continue;
558
+ seen.add(current);
559
+ candidates.push(current);
560
+ if (Array.isArray(current)) {
561
+ for (const item of current) if (item != null && !seen.has(item)) queue.push(item);
562
+ continue;
563
+ }
564
+ if (typeof current !== "object") continue;
565
+ const record = current;
566
+ for (const nested of [
567
+ record.cause,
568
+ record.reason,
569
+ record.original,
570
+ record.error
571
+ ]) if (nested != null && !seen.has(nested)) queue.push(nested);
572
+ if (Array.isArray(record.errors)) {
573
+ for (const nested of record.errors) if (nested != null && !seen.has(nested)) queue.push(nested);
574
+ }
575
+ }
576
+ return candidates;
577
+ }
578
+ function readErrorName(err) {
579
+ if (!err || typeof err !== "object") return;
580
+ const name = err.name;
581
+ return typeof name === "string" && name.trim() ? name : void 0;
582
+ }
583
+ function readErrorCode(err) {
584
+ if (!err || typeof err !== "object") return;
585
+ const code = err.code;
586
+ return typeof code === "string" || typeof code === "number" ? code : void 0;
587
+ }
588
+ function findOtlpExporterError(reason) {
589
+ for (const candidate of collectNestedErrorCandidates(reason)) if (readErrorName(candidate) === "OTLPExporterError" && candidate && typeof candidate === "object") return candidate;
590
+ }
591
+ //#endregion
592
+ //#region extensions/diagnostics-otel/src/service-events.ts
593
+ function createDiagnosticsEventHandler(params) {
594
+ const { logger, recorders, recordLogRecord, recordSecurityEvent } = params;
595
+ const { recordModelUsage, recordWebhookReceived, recordWebhookProcessed, recordWebhookError, recordMessageQueued, recordMessageReceived, recordMessageDispatchStarted, recordMessageDispatchCompleted, recordMessageProcessed, recordMessageDeliveryStarted, recordMessageDeliveryCompleted, recordMessageDeliveryError, recordTalkEvent, recordLaneEnqueue, recordLaneDequeue, recordSessionState, recordSessionTurnCreated, recordSessionStuck, recordSessionRecoveryRequested, recordSessionRecoveryCompleted, recordRunAttempt, recordHeartbeat, recordLivenessWarning, recordDiagnosticPhaseCompleted, recordRunStarted, recordRunCompleted, recordHarnessRunStarted, recordHarnessRunCompleted, recordHarnessRunError, recordContextAssembled, recordModelCallStarted, recordModelCallCompleted, recordModelCallError, recordToolExecutionStarted, recordToolExecutionCompleted, recordToolExecutionError, recordToolExecutionBlocked, recordSkillUsed, recordExecProcessCompleted, recordToolLoop, recordMemorySample, recordMemoryPressure, recordAsyncQueueDropped, recordTelemetryExporter, recordPayloadLarge, recordModelFailover } = recorders;
596
+ return (evt, metadata, privateData) => {
597
+ try {
598
+ switch (evt.type) {
599
+ case "model.usage":
600
+ recordModelUsage(evt, metadata);
601
+ return;
602
+ case "webhook.received":
603
+ recordWebhookReceived(evt);
604
+ return;
605
+ case "webhook.processed":
606
+ recordWebhookProcessed(evt);
607
+ return;
608
+ case "webhook.error":
609
+ recordWebhookError(evt);
610
+ return;
611
+ case "message.queued":
612
+ recordMessageQueued(evt);
613
+ return;
614
+ case "message.received":
615
+ recordMessageReceived(evt);
616
+ return;
617
+ case "message.dispatch.started":
618
+ recordMessageDispatchStarted(evt, metadata);
619
+ return;
620
+ case "message.dispatch.completed":
621
+ recordMessageDispatchCompleted(evt);
622
+ return;
623
+ case "message.processed":
624
+ recordMessageProcessed(evt, metadata);
625
+ return;
626
+ case "message.delivery.started":
627
+ recordMessageDeliveryStarted(evt);
628
+ return;
629
+ case "message.delivery.completed":
630
+ recordMessageDeliveryCompleted(evt, metadata);
631
+ return;
632
+ case "message.delivery.error":
633
+ recordMessageDeliveryError(evt, metadata);
634
+ return;
635
+ case "talk.event":
636
+ recordTalkEvent(evt, metadata);
637
+ return;
638
+ case "queue.lane.enqueue":
639
+ recordLaneEnqueue(evt);
640
+ return;
641
+ case "queue.lane.dequeue":
642
+ recordLaneDequeue(evt);
643
+ return;
644
+ case "session.state":
645
+ recordSessionState(evt);
646
+ break;
647
+ case "session.long_running":
648
+ case "session.stalled": break;
649
+ case "session.turn.created":
650
+ recordSessionTurnCreated(evt);
651
+ return;
652
+ case "session.stuck":
653
+ recordSessionStuck(evt);
654
+ return;
655
+ case "session.recovery.requested":
656
+ recordSessionRecoveryRequested(evt);
657
+ return;
658
+ case "session.recovery.completed":
659
+ recordSessionRecoveryCompleted(evt);
660
+ return;
661
+ case "run.attempt":
662
+ recordRunAttempt(evt);
663
+ break;
664
+ case "run.progress": break;
665
+ case "diagnostic.heartbeat":
666
+ recordHeartbeat(evt);
667
+ return;
668
+ case "diagnostic.liveness.warning":
669
+ recordLivenessWarning(evt);
670
+ return;
671
+ case "diagnostic.phase.completed":
672
+ recordDiagnosticPhaseCompleted(evt);
673
+ return;
674
+ case "run.started":
675
+ recordRunStarted(evt, metadata);
676
+ return;
677
+ case "run.completed":
678
+ recordRunCompleted(evt, metadata, privateData);
679
+ return;
680
+ case "harness.run.started":
681
+ recordHarnessRunStarted(evt, metadata);
682
+ return;
683
+ case "harness.run.completed":
684
+ recordHarnessRunCompleted(evt, metadata, privateData);
685
+ return;
686
+ case "harness.run.error":
687
+ recordHarnessRunError(evt, metadata, privateData);
688
+ return;
689
+ case "context.assembled":
690
+ recordContextAssembled(evt, metadata);
691
+ return;
692
+ case "model.call.started":
693
+ recordModelCallStarted(evt, metadata);
694
+ return;
695
+ case "model.call.completed":
696
+ recordModelCallCompleted(evt, metadata, privateData.modelContent);
697
+ return;
698
+ case "model.call.error":
699
+ recordModelCallError(evt, metadata, privateData.modelContent);
700
+ return;
701
+ case "tool.execution.started":
702
+ recordToolExecutionStarted(evt, metadata);
703
+ return;
704
+ case "tool.execution.completed":
705
+ recordToolExecutionCompleted(evt, metadata, privateData.toolContent);
706
+ return;
707
+ case "tool.execution.error":
708
+ recordToolExecutionError(evt, metadata, privateData.toolContent);
709
+ return;
710
+ case "tool.execution.blocked":
711
+ recordToolExecutionBlocked(evt, metadata);
712
+ return;
713
+ case "skill.used":
714
+ recordSkillUsed(evt, metadata);
715
+ return;
716
+ case "exec.process.completed":
717
+ recordExecProcessCompleted(evt);
718
+ break;
719
+ case "exec.approval.followup_suppressed": break;
720
+ case "log.record":
721
+ recordLogRecord?.(evt, metadata);
722
+ return;
723
+ case "security.event":
724
+ recordSecurityEvent?.(evt, metadata);
725
+ return;
726
+ case "tool.loop":
727
+ recordToolLoop(evt);
728
+ return;
729
+ case "diagnostic.memory.sample":
730
+ recordMemorySample(evt);
731
+ return;
732
+ case "diagnostic.memory.pressure":
733
+ recordMemoryPressure(evt);
734
+ return;
735
+ case "diagnostic.async_queue.dropped":
736
+ recordAsyncQueueDropped(evt);
737
+ return;
738
+ case "telemetry.exporter":
739
+ recordTelemetryExporter(evt, metadata);
740
+ return;
741
+ case "payload.large":
742
+ recordPayloadLarge(evt);
743
+ return;
744
+ case "model.failover": recordModelFailover(evt, metadata);
745
+ }
746
+ } catch (err) {
747
+ logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
748
+ }
749
+ };
750
+ }
751
+ //#endregion
752
+ //#region extensions/diagnostics-otel/src/service-trace-context.ts
753
+ function normalizeTraceContext(value) {
754
+ if (!value || typeof value !== "object" || Array.isArray(value)) return;
755
+ const candidate = value;
756
+ if (!isValidDiagnosticTraceId(candidate.traceId)) return;
757
+ if (candidate.spanId !== void 0 && !isValidDiagnosticSpanId(candidate.spanId)) return;
758
+ if (candidate.parentSpanId !== void 0 && !isValidDiagnosticSpanId(candidate.parentSpanId)) return;
759
+ if (candidate.traceFlags !== void 0 && !isValidDiagnosticTraceFlags(candidate.traceFlags)) return;
760
+ return {
761
+ traceId: candidate.traceId,
762
+ ...candidate.spanId ? { spanId: candidate.spanId } : {},
763
+ ...candidate.parentSpanId ? { parentSpanId: candidate.parentSpanId } : {},
764
+ ...candidate.traceFlags ? { traceFlags: candidate.traceFlags } : {}
765
+ };
766
+ }
767
+ function traceFlagsToOtel(traceFlags) {
768
+ return (Number.parseInt(traceFlags ?? "00", 16) & TraceFlags.SAMPLED) !== 0 ? TraceFlags.SAMPLED : TraceFlags.NONE;
769
+ }
770
+ function contextForTraceContext(traceContext) {
771
+ const normalized = normalizeTraceContext(traceContext);
772
+ if (!normalized?.spanId) return;
773
+ return trace.setSpanContext(context.active(), {
774
+ traceId: normalized.traceId,
775
+ spanId: normalized.spanId,
776
+ traceFlags: traceFlagsToOtel(normalized.traceFlags),
777
+ isRemote: true
778
+ });
779
+ }
780
+ function contextForTrustedTraceContext(evt, metadata) {
781
+ return metadata.trusted || metadata.trustedTraceContext === true ? contextForTraceContext(evt.trace) : void 0;
782
+ }
783
+ function normalizedTrustedTraceContext(evt, metadata) {
784
+ return metadata.trusted || metadata.trustedTraceContext === true ? normalizeTraceContext(evt.trace) : void 0;
785
+ }
786
+ function addTraceAttributes(attributes, traceContext) {
787
+ const normalized = normalizeTraceContext(traceContext);
788
+ if (!normalized) return;
789
+ attributes["openclaw.traceId"] = normalized.traceId;
790
+ if (normalized.spanId) attributes["openclaw.spanId"] = normalized.spanId;
791
+ if (normalized.parentSpanId) attributes["openclaw.parentSpanId"] = normalized.parentSpanId;
792
+ if (normalized.traceFlags) attributes["openclaw.traceFlags"] = normalized.traceFlags;
793
+ }
794
+ //#endregion
795
+ //#region extensions/diagnostics-otel/src/service-logs.ts
796
+ const LOG_SEVERITY_MAP = {
797
+ TRACE: 1,
798
+ DEBUG: 5,
799
+ INFO: 9,
800
+ WARN: 13,
801
+ ERROR: 17,
802
+ FATAL: 21
803
+ };
804
+ function createDiagnosticsLogExporter(params) {
805
+ const { contentCapturePolicy, emitExporterEvent, flushIntervalMs, headers, logger, logsEnabled, logsToOtlp, logsToStdout, logUrl, resource, serviceName } = params;
806
+ let logProvider = null;
807
+ const logSeverityMap = LOG_SEVERITY_MAP;
808
+ let recordLogRecord;
809
+ let recordSecurityEvent;
810
+ if (logsEnabled) {
811
+ let logRecordExportFailureLastReportedAt = Number.NEGATIVE_INFINITY;
812
+ let otelLogger;
813
+ if (logsToOtlp) {
814
+ const logHttpAgentOptions = resolveOtelHttpAgentOptions({
815
+ url: logUrl,
816
+ signalIdentifier: "LOGS",
817
+ logger
818
+ });
819
+ logProvider = new LoggerProvider({
820
+ resource,
821
+ processors: [new BatchLogRecordProcessor(new OTLPLogExporter({
822
+ ...logUrl ? { url: logUrl } : {},
823
+ ...headers ? { headers } : {},
824
+ ...logHttpAgentOptions ? { httpAgentOptions: logHttpAgentOptions } : {}
825
+ }), typeof flushIntervalMs === "number" ? { scheduledDelayMillis: Math.max(1e3, flushIntervalMs) } : {})]
826
+ });
827
+ otelLogger = logProvider.getLogger("openclaw");
828
+ }
829
+ const reportLogExportFailure = (err, label) => {
830
+ emitExporterEvent({
831
+ exporter: "diagnostics-otel",
832
+ signal: "logs",
833
+ status: "failure",
834
+ reason: "emit_failed",
835
+ errorCategory: errorCategory(err)
836
+ });
837
+ const now = Date.now();
838
+ if (now - logRecordExportFailureLastReportedAt >= 6e4) {
839
+ logRecordExportFailureLastReportedAt = now;
840
+ logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`);
841
+ }
842
+ };
843
+ const emitLogRecord = ({ logRecord, traceContext }) => {
844
+ if (logsToOtlp) otelLogger?.emit(logRecord);
845
+ if (logsToStdout) writeStdoutDiagnosticLogRecord({
846
+ logRecord,
847
+ serviceName,
848
+ ...traceContext ? { traceContext } : {}
849
+ });
850
+ };
851
+ const buildDiagnosticLogRecord = (evt, metadata) => {
852
+ const logLevelName = evt.level || "INFO";
853
+ const severityNumber = logSeverityMap[logLevelName] ?? 9;
854
+ const body = shouldCaptureOtelLogBody(contentCapturePolicy) ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
855
+ const attributes = Object.create(null);
856
+ assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName);
857
+ if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
858
+ if (evt.loggerParents?.length) assignOtelLogAttribute(attributes, "openclaw.logger.parents", evt.loggerParents.join("."));
859
+ assignOtelLogEventAttributes(attributes, evt.attributes);
860
+ if (evt.code?.line) assignOtelLogAttribute(attributes, "code.lineno", evt.code.line);
861
+ if (evt.code?.functionName) assignOtelLogAttribute(attributes, "code.function", evt.code.functionName);
862
+ const traceContext = normalizedTrustedTraceContext(evt, metadata);
863
+ addTraceAttributes(attributes, traceContext);
864
+ const logRecord = {
865
+ body,
866
+ severityText: logLevelName,
867
+ severityNumber,
868
+ attributes: redactOtelAttributes(attributes),
869
+ timestamp: evt.ts
870
+ };
871
+ const logContext = contextForTrustedTraceContext(evt, metadata);
872
+ if (logContext) logRecord.context = logContext;
873
+ return {
874
+ logRecord,
875
+ ...traceContext ? { traceContext } : {}
876
+ };
877
+ };
878
+ const buildSecurityLogRecord = (evt, metadata) => {
879
+ const severityText = securitySeverityText(evt.severity);
880
+ const attributes = Object.create(null);
881
+ assignOtelSecurityAttributes(attributes, evt);
882
+ const traceContext = normalizedTrustedTraceContext(evt, metadata);
883
+ const logRecord = {
884
+ body: "openclaw.security.event",
885
+ severityText,
886
+ severityNumber: logSeverityMap[severityText] ?? 9,
887
+ attributes: redactOtelAttributes(attributes),
888
+ timestamp: evt.ts
889
+ };
890
+ const logContext = contextForTrustedTraceContext(evt, metadata);
891
+ if (logContext) logRecord.context = logContext;
892
+ return {
893
+ logRecord,
894
+ ...traceContext ? { traceContext } : {}
895
+ };
896
+ };
897
+ recordLogRecord = (evt, metadata) => {
898
+ try {
899
+ emitLogRecord(buildDiagnosticLogRecord(evt, metadata));
900
+ } catch (err) {
901
+ reportLogExportFailure(err, "log record");
902
+ }
903
+ };
904
+ recordSecurityEvent = (evt, metadata) => {
905
+ if (!metadata.trusted) return;
906
+ try {
907
+ emitLogRecord(buildSecurityLogRecord(evt, metadata));
908
+ } catch (err) {
909
+ reportLogExportFailure(err, "security event");
910
+ }
911
+ };
912
+ }
913
+ return {
914
+ logProvider,
915
+ recordLogRecord,
916
+ recordSecurityEvent
917
+ };
918
+ }
919
+ //#endregion
920
+ //#region extensions/diagnostics-otel/src/service-metrics.ts
921
+ function createDiagnosticsMetrics(meter) {
922
+ return {
923
+ tokensCounter: meter.createCounter("openclaw.tokens", {
924
+ unit: "1",
925
+ description: "Token usage by type"
926
+ }),
927
+ genAiTokenUsageHistogram: meter.createHistogram("gen_ai.client.token.usage", {
928
+ unit: "{token}",
929
+ description: "Number of input and output tokens used by GenAI client operations",
930
+ advice: { explicitBucketBoundaries: GEN_AI_TOKEN_USAGE_BUCKETS }
931
+ }),
932
+ genAiOperationDurationHistogram: meter.createHistogram("gen_ai.client.operation.duration", {
933
+ unit: "s",
934
+ description: "GenAI client operation duration",
935
+ advice: { explicitBucketBoundaries: GEN_AI_OPERATION_DURATION_BUCKETS }
936
+ }),
937
+ costCounter: meter.createCounter("openclaw.cost.usd", {
938
+ unit: "1",
939
+ description: "Estimated model cost (USD)"
940
+ }),
941
+ durationHistogram: meter.createHistogram("openclaw.run.duration_ms", {
942
+ unit: "ms",
943
+ description: "Agent run duration",
944
+ advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
945
+ }),
946
+ harnessDurationHistogram: meter.createHistogram("openclaw.harness.duration_ms", {
947
+ unit: "ms",
948
+ description: "Agent harness lifecycle duration",
949
+ advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
950
+ }),
951
+ contextHistogram: meter.createHistogram("openclaw.context.tokens", {
952
+ unit: "1",
953
+ description: "Context window size and usage",
954
+ advice: { explicitBucketBoundaries: CONTEXT_TOKENS_BUCKETS }
955
+ }),
956
+ webhookReceivedCounter: meter.createCounter("openclaw.webhook.received", {
957
+ unit: "1",
958
+ description: "Webhook requests received"
959
+ }),
960
+ webhookErrorCounter: meter.createCounter("openclaw.webhook.error", {
961
+ unit: "1",
962
+ description: "Webhook processing errors"
963
+ }),
964
+ webhookDurationHistogram: meter.createHistogram("openclaw.webhook.duration_ms", {
965
+ unit: "ms",
966
+ description: "Webhook processing duration"
967
+ }),
968
+ messageQueuedCounter: meter.createCounter("openclaw.message.queued", {
969
+ unit: "1",
970
+ description: "Messages queued for processing"
971
+ }),
972
+ messageReceivedCounter: meter.createCounter("openclaw.message.received", {
973
+ unit: "1",
974
+ description: "Inbound messages received"
975
+ }),
976
+ messageDispatchStartedCounter: meter.createCounter("openclaw.message.dispatch.started", {
977
+ unit: "1",
978
+ description: "Inbound message dispatch attempts started"
979
+ }),
980
+ messageDispatchCompletedCounter: meter.createCounter("openclaw.message.dispatch.completed", {
981
+ unit: "1",
982
+ description: "Inbound message dispatch attempts completed"
983
+ }),
984
+ messageDispatchDurationHistogram: meter.createHistogram("openclaw.message.dispatch.duration_ms", {
985
+ unit: "ms",
986
+ description: "Inbound message dispatch duration"
987
+ }),
988
+ messageProcessedCounter: meter.createCounter("openclaw.message.processed", {
989
+ unit: "1",
990
+ description: "Messages processed by outcome"
991
+ }),
992
+ messageDurationHistogram: meter.createHistogram("openclaw.message.duration_ms", {
993
+ unit: "ms",
994
+ description: "Message processing duration"
995
+ }),
996
+ messageDeliveryStartedCounter: meter.createCounter("openclaw.message.delivery.started", {
997
+ unit: "1",
998
+ description: "Outbound message delivery attempts started"
999
+ }),
1000
+ messageDeliveryDurationHistogram: meter.createHistogram("openclaw.message.delivery.duration_ms", {
1001
+ unit: "ms",
1002
+ description: "Outbound message delivery duration"
1003
+ }),
1004
+ queueDepthHistogram: meter.createHistogram("openclaw.queue.depth", {
1005
+ unit: "1",
1006
+ description: "Queue depth on enqueue/dequeue"
1007
+ }),
1008
+ queueWaitHistogram: meter.createHistogram("openclaw.queue.wait_ms", {
1009
+ unit: "ms",
1010
+ description: "Queue wait time before execution"
1011
+ }),
1012
+ laneEnqueueCounter: meter.createCounter("openclaw.queue.lane.enqueue", {
1013
+ unit: "1",
1014
+ description: "Command queue lane enqueue events"
1015
+ }),
1016
+ laneDequeueCounter: meter.createCounter("openclaw.queue.lane.dequeue", {
1017
+ unit: "1",
1018
+ description: "Command queue lane dequeue events"
1019
+ }),
1020
+ sessionStateCounter: meter.createCounter("openclaw.session.state", {
1021
+ unit: "1",
1022
+ description: "Session state transitions"
1023
+ }),
1024
+ sessionTurnCreatedCounter: meter.createCounter("openclaw.session.turn.created", {
1025
+ unit: "1",
1026
+ description: "Agent session turns created"
1027
+ }),
1028
+ sessionStuckCounter: meter.createCounter("openclaw.session.stuck", {
1029
+ unit: "1",
1030
+ description: "Sessions stuck in processing"
1031
+ }),
1032
+ sessionStuckAgeHistogram: meter.createHistogram("openclaw.session.stuck_age_ms", {
1033
+ unit: "ms",
1034
+ description: "Age of stuck sessions"
1035
+ }),
1036
+ sessionRecoveryRequestedCounter: meter.createCounter("openclaw.session.recovery.requested", {
1037
+ unit: "1",
1038
+ description: "Session recovery attempts requested"
1039
+ }),
1040
+ sessionRecoveryCompletedCounter: meter.createCounter("openclaw.session.recovery.completed", {
1041
+ unit: "1",
1042
+ description: "Session recovery attempts completed"
1043
+ }),
1044
+ sessionRecoveryAgeHistogram: meter.createHistogram("openclaw.session.recovery.age_ms", {
1045
+ unit: "ms",
1046
+ description: "Age of sessions selected for recovery"
1047
+ }),
1048
+ talkEventCounter: meter.createCounter("openclaw.talk.event", {
1049
+ unit: "1",
1050
+ description: "Talk events emitted by type"
1051
+ }),
1052
+ talkEventDurationHistogram: meter.createHistogram("openclaw.talk.event.duration_ms", {
1053
+ unit: "ms",
1054
+ description: "Talk event duration when reported"
1055
+ }),
1056
+ talkAudioBytesHistogram: meter.createHistogram("openclaw.talk.audio.bytes", {
1057
+ unit: "By",
1058
+ description: "Talk audio frame byte lengths"
1059
+ }),
1060
+ runAttemptCounter: meter.createCounter("openclaw.run.attempt", {
1061
+ unit: "1",
1062
+ description: "Run attempts"
1063
+ }),
1064
+ toolLoopCounter: meter.createCounter("openclaw.tool.loop", {
1065
+ unit: "1",
1066
+ description: "Detected repetitive tool-call loop events"
1067
+ }),
1068
+ skillUsedCounter: meter.createCounter("openclaw.skill.used", {
1069
+ unit: "1",
1070
+ description: "Skills used by agent runs"
1071
+ }),
1072
+ modelCallDurationHistogram: meter.createHistogram("openclaw.model_call.duration_ms", {
1073
+ unit: "ms",
1074
+ description: "Model call duration"
1075
+ }),
1076
+ modelCallRequestBytesHistogram: meter.createHistogram("openclaw.model_call.request_bytes", {
1077
+ unit: "By",
1078
+ description: "UTF-8 byte size of sanitized model request payloads"
1079
+ }),
1080
+ modelCallResponseBytesHistogram: meter.createHistogram("openclaw.model_call.response_bytes", {
1081
+ unit: "By",
1082
+ description: "UTF-8 byte size of bounded streamed model response payloads"
1083
+ }),
1084
+ modelCallTimeToFirstByteHistogram: meter.createHistogram("openclaw.model_call.time_to_first_byte_ms", {
1085
+ unit: "ms",
1086
+ description: "Elapsed time before the first streamed model response event"
1087
+ }),
1088
+ modelFailoverCounter: meter.createCounter("openclaw.model.failover", {
1089
+ unit: "1",
1090
+ description: "Model failovers by source, destination, lane, and reason"
1091
+ }),
1092
+ toolExecutionDurationHistogram: meter.createHistogram("openclaw.tool.execution.duration_ms", {
1093
+ unit: "ms",
1094
+ description: "Tool execution duration"
1095
+ }),
1096
+ toolExecutionBlockedCounter: meter.createCounter("openclaw.tool.execution.blocked", {
1097
+ unit: "1",
1098
+ description: "Tool executions blocked by policy or sandbox diagnostics"
1099
+ }),
1100
+ execProcessDurationHistogram: meter.createHistogram("openclaw.exec.duration_ms", {
1101
+ unit: "ms",
1102
+ description: "Exec process duration"
1103
+ }),
1104
+ memoryRssHistogram: meter.createHistogram("openclaw.memory.rss_bytes", {
1105
+ unit: "By",
1106
+ description: "Resident set size reported by diagnostic memory samples"
1107
+ }),
1108
+ memoryHeapUsedHistogram: meter.createHistogram("openclaw.memory.heap_used_bytes", {
1109
+ unit: "By",
1110
+ description: "Heap used bytes reported by diagnostic memory samples"
1111
+ }),
1112
+ memoryHeapTotalHistogram: meter.createHistogram("openclaw.memory.heap_total_bytes", {
1113
+ unit: "By",
1114
+ description: "Heap total bytes reported by diagnostic memory samples"
1115
+ }),
1116
+ memoryExternalHistogram: meter.createHistogram("openclaw.memory.external_bytes", {
1117
+ unit: "By",
1118
+ description: "External memory bytes reported by diagnostic memory samples"
1119
+ }),
1120
+ memoryArrayBuffersHistogram: meter.createHistogram("openclaw.memory.array_buffers_bytes", {
1121
+ unit: "By",
1122
+ description: "ArrayBuffer bytes reported by diagnostic memory samples"
1123
+ }),
1124
+ memoryPressureCounter: meter.createCounter("openclaw.memory.pressure", {
1125
+ unit: "1",
1126
+ description: "Diagnostic memory pressure events"
1127
+ }),
1128
+ asyncQueueDroppedCounter: meter.createCounter("openclaw.diagnostic.async_queue.dropped", {
1129
+ unit: "1",
1130
+ description: "Async diagnostic queue drops by dropped event class"
1131
+ }),
1132
+ payloadLargeCounter: meter.createCounter("openclaw.payload.large", {
1133
+ unit: "1",
1134
+ description: "Oversized payload diagnostics by surface and action"
1135
+ }),
1136
+ payloadLargeBytesHistogram: meter.createHistogram("openclaw.payload.large_bytes", {
1137
+ unit: "By",
1138
+ description: "Oversized payload byte sizes by surface and action"
1139
+ }),
1140
+ livenessWarningCounter: meter.createCounter("openclaw.liveness.warning", {
1141
+ unit: "1",
1142
+ description: "Diagnostic liveness warning events"
1143
+ }),
1144
+ livenessEventLoopDelayP99Histogram: meter.createHistogram("openclaw.liveness.event_loop_delay_p99_ms", {
1145
+ unit: "ms",
1146
+ description: "P99 event-loop delay reported by diagnostic liveness warnings"
1147
+ }),
1148
+ livenessEventLoopDelayMaxHistogram: meter.createHistogram("openclaw.liveness.event_loop_delay_max_ms", {
1149
+ unit: "ms",
1150
+ description: "Maximum event-loop delay reported by diagnostic liveness warnings"
1151
+ }),
1152
+ livenessEventLoopUtilizationHistogram: meter.createHistogram("openclaw.liveness.event_loop_utilization", {
1153
+ unit: "1",
1154
+ description: "Event-loop utilization reported by diagnostic liveness warnings"
1155
+ }),
1156
+ livenessCpuCoreRatioHistogram: meter.createHistogram("openclaw.liveness.cpu_core_ratio", {
1157
+ unit: "1",
1158
+ description: "CPU core ratio reported by diagnostic liveness warnings"
1159
+ }),
1160
+ telemetryExporterCounter: meter.createCounter("openclaw.telemetry.exporter.events", {
1161
+ unit: "1",
1162
+ description: "Diagnostic telemetry exporter lifecycle and failure events"
1163
+ })
1164
+ };
1165
+ }
1166
+ //#endregion
1167
+ //#region extensions/diagnostics-otel/src/service-recorder-runtime.ts
1168
+ function createDiagnosticsRecorderRuntime(params) {
1169
+ return {
1170
+ ...params.metrics,
1171
+ ...params.traces,
1172
+ contentCapturePolicy: params.contentCapturePolicy,
1173
+ tracesEnabled: params.tracesEnabled
1174
+ };
1175
+ }
1176
+ //#endregion
1177
+ //#region extensions/diagnostics-otel/src/service-recorders-harness.ts
1178
+ function createHarnessRecorders(runtime) {
1179
+ const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
1180
+ const harnessRunMetricAttrs = (evt) => ({
1181
+ "openclaw.harness.id": lowCardinalityAttr(evt.harnessId, "unknown"),
1182
+ "openclaw.harness.plugin": lowCardinalityAttr(evt.pluginId),
1183
+ ...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
1184
+ "openclaw.provider": lowCardinalityAttr(evt.provider, "unknown"),
1185
+ "openclaw.model": lowCardinalityAttr(evt.model, "unknown"),
1186
+ ...evt.channel ? { "openclaw.channel": lowCardinalityAttr(evt.channel) } : {}
1187
+ });
1188
+ const recordHarnessRunStarted = (evt, metadata) => {
1189
+ if (!tracesEnabled || !metadata.trusted) return;
1190
+ trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", harnessRunMetricAttrs(evt), void 0, {
1191
+ parentContext: activeTrustedParentContext(evt, metadata),
1192
+ startTimeMs: evt.ts
1193
+ }));
1194
+ };
1195
+ const recordHarnessRunCompleted = (evt, metadata, privateData) => {
1196
+ harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
1197
+ if (!tracesEnabled) return;
1198
+ const spanAttrs = { ...harnessRunMetricAttrs(evt) };
1199
+ if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = lowCardinalityAttr(evt.resultClassification);
1200
+ if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
1201
+ if (evt.itemLifecycle) {
1202
+ spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
1203
+ spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
1204
+ spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
1205
+ }
1206
+ const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
1207
+ if (redactedError) spanAttrs["openclaw.error"] = redactedError;
1208
+ const trustedTrace = trustedTraceContext(evt, metadata);
1209
+ const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
1210
+ const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
1211
+ parentContext: activeTrustedParentContext(evt, metadata),
1212
+ endTimeMs: evt.ts
1213
+ });
1214
+ setSpanAttrs(span, spanAttrs);
1215
+ if (evt.outcome === "error") span.setStatus({
1216
+ code: SpanStatusCode.ERROR,
1217
+ message: redactedError ?? "error"
1218
+ });
1219
+ if (trackedSpan && trustedTrace?.spanId) {
1220
+ completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
1221
+ return;
1222
+ }
1223
+ span.end(evt.ts);
1224
+ };
1225
+ const recordHarnessRunError = (evt, metadata, privateData) => {
1226
+ const errorType = lowCardinalityAttr(evt.errorCategory, "other");
1227
+ const attrs = {
1228
+ ...harnessRunMetricAttrs(evt),
1229
+ "openclaw.harness.phase": evt.phase,
1230
+ "openclaw.errorCategory": errorType
1231
+ };
1232
+ harnessDurationHistogram.record(evt.durationMs, attrs);
1233
+ if (!tracesEnabled) return;
1234
+ const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
1235
+ const spanAttrs = {
1236
+ ...attrs,
1237
+ "error.type": errorType,
1238
+ ...redactedError ? { "openclaw.error": redactedError } : {},
1239
+ ...evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}
1240
+ };
1241
+ const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
1242
+ parentContext: activeTrustedParentContext(evt, metadata),
1243
+ endTimeMs: evt.ts
1244
+ });
1245
+ setSpanAttrs(span, spanAttrs);
1246
+ span.setStatus({
1247
+ code: SpanStatusCode.ERROR,
1248
+ message: redactedError ?? errorType
1249
+ });
1250
+ span.end(evt.ts);
1251
+ };
1252
+ const recordContextAssembled = (evt, metadata) => {
1253
+ if (!tracesEnabled) return;
1254
+ const spanAttrs = {
1255
+ "openclaw.context.message_count": evt.messageCount,
1256
+ "openclaw.context.history_text_chars": evt.historyTextChars,
1257
+ "openclaw.context.history_image_blocks": evt.historyImageBlocks,
1258
+ "openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
1259
+ "openclaw.context.system_prompt_chars": evt.systemPromptChars,
1260
+ "openclaw.context.prompt_chars": evt.promptChars,
1261
+ "openclaw.context.prompt_images": evt.promptImages
1262
+ };
1263
+ addRunAttrs(spanAttrs, evt);
1264
+ if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
1265
+ if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
1266
+ spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
1267
+ parentContext: activeTrustedParentContext(evt, metadata),
1268
+ endTimeMs: evt.ts
1269
+ }).end(evt.ts);
1270
+ };
1271
+ const recordModelFailover = (evt, metadata) => {
1272
+ const metricAttrs = {
1273
+ "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown"),
1274
+ "openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
1275
+ "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane, "unknown"),
1276
+ "openclaw.model": lowCardinalityAttr(evt.fromModel),
1277
+ "openclaw.provider": lowCardinalityAttr(evt.fromProvider),
1278
+ "openclaw.failover.to_model": lowCardinalityAttr(evt.toModel),
1279
+ "openclaw.failover.to_provider": lowCardinalityAttr(evt.toProvider)
1280
+ };
1281
+ modelFailoverCounter.add(1, metricAttrs);
1282
+ if (!tracesEnabled) return;
1283
+ const spanAttrs = { "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown") };
1284
+ if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
1285
+ if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
1286
+ if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
1287
+ if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
1288
+ if (evt.lane) spanAttrs["openclaw.lane"] = lowCardinalityQueueLaneAttr(evt.lane, "unknown");
1289
+ if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
1290
+ if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
1291
+ spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
1292
+ parentContext: activeTrustedParentContext(evt, metadata),
1293
+ endTimeMs: evt.ts
1294
+ }).end(evt.ts);
1295
+ };
1296
+ return {
1297
+ recordHarnessRunStarted,
1298
+ recordHarnessRunCompleted,
1299
+ recordHarnessRunError,
1300
+ recordContextAssembled,
1301
+ recordModelFailover
1302
+ };
1303
+ }
1304
+ //#endregion
1305
+ //#region extensions/diagnostics-otel/src/service-genai-attributes.ts
1306
+ function hasOtelSemconvOptIn(value, optIn) {
1307
+ return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
1308
+ }
1309
+ function emitLatestGenAiSemconv() {
1310
+ return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
1311
+ }
1312
+ function genAiOperationName(api) {
1313
+ const normalized = api?.trim().toLowerCase();
1314
+ if (!normalized) return "chat";
1315
+ if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
1316
+ if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
1317
+ return "chat";
1318
+ }
1319
+ function positiveFiniteNumber(value) {
1320
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : void 0;
1321
+ }
1322
+ function assignPositiveNumberAttr(attrs, key, value) {
1323
+ const normalized = positiveFiniteNumber(value);
1324
+ if (normalized !== void 0) attrs[key] = normalized;
1325
+ }
1326
+ function assignModelCallSizeTimingAttrs(attrs, evt) {
1327
+ assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
1328
+ assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
1329
+ assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
1330
+ }
1331
+ function assignNumberAttr(attrs, key, value) {
1332
+ if (typeof value === "number" && Number.isFinite(value)) attrs[key] = value;
1333
+ }
1334
+ function modelCallPromptTokens(usage) {
1335
+ if (typeof usage.promptTokens === "number" && Number.isFinite(usage.promptTokens)) return usage.promptTokens;
1336
+ const input = usage.input ?? 0;
1337
+ const cacheRead = usage.cacheRead ?? 0;
1338
+ const cacheWrite = usage.cacheWrite ?? 0;
1339
+ const total = input + cacheRead + cacheWrite;
1340
+ return total > 0 ? total : void 0;
1341
+ }
1342
+ function assignModelCallPromptStatsAttrs(attrs, evt) {
1343
+ const stats = evt.promptStats;
1344
+ if (!stats) return;
1345
+ for (const [key, value] of [
1346
+ ["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
1347
+ ["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
1348
+ ["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
1349
+ ["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
1350
+ ["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
1351
+ ["openclaw.model_call.prompt.total_chars", stats.totalChars]
1352
+ ]) assignNumberAttr(attrs, key, value);
1353
+ }
1354
+ function assignModelCallUsageAttrs(attrs, evt) {
1355
+ const usage = evt.usage;
1356
+ if (!usage) return;
1357
+ const promptTokens = modelCallPromptTokens(usage);
1358
+ for (const [key, value] of [
1359
+ ["openclaw.model_call.usage.input_tokens", usage.input],
1360
+ ["openclaw.model_call.usage.output_tokens", usage.output],
1361
+ ["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
1362
+ ["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
1363
+ ["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
1364
+ ["openclaw.model_call.usage.prompt_tokens", promptTokens],
1365
+ ["openclaw.model_call.usage.total_tokens", usage.total],
1366
+ ["gen_ai.usage.input_tokens", promptTokens],
1367
+ ["gen_ai.usage.output_tokens", usage.output],
1368
+ ["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
1369
+ ["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
1370
+ ]) assignPositiveNumberAttr(attrs, key, value);
1371
+ }
1372
+ function assignGenAiSpanIdentityAttrs(attrs, input) {
1373
+ if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = lowCardinalityAttr(input.provider);
1374
+ else attrs["gen_ai.system"] = lowCardinalityAttr(input.provider);
1375
+ if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
1376
+ attrs["gen_ai.operation.name"] = genAiOperationName(input.api);
1377
+ }
1378
+ function assignGenAiModelCallAttrs(attrs, evt) {
1379
+ assignGenAiSpanIdentityAttrs(attrs, evt);
1380
+ }
1381
+ function modelCallSpanName(evt) {
1382
+ if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
1383
+ return `${genAiOperationName(evt.api)} ${lowCardinalityAttr(evt.model)}`;
1384
+ }
1385
+ function modelCallSpanKind() {
1386
+ return emitLatestGenAiSemconv() ? SpanKind.CLIENT : void 0;
1387
+ }
1388
+ function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
1389
+ if (!upstreamRequestIdHash) return;
1390
+ const boundedHash = lowCardinalityAttr(upstreamRequestIdHash);
1391
+ if (boundedHash === "unknown") return;
1392
+ span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
1393
+ }
1394
+ //#endregion
1395
+ //#region extensions/diagnostics-otel/src/service-genai-content.ts
1396
+ function textPart(content) {
1397
+ return {
1398
+ type: "text",
1399
+ content
1400
+ };
1401
+ }
1402
+ function textPartContent(part) {
1403
+ if (part.type !== "text") return;
1404
+ if (typeof part.text === "string") return part.text;
1405
+ return typeof part.content === "string" ? part.content : void 0;
1406
+ }
1407
+ function toolCallResponseValue(value) {
1408
+ if (!Array.isArray(value)) return value;
1409
+ const textItems = [];
1410
+ for (const item of value) {
1411
+ const text = typeof item === "string" ? item : isRecord(item) ? textPartContent(item) : void 0;
1412
+ if (typeof text !== "string") return value;
1413
+ textItems.push(text);
1414
+ }
1415
+ const kept = textItems.slice(0, 200);
1416
+ const joined = kept.filter((text) => text.length > 0).join("\n");
1417
+ if (joined.length === 0) return value;
1418
+ const omitted = textItems.length - kept.length;
1419
+ return omitted > 0 ? `${joined}\n...(${omitted} more text parts omitted)` : joined;
1420
+ }
1421
+ function toolCallResponsePart(part) {
1422
+ return {
1423
+ type: "tool_call_response",
1424
+ ...typeof part.id === "string" ? { id: part.id } : {},
1425
+ response: toolCallResponseValue(part.response ?? part.result ?? part.content ?? part.details ?? "")
1426
+ };
1427
+ }
1428
+ function contentParts(value) {
1429
+ if (typeof value === "string") return value.length > 0 ? [textPart(value)] : [];
1430
+ if (!Array.isArray(value)) {
1431
+ if (value === void 0 || value === null) return [];
1432
+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return [textPart(String(value))];
1433
+ const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
1434
+ return json ? [textPart(json)] : [];
1435
+ }
1436
+ const parts = [];
1437
+ for (const part of value) {
1438
+ if (typeof part === "string") {
1439
+ if (part.length > 0) parts.push(textPart(part));
1440
+ continue;
1441
+ }
1442
+ if (!isRecord(part)) continue;
1443
+ const text = textPartContent(part);
1444
+ if (text !== void 0) parts.push(textPart(text));
1445
+ else if (part.type === "thinking" && typeof part.thinking === "string") parts.push({
1446
+ type: "reasoning",
1447
+ content: part.thinking
1448
+ });
1449
+ else if (part.type === "toolCall" && typeof part.name === "string") parts.push({
1450
+ type: "tool_call",
1451
+ name: part.name,
1452
+ ...typeof part.id === "string" ? { id: part.id } : {},
1453
+ ...part.arguments !== void 0 ? { arguments: part.arguments } : {}
1454
+ });
1455
+ else if (part.type === "tool_call" && typeof part.name === "string") parts.push({
1456
+ type: "tool_call",
1457
+ name: part.name,
1458
+ ...typeof part.id === "string" ? { id: part.id } : {},
1459
+ ...part.arguments !== void 0 ? { arguments: part.arguments } : {}
1460
+ });
1461
+ else if (part.type === "tool_call_response") parts.push(toolCallResponsePart(part));
1462
+ else if (part.type === "image") {
1463
+ const data = typeof part.data === "string" ? part.data : void 0;
1464
+ parts.push({
1465
+ type: "blob",
1466
+ modality: "image",
1467
+ ...typeof part.mimeType === "string" ? { mime_type: part.mimeType } : {},
1468
+ ...typeof part.mime_type === "string" ? { mime_type: part.mime_type } : {},
1469
+ ...data ? { content: data } : {}
1470
+ });
1471
+ }
1472
+ }
1473
+ return parts;
1474
+ }
1475
+ function normalizeGenAiMessage(value, fallbackRole = "user") {
1476
+ if (typeof value === "string") return {
1477
+ role: fallbackRole,
1478
+ parts: [textPart(value)]
1479
+ };
1480
+ if (!isRecord(value)) return;
1481
+ const rawRole = typeof value.role === "string" ? value.role : fallbackRole;
1482
+ const role = rawRole === "toolResult" ? "tool" : rawRole;
1483
+ let parts;
1484
+ if (role === "tool") {
1485
+ const explicitParts = contentParts(value.parts);
1486
+ parts = explicitParts.length > 0 ? explicitParts : [toolCallResponsePart({
1487
+ id: value.toolCallId,
1488
+ response: value.content ?? value.details ?? ""
1489
+ })];
1490
+ } else parts = contentParts(value.parts ?? value.content);
1491
+ if (parts.length === 0) return;
1492
+ return {
1493
+ role,
1494
+ parts,
1495
+ ...typeof value.name === "string" ? { name: value.name } : {},
1496
+ ...typeof value.finish_reason === "string" ? { finish_reason: value.finish_reason } : {},
1497
+ ...typeof value.stopReason === "string" ? { finish_reason: value.stopReason } : {}
1498
+ };
1499
+ }
1500
+ function normalizeGenAiMessages(value, fallbackRole) {
1501
+ const source = Array.isArray(value) ? value : value === void 0 ? [] : [value];
1502
+ const messages = [];
1503
+ for (const item of source.slice(0, 200)) {
1504
+ const message = normalizeGenAiMessage(item, fallbackRole);
1505
+ if (message) messages.push(message);
1506
+ }
1507
+ return messages;
1508
+ }
1509
+ function normalizeGenAiToolDefinition(value) {
1510
+ if (!isRecord(value) || typeof value.name !== "string" || value.name.trim().length === 0) return;
1511
+ return {
1512
+ type: typeof value.type === "string" ? value.type : "function",
1513
+ name: value.name,
1514
+ ...typeof value.description === "string" ? { description: value.description } : {},
1515
+ ...value.parameters !== void 0 ? { parameters: value.parameters } : {}
1516
+ };
1517
+ }
1518
+ function normalizeGenAiToolDefinitions(value) {
1519
+ if (!Array.isArray(value)) return [];
1520
+ const definitions = [];
1521
+ for (const item of value.slice(0, 200)) {
1522
+ const definition = normalizeGenAiToolDefinition(item);
1523
+ if (definition) definitions.push(definition);
1524
+ }
1525
+ return definitions;
1526
+ }
1527
+ function assignJsonAttribute(attributes, key, value) {
1528
+ const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
1529
+ if (json) attributes[key] = json;
1530
+ }
1531
+ function assignGenAiModelContentAttributes(attributes, content, policy) {
1532
+ if (policy.systemPrompt && typeof content?.systemPrompt === "string") assignJsonAttribute(attributes, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, [textPart(content.systemPrompt)]);
1533
+ if (policy.inputMessages) {
1534
+ const inputMessages = normalizeGenAiMessages(content?.inputMessages, "user");
1535
+ if (inputMessages.length > 0) {
1536
+ assignJsonAttribute(attributes, ATTR_GEN_AI_INPUT_MESSAGES, inputMessages);
1537
+ assignJsonAttribute(attributes, "input.value", inputMessages);
1538
+ attributes["input.mime_type"] = "application/json";
1539
+ }
1540
+ }
1541
+ if (policy.toolDefinitions) {
1542
+ const toolDefinitions = normalizeGenAiToolDefinitions(content?.toolDefinitions);
1543
+ if (toolDefinitions.length > 0) assignJsonAttribute(attributes, ATTR_GEN_AI_TOOL_DEFINITIONS, toolDefinitions);
1544
+ }
1545
+ if (policy.outputMessages) {
1546
+ const outputMessages = normalizeGenAiMessages(content?.outputMessages, "assistant");
1547
+ if (outputMessages.length > 0) {
1548
+ assignJsonAttribute(attributes, ATTR_GEN_AI_OUTPUT_MESSAGES, outputMessages);
1549
+ assignJsonAttribute(attributes, "output.value", outputMessages);
1550
+ attributes["output.mime_type"] = "application/json";
1551
+ }
1552
+ }
1553
+ }
1554
+ function assignOtelContentAttribute(attributes, key, value) {
1555
+ const normalized = normalizeOtelContentValue(value);
1556
+ if (normalized) attributes[key] = normalized;
1557
+ }
1558
+ function assignOtelToolIdentityAttributes(attributes, evt) {
1559
+ attributes["gen_ai.operation.name"] = GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL;
1560
+ const toolCallId = evt.toolCallId?.trim();
1561
+ if (toolCallId) attributes[ATTR_GEN_AI_TOOL_CALL_ID] = toolCallId;
1562
+ }
1563
+ function assignOtelModelContentAttributes(attributes, content, policy) {
1564
+ assignGenAiModelContentAttributes(attributes, content, policy);
1565
+ if (policy.inputMessages) assignOtelContentAttribute(attributes, "openclaw.content.input_messages", content?.inputMessages);
1566
+ if (policy.toolDefinitions) assignOtelContentAttribute(attributes, "openclaw.content.tool_definitions", content?.toolDefinitions);
1567
+ if (policy.outputMessages) assignOtelContentAttribute(attributes, "openclaw.content.output_messages", content?.outputMessages);
1568
+ if (policy.systemPrompt) assignOtelContentAttribute(attributes, "openclaw.content.system_prompt", content?.systemPrompt);
1569
+ }
1570
+ function assignOtelToolContentAttributes(attributes, content, policy) {
1571
+ if (policy.toolInputs) {
1572
+ const toolInput = normalizeOtelContentValue(content?.toolInput);
1573
+ if (toolInput) {
1574
+ attributes[ATTR_GEN_AI_TOOL_CALL_ARGUMENTS] = toolInput;
1575
+ attributes["openclaw.content.tool_input"] = toolInput;
1576
+ }
1577
+ }
1578
+ if (policy.toolOutputs) {
1579
+ const toolOutput = normalizeOtelContentValue(content?.toolOutput);
1580
+ if (toolOutput) {
1581
+ attributes[ATTR_GEN_AI_TOOL_CALL_RESULT] = toolOutput;
1582
+ attributes["openclaw.content.tool_output"] = toolOutput;
1583
+ }
1584
+ }
1585
+ }
1586
+ //#endregion
1587
+ //#region extensions/diagnostics-otel/src/service-recorders-model.ts
1588
+ function createModelRecorders(runtime) {
1589
+ const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, contentCapturePolicy, tracesEnabled } = runtime;
1590
+ const modelCallMetricAttrs = (evt) => ({
1591
+ "openclaw.provider": evt.provider,
1592
+ "openclaw.model": evt.model,
1593
+ "openclaw.api": lowCardinalityAttr(evt.api),
1594
+ "openclaw.transport": lowCardinalityAttr(evt.transport)
1595
+ });
1596
+ const genAiModelCallMetricAttrs = (evt, errorType) => ({
1597
+ "gen_ai.operation.name": genAiOperationName(evt.api),
1598
+ "gen_ai.provider.name": lowCardinalityAttr(evt.provider),
1599
+ "gen_ai.request.model": lowCardinalityAttr(evt.model),
1600
+ ...errorType ? { "error.type": errorType } : {}
1601
+ });
1602
+ const recordModelCallSizeTimingMetrics = (evt, attrs) => {
1603
+ const requestPayloadBytes = positiveFiniteNumber(evt.requestPayloadBytes);
1604
+ if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
1605
+ const responseStreamBytes = positiveFiniteNumber(evt.responseStreamBytes);
1606
+ if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
1607
+ const timeToFirstByteMs = positiveFiniteNumber(evt.timeToFirstByteMs);
1608
+ if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
1609
+ };
1610
+ const recordModelCallStarted = (evt, metadata) => {
1611
+ if (!tracesEnabled || !metadata.trusted) return;
1612
+ const spanAttrs = {
1613
+ "openclaw.provider": evt.provider,
1614
+ "openclaw.model": evt.model
1615
+ };
1616
+ assignGenAiModelCallAttrs(spanAttrs, evt);
1617
+ if (evt.api) spanAttrs["openclaw.api"] = evt.api;
1618
+ if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
1619
+ assignModelCallPromptStatsAttrs(spanAttrs, evt);
1620
+ trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
1621
+ kind: modelCallSpanKind(),
1622
+ parentContext: activeTrustedParentContext(evt, metadata),
1623
+ startTimeMs: evt.ts
1624
+ }));
1625
+ };
1626
+ const recordModelCallCompleted = (evt, metadata, modelContent) => {
1627
+ const metricAttrs = modelCallMetricAttrs(evt);
1628
+ modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
1629
+ recordModelCallSizeTimingMetrics(evt, metricAttrs);
1630
+ genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt));
1631
+ if (!tracesEnabled) return;
1632
+ const spanAttrs = {
1633
+ "openclaw.provider": evt.provider,
1634
+ "openclaw.model": evt.model
1635
+ };
1636
+ assignGenAiModelCallAttrs(spanAttrs, evt);
1637
+ if (evt.api) spanAttrs["openclaw.api"] = evt.api;
1638
+ if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
1639
+ assignModelCallSizeTimingAttrs(spanAttrs, evt);
1640
+ assignModelCallPromptStatsAttrs(spanAttrs, evt);
1641
+ assignModelCallUsageAttrs(spanAttrs, evt);
1642
+ assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
1643
+ const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
1644
+ kind: modelCallSpanKind(),
1645
+ parentContext: activeTrustedParentContext(evt, metadata),
1646
+ endTimeMs: evt.ts
1647
+ });
1648
+ setSpanAttrs(span, spanAttrs);
1649
+ addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
1650
+ span.end(evt.ts);
1651
+ };
1652
+ const recordModelCallError = (evt, metadata, modelContent) => {
1653
+ const errorType = lowCardinalityAttr(evt.errorCategory, "other");
1654
+ const metricAttrs = {
1655
+ ...modelCallMetricAttrs(evt),
1656
+ "openclaw.errorCategory": errorType,
1657
+ ...evt.failureKind ? { "openclaw.failureKind": lowCardinalityAttr(evt.failureKind, "other") } : {}
1658
+ };
1659
+ modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
1660
+ recordModelCallSizeTimingMetrics(evt, metricAttrs);
1661
+ genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt, errorType));
1662
+ if (!tracesEnabled) return;
1663
+ const spanAttrs = {
1664
+ "openclaw.provider": evt.provider,
1665
+ "openclaw.model": evt.model,
1666
+ "openclaw.errorCategory": errorType,
1667
+ "error.type": errorType
1668
+ };
1669
+ if (evt.failureKind) spanAttrs["openclaw.failureKind"] = lowCardinalityAttr(evt.failureKind, "other");
1670
+ assignGenAiModelCallAttrs(spanAttrs, evt);
1671
+ if (evt.api) spanAttrs["openclaw.api"] = evt.api;
1672
+ if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
1673
+ assignModelCallSizeTimingAttrs(spanAttrs, evt);
1674
+ assignModelCallPromptStatsAttrs(spanAttrs, evt);
1675
+ assignModelCallUsageAttrs(spanAttrs, evt);
1676
+ assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
1677
+ const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
1678
+ kind: modelCallSpanKind(),
1679
+ parentContext: activeTrustedParentContext(evt, metadata),
1680
+ endTimeMs: evt.ts
1681
+ });
1682
+ setSpanAttrs(span, spanAttrs);
1683
+ addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
1684
+ span.setStatus({
1685
+ code: SpanStatusCode.ERROR,
1686
+ message: redactSensitiveText(evt.errorCategory)
1687
+ });
1688
+ span.end(evt.ts);
1689
+ };
1690
+ return {
1691
+ recordModelCallSizeTimingMetrics,
1692
+ recordModelCallStarted,
1693
+ recordModelCallCompleted,
1694
+ recordModelCallError
1695
+ };
1696
+ }
1697
+ //#endregion
1698
+ //#region extensions/diagnostics-otel/src/service-recorders-operations.ts
1699
+ function createOperationsRecorders(runtime) {
1700
+ const { durationHistogram, queueDepthHistogram, queueWaitHistogram, laneEnqueueCounter, laneDequeueCounter, sessionStateCounter, sessionTurnCreatedCounter, sessionStuckCounter, sessionStuckAgeHistogram, sessionRecoveryRequestedCounter, sessionRecoveryCompletedCounter, sessionRecoveryAgeHistogram, talkEventCounter, talkEventDurationHistogram, talkAudioBytesHistogram, runAttemptCounter, toolLoopCounter, memoryRssHistogram, memoryHeapUsedHistogram, memoryHeapTotalHistogram, memoryExternalHistogram, memoryArrayBuffersHistogram, memoryPressureCounter, asyncQueueDroppedCounter, tracer, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
1701
+ const recordLaneEnqueue = (evt) => {
1702
+ const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
1703
+ laneEnqueueCounter.add(1, attrs);
1704
+ queueDepthHistogram.record(evt.queueSize, attrs);
1705
+ };
1706
+ const recordLaneDequeue = (evt) => {
1707
+ const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
1708
+ laneDequeueCounter.add(1, attrs);
1709
+ queueDepthHistogram.record(evt.queueSize, attrs);
1710
+ if (typeof evt.waitMs === "number") queueWaitHistogram.record(evt.waitMs, attrs);
1711
+ };
1712
+ const recordSessionState = (evt) => {
1713
+ const attrs = { "openclaw.state": evt.state };
1714
+ if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
1715
+ sessionStateCounter.add(1, attrs);
1716
+ };
1717
+ const recordSessionTurnCreated = (evt) => {
1718
+ sessionTurnCreatedCounter.add(1, {
1719
+ "openclaw.agent": lowCardinalityAttr(evt.agentId, "unknown"),
1720
+ "openclaw.channel": lowCardinalityAttr(evt.channel, "unknown"),
1721
+ "openclaw.trigger": evt.trigger
1722
+ });
1723
+ };
1724
+ const recordSessionStuck = (evt) => {
1725
+ const attrs = { "openclaw.state": evt.state };
1726
+ sessionStuckCounter.add(1, attrs);
1727
+ if (typeof evt.ageMs === "number") sessionStuckAgeHistogram.record(evt.ageMs, attrs);
1728
+ if (!tracesEnabled) return;
1729
+ const spanAttrs = { ...attrs };
1730
+ spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
1731
+ spanAttrs["openclaw.ageMs"] = evt.ageMs;
1732
+ const span = tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
1733
+ span.setStatus({
1734
+ code: SpanStatusCode.ERROR,
1735
+ message: "session stuck"
1736
+ });
1737
+ span.end();
1738
+ };
1739
+ const sessionRecoveryAttrs = (evt) => {
1740
+ const attrs = { "openclaw.state": evt.state };
1741
+ if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
1742
+ if (evt.activeWorkKind) attrs["openclaw.active_work_kind"] = evt.activeWorkKind;
1743
+ return attrs;
1744
+ };
1745
+ const recordSessionRecoveryRequested = (evt) => {
1746
+ const attrs = sessionRecoveryAttrs(evt);
1747
+ attrs["openclaw.action"] = evt.allowActiveAbort ? "abort" : "recover";
1748
+ sessionRecoveryRequestedCounter.add(1, attrs);
1749
+ sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
1750
+ };
1751
+ const recordSessionRecoveryCompleted = (evt) => {
1752
+ const attrs = sessionRecoveryAttrs(evt);
1753
+ attrs["openclaw.status"] = evt.status;
1754
+ attrs["openclaw.action"] = lowCardinalityAttr(evt.action, "unknown");
1755
+ if (evt.outcomeReason) attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason);
1756
+ sessionRecoveryCompletedCounter.add(1, attrs);
1757
+ sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
1758
+ };
1759
+ const talkEventAttrs = (evt) => ({
1760
+ "openclaw.talk.brain": lowCardinalityAttr(evt.brain),
1761
+ "openclaw.talk.event_type": lowCardinalityAttr(evt.talkEventType),
1762
+ "openclaw.talk.mode": lowCardinalityAttr(evt.mode),
1763
+ "openclaw.talk.provider": lowCardinalityAttr(evt.provider),
1764
+ "openclaw.talk.transport": lowCardinalityAttr(evt.transport)
1765
+ });
1766
+ const recordTalkEvent = (evt, metadata) => {
1767
+ if (!metadata.trusted) return;
1768
+ const attrs = talkEventAttrs(evt);
1769
+ talkEventCounter.add(1, attrs);
1770
+ if (typeof evt.durationMs === "number") talkEventDurationHistogram.record(evt.durationMs, attrs);
1771
+ if (typeof evt.byteLength === "number") talkAudioBytesHistogram.record(evt.byteLength, attrs);
1772
+ };
1773
+ const recordRunAttempt = (evt) => {
1774
+ runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
1775
+ };
1776
+ const toolLoopAttrs = (evt) => ({
1777
+ "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool"),
1778
+ "openclaw.loop.level": evt.level,
1779
+ "openclaw.loop.action": evt.action,
1780
+ "openclaw.loop.detector": evt.detector,
1781
+ "openclaw.loop.count": evt.count,
1782
+ ...evt.pairedToolName ? { "openclaw.loop.paired_tool": lowCardinalityAttr(evt.pairedToolName, "tool") } : {}
1783
+ });
1784
+ const recordToolLoop = (evt) => {
1785
+ const attrs = toolLoopAttrs(evt);
1786
+ toolLoopCounter.add(1, attrs);
1787
+ if (!tracesEnabled) return;
1788
+ const span = spanWithDuration("openclaw.tool.loop", attrs, 0, { endTimeMs: evt.ts });
1789
+ if (evt.level === "critical" || evt.action === "block") span.setStatus({
1790
+ code: SpanStatusCode.ERROR,
1791
+ message: `${evt.detector}:${evt.action}`
1792
+ });
1793
+ span.end(evt.ts);
1794
+ };
1795
+ const recordMemoryUsageMetrics = (evt, attrs = {}) => {
1796
+ memoryRssHistogram.record(evt.memory.rssBytes, attrs);
1797
+ memoryHeapUsedHistogram.record(evt.memory.heapUsedBytes, attrs);
1798
+ memoryHeapTotalHistogram.record(evt.memory.heapTotalBytes, attrs);
1799
+ memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
1800
+ memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
1801
+ };
1802
+ const recordMemorySample = (evt) => {
1803
+ recordMemoryUsageMetrics(evt);
1804
+ };
1805
+ const recordMemoryPressure = (evt) => {
1806
+ const attrs = {
1807
+ "openclaw.memory.level": evt.level,
1808
+ "openclaw.memory.reason": evt.reason
1809
+ };
1810
+ memoryPressureCounter.add(1, attrs);
1811
+ recordMemoryUsageMetrics(evt, attrs);
1812
+ if (!tracesEnabled) return;
1813
+ const spanAttrs = {
1814
+ ...attrs,
1815
+ "openclaw.memory.rss_bytes": evt.memory.rssBytes,
1816
+ "openclaw.memory.heap_used_bytes": evt.memory.heapUsedBytes,
1817
+ "openclaw.memory.heap_total_bytes": evt.memory.heapTotalBytes,
1818
+ "openclaw.memory.external_bytes": evt.memory.externalBytes,
1819
+ "openclaw.memory.array_buffers_bytes": evt.memory.arrayBuffersBytes,
1820
+ ...evt.thresholdBytes !== void 0 ? { "openclaw.memory.threshold_bytes": evt.thresholdBytes } : {},
1821
+ ...evt.rssGrowthBytes !== void 0 ? { "openclaw.memory.rss_growth_bytes": evt.rssGrowthBytes } : {},
1822
+ ...evt.windowMs !== void 0 ? { "openclaw.memory.window_ms": evt.windowMs } : {}
1823
+ };
1824
+ const span = spanWithDuration("openclaw.memory.pressure", spanAttrs, 0, { endTimeMs: evt.ts });
1825
+ if (evt.level === "critical") span.setStatus({
1826
+ code: SpanStatusCode.ERROR,
1827
+ message: evt.reason
1828
+ });
1829
+ span.end(evt.ts);
1830
+ };
1831
+ const recordAsyncQueueDropped = (evt) => {
1832
+ asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
1833
+ if (evt.droppedTrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedTrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "trusted" });
1834
+ if (evt.droppedUntrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedUntrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "untrusted" });
1835
+ if (evt.droppedPriorityEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedPriorityEvents, { "openclaw.diagnostic.async_queue.drop_class": "priority" });
1836
+ };
1837
+ const recordRunCompleted = (evt, metadata, privateData) => {
1838
+ const attrs = {
1839
+ "openclaw.outcome": evt.outcome,
1840
+ "openclaw.provider": evt.provider ?? "unknown",
1841
+ "openclaw.model": evt.model ?? "unknown"
1842
+ };
1843
+ if (evt.channel) attrs["openclaw.channel"] = evt.channel;
1844
+ if (evt.blockedBy) attrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
1845
+ durationHistogram.record(evt.durationMs, attrs);
1846
+ if (!tracesEnabled) return;
1847
+ const spanAttrs = { "openclaw.outcome": evt.outcome };
1848
+ addRunAttrs(spanAttrs, evt);
1849
+ if (evt.blockedBy) spanAttrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
1850
+ if (evt.errorCategory) spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other");
1851
+ const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
1852
+ if (redactedError) spanAttrs["openclaw.error"] = redactedError;
1853
+ const trustedTrace = trustedTraceContext(evt, metadata);
1854
+ const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
1855
+ const span = trackedSpan ?? spanWithDuration("openclaw.run", spanAttrs, evt.durationMs, {
1856
+ parentContext: activeTrustedParentContext(evt, metadata),
1857
+ endTimeMs: evt.ts
1858
+ });
1859
+ setSpanAttrs(span, spanAttrs);
1860
+ if (evt.outcome === "error") {
1861
+ const message = redactedError ?? (evt.errorCategory ? redactSensitiveText(evt.errorCategory) : void 0);
1862
+ span.setStatus({
1863
+ code: SpanStatusCode.ERROR,
1864
+ ...message ? { message } : {}
1865
+ });
1866
+ }
1867
+ if (trackedSpan && trustedTrace?.spanId) {
1868
+ completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
1869
+ return;
1870
+ }
1871
+ span.end(evt.ts);
1872
+ };
1873
+ return {
1874
+ recordLaneEnqueue,
1875
+ recordLaneDequeue,
1876
+ recordSessionState,
1877
+ recordSessionTurnCreated,
1878
+ recordSessionStuck,
1879
+ recordSessionRecoveryRequested,
1880
+ recordSessionRecoveryCompleted,
1881
+ recordTalkEvent,
1882
+ recordRunAttempt,
1883
+ recordToolLoop,
1884
+ recordMemoryUsageMetrics,
1885
+ recordMemorySample,
1886
+ recordMemoryPressure,
1887
+ recordAsyncQueueDropped,
1888
+ recordRunCompleted
1889
+ };
1890
+ }
1891
+ //#endregion
1892
+ //#region extensions/diagnostics-otel/src/service-recorders-tools.ts
1893
+ function createToolAndSystemRecorders(runtime) {
1894
+ const { queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
1895
+ const toolExecutionBaseAttrs = (evt) => ({
1896
+ "openclaw.toolName": evt.toolName,
1897
+ "openclaw.tool.source": lowCardinalityAttr(evt.toolSource, "core"),
1898
+ "gen_ai.tool.name": evt.toolName,
1899
+ ...evt.toolOwner ? { "openclaw.tool.owner": lowCardinalityAttr(evt.toolOwner) } : {},
1900
+ ...paramsSummaryAttrs(evt.paramsSummary)
1901
+ });
1902
+ const skillUsedAttrs = (evt) => ({
1903
+ "openclaw.skill.name": lowCardinalityAttr(evt.skillName, "skill"),
1904
+ "openclaw.skill.source": lowCardinalityAttr(evt.skillSource),
1905
+ "openclaw.skill.activation": lowCardinalityAttr(evt.activation),
1906
+ ...evt.agentId ? { "openclaw.agent": lowCardinalityAttr(evt.agentId) } : {},
1907
+ ...evt.toolName ? { "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool") } : {}
1908
+ });
1909
+ const recordSkillUsed = (evt, metadata) => {
1910
+ if (!metadata.trusted) return;
1911
+ const attrs = skillUsedAttrs(evt);
1912
+ skillUsedCounter.add(1, attrs);
1913
+ if (!tracesEnabled) return;
1914
+ const spanAttrs = { ...attrs };
1915
+ addRunAttrs(spanAttrs, evt);
1916
+ const span = spanWithDuration("openclaw.skill.used", spanAttrs, 0, {
1917
+ parentContext: activeTrustedParentContext(evt, metadata),
1918
+ endTimeMs: evt.ts
1919
+ });
1920
+ setSpanAttrs(span, spanAttrs);
1921
+ span.end(evt.ts);
1922
+ };
1923
+ const recordToolExecutionStarted = (evt, metadata) => {
1924
+ if (!tracesEnabled || !metadata.trusted) return;
1925
+ const spanAttrs = toolExecutionBaseAttrs(evt);
1926
+ assignOtelToolIdentityAttributes(spanAttrs, evt);
1927
+ trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.tool.execution", spanAttrs, void 0, {
1928
+ parentContext: activeTrustedParentContext(evt, metadata),
1929
+ startTimeMs: evt.ts
1930
+ }));
1931
+ };
1932
+ const recordToolExecutionCompleted = (evt, metadata, toolContent) => {
1933
+ const attrs = toolExecutionBaseAttrs(evt);
1934
+ toolExecutionDurationHistogram.record(evt.durationMs, attrs);
1935
+ if (!tracesEnabled) return;
1936
+ const spanAttrs = { ...attrs };
1937
+ addRunAttrs(spanAttrs, evt);
1938
+ assignOtelToolIdentityAttributes(spanAttrs, evt);
1939
+ assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
1940
+ const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
1941
+ parentContext: activeTrustedParentContext(evt, metadata),
1942
+ endTimeMs: evt.ts
1943
+ });
1944
+ setSpanAttrs(span, spanAttrs);
1945
+ span.end(evt.ts);
1946
+ };
1947
+ const recordToolExecutionError = (evt, metadata, toolContent) => {
1948
+ const attrs = {
1949
+ ...toolExecutionBaseAttrs(evt),
1950
+ "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
1951
+ };
1952
+ toolExecutionDurationHistogram.record(evt.durationMs, attrs);
1953
+ if (!tracesEnabled) return;
1954
+ const spanAttrs = { ...attrs };
1955
+ addRunAttrs(spanAttrs, evt);
1956
+ assignOtelToolIdentityAttributes(spanAttrs, evt);
1957
+ if (evt.errorCode) spanAttrs["openclaw.errorCode"] = lowCardinalityAttr(evt.errorCode, "other");
1958
+ assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
1959
+ const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
1960
+ parentContext: activeTrustedParentContext(evt, metadata),
1961
+ endTimeMs: evt.ts
1962
+ });
1963
+ setSpanAttrs(span, spanAttrs);
1964
+ span.setStatus({
1965
+ code: SpanStatusCode.ERROR,
1966
+ message: redactSensitiveText(evt.errorCategory)
1967
+ });
1968
+ span.end(evt.ts);
1969
+ };
1970
+ const recordToolExecutionBlocked = (evt, metadata) => {
1971
+ toolExecutionBlockedCounter.add(1, {
1972
+ ...toolExecutionBaseAttrs(evt),
1973
+ "openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
1974
+ });
1975
+ if (!tracesEnabled) return;
1976
+ const spanAttrs = {
1977
+ ...toolExecutionBaseAttrs(evt),
1978
+ "openclaw.outcome": "blocked",
1979
+ "openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
1980
+ };
1981
+ addRunAttrs(spanAttrs, evt);
1982
+ assignOtelToolIdentityAttributes(spanAttrs, evt);
1983
+ const span = spanWithDuration("openclaw.tool.execution", spanAttrs, 0, {
1984
+ parentContext: activeTrustedParentContext(evt, metadata),
1985
+ endTimeMs: evt.ts
1986
+ });
1987
+ setSpanAttrs(span, spanAttrs);
1988
+ span.end(evt.ts);
1989
+ };
1990
+ const recordPayloadLarge = (evt) => {
1991
+ const attrs = {
1992
+ "openclaw.payload.action": evt.action,
1993
+ "openclaw.payload.surface": lowCardinalityAttr(evt.surface, "unknown"),
1994
+ "openclaw.channel": lowCardinalityAttr(evt.channel, "none"),
1995
+ "openclaw.plugin": lowCardinalityAttr(evt.pluginId, "none"),
1996
+ "openclaw.reason": lowCardinalityAttr(evt.reason, "none")
1997
+ };
1998
+ payloadLargeCounter.add(1, attrs);
1999
+ const bytes = positiveFiniteNumber(evt.bytes);
2000
+ if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
2001
+ };
2002
+ const recordExecProcessCompleted = (evt) => {
2003
+ const attrs = {
2004
+ "openclaw.exec.target": evt.target,
2005
+ "openclaw.exec.mode": evt.mode,
2006
+ "openclaw.outcome": evt.outcome
2007
+ };
2008
+ if (evt.failureKind) attrs["openclaw.failureKind"] = evt.failureKind;
2009
+ execProcessDurationHistogram.record(evt.durationMs, attrs);
2010
+ if (!tracesEnabled) return;
2011
+ const spanAttrs = {
2012
+ ...attrs,
2013
+ "openclaw.exec.command_length": evt.commandLength
2014
+ };
2015
+ if (typeof evt.exitCode === "number") spanAttrs["openclaw.exec.exit_code"] = evt.exitCode;
2016
+ if (evt.exitSignal) spanAttrs["openclaw.exec.exit_signal"] = lowCardinalityAttr(evt.exitSignal, "other");
2017
+ if (evt.timedOut !== void 0) spanAttrs["openclaw.exec.timed_out"] = evt.timedOut;
2018
+ const span = spanWithDuration("openclaw.exec", spanAttrs, evt.durationMs, { endTimeMs: evt.ts });
2019
+ if (evt.outcome === "failed") span.setStatus({
2020
+ code: SpanStatusCode.ERROR,
2021
+ ...evt.failureKind ? { message: evt.failureKind } : {}
2022
+ });
2023
+ span.end(evt.ts);
2024
+ };
2025
+ const recordHeartbeat = (evt) => {
2026
+ queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
2027
+ };
2028
+ const recordLivenessWarning = (evt) => {
2029
+ const reason = evt.reasons.join(":");
2030
+ const attrs = { "openclaw.liveness.reason": lowCardinalityAttr(reason, "unknown") };
2031
+ livenessWarningCounter.add(1, attrs);
2032
+ queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" });
2033
+ if (evt.eventLoopDelayP99Ms !== void 0) livenessEventLoopDelayP99Histogram.record(evt.eventLoopDelayP99Ms, attrs);
2034
+ if (evt.eventLoopDelayMaxMs !== void 0) livenessEventLoopDelayMaxHistogram.record(evt.eventLoopDelayMaxMs, attrs);
2035
+ if (evt.eventLoopUtilization !== void 0) livenessEventLoopUtilizationHistogram.record(evt.eventLoopUtilization, attrs);
2036
+ if (evt.cpuCoreRatio !== void 0) livenessCpuCoreRatioHistogram.record(evt.cpuCoreRatio, attrs);
2037
+ if (!tracesEnabled) return;
2038
+ const spanAttrs = {
2039
+ ...attrs,
2040
+ "openclaw.liveness.active": evt.active,
2041
+ "openclaw.liveness.waiting": evt.waiting,
2042
+ "openclaw.liveness.queued": evt.queued,
2043
+ "openclaw.liveness.interval_ms": evt.intervalMs,
2044
+ ...evt.eventLoopDelayP99Ms !== void 0 ? { "openclaw.liveness.event_loop_delay_p99_ms": evt.eventLoopDelayP99Ms } : {},
2045
+ ...evt.eventLoopDelayMaxMs !== void 0 ? { "openclaw.liveness.event_loop_delay_max_ms": evt.eventLoopDelayMaxMs } : {},
2046
+ ...evt.eventLoopUtilization !== void 0 ? { "openclaw.liveness.event_loop_utilization": evt.eventLoopUtilization } : {},
2047
+ ...evt.cpuUserMs !== void 0 ? { "openclaw.liveness.cpu_user_ms": evt.cpuUserMs } : {},
2048
+ ...evt.cpuSystemMs !== void 0 ? { "openclaw.liveness.cpu_system_ms": evt.cpuSystemMs } : {},
2049
+ ...evt.cpuTotalMs !== void 0 ? { "openclaw.liveness.cpu_total_ms": evt.cpuTotalMs } : {},
2050
+ ...evt.cpuCoreRatio !== void 0 ? { "openclaw.liveness.cpu_core_ratio": evt.cpuCoreRatio } : {}
2051
+ };
2052
+ const span = spanWithDuration("openclaw.liveness.warning", spanAttrs, 0, { endTimeMs: evt.ts });
2053
+ span.setStatus({
2054
+ code: SpanStatusCode.ERROR,
2055
+ message: reason
2056
+ });
2057
+ span.end(evt.ts);
2058
+ };
2059
+ const recordDiagnosticPhaseCompleted = (evt) => {
2060
+ if (!tracesEnabled) return;
2061
+ const spanAttrs = {
2062
+ "openclaw.phase": lowCardinalityAttr(evt.name, "unknown"),
2063
+ ...evt.cpuUserMs !== void 0 ? { "openclaw.phase.cpu_user_ms": evt.cpuUserMs } : {},
2064
+ ...evt.cpuSystemMs !== void 0 ? { "openclaw.phase.cpu_system_ms": evt.cpuSystemMs } : {},
2065
+ ...evt.cpuTotalMs !== void 0 ? { "openclaw.phase.cpu_total_ms": evt.cpuTotalMs } : {},
2066
+ ...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
2067
+ };
2068
+ for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
2069
+ spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, { endTimeMs: evt.ts }).end(evt.ts);
2070
+ };
2071
+ const recordTelemetryExporter = (evt, metadata) => {
2072
+ if (!metadata.trusted) return;
2073
+ telemetryExporterCounter.add(1, {
2074
+ "openclaw.exporter": lowCardinalityAttr(evt.exporter, "unknown"),
2075
+ "openclaw.signal": evt.signal,
2076
+ "openclaw.status": evt.status,
2077
+ ...evt.reason ? { "openclaw.reason": evt.reason } : {},
2078
+ ...evt.errorCategory ? { "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other") } : {}
2079
+ });
2080
+ };
2081
+ return {
2082
+ recordSkillUsed,
2083
+ recordToolExecutionStarted,
2084
+ recordToolExecutionCompleted,
2085
+ recordToolExecutionError,
2086
+ recordToolExecutionBlocked,
2087
+ recordPayloadLarge,
2088
+ recordExecProcessCompleted,
2089
+ recordHeartbeat,
2090
+ recordLivenessWarning,
2091
+ recordDiagnosticPhaseCompleted,
2092
+ recordTelemetryExporter
2093
+ };
2094
+ }
2095
+ //#endregion
2096
+ //#region extensions/diagnostics-otel/src/service-recorders-usage.ts
2097
+ function createUsageRecorders(runtime) {
2098
+ const { tokensCounter, genAiTokenUsageHistogram, costCounter, durationHistogram, contextHistogram, webhookReceivedCounter, webhookErrorCounter, webhookDurationHistogram, messageQueuedCounter, messageReceivedCounter, messageDispatchStartedCounter, messageDispatchCompletedCounter, messageDispatchDurationHistogram, messageProcessedCounter, messageDurationHistogram, messageDeliveryStartedCounter, messageDeliveryDurationHistogram, queueDepthHistogram, tracer, activeTrustedSpans, activeTrustedSpanAliases, trustedSpanAliasKey, spanWithDuration, trustedTraceContext, internalOrTrustedTraceContext, internalOrTrustedExplicitParentContext, activeTrustedParentContext, activeInternalOrTrustedContext, trackTrustedSpan, trackInternalOrTrustedSpan, getTrackedInternalOrTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
2099
+ const recordModelUsage = (evt, metadata) => {
2100
+ const attrs = {
2101
+ "openclaw.channel": evt.channel ?? "unknown",
2102
+ "openclaw.agent": lowCardinalityAttr(evt.agentId),
2103
+ "openclaw.provider": evt.provider ?? "unknown",
2104
+ "openclaw.model": evt.model ?? "unknown"
2105
+ };
2106
+ const genAiAttrs = {
2107
+ "gen_ai.operation.name": "chat",
2108
+ "gen_ai.provider.name": lowCardinalityAttr(evt.provider),
2109
+ "gen_ai.request.model": lowCardinalityAttr(evt.model)
2110
+ };
2111
+ const usage = evt.usage;
2112
+ if (usage.input) {
2113
+ tokensCounter.add(usage.input, {
2114
+ ...attrs,
2115
+ "openclaw.token": "input"
2116
+ });
2117
+ genAiTokenUsageHistogram.record(usage.input, {
2118
+ ...genAiAttrs,
2119
+ "gen_ai.token.type": "input"
2120
+ });
2121
+ }
2122
+ if (usage.output) {
2123
+ tokensCounter.add(usage.output, {
2124
+ ...attrs,
2125
+ "openclaw.token": "output"
2126
+ });
2127
+ genAiTokenUsageHistogram.record(usage.output, {
2128
+ ...genAiAttrs,
2129
+ "gen_ai.token.type": "output"
2130
+ });
2131
+ }
2132
+ if (usage.cacheRead) tokensCounter.add(usage.cacheRead, {
2133
+ ...attrs,
2134
+ "openclaw.token": "cache_read"
2135
+ });
2136
+ if (usage.cacheWrite) tokensCounter.add(usage.cacheWrite, {
2137
+ ...attrs,
2138
+ "openclaw.token": "cache_write"
2139
+ });
2140
+ if (usage.promptTokens) tokensCounter.add(usage.promptTokens, {
2141
+ ...attrs,
2142
+ "openclaw.token": "prompt"
2143
+ });
2144
+ if (usage.total) tokensCounter.add(usage.total, {
2145
+ ...attrs,
2146
+ "openclaw.token": "total"
2147
+ });
2148
+ if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
2149
+ if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
2150
+ if (evt.context?.limit) contextHistogram.record(evt.context.limit, {
2151
+ ...attrs,
2152
+ "openclaw.context": "limit"
2153
+ });
2154
+ if (evt.context?.used) contextHistogram.record(evt.context.used, {
2155
+ ...attrs,
2156
+ "openclaw.context": "used"
2157
+ });
2158
+ if (!tracesEnabled) return;
2159
+ const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
2160
+ const spanAttrs = {
2161
+ ...attrs,
2162
+ "openclaw.tokens.input": usage.input ?? 0,
2163
+ "openclaw.tokens.output": usage.output ?? 0,
2164
+ "openclaw.tokens.cache_read": usage.cacheRead ?? 0,
2165
+ "openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
2166
+ "openclaw.tokens.total": usage.total ?? 0
2167
+ };
2168
+ assignGenAiSpanIdentityAttrs(spanAttrs, evt);
2169
+ assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
2170
+ assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
2171
+ assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
2172
+ assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite);
2173
+ spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs, {
2174
+ parentContext: activeTrustedParentContext(evt, metadata),
2175
+ endTimeMs: evt.ts
2176
+ }).end(evt.ts);
2177
+ };
2178
+ const recordWebhookReceived = (evt) => {
2179
+ const attrs = {
2180
+ "openclaw.channel": evt.channel ?? "unknown",
2181
+ "openclaw.webhook": evt.updateType ?? "unknown"
2182
+ };
2183
+ webhookReceivedCounter.add(1, attrs);
2184
+ };
2185
+ const recordWebhookProcessed = (evt) => {
2186
+ const attrs = {
2187
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2188
+ "openclaw.webhook": lowCardinalityAttr(evt.updateType)
2189
+ };
2190
+ if (typeof evt.durationMs === "number") webhookDurationHistogram.record(evt.durationMs, attrs);
2191
+ if (!tracesEnabled) return;
2192
+ const spanAttrs = { ...attrs };
2193
+ spanWithDuration("openclaw.webhook.processed", spanAttrs, evt.durationMs).end();
2194
+ };
2195
+ const recordWebhookError = (evt) => {
2196
+ const attrs = {
2197
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2198
+ "openclaw.webhook": lowCardinalityAttr(evt.updateType)
2199
+ };
2200
+ webhookErrorCounter.add(1, attrs);
2201
+ if (!tracesEnabled) return;
2202
+ const redactedError = redactSensitiveText(evt.error);
2203
+ const spanAttrs = {
2204
+ ...attrs,
2205
+ "openclaw.error": redactedError
2206
+ };
2207
+ const span = tracer.startSpan("openclaw.webhook.error", { attributes: spanAttrs });
2208
+ span.setStatus({
2209
+ code: SpanStatusCode.ERROR,
2210
+ message: redactedError
2211
+ });
2212
+ span.end();
2213
+ };
2214
+ const recordMessageQueued = (evt) => {
2215
+ const attrs = {
2216
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2217
+ "openclaw.source": lowCardinalityAttr(evt.source)
2218
+ };
2219
+ messageQueuedCounter.add(1, attrs);
2220
+ if (typeof evt.queueDepth === "number") queueDepthHistogram.record(evt.queueDepth, attrs);
2221
+ };
2222
+ const recordMessageReceived = (evt) => {
2223
+ messageReceivedCounter.add(1, {
2224
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2225
+ "openclaw.source": lowCardinalityAttr(evt.source)
2226
+ });
2227
+ };
2228
+ const recordMessageDispatchStarted = (evt, metadata) => {
2229
+ const attrs = {
2230
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2231
+ "openclaw.source": lowCardinalityAttr(evt.source)
2232
+ };
2233
+ messageDispatchStartedCounter.add(1, attrs);
2234
+ if (!tracesEnabled) return;
2235
+ const traceContext = internalOrTrustedTraceContext(evt, metadata);
2236
+ if (!traceContext?.spanId || activeTrustedSpans.has(traceContext.spanId)) return;
2237
+ trackInternalOrTrustedSpan(evt, metadata, spanWithDuration("openclaw.message.processed", attrs, void 0, {
2238
+ parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
2239
+ startTimeMs: evt.ts
2240
+ }));
2241
+ };
2242
+ const recordMessageDispatchCompleted = (evt) => {
2243
+ const attrs = {
2244
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2245
+ "openclaw.outcome": evt.outcome,
2246
+ "openclaw.reason": lowCardinalityAttr(evt.reason, "none"),
2247
+ "openclaw.source": lowCardinalityAttr(evt.source)
2248
+ };
2249
+ messageDispatchCompletedCounter.add(1, attrs);
2250
+ messageDispatchDurationHistogram.record(evt.durationMs, attrs);
2251
+ };
2252
+ const recordMessageProcessed = (evt, metadata) => {
2253
+ const attrs = {
2254
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2255
+ "openclaw.outcome": evt.outcome ?? "unknown"
2256
+ };
2257
+ messageProcessedCounter.add(1, attrs);
2258
+ if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
2259
+ if (!tracesEnabled) return;
2260
+ const spanAttrs = { ...attrs };
2261
+ if (evt.reason) spanAttrs["openclaw.reason"] = lowCardinalityAttr(evt.reason, "unknown");
2262
+ const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
2263
+ const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
2264
+ parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
2265
+ endTimeMs: evt.ts
2266
+ });
2267
+ setSpanAttrs(span, spanAttrs);
2268
+ if (evt.outcome === "error" && evt.error) span.setStatus({
2269
+ code: SpanStatusCode.ERROR,
2270
+ message: redactSensitiveText(evt.error)
2271
+ });
2272
+ const traceContext = internalOrTrustedTraceContext(evt, metadata);
2273
+ if (trackedSpan && traceContext?.spanId) {
2274
+ completeTrackedLifecycleSpan(traceContext.spanId, trackedSpan, evt.ts);
2275
+ return;
2276
+ }
2277
+ span.end(evt.ts);
2278
+ };
2279
+ const messageDeliveryAttrs = (evt) => ({
2280
+ "openclaw.channel": lowCardinalityAttr(evt.channel),
2281
+ "openclaw.delivery.kind": lowCardinalityAttr(evt.deliveryKind, "other")
2282
+ });
2283
+ const recordMessageDeliveryStarted = (evt) => {
2284
+ messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
2285
+ };
2286
+ const recordMessageDeliveryCompleted = (evt, metadata) => {
2287
+ const attrs = {
2288
+ ...messageDeliveryAttrs(evt),
2289
+ "openclaw.outcome": "completed"
2290
+ };
2291
+ messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
2292
+ if (!tracesEnabled) return;
2293
+ spanWithDuration("openclaw.message.delivery", {
2294
+ ...attrs,
2295
+ "openclaw.delivery.result_count": evt.resultCount
2296
+ }, evt.durationMs, {
2297
+ parentContext: activeInternalOrTrustedContext(evt, metadata),
2298
+ endTimeMs: evt.ts
2299
+ }).end(evt.ts);
2300
+ };
2301
+ const recordMessageDeliveryError = (evt, metadata) => {
2302
+ const attrs = {
2303
+ ...messageDeliveryAttrs(evt),
2304
+ "openclaw.outcome": "error",
2305
+ "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
2306
+ };
2307
+ messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
2308
+ if (!tracesEnabled) return;
2309
+ const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, {
2310
+ parentContext: activeInternalOrTrustedContext(evt, metadata),
2311
+ endTimeMs: evt.ts
2312
+ });
2313
+ span.setStatus({
2314
+ code: SpanStatusCode.ERROR,
2315
+ message: redactSensitiveText(evt.errorCategory)
2316
+ });
2317
+ span.end(evt.ts);
2318
+ };
2319
+ const recordRunStarted = (evt, metadata) => {
2320
+ if (!tracesEnabled || !metadata.trusted) return;
2321
+ const spanAttrs = {};
2322
+ addRunAttrs(spanAttrs, evt);
2323
+ const span = trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.run", spanAttrs, void 0, {
2324
+ parentContext: activeTrustedParentContext(evt, metadata),
2325
+ startTimeMs: evt.ts
2326
+ }));
2327
+ const parentSpanId = trustedTraceContext(evt, metadata)?.parentSpanId;
2328
+ if (parentSpanId && !activeTrustedSpans.has(parentSpanId)) {
2329
+ const owner = {
2330
+ kind: "run",
2331
+ id: evt.runId
2332
+ };
2333
+ activeTrustedSpanAliases.set(trustedSpanAliasKey(parentSpanId, owner), {
2334
+ span,
2335
+ spanId: parentSpanId,
2336
+ owner
2337
+ });
2338
+ }
2339
+ };
2340
+ return {
2341
+ recordModelUsage,
2342
+ recordWebhookReceived,
2343
+ recordWebhookProcessed,
2344
+ recordWebhookError,
2345
+ recordMessageQueued,
2346
+ recordMessageReceived,
2347
+ recordMessageDispatchStarted,
2348
+ recordMessageDispatchCompleted,
2349
+ recordMessageProcessed,
2350
+ recordMessageDeliveryStarted,
2351
+ recordMessageDeliveryCompleted,
2352
+ recordMessageDeliveryError,
2353
+ recordRunStarted
2354
+ };
2355
+ }
2356
+ //#endregion
2357
+ //#region extensions/diagnostics-otel/src/service-traces.ts
2358
+ function createDiagnosticsTraceRuntime(tracer) {
2359
+ const activeTrustedSpans = /* @__PURE__ */ new Map();
2360
+ const activeTrustedSpanAliases = /* @__PURE__ */ new Map();
2361
+ const retainedTrustedSpanContexts = /* @__PURE__ */ new Map();
2362
+ const retainedTrustedSpanContextCleanupTimers = /* @__PURE__ */ new Set();
2363
+ const stopActiveTrustedSpans = () => {
2364
+ const stopAt = Date.now();
2365
+ for (const handle of retainedTrustedSpanContextCleanupTimers) clearTimeout(handle);
2366
+ retainedTrustedSpanContextCleanupTimers.clear();
2367
+ retainedTrustedSpanContexts.clear();
2368
+ for (const span of /* @__PURE__ */ new Set([...activeTrustedSpans.values(), ...Array.from(activeTrustedSpanAliases.values(), (entry) => entry.span)])) span.end(stopAt);
2369
+ activeTrustedSpans.clear();
2370
+ activeTrustedSpanAliases.clear();
2371
+ };
2372
+ const spanWithDuration = (name, attributes, durationMs, options = {}) => {
2373
+ const endTimeMs = options.endTimeMs ?? Date.now();
2374
+ const startTime = typeof options.startTimeMs === "number" ? options.startTimeMs : typeof durationMs === "number" && durationMs >= 0 ? endTimeMs - durationMs : void 0;
2375
+ const parentContext = "parentContext" in options ? options.parentContext ?? void 0 : void 0;
2376
+ return tracer.startSpan(name, {
2377
+ attributes: redactOtelAttributes(attributes),
2378
+ ...options.kind !== void 0 ? { kind: options.kind } : {},
2379
+ ...startTime !== void 0 ? { startTime } : {}
2380
+ }, parentContext);
2381
+ };
2382
+ const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
2383
+ const internalOrTrustedTraceContext = (evt, metadata) => metadata.trusted || metadata.internal ? normalizeTraceContext(evt.trace) : void 0;
2384
+ const trustedSpanAliasOwner = (evt) => {
2385
+ if ("runId" in evt && evt.runId) return {
2386
+ kind: "run",
2387
+ id: evt.runId
2388
+ };
2389
+ };
2390
+ const sameTrustedSpanAliasOwner = (left, right) => Boolean(left && right && left.kind === right.kind && left.id === right.id);
2391
+ const trustedSpanAliasKey = (spanId, owner) => `${spanId}:${owner.kind}:${owner.id}`;
2392
+ const retainedTrustedSpanContextKey = (traceId, spanId, owner) => `${traceId}:${owner ? trustedSpanAliasKey(spanId, owner) : spanId}`;
2393
+ const retainedTrustedSpanContext = (traceContext, spanId, owner) => {
2394
+ if (!traceContext?.traceId || !spanId) return;
2395
+ const retained = (owner ? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId, owner)) : void 0) ?? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId));
2396
+ if (retained?.spanContext.traceId !== traceContext.traceId) return;
2397
+ if (retained.owner && !sameTrustedSpanAliasOwner(retained.owner, owner)) return;
2398
+ return retained.spanContext;
2399
+ };
2400
+ const activeTrustedSpanAlias = (spanId, owner) => {
2401
+ if (!owner) return;
2402
+ const alias = activeTrustedSpanAliases.get(trustedSpanAliasKey(spanId, owner));
2403
+ if (!alias || !sameTrustedSpanAliasOwner(alias.owner, owner)) return;
2404
+ return alias.span;
2405
+ };
2406
+ const internalOrTrustedParentContext = (evt, metadata) => {
2407
+ const traceContext = internalOrTrustedTraceContext(evt, metadata);
2408
+ const parentSpanId = traceContext?.parentSpanId ?? traceContext?.spanId;
2409
+ if (!traceContext || !parentSpanId) return;
2410
+ return contextForTraceContext({
2411
+ ...traceContext,
2412
+ spanId: parentSpanId
2413
+ });
2414
+ };
2415
+ const internalOrTrustedExplicitParentContext = (evt, metadata) => {
2416
+ const traceContext = internalOrTrustedTraceContext(evt, metadata);
2417
+ if (!traceContext?.parentSpanId) return;
2418
+ return contextForTraceContext({
2419
+ ...traceContext,
2420
+ spanId: traceContext.parentSpanId
2421
+ });
2422
+ };
2423
+ const activeTrustedParentContext = (evt, metadata) => {
2424
+ const traceContext = trustedTraceContext(evt, metadata);
2425
+ const parentSpanId = traceContext?.parentSpanId;
2426
+ if (!parentSpanId) return;
2427
+ const owner = trustedSpanAliasOwner(evt);
2428
+ const spanContext = (activeTrustedSpans.get(parentSpanId) ?? activeTrustedSpanAlias(parentSpanId, owner))?.spanContext() ?? retainedTrustedSpanContext(traceContext, parentSpanId, owner);
2429
+ if (!spanContext) return;
2430
+ return trace.setSpanContext(context.active(), spanContext);
2431
+ };
2432
+ const activeInternalOrTrustedContext = (evt, metadata) => {
2433
+ const traceContext = internalOrTrustedTraceContext(evt, metadata);
2434
+ if (!traceContext) return;
2435
+ const owner = trustedSpanAliasOwner(evt);
2436
+ const activeSpan = (traceContext.spanId ? activeTrustedSpans.get(traceContext.spanId) ?? activeTrustedSpanAlias(traceContext.spanId, owner) : void 0) ?? (traceContext.parentSpanId ? activeTrustedSpans.get(traceContext.parentSpanId) ?? activeTrustedSpanAlias(traceContext.parentSpanId, owner) : void 0);
2437
+ if (activeSpan) return trace.setSpanContext(context.active(), activeSpan.spanContext());
2438
+ const retainedSpanContext = retainedTrustedSpanContext(traceContext, traceContext.spanId, owner) ?? retainedTrustedSpanContext(traceContext, traceContext.parentSpanId, owner);
2439
+ if (retainedSpanContext) return trace.setSpanContext(context.active(), retainedSpanContext);
2440
+ return internalOrTrustedParentContext(evt, metadata);
2441
+ };
2442
+ const trackTrustedSpan = (evt, metadata, span) => {
2443
+ const spanId = trustedTraceContext(evt, metadata)?.spanId;
2444
+ if (spanId) activeTrustedSpans.set(spanId, span);
2445
+ return span;
2446
+ };
2447
+ const trackInternalOrTrustedSpan = (evt, metadata, span) => {
2448
+ const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
2449
+ if (spanId) activeTrustedSpans.set(spanId, span);
2450
+ return span;
2451
+ };
2452
+ const takeTrackedTrustedSpan = (evt, metadata) => {
2453
+ const spanId = trustedTraceContext(evt, metadata)?.spanId;
2454
+ if (!spanId) return;
2455
+ const span = activeTrustedSpans.get(spanId);
2456
+ if (span) activeTrustedSpans.delete(spanId);
2457
+ return span;
2458
+ };
2459
+ const getTrackedInternalOrTrustedSpan = (evt, metadata) => {
2460
+ const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
2461
+ if (!spanId) return;
2462
+ return activeTrustedSpans.get(spanId);
2463
+ };
2464
+ const setSpanAttrs = (span, attributes) => {
2465
+ span.setAttributes?.(redactOtelAttributes(attributes));
2466
+ };
2467
+ const retainTrustedSpanContext = (traceId, spanId, spanContext, retentionMarker, owner) => {
2468
+ retainedTrustedSpanContexts.set(retainedTrustedSpanContextKey(traceId, spanId, owner), {
2469
+ spanContext,
2470
+ retentionMarker,
2471
+ ...owner ? { owner } : {}
2472
+ });
2473
+ while (retainedTrustedSpanContexts.size > MAX_RETAINED_TRUSTED_SPAN_CONTEXTS) {
2474
+ const oldestKey = retainedTrustedSpanContexts.keys().next().value;
2475
+ if (!oldestKey) break;
2476
+ retainedTrustedSpanContexts.delete(oldestKey);
2477
+ }
2478
+ };
2479
+ const scheduleRetainedTrustedSpanContextCleanup = (retentionMarker) => {
2480
+ let drainHandle;
2481
+ let timeoutHandle;
2482
+ const cleanup = () => {
2483
+ if (drainHandle) {
2484
+ clearTimeout(drainHandle);
2485
+ retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
2486
+ drainHandle = void 0;
2487
+ }
2488
+ if (timeoutHandle) {
2489
+ clearTimeout(timeoutHandle);
2490
+ retainedTrustedSpanContextCleanupTimers.delete(timeoutHandle);
2491
+ timeoutHandle = void 0;
2492
+ }
2493
+ for (const [key, retained] of retainedTrustedSpanContexts) if (retained.retentionMarker === retentionMarker) retainedTrustedSpanContexts.delete(key);
2494
+ };
2495
+ drainHandle = setTimeout(() => {
2496
+ if (drainHandle) {
2497
+ retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
2498
+ drainHandle = void 0;
2499
+ }
2500
+ waitForDiagnosticEventsDrained().then(cleanup, cleanup);
2501
+ }, 0);
2502
+ drainHandle.unref?.();
2503
+ retainedTrustedSpanContextCleanupTimers.add(drainHandle);
2504
+ timeoutHandle = setTimeout(cleanup, RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS);
2505
+ timeoutHandle.unref?.();
2506
+ retainedTrustedSpanContextCleanupTimers.add(timeoutHandle);
2507
+ };
2508
+ const completeTrackedLifecycleSpan = (spanId, span, endTimeMs) => {
2509
+ const spanContext = span.spanContext();
2510
+ const retainedKeys = [{ spanId }];
2511
+ const retainedAliasKeys = [];
2512
+ for (const [aliasKey, alias] of activeTrustedSpanAliases) if (alias.span === span) {
2513
+ retainedKeys.push({
2514
+ spanId: alias.spanId,
2515
+ owner: alias.owner
2516
+ });
2517
+ retainedAliasKeys.push(aliasKey);
2518
+ }
2519
+ if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
2520
+ for (const aliasKey of retainedAliasKeys) if (activeTrustedSpanAliases.get(aliasKey)?.span === span) activeTrustedSpanAliases.delete(aliasKey);
2521
+ span.end(endTimeMs);
2522
+ const retentionMarker = Symbol("retainedTrustedSpanContext");
2523
+ for (const retainedKey of retainedKeys) retainTrustedSpanContext(spanContext.traceId, retainedKey.spanId, spanContext, retentionMarker, retainedKey.owner);
2524
+ scheduleRetainedTrustedSpanContextCleanup(retentionMarker);
2525
+ };
2526
+ const addRunAttrs = (spanAttrs, evt) => {
2527
+ if (evt.provider) spanAttrs["openclaw.provider"] = evt.provider;
2528
+ if (evt.model) spanAttrs["openclaw.model"] = evt.model;
2529
+ if (evt.channel) spanAttrs["openclaw.channel"] = evt.channel;
2530
+ if (evt.trigger) spanAttrs["openclaw.trigger"] = evt.trigger;
2531
+ };
2532
+ const paramsSummaryAttrs = (summary) => {
2533
+ if (!summary) return {};
2534
+ return {
2535
+ "openclaw.tool.params.kind": summary.kind,
2536
+ ..."length" in summary ? { "openclaw.tool.params.length": summary.length } : {}
2537
+ };
2538
+ };
2539
+ return {
2540
+ tracer,
2541
+ activeTrustedSpans,
2542
+ activeTrustedSpanAliases,
2543
+ trustedSpanAliasKey,
2544
+ trustedSpanAliasOwner,
2545
+ spanWithDuration,
2546
+ trustedTraceContext,
2547
+ internalOrTrustedTraceContext,
2548
+ internalOrTrustedParentContext,
2549
+ internalOrTrustedExplicitParentContext,
2550
+ activeTrustedParentContext,
2551
+ activeInternalOrTrustedContext,
2552
+ trackTrustedSpan,
2553
+ trackInternalOrTrustedSpan,
2554
+ takeTrackedTrustedSpan,
2555
+ getTrackedInternalOrTrustedSpan,
2556
+ setSpanAttrs,
2557
+ completeTrackedLifecycleSpan,
2558
+ addRunAttrs,
2559
+ paramsSummaryAttrs,
2560
+ stopActiveTrustedSpans
2561
+ };
2562
+ }
2563
+ //#endregion
2564
+ //#region extensions/diagnostics-otel/src/service.ts
2565
+ function createDiagnosticsOtelService() {
2566
+ let sdk = null;
2567
+ let logProvider = null;
2568
+ let unsubscribe = null;
2569
+ let stopActiveTrustedSpans = null;
2570
+ let unregisterUnhandledRejectionHandler = null;
2571
+ const stopStarted = async () => {
2572
+ const currentUnsubscribe = unsubscribe;
2573
+ const currentLogProvider = logProvider;
2574
+ const currentSdk = sdk;
2575
+ const currentStopActiveTrustedSpans = stopActiveTrustedSpans;
2576
+ const currentUnregisterUnhandledRejectionHandler = unregisterUnhandledRejectionHandler;
2577
+ unsubscribe = null;
2578
+ logProvider = null;
2579
+ sdk = null;
2580
+ stopActiveTrustedSpans = null;
2581
+ unregisterUnhandledRejectionHandler = null;
2582
+ currentUnregisterUnhandledRejectionHandler?.();
2583
+ currentUnsubscribe?.();
2584
+ currentStopActiveTrustedSpans?.();
2585
+ if (currentLogProvider) await currentLogProvider.shutdown().catch(() => void 0);
2586
+ if (currentSdk) await currentSdk.shutdown().catch(() => void 0);
2587
+ };
2588
+ return {
2589
+ id: "diagnostics-otel",
2590
+ async start(ctx) {
2591
+ await stopStarted();
2592
+ const cfg = ctx.config.diagnostics;
2593
+ const otel = cfg?.otel;
2594
+ if (!cfg || cfg.enabled === false || !otel?.enabled) return;
2595
+ const emitExporterEvent = (event) => {
2596
+ try {
2597
+ ctx.internalDiagnostics?.emit({
2598
+ type: "telemetry.exporter",
2599
+ ...event
2600
+ });
2601
+ } catch {}
2602
+ };
2603
+ const emitForSignals = (signals, event) => {
2604
+ for (const signal of signals) emitExporterEvent({
2605
+ signal,
2606
+ ...event
2607
+ });
2608
+ };
2609
+ const tracesEnabled = otel.traces !== false;
2610
+ const metricsEnabled = otel.metrics !== false;
2611
+ const logsEnabled = otel.logs === true;
2612
+ const logsExporter = otel.logsExporter ?? "otlp";
2613
+ const logsToOtlp = logsEnabled && (logsExporter === "otlp" || logsExporter === "both");
2614
+ const logsToStdout = logsEnabled && (logsExporter === "stdout" || logsExporter === "both");
2615
+ const otlpSignals = [
2616
+ ...tracesEnabled ? ["traces"] : [],
2617
+ ...metricsEnabled ? ["metrics"] : [],
2618
+ ...logsToOtlp ? ["logs"] : []
2619
+ ];
2620
+ const enabledSignals = [
2621
+ ...tracesEnabled ? ["traces"] : [],
2622
+ ...metricsEnabled ? ["metrics"] : [],
2623
+ ...logsEnabled ? ["logs"] : []
2624
+ ];
2625
+ if (enabledSignals.length === 0) return;
2626
+ const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
2627
+ if (otlpSignals.length > 0 && protocol !== "http/protobuf") {
2628
+ emitForSignals(otlpSignals, {
2629
+ exporter: "diagnostics-otel",
2630
+ status: "failure",
2631
+ reason: "unsupported_protocol"
2632
+ });
2633
+ ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
2634
+ return;
2635
+ }
2636
+ const endpoint = normalizeEndpoint(otel.endpoint ?? process.env["OTEL_EXPORTER_OTLP_ENDPOINT"]);
2637
+ const headers = otel.headers ?? void 0;
2638
+ const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || "openclaw";
2639
+ const sampleRate = resolveSampleRate(otel.sampleRate);
2640
+ const contentCapturePolicy = resolveContentCapturePolicy(otel.captureContent);
2641
+ const sdkPreloaded = hasPreloadedOtelSdk();
2642
+ const resource = resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
2643
+ const logUrl = resolveSignalOtelUrl({
2644
+ signalEndpoint: otel.logsEndpoint,
2645
+ signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV],
2646
+ endpoint,
2647
+ path: "v1/logs"
2648
+ });
2649
+ if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) {
2650
+ const traceUrl = resolveSignalOtelUrl({
2651
+ signalEndpoint: otel.tracesEndpoint,
2652
+ signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV],
2653
+ endpoint,
2654
+ path: "v1/traces"
2655
+ });
2656
+ const metricUrl = resolveSignalOtelUrl({
2657
+ signalEndpoint: otel.metricsEndpoint,
2658
+ signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV],
2659
+ endpoint,
2660
+ path: "v1/metrics"
2661
+ });
2662
+ const traceHttpAgentOptions = resolveOtelHttpAgentOptions({
2663
+ url: traceUrl,
2664
+ signalIdentifier: "TRACES",
2665
+ logger: ctx.logger
2666
+ });
2667
+ const metricHttpAgentOptions = resolveOtelHttpAgentOptions({
2668
+ url: metricUrl,
2669
+ signalIdentifier: "METRICS",
2670
+ logger: ctx.logger
2671
+ });
2672
+ const traceExporter = tracesEnabled ? new OTLPTraceExporter({
2673
+ ...traceUrl ? { url: traceUrl } : {},
2674
+ ...headers ? { headers } : {},
2675
+ ...traceHttpAgentOptions ? { httpAgentOptions: traceHttpAgentOptions } : {}
2676
+ }) : void 0;
2677
+ const spanProcessors = traceExporter && typeof otel.flushIntervalMs === "number" ? [new BatchSpanProcessor(traceExporter, { scheduledDelayMillis: Math.max(1e3, otel.flushIntervalMs) })] : void 0;
2678
+ const metricExporter = metricsEnabled ? new OTLPMetricExporter({
2679
+ ...metricUrl ? { url: metricUrl } : {},
2680
+ ...headers ? { headers } : {},
2681
+ ...metricHttpAgentOptions ? { httpAgentOptions: metricHttpAgentOptions } : {}
2682
+ }) : void 0;
2683
+ const metricReader = metricExporter ? new PeriodicExportingMetricReader({
2684
+ exporter: metricExporter,
2685
+ ...typeof otel.flushIntervalMs === "number" ? { exportIntervalMillis: Math.max(1e3, otel.flushIntervalMs) } : {}
2686
+ }) : void 0;
2687
+ sdk = new NodeSDK({
2688
+ resource,
2689
+ ...spanProcessors ? { spanProcessors } : traceExporter ? { traceExporter } : {},
2690
+ ...metricReader ? { metricReader } : {},
2691
+ ...sampleRate !== void 0 ? { sampler: new ParentBasedSampler({ root: new TraceIdRatioBasedSampler(sampleRate) }) } : {}
2692
+ });
2693
+ try {
2694
+ sdk.start();
2695
+ } catch (err) {
2696
+ emitForSignals([...tracesEnabled ? ["traces"] : [], ...metricsEnabled ? ["metrics"] : []], {
2697
+ exporter: "diagnostics-otel",
2698
+ status: "failure",
2699
+ reason: "start_failed",
2700
+ errorCategory: errorCategory(err)
2701
+ });
2702
+ await stopStarted();
2703
+ ctx.logger.error(`diagnostics-otel: failed to start SDK: ${formatError(err)}`);
2704
+ throw err;
2705
+ }
2706
+ } else if (sdkPreloaded && (tracesEnabled || metricsEnabled)) ctx.logger.info("diagnostics-otel: using preloaded OpenTelemetry SDK");
2707
+ const meter = metrics.getMeter("openclaw");
2708
+ const diagnosticsTrace = createDiagnosticsTraceRuntime(trace.getTracer("openclaw"));
2709
+ stopActiveTrustedSpans = diagnosticsTrace.stopActiveTrustedSpans;
2710
+ const diagnosticMetrics = createDiagnosticsMetrics(meter);
2711
+ const diagnosticsLogs = createDiagnosticsLogExporter({
2712
+ contentCapturePolicy,
2713
+ emitExporterEvent,
2714
+ flushIntervalMs: otel.flushIntervalMs,
2715
+ headers,
2716
+ logger: ctx.logger,
2717
+ logsEnabled,
2718
+ logsToOtlp,
2719
+ logsToStdout,
2720
+ logUrl,
2721
+ resource,
2722
+ serviceName
2723
+ });
2724
+ logProvider = diagnosticsLogs.logProvider;
2725
+ const { recordLogRecord, recordSecurityEvent } = diagnosticsLogs;
2726
+ const recorderRuntime = createDiagnosticsRecorderRuntime({
2727
+ contentCapturePolicy,
2728
+ metrics: diagnosticMetrics,
2729
+ traces: diagnosticsTrace,
2730
+ tracesEnabled
2731
+ });
2732
+ const recorders = {
2733
+ ...createUsageRecorders(recorderRuntime),
2734
+ ...createOperationsRecorders(recorderRuntime),
2735
+ ...createHarnessRecorders(recorderRuntime),
2736
+ ...createModelRecorders(recorderRuntime),
2737
+ ...createToolAndSystemRecorders(recorderRuntime)
2738
+ };
2739
+ const subscribe = ctx.internalDiagnostics?.onEvent;
2740
+ if (!subscribe) {
2741
+ ctx.logger.error("diagnostics-otel: internal diagnostics capability unavailable");
2742
+ return;
2743
+ }
2744
+ unsubscribe = subscribe(createDiagnosticsEventHandler({
2745
+ logger: ctx.logger,
2746
+ recorders,
2747
+ recordLogRecord,
2748
+ recordSecurityEvent
2749
+ }));
2750
+ unregisterUnhandledRejectionHandler = registerUnhandledRejectionHandler((reason) => {
2751
+ const otlpError = findOtlpExporterError(reason);
2752
+ if (!otlpError) return false;
2753
+ const code = readErrorCode(otlpError) ?? "unknown";
2754
+ ctx.logger.warn(`diagnostics-otel: suppressed OTLP exporter unhandled rejection (code=${String(code)})`);
2755
+ return true;
2756
+ });
2757
+ emitForSignals(enabledSignals, {
2758
+ exporter: "diagnostics-otel",
2759
+ status: "started",
2760
+ reason: "configured"
2761
+ });
2762
+ if (logsEnabled) {
2763
+ const label = logsExporter === "both" ? "OTLP/Protobuf + stdout JSONL" : logsExporter === "stdout" ? "stdout JSONL" : "OTLP/Protobuf";
2764
+ ctx.logger.info(`diagnostics-otel: logs exporter enabled (${label})`);
2765
+ }
2766
+ },
2767
+ async stop() {
2768
+ await stopStarted();
2769
+ }
2770
+ };
2771
+ }
2772
+ //#endregion
2773
+ export { createDiagnosticsOtelService as t };