@openclaw/diagnostics-otel 2026.7.1-beta.6 → 2026.7.2-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -1
- package/dist/runtime-api-DHI2xBKF.js +2773 -0
- package/dist/runtime-api.js +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +4 -4
- package/dist/runtime-api-BUlvn2ET.js +0 -2589
|
@@ -0,0 +1,2773 @@
|
|
|
1
|
+
import { isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, redactSensitiveText } from "./api.js";
|
|
2
|
+
import { SpanKind, SpanStatusCode, TraceFlags, context, metrics, trace } from "@opentelemetry/api";
|
|
3
|
+
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
|
|
4
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
5
|
+
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
6
|
+
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
7
|
+
import { NodeSDK } from "@opentelemetry/sdk-node";
|
|
8
|
+
import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
|
9
|
+
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
10
|
+
import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
|
|
11
|
+
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
12
|
+
import { waitForDiagnosticEventsDrained } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
13
|
+
import { readFileSync } from "node:fs";
|
|
14
|
+
import nodePath from "node:path";
|
|
15
|
+
import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime";
|
|
16
|
+
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
17
|
+
import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
18
|
+
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_TOOL_DEFINITIONS, GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL } from "@opentelemetry/semantic-conventions/incubating";
|
|
19
|
+
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
20
|
+
const DROPPED_OTEL_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
|
|
21
|
+
"openclaw.callId",
|
|
22
|
+
"openclaw.call_id",
|
|
23
|
+
"openclaw.chatId",
|
|
24
|
+
"openclaw.chat_id",
|
|
25
|
+
"openclaw.messageId",
|
|
26
|
+
"openclaw.message_id",
|
|
27
|
+
"openclaw.parentSpanId",
|
|
28
|
+
"openclaw.parent_span_id",
|
|
29
|
+
"openclaw.runId",
|
|
30
|
+
"openclaw.run_id",
|
|
31
|
+
"openclaw.sessionId",
|
|
32
|
+
"openclaw.session_id",
|
|
33
|
+
"openclaw.sessionKey",
|
|
34
|
+
"openclaw.session_key",
|
|
35
|
+
"openclaw.spanId",
|
|
36
|
+
"openclaw.span_id",
|
|
37
|
+
"openclaw.toolCallId",
|
|
38
|
+
"openclaw.tool_call_id",
|
|
39
|
+
"openclaw.traceId",
|
|
40
|
+
"openclaw.trace_id"
|
|
41
|
+
]);
|
|
42
|
+
const LOW_CARDINALITY_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u;
|
|
43
|
+
const SECURITY_TARGET_NAME_VALUE_RE = /^[A-Za-z0-9@/_.:-]{1,256}$/u;
|
|
44
|
+
const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024;
|
|
45
|
+
const MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS = 4 * 1024;
|
|
46
|
+
const OTEL_LOG_RAW_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,64}$/u;
|
|
47
|
+
const OTEL_LOG_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,96}$/u;
|
|
48
|
+
const BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
|
|
49
|
+
"__proto__",
|
|
50
|
+
"prototype",
|
|
51
|
+
"constructor"
|
|
52
|
+
]);
|
|
53
|
+
const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
|
|
54
|
+
const OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT";
|
|
55
|
+
const OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT";
|
|
56
|
+
const OTEL_EXPORTER_OTLP_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CERTIFICATE";
|
|
57
|
+
const OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE";
|
|
58
|
+
const OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV = "OTEL_EXPORTER_OTLP_CLIENT_KEY";
|
|
59
|
+
const OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN";
|
|
60
|
+
const GEN_AI_LATEST_EXPERIMENTAL_OPT_IN = "gen_ai_latest_experimental";
|
|
61
|
+
const GEN_AI_TOKEN_USAGE_BUCKETS = [
|
|
62
|
+
1,
|
|
63
|
+
4,
|
|
64
|
+
16,
|
|
65
|
+
64,
|
|
66
|
+
256,
|
|
67
|
+
1024,
|
|
68
|
+
4096,
|
|
69
|
+
16384,
|
|
70
|
+
65536,
|
|
71
|
+
262144,
|
|
72
|
+
1048576,
|
|
73
|
+
4194304,
|
|
74
|
+
16777216,
|
|
75
|
+
67108864
|
|
76
|
+
];
|
|
77
|
+
const GEN_AI_OPERATION_DURATION_BUCKETS = [
|
|
78
|
+
.01,
|
|
79
|
+
.02,
|
|
80
|
+
.04,
|
|
81
|
+
.08,
|
|
82
|
+
.16,
|
|
83
|
+
.32,
|
|
84
|
+
.64,
|
|
85
|
+
1.28,
|
|
86
|
+
2.56,
|
|
87
|
+
5.12,
|
|
88
|
+
10.24,
|
|
89
|
+
20.48,
|
|
90
|
+
40.96,
|
|
91
|
+
81.92
|
|
92
|
+
];
|
|
93
|
+
const OTEL_DEFAULT_HISTOGRAM_BUCKETS = [
|
|
94
|
+
0,
|
|
95
|
+
5,
|
|
96
|
+
10,
|
|
97
|
+
25,
|
|
98
|
+
50,
|
|
99
|
+
75,
|
|
100
|
+
100,
|
|
101
|
+
250,
|
|
102
|
+
500,
|
|
103
|
+
750,
|
|
104
|
+
1e3,
|
|
105
|
+
2500,
|
|
106
|
+
5e3,
|
|
107
|
+
7500,
|
|
108
|
+
1e4
|
|
109
|
+
];
|
|
110
|
+
const AGENT_DURATION_MS_BUCKETS = [
|
|
111
|
+
...OTEL_DEFAULT_HISTOGRAM_BUCKETS,
|
|
112
|
+
15e3,
|
|
113
|
+
2e4,
|
|
114
|
+
3e4,
|
|
115
|
+
45e3,
|
|
116
|
+
6e4,
|
|
117
|
+
12e4,
|
|
118
|
+
18e4,
|
|
119
|
+
24e4,
|
|
120
|
+
3e5,
|
|
121
|
+
6e5,
|
|
122
|
+
9e5,
|
|
123
|
+
18e5,
|
|
124
|
+
36e5
|
|
125
|
+
];
|
|
126
|
+
const CONTEXT_TOKENS_BUCKETS = [
|
|
127
|
+
...OTEL_DEFAULT_HISTOGRAM_BUCKETS,
|
|
128
|
+
16e3,
|
|
129
|
+
32e3,
|
|
130
|
+
64e3,
|
|
131
|
+
128e3,
|
|
132
|
+
2e5,
|
|
133
|
+
4e5,
|
|
134
|
+
1e6,
|
|
135
|
+
2e6
|
|
136
|
+
];
|
|
137
|
+
const MAX_RETAINED_TRUSTED_SPAN_CONTEXTS = 1024;
|
|
138
|
+
const RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS = 5e3;
|
|
139
|
+
//#endregion
|
|
140
|
+
//#region extensions/diagnostics-otel/src/service-content-normalization.ts
|
|
141
|
+
const MAX_OTEL_CONTENT_ATTRIBUTE_CHARS = 128 * 1024;
|
|
142
|
+
const MAX_OTEL_ERROR_MESSAGE_CHARS = 4 * 1024;
|
|
143
|
+
const PRELOADED_OTEL_SDK_ENV = "OPENCLAW_OTEL_PRELOADED";
|
|
144
|
+
const NO_CONTENT_CAPTURE = {
|
|
145
|
+
inputMessages: false,
|
|
146
|
+
outputMessages: false,
|
|
147
|
+
toolInputs: false,
|
|
148
|
+
toolOutputs: false,
|
|
149
|
+
systemPrompt: false,
|
|
150
|
+
toolDefinitions: false,
|
|
151
|
+
logBodies: false
|
|
152
|
+
};
|
|
153
|
+
function clampOtelLogText(value, maxChars) {
|
|
154
|
+
return value.length > maxChars ? `${truncateUtf16Safe(value, maxChars)}...(truncated)` : value;
|
|
155
|
+
}
|
|
156
|
+
function normalizeOtelLogString(value, maxChars) {
|
|
157
|
+
return clampOtelLogText(redactSensitiveText(value), maxChars);
|
|
158
|
+
}
|
|
159
|
+
function normalizeOtelErrorMessage(value) {
|
|
160
|
+
if (!value) return;
|
|
161
|
+
return normalizeOtelLogString(value.trim(), MAX_OTEL_ERROR_MESSAGE_CHARS) || void 0;
|
|
162
|
+
}
|
|
163
|
+
function resolveContentCapturePolicy(value) {
|
|
164
|
+
if (value === true) return {
|
|
165
|
+
inputMessages: true,
|
|
166
|
+
outputMessages: true,
|
|
167
|
+
toolInputs: true,
|
|
168
|
+
toolOutputs: true,
|
|
169
|
+
systemPrompt: false,
|
|
170
|
+
toolDefinitions: true,
|
|
171
|
+
logBodies: true
|
|
172
|
+
};
|
|
173
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return NO_CONTENT_CAPTURE;
|
|
174
|
+
const config = value;
|
|
175
|
+
if (config.enabled !== true) return NO_CONTENT_CAPTURE;
|
|
176
|
+
return {
|
|
177
|
+
inputMessages: config.inputMessages === true,
|
|
178
|
+
outputMessages: config.outputMessages === true,
|
|
179
|
+
toolInputs: config.toolInputs === true,
|
|
180
|
+
toolOutputs: config.toolOutputs === true,
|
|
181
|
+
systemPrompt: config.systemPrompt === true,
|
|
182
|
+
toolDefinitions: config.toolDefinitions === true,
|
|
183
|
+
logBodies: false
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
function hasPreloadedOtelSdk() {
|
|
187
|
+
return process.env[PRELOADED_OTEL_SDK_ENV] === "1";
|
|
188
|
+
}
|
|
189
|
+
function normalizeOtelContentValue(value) {
|
|
190
|
+
if (typeof value === "string") return normalizeOtelLogString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
191
|
+
if (Array.isArray(value)) {
|
|
192
|
+
const items = [];
|
|
193
|
+
for (const item of value.slice(0, 200)) if (typeof item === "string") items.push(item);
|
|
194
|
+
if (items.length > 0) return normalizeOtelLogString(items.join("\n"), MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
195
|
+
}
|
|
196
|
+
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
197
|
+
if (json) return json;
|
|
198
|
+
}
|
|
199
|
+
const TRUNCATED_JSON_TEXT_SUFFIX = "...(truncated)";
|
|
200
|
+
const JSON_TRUNCATION_STRING_BUDGETS = [
|
|
201
|
+
8192,
|
|
202
|
+
4096,
|
|
203
|
+
2048,
|
|
204
|
+
1024,
|
|
205
|
+
512,
|
|
206
|
+
256,
|
|
207
|
+
128,
|
|
208
|
+
64,
|
|
209
|
+
32
|
|
210
|
+
];
|
|
211
|
+
const JSON_TRUNCATION_ARRAY_ITEM_BUDGETS = [
|
|
212
|
+
200,
|
|
213
|
+
100,
|
|
214
|
+
50,
|
|
215
|
+
25,
|
|
216
|
+
10,
|
|
217
|
+
5,
|
|
218
|
+
1
|
|
219
|
+
];
|
|
220
|
+
const JSON_TRUNCATION_MAX_OBJECT_FIELDS = 64;
|
|
221
|
+
const JSON_TRUNCATION_MAX_DEPTH = 8;
|
|
222
|
+
function safeJsonString(value, maxChars) {
|
|
223
|
+
if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
|
|
224
|
+
const exact = stringifyJsonForOtelAttribute(value);
|
|
225
|
+
if (exact && exact.length <= maxChars) return exact;
|
|
226
|
+
for (const maxArrayItems of JSON_TRUNCATION_ARRAY_ITEM_BUDGETS) for (const maxStringChars of JSON_TRUNCATION_STRING_BUDGETS) {
|
|
227
|
+
const json = stringifyJsonForOtelAttribute(truncateJsonValueForOtelAttribute(value, {
|
|
228
|
+
maxArrayItems,
|
|
229
|
+
maxDepth: JSON_TRUNCATION_MAX_DEPTH,
|
|
230
|
+
maxObjectFields: JSON_TRUNCATION_MAX_OBJECT_FIELDS,
|
|
231
|
+
maxStringChars,
|
|
232
|
+
seen: /* @__PURE__ */ new WeakSet()
|
|
233
|
+
}));
|
|
234
|
+
if (json && json.length <= maxChars) return json;
|
|
235
|
+
}
|
|
236
|
+
const summary = stringifyJsonForOtelAttribute({
|
|
237
|
+
truncated: true,
|
|
238
|
+
reason: exact ? "max_attribute_size" : "unserializable_value",
|
|
239
|
+
type: describeJsonValue(value)
|
|
240
|
+
});
|
|
241
|
+
return summary && summary.length <= maxChars ? summary : void 0;
|
|
242
|
+
}
|
|
243
|
+
function stringifyJsonForOtelAttribute(value) {
|
|
244
|
+
try {
|
|
245
|
+
const json = JSON.stringify(value);
|
|
246
|
+
if (!json) return;
|
|
247
|
+
return redactSensitiveText(json);
|
|
248
|
+
} catch {
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function truncateJsonValueForOtelAttribute(value, options) {
|
|
253
|
+
if (typeof value === "string") return truncateJsonTextForOtelAttribute(value, options.maxStringChars);
|
|
254
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) return value;
|
|
255
|
+
if (typeof value === "bigint") return truncateJsonTextForOtelAttribute(String(value), options.maxStringChars);
|
|
256
|
+
if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
|
|
257
|
+
if (options.maxDepth <= 0) return {
|
|
258
|
+
truncated: true,
|
|
259
|
+
reason: "max_depth"
|
|
260
|
+
};
|
|
261
|
+
if (Array.isArray(value)) return truncateJsonArrayForOtelAttribute(value, options);
|
|
262
|
+
if (typeof value === "object") return truncateJsonObjectForOtelAttribute(value, options);
|
|
263
|
+
}
|
|
264
|
+
function truncateJsonArrayForOtelAttribute(value, options) {
|
|
265
|
+
if (options.seen.has(value)) return [{
|
|
266
|
+
truncated: true,
|
|
267
|
+
reason: "circular_reference"
|
|
268
|
+
}];
|
|
269
|
+
options.seen.add(value);
|
|
270
|
+
const nextOptions = {
|
|
271
|
+
...options,
|
|
272
|
+
maxDepth: options.maxDepth - 1
|
|
273
|
+
};
|
|
274
|
+
const items = value.slice(0, options.maxArrayItems).map((item) => truncateJsonValueForOtelAttribute(item, nextOptions));
|
|
275
|
+
if (value.length > items.length) items.push({
|
|
276
|
+
truncated: true,
|
|
277
|
+
omittedItems: value.length - items.length
|
|
278
|
+
});
|
|
279
|
+
options.seen.delete(value);
|
|
280
|
+
return items;
|
|
281
|
+
}
|
|
282
|
+
function truncateJsonObjectForOtelAttribute(value, options) {
|
|
283
|
+
if (options.seen.has(value)) return {
|
|
284
|
+
truncated: true,
|
|
285
|
+
reason: "circular_reference"
|
|
286
|
+
};
|
|
287
|
+
options.seen.add(value);
|
|
288
|
+
const nextOptions = {
|
|
289
|
+
...options,
|
|
290
|
+
maxDepth: options.maxDepth - 1
|
|
291
|
+
};
|
|
292
|
+
const result = {};
|
|
293
|
+
const entries = Object.entries(value).filter(([, field]) => field !== void 0 && typeof field !== "function" && typeof field !== "symbol");
|
|
294
|
+
for (const [key, field] of entries.slice(0, options.maxObjectFields)) result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
|
|
295
|
+
if (entries.length > options.maxObjectFields) {
|
|
296
|
+
result.truncated = true;
|
|
297
|
+
result.omittedFields = entries.length - options.maxObjectFields;
|
|
298
|
+
}
|
|
299
|
+
options.seen.delete(value);
|
|
300
|
+
return result;
|
|
301
|
+
}
|
|
302
|
+
function truncateJsonTextForOtelAttribute(value, maxChars) {
|
|
303
|
+
const redacted = redactSensitiveText(value);
|
|
304
|
+
if (redacted.length <= maxChars) return redacted;
|
|
305
|
+
const suffixBudget = Math.min(14, maxChars);
|
|
306
|
+
return `${truncateUtf16Safe(redacted, Math.max(0, maxChars - suffixBudget))}${TRUNCATED_JSON_TEXT_SUFFIX.slice(14 - suffixBudget)}`;
|
|
307
|
+
}
|
|
308
|
+
function describeJsonValue(value) {
|
|
309
|
+
if (Array.isArray(value)) return "array";
|
|
310
|
+
if (value === null) return "null";
|
|
311
|
+
return typeof value;
|
|
312
|
+
}
|
|
313
|
+
//#endregion
|
|
314
|
+
//#region extensions/diagnostics-otel/src/service-attributes.ts
|
|
315
|
+
function redactOtelAttributes(attributes) {
|
|
316
|
+
const redactedAttributes = {};
|
|
317
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
318
|
+
if (DROPPED_OTEL_ATTRIBUTE_KEYS.has(key)) continue;
|
|
319
|
+
redactedAttributes[key] = typeof value === "string" ? redactSensitiveText(value) : value;
|
|
320
|
+
}
|
|
321
|
+
return redactedAttributes;
|
|
322
|
+
}
|
|
323
|
+
function lowCardinalityAttr(value, fallback = "unknown") {
|
|
324
|
+
if (!value) return fallback;
|
|
325
|
+
const redacted = redactSensitiveText(value.trim());
|
|
326
|
+
const redactedLower = redacted.toLowerCase();
|
|
327
|
+
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
328
|
+
return LOW_CARDINALITY_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
329
|
+
}
|
|
330
|
+
function securityTargetNameAttr(value, fallback = "unknown") {
|
|
331
|
+
if (!value) return fallback;
|
|
332
|
+
const redacted = redactSensitiveText(value.trim());
|
|
333
|
+
const redactedLower = redacted.toLowerCase();
|
|
334
|
+
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
335
|
+
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
336
|
+
}
|
|
337
|
+
function lowCardinalityQueueLaneAttr(value, fallback = "unknown") {
|
|
338
|
+
if (!value) return fallback;
|
|
339
|
+
const redacted = redactSensitiveText(value.trim());
|
|
340
|
+
if (redacted.toLowerCase().startsWith("agent:")) return fallback;
|
|
341
|
+
const scopedLaneIndex = redacted.indexOf(":");
|
|
342
|
+
const lane = scopedLaneIndex >= 0 ? redacted.slice(0, scopedLaneIndex) : redacted;
|
|
343
|
+
return LOW_CARDINALITY_VALUE_RE.test(lane) ? lane : fallback;
|
|
344
|
+
}
|
|
345
|
+
function shouldCaptureOtelLogBody(policy) {
|
|
346
|
+
return policy.logBodies;
|
|
347
|
+
}
|
|
348
|
+
function otelLogTimestampIso(timestamp) {
|
|
349
|
+
if (timestamp instanceof Date) return timestamp.toISOString();
|
|
350
|
+
if (typeof timestamp === "number" && Number.isFinite(timestamp)) return new Date(timestamp).toISOString();
|
|
351
|
+
if (Array.isArray(timestamp)) {
|
|
352
|
+
const [seconds, nanoseconds] = timestamp;
|
|
353
|
+
if (Number.isFinite(seconds) && Number.isFinite(nanoseconds)) return new Date(seconds * 1e3 + Math.trunc(nanoseconds / 1e6)).toISOString();
|
|
354
|
+
}
|
|
355
|
+
return (/* @__PURE__ */ new Date()).toISOString();
|
|
356
|
+
}
|
|
357
|
+
function writeStdoutDiagnosticLogRecord(params) {
|
|
358
|
+
const { logRecord, serviceName, traceContext } = params;
|
|
359
|
+
const line = {
|
|
360
|
+
ts: otelLogTimestampIso(logRecord.timestamp),
|
|
361
|
+
signal: "openclaw.diagnostic.log",
|
|
362
|
+
"service.name": serviceName,
|
|
363
|
+
severityText: logRecord.severityText,
|
|
364
|
+
severityNumber: logRecord.severityNumber,
|
|
365
|
+
body: logRecord.body,
|
|
366
|
+
attributes: logRecord.attributes ?? {},
|
|
367
|
+
...traceContext?.traceId ? { trace_id: traceContext.traceId } : {},
|
|
368
|
+
...traceContext?.spanId ? { span_id: traceContext.spanId } : {},
|
|
369
|
+
...traceContext?.traceFlags ? { trace_flags: traceContext.traceFlags } : {}
|
|
370
|
+
};
|
|
371
|
+
process.stdout.write(`${JSON.stringify(line)}\n`);
|
|
372
|
+
}
|
|
373
|
+
function assignOtelLogAttribute(attributes, key, value) {
|
|
374
|
+
if (Object.keys(attributes).length >= 64) return;
|
|
375
|
+
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) return;
|
|
376
|
+
if (redactSensitiveText(key) !== key) return;
|
|
377
|
+
if (!OTEL_LOG_ATTRIBUTE_KEY_RE.test(key)) return;
|
|
378
|
+
if (typeof value === "string") {
|
|
379
|
+
attributes[key] = normalizeOtelLogString(value, MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS);
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
383
|
+
attributes[key] = value;
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
if (typeof value === "boolean") attributes[key] = value;
|
|
387
|
+
}
|
|
388
|
+
function assignOtelLogEventAttributes(attributes, eventAttributes) {
|
|
389
|
+
if (!eventAttributes) return;
|
|
390
|
+
for (const [rawKey, value] of Object.entries(eventAttributes)) {
|
|
391
|
+
if (Object.keys(attributes).length >= 64) break;
|
|
392
|
+
const key = rawKey.trim();
|
|
393
|
+
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
|
|
394
|
+
if (redactSensitiveText(key) !== key) continue;
|
|
395
|
+
if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
|
|
396
|
+
assignOtelLogAttribute(attributes, `openclaw.${key}`, value);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function assignOtelSecurityEventAttributes(attributes, eventAttributes) {
|
|
400
|
+
if (!eventAttributes) return;
|
|
401
|
+
for (const [rawKey, value] of Object.entries(eventAttributes)) {
|
|
402
|
+
if (Object.keys(attributes).length >= 64) break;
|
|
403
|
+
const key = rawKey.trim();
|
|
404
|
+
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
|
|
405
|
+
if (redactSensitiveText(key) !== key) continue;
|
|
406
|
+
if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
|
|
407
|
+
assignOtelLogAttribute(attributes, `openclaw.security.attribute.${key}`, typeof value === "string" ? lowCardinalityAttr(value) : value);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function securitySeverityText(severity) {
|
|
411
|
+
switch (severity) {
|
|
412
|
+
case "critical": return "FATAL";
|
|
413
|
+
case "high": return "ERROR";
|
|
414
|
+
case "medium": return "WARN";
|
|
415
|
+
case "info":
|
|
416
|
+
case "low": return "INFO";
|
|
417
|
+
}
|
|
418
|
+
return severity;
|
|
419
|
+
}
|
|
420
|
+
function assignOtelSecurityAttributes(attributes, evt) {
|
|
421
|
+
assignOtelLogAttribute(attributes, "openclaw.security.event_id", evt.eventId);
|
|
422
|
+
assignOtelLogAttribute(attributes, "openclaw.security.category", evt.category);
|
|
423
|
+
assignOtelLogAttribute(attributes, "openclaw.security.action", lowCardinalityAttr(evt.action));
|
|
424
|
+
assignOtelLogAttribute(attributes, "openclaw.security.outcome", evt.outcome);
|
|
425
|
+
assignOtelLogAttribute(attributes, "openclaw.security.severity", evt.severity);
|
|
426
|
+
if (evt.reason) assignOtelLogAttribute(attributes, "openclaw.security.reason", lowCardinalityAttr(evt.reason));
|
|
427
|
+
if (evt.actor) {
|
|
428
|
+
assignOtelLogAttribute(attributes, "openclaw.security.actor.kind", evt.actor.kind);
|
|
429
|
+
if (evt.actor.idHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.id_hash", lowCardinalityAttr(evt.actor.idHash));
|
|
430
|
+
if (evt.actor.deviceIdHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.device_id_hash", lowCardinalityAttr(evt.actor.deviceIdHash));
|
|
431
|
+
if (evt.actor.channel) assignOtelLogAttribute(attributes, "openclaw.security.actor.channel", lowCardinalityAttr(evt.actor.channel));
|
|
432
|
+
if (evt.actor.role) assignOtelLogAttribute(attributes, "openclaw.security.actor.role", lowCardinalityAttr(evt.actor.role));
|
|
433
|
+
if (evt.actor.scopes?.length) assignOtelLogAttribute(attributes, "openclaw.security.actor.scopes", evt.actor.scopes.map((scope) => lowCardinalityAttr(scope)).join(","));
|
|
434
|
+
}
|
|
435
|
+
if (evt.target) {
|
|
436
|
+
assignOtelLogAttribute(attributes, "openclaw.security.target.kind", evt.target.kind);
|
|
437
|
+
if (evt.target.idHash) assignOtelLogAttribute(attributes, "openclaw.security.target.id_hash", lowCardinalityAttr(evt.target.idHash));
|
|
438
|
+
if (evt.target.name) assignOtelLogAttribute(attributes, "openclaw.security.target.name", securityTargetNameAttr(evt.target.name));
|
|
439
|
+
if (evt.target.owner) assignOtelLogAttribute(attributes, "openclaw.security.target.owner", lowCardinalityAttr(evt.target.owner));
|
|
440
|
+
}
|
|
441
|
+
if (evt.policy) {
|
|
442
|
+
if (evt.policy.id) assignOtelLogAttribute(attributes, "openclaw.security.policy.id", lowCardinalityAttr(evt.policy.id));
|
|
443
|
+
if (evt.policy.decision) assignOtelLogAttribute(attributes, "openclaw.security.policy.decision", evt.policy.decision);
|
|
444
|
+
if (evt.policy.reason) assignOtelLogAttribute(attributes, "openclaw.security.policy.reason", lowCardinalityAttr(evt.policy.reason));
|
|
445
|
+
}
|
|
446
|
+
if (evt.control) {
|
|
447
|
+
if (evt.control.id) assignOtelLogAttribute(attributes, "openclaw.security.control.id", lowCardinalityAttr(evt.control.id));
|
|
448
|
+
if (evt.control.family) assignOtelLogAttribute(attributes, "openclaw.security.control.family", evt.control.family);
|
|
449
|
+
}
|
|
450
|
+
assignOtelSecurityEventAttributes(attributes, evt.attributes);
|
|
451
|
+
}
|
|
452
|
+
//#endregion
|
|
453
|
+
//#region extensions/diagnostics-otel/src/service-exporter.ts
|
|
454
|
+
function normalizeEndpoint(endpoint) {
|
|
455
|
+
const trimmed = endpoint?.trim();
|
|
456
|
+
return trimmed ? trimmed.replace(/\/+$/, "") : void 0;
|
|
457
|
+
}
|
|
458
|
+
function resolveOtelUrl(endpoint, path) {
|
|
459
|
+
if (!endpoint) return;
|
|
460
|
+
const endpointWithoutQueryOrFragment = endpoint.split(/[?#]/, 1)[0] ?? endpoint;
|
|
461
|
+
if (/\/v1\/(?:traces|metrics|logs)$/i.test(endpointWithoutQueryOrFragment)) return endpoint;
|
|
462
|
+
if (/[?#]/u.test(endpoint)) try {
|
|
463
|
+
const url = new URL(endpoint);
|
|
464
|
+
url.pathname = `${url.pathname.replace(/\/+$/u, "")}/${path}`;
|
|
465
|
+
return url.toString();
|
|
466
|
+
} catch {}
|
|
467
|
+
return `${endpoint}/${path}`;
|
|
468
|
+
}
|
|
469
|
+
function resolveSignalOtelUrl(params) {
|
|
470
|
+
return resolveOtelUrl(normalizeEndpoint(params.signalEndpoint ?? params.signalEnvEndpoint) ?? params.endpoint, params.path);
|
|
471
|
+
}
|
|
472
|
+
function readOtelEnvFile(params) {
|
|
473
|
+
const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`;
|
|
474
|
+
const filePath = normalizeOtelEnvValue(process.env[signalEnvName]) ?? normalizeOtelEnvValue(process.env[params.sharedEnvName]);
|
|
475
|
+
if (!filePath) return;
|
|
476
|
+
try {
|
|
477
|
+
return readFileSync(nodePath.resolve(process.cwd(), filePath));
|
|
478
|
+
} catch {
|
|
479
|
+
params.logger.warn(`diagnostics-otel: ${params.warning}`);
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
function normalizeOtelEnvValue(value) {
|
|
484
|
+
const trimmed = value?.trim();
|
|
485
|
+
return trimmed ? trimmed : void 0;
|
|
486
|
+
}
|
|
487
|
+
function resolveOtelHttpAgentOptions(params) {
|
|
488
|
+
const { url, signalIdentifier, logger } = params;
|
|
489
|
+
if (!url) return;
|
|
490
|
+
const ca = readOtelEnvFile({
|
|
491
|
+
signalIdentifier,
|
|
492
|
+
signalSuffix: "CERTIFICATE",
|
|
493
|
+
sharedEnvName: OTEL_EXPORTER_OTLP_CERTIFICATE_ENV,
|
|
494
|
+
logger,
|
|
495
|
+
warning: "failed to read root certificate file"
|
|
496
|
+
});
|
|
497
|
+
const cert = readOtelEnvFile({
|
|
498
|
+
signalIdentifier,
|
|
499
|
+
signalSuffix: "CLIENT_CERTIFICATE",
|
|
500
|
+
sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV,
|
|
501
|
+
logger,
|
|
502
|
+
warning: "failed to read client certificate chain file"
|
|
503
|
+
});
|
|
504
|
+
const key = readOtelEnvFile({
|
|
505
|
+
signalIdentifier,
|
|
506
|
+
signalSuffix: "CLIENT_KEY",
|
|
507
|
+
sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV,
|
|
508
|
+
logger,
|
|
509
|
+
warning: "failed to read client certificate private key file"
|
|
510
|
+
});
|
|
511
|
+
const agentOptions = {
|
|
512
|
+
keepAlive: true,
|
|
513
|
+
...ca !== void 0 ? { ca } : {},
|
|
514
|
+
...cert !== void 0 ? { cert } : {},
|
|
515
|
+
...key !== void 0 ? { key } : {}
|
|
516
|
+
};
|
|
517
|
+
try {
|
|
518
|
+
const agent = createNodeProxyAgent({
|
|
519
|
+
mode: "env",
|
|
520
|
+
targetUrl: url,
|
|
521
|
+
agentOptions
|
|
522
|
+
});
|
|
523
|
+
return agent ? () => agent : void 0;
|
|
524
|
+
} catch {
|
|
525
|
+
logger.warn(`diagnostics-otel: env proxy agent unavailable for OTLP ${signalIdentifier.toLowerCase()} exporter; falling back to default Node agent`);
|
|
526
|
+
return;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
function resolveSampleRate(value) {
|
|
530
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return;
|
|
531
|
+
if (value < 0 || value > 1) return;
|
|
532
|
+
return value;
|
|
533
|
+
}
|
|
534
|
+
function formatError(err) {
|
|
535
|
+
if (err instanceof Error) return err.stack ?? err.message;
|
|
536
|
+
if (typeof err === "string") return err;
|
|
537
|
+
try {
|
|
538
|
+
return JSON.stringify(err);
|
|
539
|
+
} catch {
|
|
540
|
+
return String(err);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
function errorCategory(err) {
|
|
544
|
+
try {
|
|
545
|
+
if (err instanceof Error && typeof err.name === "string" && err.name.trim()) return lowCardinalityAttr(err.name, "Error");
|
|
546
|
+
return lowCardinalityAttr(typeof err, "unknown");
|
|
547
|
+
} catch {
|
|
548
|
+
return "unknown";
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
function collectNestedErrorCandidates(err) {
|
|
552
|
+
const queue = [err];
|
|
553
|
+
const seen = /* @__PURE__ */ new Set();
|
|
554
|
+
const candidates = [];
|
|
555
|
+
while (queue.length > 0) {
|
|
556
|
+
const current = queue.shift();
|
|
557
|
+
if (current == null || seen.has(current)) continue;
|
|
558
|
+
seen.add(current);
|
|
559
|
+
candidates.push(current);
|
|
560
|
+
if (Array.isArray(current)) {
|
|
561
|
+
for (const item of current) if (item != null && !seen.has(item)) queue.push(item);
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
if (typeof current !== "object") continue;
|
|
565
|
+
const record = current;
|
|
566
|
+
for (const nested of [
|
|
567
|
+
record.cause,
|
|
568
|
+
record.reason,
|
|
569
|
+
record.original,
|
|
570
|
+
record.error
|
|
571
|
+
]) if (nested != null && !seen.has(nested)) queue.push(nested);
|
|
572
|
+
if (Array.isArray(record.errors)) {
|
|
573
|
+
for (const nested of record.errors) if (nested != null && !seen.has(nested)) queue.push(nested);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
return candidates;
|
|
577
|
+
}
|
|
578
|
+
function readErrorName(err) {
|
|
579
|
+
if (!err || typeof err !== "object") return;
|
|
580
|
+
const name = err.name;
|
|
581
|
+
return typeof name === "string" && name.trim() ? name : void 0;
|
|
582
|
+
}
|
|
583
|
+
function readErrorCode(err) {
|
|
584
|
+
if (!err || typeof err !== "object") return;
|
|
585
|
+
const code = err.code;
|
|
586
|
+
return typeof code === "string" || typeof code === "number" ? code : void 0;
|
|
587
|
+
}
|
|
588
|
+
function findOtlpExporterError(reason) {
|
|
589
|
+
for (const candidate of collectNestedErrorCandidates(reason)) if (readErrorName(candidate) === "OTLPExporterError" && candidate && typeof candidate === "object") return candidate;
|
|
590
|
+
}
|
|
591
|
+
//#endregion
|
|
592
|
+
//#region extensions/diagnostics-otel/src/service-events.ts
|
|
593
|
+
function createDiagnosticsEventHandler(params) {
|
|
594
|
+
const { logger, recorders, recordLogRecord, recordSecurityEvent } = params;
|
|
595
|
+
const { recordModelUsage, recordWebhookReceived, recordWebhookProcessed, recordWebhookError, recordMessageQueued, recordMessageReceived, recordMessageDispatchStarted, recordMessageDispatchCompleted, recordMessageProcessed, recordMessageDeliveryStarted, recordMessageDeliveryCompleted, recordMessageDeliveryError, recordTalkEvent, recordLaneEnqueue, recordLaneDequeue, recordSessionState, recordSessionTurnCreated, recordSessionStuck, recordSessionRecoveryRequested, recordSessionRecoveryCompleted, recordRunAttempt, recordHeartbeat, recordLivenessWarning, recordDiagnosticPhaseCompleted, recordRunStarted, recordRunCompleted, recordHarnessRunStarted, recordHarnessRunCompleted, recordHarnessRunError, recordContextAssembled, recordModelCallStarted, recordModelCallCompleted, recordModelCallError, recordToolExecutionStarted, recordToolExecutionCompleted, recordToolExecutionError, recordToolExecutionBlocked, recordSkillUsed, recordExecProcessCompleted, recordToolLoop, recordMemorySample, recordMemoryPressure, recordAsyncQueueDropped, recordTelemetryExporter, recordPayloadLarge, recordModelFailover } = recorders;
|
|
596
|
+
return (evt, metadata, privateData) => {
|
|
597
|
+
try {
|
|
598
|
+
switch (evt.type) {
|
|
599
|
+
case "model.usage":
|
|
600
|
+
recordModelUsage(evt, metadata);
|
|
601
|
+
return;
|
|
602
|
+
case "webhook.received":
|
|
603
|
+
recordWebhookReceived(evt);
|
|
604
|
+
return;
|
|
605
|
+
case "webhook.processed":
|
|
606
|
+
recordWebhookProcessed(evt);
|
|
607
|
+
return;
|
|
608
|
+
case "webhook.error":
|
|
609
|
+
recordWebhookError(evt);
|
|
610
|
+
return;
|
|
611
|
+
case "message.queued":
|
|
612
|
+
recordMessageQueued(evt);
|
|
613
|
+
return;
|
|
614
|
+
case "message.received":
|
|
615
|
+
recordMessageReceived(evt);
|
|
616
|
+
return;
|
|
617
|
+
case "message.dispatch.started":
|
|
618
|
+
recordMessageDispatchStarted(evt, metadata);
|
|
619
|
+
return;
|
|
620
|
+
case "message.dispatch.completed":
|
|
621
|
+
recordMessageDispatchCompleted(evt);
|
|
622
|
+
return;
|
|
623
|
+
case "message.processed":
|
|
624
|
+
recordMessageProcessed(evt, metadata);
|
|
625
|
+
return;
|
|
626
|
+
case "message.delivery.started":
|
|
627
|
+
recordMessageDeliveryStarted(evt);
|
|
628
|
+
return;
|
|
629
|
+
case "message.delivery.completed":
|
|
630
|
+
recordMessageDeliveryCompleted(evt, metadata);
|
|
631
|
+
return;
|
|
632
|
+
case "message.delivery.error":
|
|
633
|
+
recordMessageDeliveryError(evt, metadata);
|
|
634
|
+
return;
|
|
635
|
+
case "talk.event":
|
|
636
|
+
recordTalkEvent(evt, metadata);
|
|
637
|
+
return;
|
|
638
|
+
case "queue.lane.enqueue":
|
|
639
|
+
recordLaneEnqueue(evt);
|
|
640
|
+
return;
|
|
641
|
+
case "queue.lane.dequeue":
|
|
642
|
+
recordLaneDequeue(evt);
|
|
643
|
+
return;
|
|
644
|
+
case "session.state":
|
|
645
|
+
recordSessionState(evt);
|
|
646
|
+
break;
|
|
647
|
+
case "session.long_running":
|
|
648
|
+
case "session.stalled": break;
|
|
649
|
+
case "session.turn.created":
|
|
650
|
+
recordSessionTurnCreated(evt);
|
|
651
|
+
return;
|
|
652
|
+
case "session.stuck":
|
|
653
|
+
recordSessionStuck(evt);
|
|
654
|
+
return;
|
|
655
|
+
case "session.recovery.requested":
|
|
656
|
+
recordSessionRecoveryRequested(evt);
|
|
657
|
+
return;
|
|
658
|
+
case "session.recovery.completed":
|
|
659
|
+
recordSessionRecoveryCompleted(evt);
|
|
660
|
+
return;
|
|
661
|
+
case "run.attempt":
|
|
662
|
+
recordRunAttempt(evt);
|
|
663
|
+
break;
|
|
664
|
+
case "run.progress": break;
|
|
665
|
+
case "diagnostic.heartbeat":
|
|
666
|
+
recordHeartbeat(evt);
|
|
667
|
+
return;
|
|
668
|
+
case "diagnostic.liveness.warning":
|
|
669
|
+
recordLivenessWarning(evt);
|
|
670
|
+
return;
|
|
671
|
+
case "diagnostic.phase.completed":
|
|
672
|
+
recordDiagnosticPhaseCompleted(evt);
|
|
673
|
+
return;
|
|
674
|
+
case "run.started":
|
|
675
|
+
recordRunStarted(evt, metadata);
|
|
676
|
+
return;
|
|
677
|
+
case "run.completed":
|
|
678
|
+
recordRunCompleted(evt, metadata, privateData);
|
|
679
|
+
return;
|
|
680
|
+
case "harness.run.started":
|
|
681
|
+
recordHarnessRunStarted(evt, metadata);
|
|
682
|
+
return;
|
|
683
|
+
case "harness.run.completed":
|
|
684
|
+
recordHarnessRunCompleted(evt, metadata, privateData);
|
|
685
|
+
return;
|
|
686
|
+
case "harness.run.error":
|
|
687
|
+
recordHarnessRunError(evt, metadata, privateData);
|
|
688
|
+
return;
|
|
689
|
+
case "context.assembled":
|
|
690
|
+
recordContextAssembled(evt, metadata);
|
|
691
|
+
return;
|
|
692
|
+
case "model.call.started":
|
|
693
|
+
recordModelCallStarted(evt, metadata);
|
|
694
|
+
return;
|
|
695
|
+
case "model.call.completed":
|
|
696
|
+
recordModelCallCompleted(evt, metadata, privateData.modelContent);
|
|
697
|
+
return;
|
|
698
|
+
case "model.call.error":
|
|
699
|
+
recordModelCallError(evt, metadata, privateData.modelContent);
|
|
700
|
+
return;
|
|
701
|
+
case "tool.execution.started":
|
|
702
|
+
recordToolExecutionStarted(evt, metadata);
|
|
703
|
+
return;
|
|
704
|
+
case "tool.execution.completed":
|
|
705
|
+
recordToolExecutionCompleted(evt, metadata, privateData.toolContent);
|
|
706
|
+
return;
|
|
707
|
+
case "tool.execution.error":
|
|
708
|
+
recordToolExecutionError(evt, metadata, privateData.toolContent);
|
|
709
|
+
return;
|
|
710
|
+
case "tool.execution.blocked":
|
|
711
|
+
recordToolExecutionBlocked(evt, metadata);
|
|
712
|
+
return;
|
|
713
|
+
case "skill.used":
|
|
714
|
+
recordSkillUsed(evt, metadata);
|
|
715
|
+
return;
|
|
716
|
+
case "exec.process.completed":
|
|
717
|
+
recordExecProcessCompleted(evt);
|
|
718
|
+
break;
|
|
719
|
+
case "exec.approval.followup_suppressed": break;
|
|
720
|
+
case "log.record":
|
|
721
|
+
recordLogRecord?.(evt, metadata);
|
|
722
|
+
return;
|
|
723
|
+
case "security.event":
|
|
724
|
+
recordSecurityEvent?.(evt, metadata);
|
|
725
|
+
return;
|
|
726
|
+
case "tool.loop":
|
|
727
|
+
recordToolLoop(evt);
|
|
728
|
+
return;
|
|
729
|
+
case "diagnostic.memory.sample":
|
|
730
|
+
recordMemorySample(evt);
|
|
731
|
+
return;
|
|
732
|
+
case "diagnostic.memory.pressure":
|
|
733
|
+
recordMemoryPressure(evt);
|
|
734
|
+
return;
|
|
735
|
+
case "diagnostic.async_queue.dropped":
|
|
736
|
+
recordAsyncQueueDropped(evt);
|
|
737
|
+
return;
|
|
738
|
+
case "telemetry.exporter":
|
|
739
|
+
recordTelemetryExporter(evt, metadata);
|
|
740
|
+
return;
|
|
741
|
+
case "payload.large":
|
|
742
|
+
recordPayloadLarge(evt);
|
|
743
|
+
return;
|
|
744
|
+
case "model.failover": recordModelFailover(evt, metadata);
|
|
745
|
+
}
|
|
746
|
+
} catch (err) {
|
|
747
|
+
logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
|
|
748
|
+
}
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
//#endregion
|
|
752
|
+
//#region extensions/diagnostics-otel/src/service-trace-context.ts
|
|
753
|
+
function normalizeTraceContext(value) {
|
|
754
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
755
|
+
const candidate = value;
|
|
756
|
+
if (!isValidDiagnosticTraceId(candidate.traceId)) return;
|
|
757
|
+
if (candidate.spanId !== void 0 && !isValidDiagnosticSpanId(candidate.spanId)) return;
|
|
758
|
+
if (candidate.parentSpanId !== void 0 && !isValidDiagnosticSpanId(candidate.parentSpanId)) return;
|
|
759
|
+
if (candidate.traceFlags !== void 0 && !isValidDiagnosticTraceFlags(candidate.traceFlags)) return;
|
|
760
|
+
return {
|
|
761
|
+
traceId: candidate.traceId,
|
|
762
|
+
...candidate.spanId ? { spanId: candidate.spanId } : {},
|
|
763
|
+
...candidate.parentSpanId ? { parentSpanId: candidate.parentSpanId } : {},
|
|
764
|
+
...candidate.traceFlags ? { traceFlags: candidate.traceFlags } : {}
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
function traceFlagsToOtel(traceFlags) {
|
|
768
|
+
return (Number.parseInt(traceFlags ?? "00", 16) & TraceFlags.SAMPLED) !== 0 ? TraceFlags.SAMPLED : TraceFlags.NONE;
|
|
769
|
+
}
|
|
770
|
+
function contextForTraceContext(traceContext) {
|
|
771
|
+
const normalized = normalizeTraceContext(traceContext);
|
|
772
|
+
if (!normalized?.spanId) return;
|
|
773
|
+
return trace.setSpanContext(context.active(), {
|
|
774
|
+
traceId: normalized.traceId,
|
|
775
|
+
spanId: normalized.spanId,
|
|
776
|
+
traceFlags: traceFlagsToOtel(normalized.traceFlags),
|
|
777
|
+
isRemote: true
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
function contextForTrustedTraceContext(evt, metadata) {
|
|
781
|
+
return metadata.trusted || metadata.trustedTraceContext === true ? contextForTraceContext(evt.trace) : void 0;
|
|
782
|
+
}
|
|
783
|
+
function normalizedTrustedTraceContext(evt, metadata) {
|
|
784
|
+
return metadata.trusted || metadata.trustedTraceContext === true ? normalizeTraceContext(evt.trace) : void 0;
|
|
785
|
+
}
|
|
786
|
+
function addTraceAttributes(attributes, traceContext) {
|
|
787
|
+
const normalized = normalizeTraceContext(traceContext);
|
|
788
|
+
if (!normalized) return;
|
|
789
|
+
attributes["openclaw.traceId"] = normalized.traceId;
|
|
790
|
+
if (normalized.spanId) attributes["openclaw.spanId"] = normalized.spanId;
|
|
791
|
+
if (normalized.parentSpanId) attributes["openclaw.parentSpanId"] = normalized.parentSpanId;
|
|
792
|
+
if (normalized.traceFlags) attributes["openclaw.traceFlags"] = normalized.traceFlags;
|
|
793
|
+
}
|
|
794
|
+
//#endregion
|
|
795
|
+
//#region extensions/diagnostics-otel/src/service-logs.ts
|
|
796
|
+
const LOG_SEVERITY_MAP = {
|
|
797
|
+
TRACE: 1,
|
|
798
|
+
DEBUG: 5,
|
|
799
|
+
INFO: 9,
|
|
800
|
+
WARN: 13,
|
|
801
|
+
ERROR: 17,
|
|
802
|
+
FATAL: 21
|
|
803
|
+
};
|
|
804
|
+
function createDiagnosticsLogExporter(params) {
|
|
805
|
+
const { contentCapturePolicy, emitExporterEvent, flushIntervalMs, headers, logger, logsEnabled, logsToOtlp, logsToStdout, logUrl, resource, serviceName } = params;
|
|
806
|
+
let logProvider = null;
|
|
807
|
+
const logSeverityMap = LOG_SEVERITY_MAP;
|
|
808
|
+
let recordLogRecord;
|
|
809
|
+
let recordSecurityEvent;
|
|
810
|
+
if (logsEnabled) {
|
|
811
|
+
let logRecordExportFailureLastReportedAt = Number.NEGATIVE_INFINITY;
|
|
812
|
+
let otelLogger;
|
|
813
|
+
if (logsToOtlp) {
|
|
814
|
+
const logHttpAgentOptions = resolveOtelHttpAgentOptions({
|
|
815
|
+
url: logUrl,
|
|
816
|
+
signalIdentifier: "LOGS",
|
|
817
|
+
logger
|
|
818
|
+
});
|
|
819
|
+
logProvider = new LoggerProvider({
|
|
820
|
+
resource,
|
|
821
|
+
processors: [new BatchLogRecordProcessor(new OTLPLogExporter({
|
|
822
|
+
...logUrl ? { url: logUrl } : {},
|
|
823
|
+
...headers ? { headers } : {},
|
|
824
|
+
...logHttpAgentOptions ? { httpAgentOptions: logHttpAgentOptions } : {}
|
|
825
|
+
}), typeof flushIntervalMs === "number" ? { scheduledDelayMillis: Math.max(1e3, flushIntervalMs) } : {})]
|
|
826
|
+
});
|
|
827
|
+
otelLogger = logProvider.getLogger("openclaw");
|
|
828
|
+
}
|
|
829
|
+
const reportLogExportFailure = (err, label) => {
|
|
830
|
+
emitExporterEvent({
|
|
831
|
+
exporter: "diagnostics-otel",
|
|
832
|
+
signal: "logs",
|
|
833
|
+
status: "failure",
|
|
834
|
+
reason: "emit_failed",
|
|
835
|
+
errorCategory: errorCategory(err)
|
|
836
|
+
});
|
|
837
|
+
const now = Date.now();
|
|
838
|
+
if (now - logRecordExportFailureLastReportedAt >= 6e4) {
|
|
839
|
+
logRecordExportFailureLastReportedAt = now;
|
|
840
|
+
logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`);
|
|
841
|
+
}
|
|
842
|
+
};
|
|
843
|
+
const emitLogRecord = ({ logRecord, traceContext }) => {
|
|
844
|
+
if (logsToOtlp) otelLogger?.emit(logRecord);
|
|
845
|
+
if (logsToStdout) writeStdoutDiagnosticLogRecord({
|
|
846
|
+
logRecord,
|
|
847
|
+
serviceName,
|
|
848
|
+
...traceContext ? { traceContext } : {}
|
|
849
|
+
});
|
|
850
|
+
};
|
|
851
|
+
const buildDiagnosticLogRecord = (evt, metadata) => {
|
|
852
|
+
const logLevelName = evt.level || "INFO";
|
|
853
|
+
const severityNumber = logSeverityMap[logLevelName] ?? 9;
|
|
854
|
+
const body = shouldCaptureOtelLogBody(contentCapturePolicy) ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
855
|
+
const attributes = Object.create(null);
|
|
856
|
+
assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName);
|
|
857
|
+
if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
|
|
858
|
+
if (evt.loggerParents?.length) assignOtelLogAttribute(attributes, "openclaw.logger.parents", evt.loggerParents.join("."));
|
|
859
|
+
assignOtelLogEventAttributes(attributes, evt.attributes);
|
|
860
|
+
if (evt.code?.line) assignOtelLogAttribute(attributes, "code.lineno", evt.code.line);
|
|
861
|
+
if (evt.code?.functionName) assignOtelLogAttribute(attributes, "code.function", evt.code.functionName);
|
|
862
|
+
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
863
|
+
addTraceAttributes(attributes, traceContext);
|
|
864
|
+
const logRecord = {
|
|
865
|
+
body,
|
|
866
|
+
severityText: logLevelName,
|
|
867
|
+
severityNumber,
|
|
868
|
+
attributes: redactOtelAttributes(attributes),
|
|
869
|
+
timestamp: evt.ts
|
|
870
|
+
};
|
|
871
|
+
const logContext = contextForTrustedTraceContext(evt, metadata);
|
|
872
|
+
if (logContext) logRecord.context = logContext;
|
|
873
|
+
return {
|
|
874
|
+
logRecord,
|
|
875
|
+
...traceContext ? { traceContext } : {}
|
|
876
|
+
};
|
|
877
|
+
};
|
|
878
|
+
const buildSecurityLogRecord = (evt, metadata) => {
|
|
879
|
+
const severityText = securitySeverityText(evt.severity);
|
|
880
|
+
const attributes = Object.create(null);
|
|
881
|
+
assignOtelSecurityAttributes(attributes, evt);
|
|
882
|
+
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
883
|
+
const logRecord = {
|
|
884
|
+
body: "openclaw.security.event",
|
|
885
|
+
severityText,
|
|
886
|
+
severityNumber: logSeverityMap[severityText] ?? 9,
|
|
887
|
+
attributes: redactOtelAttributes(attributes),
|
|
888
|
+
timestamp: evt.ts
|
|
889
|
+
};
|
|
890
|
+
const logContext = contextForTrustedTraceContext(evt, metadata);
|
|
891
|
+
if (logContext) logRecord.context = logContext;
|
|
892
|
+
return {
|
|
893
|
+
logRecord,
|
|
894
|
+
...traceContext ? { traceContext } : {}
|
|
895
|
+
};
|
|
896
|
+
};
|
|
897
|
+
recordLogRecord = (evt, metadata) => {
|
|
898
|
+
try {
|
|
899
|
+
emitLogRecord(buildDiagnosticLogRecord(evt, metadata));
|
|
900
|
+
} catch (err) {
|
|
901
|
+
reportLogExportFailure(err, "log record");
|
|
902
|
+
}
|
|
903
|
+
};
|
|
904
|
+
recordSecurityEvent = (evt, metadata) => {
|
|
905
|
+
if (!metadata.trusted) return;
|
|
906
|
+
try {
|
|
907
|
+
emitLogRecord(buildSecurityLogRecord(evt, metadata));
|
|
908
|
+
} catch (err) {
|
|
909
|
+
reportLogExportFailure(err, "security event");
|
|
910
|
+
}
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
return {
|
|
914
|
+
logProvider,
|
|
915
|
+
recordLogRecord,
|
|
916
|
+
recordSecurityEvent
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
//#endregion
|
|
920
|
+
//#region extensions/diagnostics-otel/src/service-metrics.ts
|
|
921
|
+
function createDiagnosticsMetrics(meter) {
|
|
922
|
+
return {
|
|
923
|
+
tokensCounter: meter.createCounter("openclaw.tokens", {
|
|
924
|
+
unit: "1",
|
|
925
|
+
description: "Token usage by type"
|
|
926
|
+
}),
|
|
927
|
+
genAiTokenUsageHistogram: meter.createHistogram("gen_ai.client.token.usage", {
|
|
928
|
+
unit: "{token}",
|
|
929
|
+
description: "Number of input and output tokens used by GenAI client operations",
|
|
930
|
+
advice: { explicitBucketBoundaries: GEN_AI_TOKEN_USAGE_BUCKETS }
|
|
931
|
+
}),
|
|
932
|
+
genAiOperationDurationHistogram: meter.createHistogram("gen_ai.client.operation.duration", {
|
|
933
|
+
unit: "s",
|
|
934
|
+
description: "GenAI client operation duration",
|
|
935
|
+
advice: { explicitBucketBoundaries: GEN_AI_OPERATION_DURATION_BUCKETS }
|
|
936
|
+
}),
|
|
937
|
+
costCounter: meter.createCounter("openclaw.cost.usd", {
|
|
938
|
+
unit: "1",
|
|
939
|
+
description: "Estimated model cost (USD)"
|
|
940
|
+
}),
|
|
941
|
+
durationHistogram: meter.createHistogram("openclaw.run.duration_ms", {
|
|
942
|
+
unit: "ms",
|
|
943
|
+
description: "Agent run duration",
|
|
944
|
+
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
945
|
+
}),
|
|
946
|
+
harnessDurationHistogram: meter.createHistogram("openclaw.harness.duration_ms", {
|
|
947
|
+
unit: "ms",
|
|
948
|
+
description: "Agent harness lifecycle duration",
|
|
949
|
+
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
950
|
+
}),
|
|
951
|
+
contextHistogram: meter.createHistogram("openclaw.context.tokens", {
|
|
952
|
+
unit: "1",
|
|
953
|
+
description: "Context window size and usage",
|
|
954
|
+
advice: { explicitBucketBoundaries: CONTEXT_TOKENS_BUCKETS }
|
|
955
|
+
}),
|
|
956
|
+
webhookReceivedCounter: meter.createCounter("openclaw.webhook.received", {
|
|
957
|
+
unit: "1",
|
|
958
|
+
description: "Webhook requests received"
|
|
959
|
+
}),
|
|
960
|
+
webhookErrorCounter: meter.createCounter("openclaw.webhook.error", {
|
|
961
|
+
unit: "1",
|
|
962
|
+
description: "Webhook processing errors"
|
|
963
|
+
}),
|
|
964
|
+
webhookDurationHistogram: meter.createHistogram("openclaw.webhook.duration_ms", {
|
|
965
|
+
unit: "ms",
|
|
966
|
+
description: "Webhook processing duration"
|
|
967
|
+
}),
|
|
968
|
+
messageQueuedCounter: meter.createCounter("openclaw.message.queued", {
|
|
969
|
+
unit: "1",
|
|
970
|
+
description: "Messages queued for processing"
|
|
971
|
+
}),
|
|
972
|
+
messageReceivedCounter: meter.createCounter("openclaw.message.received", {
|
|
973
|
+
unit: "1",
|
|
974
|
+
description: "Inbound messages received"
|
|
975
|
+
}),
|
|
976
|
+
messageDispatchStartedCounter: meter.createCounter("openclaw.message.dispatch.started", {
|
|
977
|
+
unit: "1",
|
|
978
|
+
description: "Inbound message dispatch attempts started"
|
|
979
|
+
}),
|
|
980
|
+
messageDispatchCompletedCounter: meter.createCounter("openclaw.message.dispatch.completed", {
|
|
981
|
+
unit: "1",
|
|
982
|
+
description: "Inbound message dispatch attempts completed"
|
|
983
|
+
}),
|
|
984
|
+
messageDispatchDurationHistogram: meter.createHistogram("openclaw.message.dispatch.duration_ms", {
|
|
985
|
+
unit: "ms",
|
|
986
|
+
description: "Inbound message dispatch duration"
|
|
987
|
+
}),
|
|
988
|
+
messageProcessedCounter: meter.createCounter("openclaw.message.processed", {
|
|
989
|
+
unit: "1",
|
|
990
|
+
description: "Messages processed by outcome"
|
|
991
|
+
}),
|
|
992
|
+
messageDurationHistogram: meter.createHistogram("openclaw.message.duration_ms", {
|
|
993
|
+
unit: "ms",
|
|
994
|
+
description: "Message processing duration"
|
|
995
|
+
}),
|
|
996
|
+
messageDeliveryStartedCounter: meter.createCounter("openclaw.message.delivery.started", {
|
|
997
|
+
unit: "1",
|
|
998
|
+
description: "Outbound message delivery attempts started"
|
|
999
|
+
}),
|
|
1000
|
+
messageDeliveryDurationHistogram: meter.createHistogram("openclaw.message.delivery.duration_ms", {
|
|
1001
|
+
unit: "ms",
|
|
1002
|
+
description: "Outbound message delivery duration"
|
|
1003
|
+
}),
|
|
1004
|
+
queueDepthHistogram: meter.createHistogram("openclaw.queue.depth", {
|
|
1005
|
+
unit: "1",
|
|
1006
|
+
description: "Queue depth on enqueue/dequeue"
|
|
1007
|
+
}),
|
|
1008
|
+
queueWaitHistogram: meter.createHistogram("openclaw.queue.wait_ms", {
|
|
1009
|
+
unit: "ms",
|
|
1010
|
+
description: "Queue wait time before execution"
|
|
1011
|
+
}),
|
|
1012
|
+
laneEnqueueCounter: meter.createCounter("openclaw.queue.lane.enqueue", {
|
|
1013
|
+
unit: "1",
|
|
1014
|
+
description: "Command queue lane enqueue events"
|
|
1015
|
+
}),
|
|
1016
|
+
laneDequeueCounter: meter.createCounter("openclaw.queue.lane.dequeue", {
|
|
1017
|
+
unit: "1",
|
|
1018
|
+
description: "Command queue lane dequeue events"
|
|
1019
|
+
}),
|
|
1020
|
+
sessionStateCounter: meter.createCounter("openclaw.session.state", {
|
|
1021
|
+
unit: "1",
|
|
1022
|
+
description: "Session state transitions"
|
|
1023
|
+
}),
|
|
1024
|
+
sessionTurnCreatedCounter: meter.createCounter("openclaw.session.turn.created", {
|
|
1025
|
+
unit: "1",
|
|
1026
|
+
description: "Agent session turns created"
|
|
1027
|
+
}),
|
|
1028
|
+
sessionStuckCounter: meter.createCounter("openclaw.session.stuck", {
|
|
1029
|
+
unit: "1",
|
|
1030
|
+
description: "Sessions stuck in processing"
|
|
1031
|
+
}),
|
|
1032
|
+
sessionStuckAgeHistogram: meter.createHistogram("openclaw.session.stuck_age_ms", {
|
|
1033
|
+
unit: "ms",
|
|
1034
|
+
description: "Age of stuck sessions"
|
|
1035
|
+
}),
|
|
1036
|
+
sessionRecoveryRequestedCounter: meter.createCounter("openclaw.session.recovery.requested", {
|
|
1037
|
+
unit: "1",
|
|
1038
|
+
description: "Session recovery attempts requested"
|
|
1039
|
+
}),
|
|
1040
|
+
sessionRecoveryCompletedCounter: meter.createCounter("openclaw.session.recovery.completed", {
|
|
1041
|
+
unit: "1",
|
|
1042
|
+
description: "Session recovery attempts completed"
|
|
1043
|
+
}),
|
|
1044
|
+
sessionRecoveryAgeHistogram: meter.createHistogram("openclaw.session.recovery.age_ms", {
|
|
1045
|
+
unit: "ms",
|
|
1046
|
+
description: "Age of sessions selected for recovery"
|
|
1047
|
+
}),
|
|
1048
|
+
talkEventCounter: meter.createCounter("openclaw.talk.event", {
|
|
1049
|
+
unit: "1",
|
|
1050
|
+
description: "Talk events emitted by type"
|
|
1051
|
+
}),
|
|
1052
|
+
talkEventDurationHistogram: meter.createHistogram("openclaw.talk.event.duration_ms", {
|
|
1053
|
+
unit: "ms",
|
|
1054
|
+
description: "Talk event duration when reported"
|
|
1055
|
+
}),
|
|
1056
|
+
talkAudioBytesHistogram: meter.createHistogram("openclaw.talk.audio.bytes", {
|
|
1057
|
+
unit: "By",
|
|
1058
|
+
description: "Talk audio frame byte lengths"
|
|
1059
|
+
}),
|
|
1060
|
+
runAttemptCounter: meter.createCounter("openclaw.run.attempt", {
|
|
1061
|
+
unit: "1",
|
|
1062
|
+
description: "Run attempts"
|
|
1063
|
+
}),
|
|
1064
|
+
toolLoopCounter: meter.createCounter("openclaw.tool.loop", {
|
|
1065
|
+
unit: "1",
|
|
1066
|
+
description: "Detected repetitive tool-call loop events"
|
|
1067
|
+
}),
|
|
1068
|
+
skillUsedCounter: meter.createCounter("openclaw.skill.used", {
|
|
1069
|
+
unit: "1",
|
|
1070
|
+
description: "Skills used by agent runs"
|
|
1071
|
+
}),
|
|
1072
|
+
modelCallDurationHistogram: meter.createHistogram("openclaw.model_call.duration_ms", {
|
|
1073
|
+
unit: "ms",
|
|
1074
|
+
description: "Model call duration"
|
|
1075
|
+
}),
|
|
1076
|
+
modelCallRequestBytesHistogram: meter.createHistogram("openclaw.model_call.request_bytes", {
|
|
1077
|
+
unit: "By",
|
|
1078
|
+
description: "UTF-8 byte size of sanitized model request payloads"
|
|
1079
|
+
}),
|
|
1080
|
+
modelCallResponseBytesHistogram: meter.createHistogram("openclaw.model_call.response_bytes", {
|
|
1081
|
+
unit: "By",
|
|
1082
|
+
description: "UTF-8 byte size of bounded streamed model response payloads"
|
|
1083
|
+
}),
|
|
1084
|
+
modelCallTimeToFirstByteHistogram: meter.createHistogram("openclaw.model_call.time_to_first_byte_ms", {
|
|
1085
|
+
unit: "ms",
|
|
1086
|
+
description: "Elapsed time before the first streamed model response event"
|
|
1087
|
+
}),
|
|
1088
|
+
modelFailoverCounter: meter.createCounter("openclaw.model.failover", {
|
|
1089
|
+
unit: "1",
|
|
1090
|
+
description: "Model failovers by source, destination, lane, and reason"
|
|
1091
|
+
}),
|
|
1092
|
+
toolExecutionDurationHistogram: meter.createHistogram("openclaw.tool.execution.duration_ms", {
|
|
1093
|
+
unit: "ms",
|
|
1094
|
+
description: "Tool execution duration"
|
|
1095
|
+
}),
|
|
1096
|
+
toolExecutionBlockedCounter: meter.createCounter("openclaw.tool.execution.blocked", {
|
|
1097
|
+
unit: "1",
|
|
1098
|
+
description: "Tool executions blocked by policy or sandbox diagnostics"
|
|
1099
|
+
}),
|
|
1100
|
+
execProcessDurationHistogram: meter.createHistogram("openclaw.exec.duration_ms", {
|
|
1101
|
+
unit: "ms",
|
|
1102
|
+
description: "Exec process duration"
|
|
1103
|
+
}),
|
|
1104
|
+
memoryRssHistogram: meter.createHistogram("openclaw.memory.rss_bytes", {
|
|
1105
|
+
unit: "By",
|
|
1106
|
+
description: "Resident set size reported by diagnostic memory samples"
|
|
1107
|
+
}),
|
|
1108
|
+
memoryHeapUsedHistogram: meter.createHistogram("openclaw.memory.heap_used_bytes", {
|
|
1109
|
+
unit: "By",
|
|
1110
|
+
description: "Heap used bytes reported by diagnostic memory samples"
|
|
1111
|
+
}),
|
|
1112
|
+
memoryHeapTotalHistogram: meter.createHistogram("openclaw.memory.heap_total_bytes", {
|
|
1113
|
+
unit: "By",
|
|
1114
|
+
description: "Heap total bytes reported by diagnostic memory samples"
|
|
1115
|
+
}),
|
|
1116
|
+
memoryExternalHistogram: meter.createHistogram("openclaw.memory.external_bytes", {
|
|
1117
|
+
unit: "By",
|
|
1118
|
+
description: "External memory bytes reported by diagnostic memory samples"
|
|
1119
|
+
}),
|
|
1120
|
+
memoryArrayBuffersHistogram: meter.createHistogram("openclaw.memory.array_buffers_bytes", {
|
|
1121
|
+
unit: "By",
|
|
1122
|
+
description: "ArrayBuffer bytes reported by diagnostic memory samples"
|
|
1123
|
+
}),
|
|
1124
|
+
memoryPressureCounter: meter.createCounter("openclaw.memory.pressure", {
|
|
1125
|
+
unit: "1",
|
|
1126
|
+
description: "Diagnostic memory pressure events"
|
|
1127
|
+
}),
|
|
1128
|
+
asyncQueueDroppedCounter: meter.createCounter("openclaw.diagnostic.async_queue.dropped", {
|
|
1129
|
+
unit: "1",
|
|
1130
|
+
description: "Async diagnostic queue drops by dropped event class"
|
|
1131
|
+
}),
|
|
1132
|
+
payloadLargeCounter: meter.createCounter("openclaw.payload.large", {
|
|
1133
|
+
unit: "1",
|
|
1134
|
+
description: "Oversized payload diagnostics by surface and action"
|
|
1135
|
+
}),
|
|
1136
|
+
payloadLargeBytesHistogram: meter.createHistogram("openclaw.payload.large_bytes", {
|
|
1137
|
+
unit: "By",
|
|
1138
|
+
description: "Oversized payload byte sizes by surface and action"
|
|
1139
|
+
}),
|
|
1140
|
+
livenessWarningCounter: meter.createCounter("openclaw.liveness.warning", {
|
|
1141
|
+
unit: "1",
|
|
1142
|
+
description: "Diagnostic liveness warning events"
|
|
1143
|
+
}),
|
|
1144
|
+
livenessEventLoopDelayP99Histogram: meter.createHistogram("openclaw.liveness.event_loop_delay_p99_ms", {
|
|
1145
|
+
unit: "ms",
|
|
1146
|
+
description: "P99 event-loop delay reported by diagnostic liveness warnings"
|
|
1147
|
+
}),
|
|
1148
|
+
livenessEventLoopDelayMaxHistogram: meter.createHistogram("openclaw.liveness.event_loop_delay_max_ms", {
|
|
1149
|
+
unit: "ms",
|
|
1150
|
+
description: "Maximum event-loop delay reported by diagnostic liveness warnings"
|
|
1151
|
+
}),
|
|
1152
|
+
livenessEventLoopUtilizationHistogram: meter.createHistogram("openclaw.liveness.event_loop_utilization", {
|
|
1153
|
+
unit: "1",
|
|
1154
|
+
description: "Event-loop utilization reported by diagnostic liveness warnings"
|
|
1155
|
+
}),
|
|
1156
|
+
livenessCpuCoreRatioHistogram: meter.createHistogram("openclaw.liveness.cpu_core_ratio", {
|
|
1157
|
+
unit: "1",
|
|
1158
|
+
description: "CPU core ratio reported by diagnostic liveness warnings"
|
|
1159
|
+
}),
|
|
1160
|
+
telemetryExporterCounter: meter.createCounter("openclaw.telemetry.exporter.events", {
|
|
1161
|
+
unit: "1",
|
|
1162
|
+
description: "Diagnostic telemetry exporter lifecycle and failure events"
|
|
1163
|
+
})
|
|
1164
|
+
};
|
|
1165
|
+
}
|
|
1166
|
+
//#endregion
|
|
1167
|
+
//#region extensions/diagnostics-otel/src/service-recorder-runtime.ts
|
|
1168
|
+
function createDiagnosticsRecorderRuntime(params) {
|
|
1169
|
+
return {
|
|
1170
|
+
...params.metrics,
|
|
1171
|
+
...params.traces,
|
|
1172
|
+
contentCapturePolicy: params.contentCapturePolicy,
|
|
1173
|
+
tracesEnabled: params.tracesEnabled
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
//#endregion
|
|
1177
|
+
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1178
|
+
function createHarnessRecorders(runtime) {
|
|
1179
|
+
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1180
|
+
const harnessRunMetricAttrs = (evt) => ({
|
|
1181
|
+
"openclaw.harness.id": lowCardinalityAttr(evt.harnessId, "unknown"),
|
|
1182
|
+
"openclaw.harness.plugin": lowCardinalityAttr(evt.pluginId),
|
|
1183
|
+
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1184
|
+
"openclaw.provider": lowCardinalityAttr(evt.provider, "unknown"),
|
|
1185
|
+
"openclaw.model": lowCardinalityAttr(evt.model, "unknown"),
|
|
1186
|
+
...evt.channel ? { "openclaw.channel": lowCardinalityAttr(evt.channel) } : {}
|
|
1187
|
+
});
|
|
1188
|
+
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1189
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1190
|
+
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", harnessRunMetricAttrs(evt), void 0, {
|
|
1191
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1192
|
+
startTimeMs: evt.ts
|
|
1193
|
+
}));
|
|
1194
|
+
};
|
|
1195
|
+
const recordHarnessRunCompleted = (evt, metadata, privateData) => {
|
|
1196
|
+
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
|
|
1197
|
+
if (!tracesEnabled) return;
|
|
1198
|
+
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1199
|
+
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = lowCardinalityAttr(evt.resultClassification);
|
|
1200
|
+
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1201
|
+
if (evt.itemLifecycle) {
|
|
1202
|
+
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1203
|
+
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1204
|
+
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1205
|
+
}
|
|
1206
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1207
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1208
|
+
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1209
|
+
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1210
|
+
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1211
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1212
|
+
endTimeMs: evt.ts
|
|
1213
|
+
});
|
|
1214
|
+
setSpanAttrs(span, spanAttrs);
|
|
1215
|
+
if (evt.outcome === "error") span.setStatus({
|
|
1216
|
+
code: SpanStatusCode.ERROR,
|
|
1217
|
+
message: redactedError ?? "error"
|
|
1218
|
+
});
|
|
1219
|
+
if (trackedSpan && trustedTrace?.spanId) {
|
|
1220
|
+
completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1223
|
+
span.end(evt.ts);
|
|
1224
|
+
};
|
|
1225
|
+
const recordHarnessRunError = (evt, metadata, privateData) => {
|
|
1226
|
+
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
|
|
1227
|
+
const attrs = {
|
|
1228
|
+
...harnessRunMetricAttrs(evt),
|
|
1229
|
+
"openclaw.harness.phase": evt.phase,
|
|
1230
|
+
"openclaw.errorCategory": errorType
|
|
1231
|
+
};
|
|
1232
|
+
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1233
|
+
if (!tracesEnabled) return;
|
|
1234
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1235
|
+
const spanAttrs = {
|
|
1236
|
+
...attrs,
|
|
1237
|
+
"error.type": errorType,
|
|
1238
|
+
...redactedError ? { "openclaw.error": redactedError } : {},
|
|
1239
|
+
...evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}
|
|
1240
|
+
};
|
|
1241
|
+
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1242
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1243
|
+
endTimeMs: evt.ts
|
|
1244
|
+
});
|
|
1245
|
+
setSpanAttrs(span, spanAttrs);
|
|
1246
|
+
span.setStatus({
|
|
1247
|
+
code: SpanStatusCode.ERROR,
|
|
1248
|
+
message: redactedError ?? errorType
|
|
1249
|
+
});
|
|
1250
|
+
span.end(evt.ts);
|
|
1251
|
+
};
|
|
1252
|
+
const recordContextAssembled = (evt, metadata) => {
|
|
1253
|
+
if (!tracesEnabled) return;
|
|
1254
|
+
const spanAttrs = {
|
|
1255
|
+
"openclaw.context.message_count": evt.messageCount,
|
|
1256
|
+
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1257
|
+
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1258
|
+
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1259
|
+
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1260
|
+
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1261
|
+
"openclaw.context.prompt_images": evt.promptImages
|
|
1262
|
+
};
|
|
1263
|
+
addRunAttrs(spanAttrs, evt);
|
|
1264
|
+
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1265
|
+
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1266
|
+
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1267
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1268
|
+
endTimeMs: evt.ts
|
|
1269
|
+
}).end(evt.ts);
|
|
1270
|
+
};
|
|
1271
|
+
const recordModelFailover = (evt, metadata) => {
|
|
1272
|
+
const metricAttrs = {
|
|
1273
|
+
"openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown"),
|
|
1274
|
+
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1275
|
+
"openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane, "unknown"),
|
|
1276
|
+
"openclaw.model": lowCardinalityAttr(evt.fromModel),
|
|
1277
|
+
"openclaw.provider": lowCardinalityAttr(evt.fromProvider),
|
|
1278
|
+
"openclaw.failover.to_model": lowCardinalityAttr(evt.toModel),
|
|
1279
|
+
"openclaw.failover.to_provider": lowCardinalityAttr(evt.toProvider)
|
|
1280
|
+
};
|
|
1281
|
+
modelFailoverCounter.add(1, metricAttrs);
|
|
1282
|
+
if (!tracesEnabled) return;
|
|
1283
|
+
const spanAttrs = { "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown") };
|
|
1284
|
+
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1285
|
+
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1286
|
+
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1287
|
+
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1288
|
+
if (evt.lane) spanAttrs["openclaw.lane"] = lowCardinalityQueueLaneAttr(evt.lane, "unknown");
|
|
1289
|
+
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1290
|
+
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1291
|
+
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1292
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1293
|
+
endTimeMs: evt.ts
|
|
1294
|
+
}).end(evt.ts);
|
|
1295
|
+
};
|
|
1296
|
+
return {
|
|
1297
|
+
recordHarnessRunStarted,
|
|
1298
|
+
recordHarnessRunCompleted,
|
|
1299
|
+
recordHarnessRunError,
|
|
1300
|
+
recordContextAssembled,
|
|
1301
|
+
recordModelFailover
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
1304
|
+
//#endregion
|
|
1305
|
+
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1306
|
+
function hasOtelSemconvOptIn(value, optIn) {
|
|
1307
|
+
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1308
|
+
}
|
|
1309
|
+
function emitLatestGenAiSemconv() {
|
|
1310
|
+
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
1311
|
+
}
|
|
1312
|
+
function genAiOperationName(api) {
|
|
1313
|
+
const normalized = api?.trim().toLowerCase();
|
|
1314
|
+
if (!normalized) return "chat";
|
|
1315
|
+
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
1316
|
+
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
1317
|
+
return "chat";
|
|
1318
|
+
}
|
|
1319
|
+
function positiveFiniteNumber(value) {
|
|
1320
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : void 0;
|
|
1321
|
+
}
|
|
1322
|
+
function assignPositiveNumberAttr(attrs, key, value) {
|
|
1323
|
+
const normalized = positiveFiniteNumber(value);
|
|
1324
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1325
|
+
}
|
|
1326
|
+
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1327
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1328
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1329
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1330
|
+
}
|
|
1331
|
+
function assignNumberAttr(attrs, key, value) {
|
|
1332
|
+
if (typeof value === "number" && Number.isFinite(value)) attrs[key] = value;
|
|
1333
|
+
}
|
|
1334
|
+
function modelCallPromptTokens(usage) {
|
|
1335
|
+
if (typeof usage.promptTokens === "number" && Number.isFinite(usage.promptTokens)) return usage.promptTokens;
|
|
1336
|
+
const input = usage.input ?? 0;
|
|
1337
|
+
const cacheRead = usage.cacheRead ?? 0;
|
|
1338
|
+
const cacheWrite = usage.cacheWrite ?? 0;
|
|
1339
|
+
const total = input + cacheRead + cacheWrite;
|
|
1340
|
+
return total > 0 ? total : void 0;
|
|
1341
|
+
}
|
|
1342
|
+
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
1343
|
+
const stats = evt.promptStats;
|
|
1344
|
+
if (!stats) return;
|
|
1345
|
+
for (const [key, value] of [
|
|
1346
|
+
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
1347
|
+
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
1348
|
+
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
1349
|
+
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1350
|
+
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1351
|
+
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1352
|
+
]) assignNumberAttr(attrs, key, value);
|
|
1353
|
+
}
|
|
1354
|
+
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1355
|
+
const usage = evt.usage;
|
|
1356
|
+
if (!usage) return;
|
|
1357
|
+
const promptTokens = modelCallPromptTokens(usage);
|
|
1358
|
+
for (const [key, value] of [
|
|
1359
|
+
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
1360
|
+
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
1361
|
+
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
1362
|
+
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
1363
|
+
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
1364
|
+
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
1365
|
+
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
1366
|
+
["gen_ai.usage.input_tokens", promptTokens],
|
|
1367
|
+
["gen_ai.usage.output_tokens", usage.output],
|
|
1368
|
+
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
1369
|
+
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
1370
|
+
]) assignPositiveNumberAttr(attrs, key, value);
|
|
1371
|
+
}
|
|
1372
|
+
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
1373
|
+
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = lowCardinalityAttr(input.provider);
|
|
1374
|
+
else attrs["gen_ai.system"] = lowCardinalityAttr(input.provider);
|
|
1375
|
+
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
1376
|
+
attrs["gen_ai.operation.name"] = genAiOperationName(input.api);
|
|
1377
|
+
}
|
|
1378
|
+
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1379
|
+
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1380
|
+
}
|
|
1381
|
+
function modelCallSpanName(evt) {
|
|
1382
|
+
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
1383
|
+
return `${genAiOperationName(evt.api)} ${lowCardinalityAttr(evt.model)}`;
|
|
1384
|
+
}
|
|
1385
|
+
function modelCallSpanKind() {
|
|
1386
|
+
return emitLatestGenAiSemconv() ? SpanKind.CLIENT : void 0;
|
|
1387
|
+
}
|
|
1388
|
+
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
1389
|
+
if (!upstreamRequestIdHash) return;
|
|
1390
|
+
const boundedHash = lowCardinalityAttr(upstreamRequestIdHash);
|
|
1391
|
+
if (boundedHash === "unknown") return;
|
|
1392
|
+
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
1393
|
+
}
|
|
1394
|
+
//#endregion
|
|
1395
|
+
//#region extensions/diagnostics-otel/src/service-genai-content.ts
|
|
1396
|
+
function textPart(content) {
|
|
1397
|
+
return {
|
|
1398
|
+
type: "text",
|
|
1399
|
+
content
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
function textPartContent(part) {
|
|
1403
|
+
if (part.type !== "text") return;
|
|
1404
|
+
if (typeof part.text === "string") return part.text;
|
|
1405
|
+
return typeof part.content === "string" ? part.content : void 0;
|
|
1406
|
+
}
|
|
1407
|
+
function toolCallResponseValue(value) {
|
|
1408
|
+
if (!Array.isArray(value)) return value;
|
|
1409
|
+
const textItems = [];
|
|
1410
|
+
for (const item of value) {
|
|
1411
|
+
const text = typeof item === "string" ? item : isRecord(item) ? textPartContent(item) : void 0;
|
|
1412
|
+
if (typeof text !== "string") return value;
|
|
1413
|
+
textItems.push(text);
|
|
1414
|
+
}
|
|
1415
|
+
const kept = textItems.slice(0, 200);
|
|
1416
|
+
const joined = kept.filter((text) => text.length > 0).join("\n");
|
|
1417
|
+
if (joined.length === 0) return value;
|
|
1418
|
+
const omitted = textItems.length - kept.length;
|
|
1419
|
+
return omitted > 0 ? `${joined}\n...(${omitted} more text parts omitted)` : joined;
|
|
1420
|
+
}
|
|
1421
|
+
function toolCallResponsePart(part) {
|
|
1422
|
+
return {
|
|
1423
|
+
type: "tool_call_response",
|
|
1424
|
+
...typeof part.id === "string" ? { id: part.id } : {},
|
|
1425
|
+
response: toolCallResponseValue(part.response ?? part.result ?? part.content ?? part.details ?? "")
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
function contentParts(value) {
|
|
1429
|
+
if (typeof value === "string") return value.length > 0 ? [textPart(value)] : [];
|
|
1430
|
+
if (!Array.isArray(value)) {
|
|
1431
|
+
if (value === void 0 || value === null) return [];
|
|
1432
|
+
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return [textPart(String(value))];
|
|
1433
|
+
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
1434
|
+
return json ? [textPart(json)] : [];
|
|
1435
|
+
}
|
|
1436
|
+
const parts = [];
|
|
1437
|
+
for (const part of value) {
|
|
1438
|
+
if (typeof part === "string") {
|
|
1439
|
+
if (part.length > 0) parts.push(textPart(part));
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
if (!isRecord(part)) continue;
|
|
1443
|
+
const text = textPartContent(part);
|
|
1444
|
+
if (text !== void 0) parts.push(textPart(text));
|
|
1445
|
+
else if (part.type === "thinking" && typeof part.thinking === "string") parts.push({
|
|
1446
|
+
type: "reasoning",
|
|
1447
|
+
content: part.thinking
|
|
1448
|
+
});
|
|
1449
|
+
else if (part.type === "toolCall" && typeof part.name === "string") parts.push({
|
|
1450
|
+
type: "tool_call",
|
|
1451
|
+
name: part.name,
|
|
1452
|
+
...typeof part.id === "string" ? { id: part.id } : {},
|
|
1453
|
+
...part.arguments !== void 0 ? { arguments: part.arguments } : {}
|
|
1454
|
+
});
|
|
1455
|
+
else if (part.type === "tool_call" && typeof part.name === "string") parts.push({
|
|
1456
|
+
type: "tool_call",
|
|
1457
|
+
name: part.name,
|
|
1458
|
+
...typeof part.id === "string" ? { id: part.id } : {},
|
|
1459
|
+
...part.arguments !== void 0 ? { arguments: part.arguments } : {}
|
|
1460
|
+
});
|
|
1461
|
+
else if (part.type === "tool_call_response") parts.push(toolCallResponsePart(part));
|
|
1462
|
+
else if (part.type === "image") {
|
|
1463
|
+
const data = typeof part.data === "string" ? part.data : void 0;
|
|
1464
|
+
parts.push({
|
|
1465
|
+
type: "blob",
|
|
1466
|
+
modality: "image",
|
|
1467
|
+
...typeof part.mimeType === "string" ? { mime_type: part.mimeType } : {},
|
|
1468
|
+
...typeof part.mime_type === "string" ? { mime_type: part.mime_type } : {},
|
|
1469
|
+
...data ? { content: data } : {}
|
|
1470
|
+
});
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
return parts;
|
|
1474
|
+
}
|
|
1475
|
+
function normalizeGenAiMessage(value, fallbackRole = "user") {
|
|
1476
|
+
if (typeof value === "string") return {
|
|
1477
|
+
role: fallbackRole,
|
|
1478
|
+
parts: [textPart(value)]
|
|
1479
|
+
};
|
|
1480
|
+
if (!isRecord(value)) return;
|
|
1481
|
+
const rawRole = typeof value.role === "string" ? value.role : fallbackRole;
|
|
1482
|
+
const role = rawRole === "toolResult" ? "tool" : rawRole;
|
|
1483
|
+
let parts;
|
|
1484
|
+
if (role === "tool") {
|
|
1485
|
+
const explicitParts = contentParts(value.parts);
|
|
1486
|
+
parts = explicitParts.length > 0 ? explicitParts : [toolCallResponsePart({
|
|
1487
|
+
id: value.toolCallId,
|
|
1488
|
+
response: value.content ?? value.details ?? ""
|
|
1489
|
+
})];
|
|
1490
|
+
} else parts = contentParts(value.parts ?? value.content);
|
|
1491
|
+
if (parts.length === 0) return;
|
|
1492
|
+
return {
|
|
1493
|
+
role,
|
|
1494
|
+
parts,
|
|
1495
|
+
...typeof value.name === "string" ? { name: value.name } : {},
|
|
1496
|
+
...typeof value.finish_reason === "string" ? { finish_reason: value.finish_reason } : {},
|
|
1497
|
+
...typeof value.stopReason === "string" ? { finish_reason: value.stopReason } : {}
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
function normalizeGenAiMessages(value, fallbackRole) {
|
|
1501
|
+
const source = Array.isArray(value) ? value : value === void 0 ? [] : [value];
|
|
1502
|
+
const messages = [];
|
|
1503
|
+
for (const item of source.slice(0, 200)) {
|
|
1504
|
+
const message = normalizeGenAiMessage(item, fallbackRole);
|
|
1505
|
+
if (message) messages.push(message);
|
|
1506
|
+
}
|
|
1507
|
+
return messages;
|
|
1508
|
+
}
|
|
1509
|
+
function normalizeGenAiToolDefinition(value) {
|
|
1510
|
+
if (!isRecord(value) || typeof value.name !== "string" || value.name.trim().length === 0) return;
|
|
1511
|
+
return {
|
|
1512
|
+
type: typeof value.type === "string" ? value.type : "function",
|
|
1513
|
+
name: value.name,
|
|
1514
|
+
...typeof value.description === "string" ? { description: value.description } : {},
|
|
1515
|
+
...value.parameters !== void 0 ? { parameters: value.parameters } : {}
|
|
1516
|
+
};
|
|
1517
|
+
}
|
|
1518
|
+
function normalizeGenAiToolDefinitions(value) {
|
|
1519
|
+
if (!Array.isArray(value)) return [];
|
|
1520
|
+
const definitions = [];
|
|
1521
|
+
for (const item of value.slice(0, 200)) {
|
|
1522
|
+
const definition = normalizeGenAiToolDefinition(item);
|
|
1523
|
+
if (definition) definitions.push(definition);
|
|
1524
|
+
}
|
|
1525
|
+
return definitions;
|
|
1526
|
+
}
|
|
1527
|
+
function assignJsonAttribute(attributes, key, value) {
|
|
1528
|
+
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
1529
|
+
if (json) attributes[key] = json;
|
|
1530
|
+
}
|
|
1531
|
+
function assignGenAiModelContentAttributes(attributes, content, policy) {
|
|
1532
|
+
if (policy.systemPrompt && typeof content?.systemPrompt === "string") assignJsonAttribute(attributes, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, [textPart(content.systemPrompt)]);
|
|
1533
|
+
if (policy.inputMessages) {
|
|
1534
|
+
const inputMessages = normalizeGenAiMessages(content?.inputMessages, "user");
|
|
1535
|
+
if (inputMessages.length > 0) {
|
|
1536
|
+
assignJsonAttribute(attributes, ATTR_GEN_AI_INPUT_MESSAGES, inputMessages);
|
|
1537
|
+
assignJsonAttribute(attributes, "input.value", inputMessages);
|
|
1538
|
+
attributes["input.mime_type"] = "application/json";
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
if (policy.toolDefinitions) {
|
|
1542
|
+
const toolDefinitions = normalizeGenAiToolDefinitions(content?.toolDefinitions);
|
|
1543
|
+
if (toolDefinitions.length > 0) assignJsonAttribute(attributes, ATTR_GEN_AI_TOOL_DEFINITIONS, toolDefinitions);
|
|
1544
|
+
}
|
|
1545
|
+
if (policy.outputMessages) {
|
|
1546
|
+
const outputMessages = normalizeGenAiMessages(content?.outputMessages, "assistant");
|
|
1547
|
+
if (outputMessages.length > 0) {
|
|
1548
|
+
assignJsonAttribute(attributes, ATTR_GEN_AI_OUTPUT_MESSAGES, outputMessages);
|
|
1549
|
+
assignJsonAttribute(attributes, "output.value", outputMessages);
|
|
1550
|
+
attributes["output.mime_type"] = "application/json";
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
function assignOtelContentAttribute(attributes, key, value) {
|
|
1555
|
+
const normalized = normalizeOtelContentValue(value);
|
|
1556
|
+
if (normalized) attributes[key] = normalized;
|
|
1557
|
+
}
|
|
1558
|
+
function assignOtelToolIdentityAttributes(attributes, evt) {
|
|
1559
|
+
attributes["gen_ai.operation.name"] = GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL;
|
|
1560
|
+
const toolCallId = evt.toolCallId?.trim();
|
|
1561
|
+
if (toolCallId) attributes[ATTR_GEN_AI_TOOL_CALL_ID] = toolCallId;
|
|
1562
|
+
}
|
|
1563
|
+
function assignOtelModelContentAttributes(attributes, content, policy) {
|
|
1564
|
+
assignGenAiModelContentAttributes(attributes, content, policy);
|
|
1565
|
+
if (policy.inputMessages) assignOtelContentAttribute(attributes, "openclaw.content.input_messages", content?.inputMessages);
|
|
1566
|
+
if (policy.toolDefinitions) assignOtelContentAttribute(attributes, "openclaw.content.tool_definitions", content?.toolDefinitions);
|
|
1567
|
+
if (policy.outputMessages) assignOtelContentAttribute(attributes, "openclaw.content.output_messages", content?.outputMessages);
|
|
1568
|
+
if (policy.systemPrompt) assignOtelContentAttribute(attributes, "openclaw.content.system_prompt", content?.systemPrompt);
|
|
1569
|
+
}
|
|
1570
|
+
function assignOtelToolContentAttributes(attributes, content, policy) {
|
|
1571
|
+
if (policy.toolInputs) {
|
|
1572
|
+
const toolInput = normalizeOtelContentValue(content?.toolInput);
|
|
1573
|
+
if (toolInput) {
|
|
1574
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_ARGUMENTS] = toolInput;
|
|
1575
|
+
attributes["openclaw.content.tool_input"] = toolInput;
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
if (policy.toolOutputs) {
|
|
1579
|
+
const toolOutput = normalizeOtelContentValue(content?.toolOutput);
|
|
1580
|
+
if (toolOutput) {
|
|
1581
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_RESULT] = toolOutput;
|
|
1582
|
+
attributes["openclaw.content.tool_output"] = toolOutput;
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
//#endregion
|
|
1587
|
+
//#region extensions/diagnostics-otel/src/service-recorders-model.ts
|
|
1588
|
+
function createModelRecorders(runtime) {
|
|
1589
|
+
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1590
|
+
const modelCallMetricAttrs = (evt) => ({
|
|
1591
|
+
"openclaw.provider": evt.provider,
|
|
1592
|
+
"openclaw.model": evt.model,
|
|
1593
|
+
"openclaw.api": lowCardinalityAttr(evt.api),
|
|
1594
|
+
"openclaw.transport": lowCardinalityAttr(evt.transport)
|
|
1595
|
+
});
|
|
1596
|
+
const genAiModelCallMetricAttrs = (evt, errorType) => ({
|
|
1597
|
+
"gen_ai.operation.name": genAiOperationName(evt.api),
|
|
1598
|
+
"gen_ai.provider.name": lowCardinalityAttr(evt.provider),
|
|
1599
|
+
"gen_ai.request.model": lowCardinalityAttr(evt.model),
|
|
1600
|
+
...errorType ? { "error.type": errorType } : {}
|
|
1601
|
+
});
|
|
1602
|
+
const recordModelCallSizeTimingMetrics = (evt, attrs) => {
|
|
1603
|
+
const requestPayloadBytes = positiveFiniteNumber(evt.requestPayloadBytes);
|
|
1604
|
+
if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
|
|
1605
|
+
const responseStreamBytes = positiveFiniteNumber(evt.responseStreamBytes);
|
|
1606
|
+
if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
|
|
1607
|
+
const timeToFirstByteMs = positiveFiniteNumber(evt.timeToFirstByteMs);
|
|
1608
|
+
if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
|
|
1609
|
+
};
|
|
1610
|
+
const recordModelCallStarted = (evt, metadata) => {
|
|
1611
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1612
|
+
const spanAttrs = {
|
|
1613
|
+
"openclaw.provider": evt.provider,
|
|
1614
|
+
"openclaw.model": evt.model
|
|
1615
|
+
};
|
|
1616
|
+
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1617
|
+
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1618
|
+
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1619
|
+
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1620
|
+
trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1621
|
+
kind: modelCallSpanKind(),
|
|
1622
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1623
|
+
startTimeMs: evt.ts
|
|
1624
|
+
}));
|
|
1625
|
+
};
|
|
1626
|
+
const recordModelCallCompleted = (evt, metadata, modelContent) => {
|
|
1627
|
+
const metricAttrs = modelCallMetricAttrs(evt);
|
|
1628
|
+
modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
1629
|
+
recordModelCallSizeTimingMetrics(evt, metricAttrs);
|
|
1630
|
+
genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt));
|
|
1631
|
+
if (!tracesEnabled) return;
|
|
1632
|
+
const spanAttrs = {
|
|
1633
|
+
"openclaw.provider": evt.provider,
|
|
1634
|
+
"openclaw.model": evt.model
|
|
1635
|
+
};
|
|
1636
|
+
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1637
|
+
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1638
|
+
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1639
|
+
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
1640
|
+
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1641
|
+
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1642
|
+
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
1643
|
+
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1644
|
+
kind: modelCallSpanKind(),
|
|
1645
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1646
|
+
endTimeMs: evt.ts
|
|
1647
|
+
});
|
|
1648
|
+
setSpanAttrs(span, spanAttrs);
|
|
1649
|
+
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
1650
|
+
span.end(evt.ts);
|
|
1651
|
+
};
|
|
1652
|
+
const recordModelCallError = (evt, metadata, modelContent) => {
|
|
1653
|
+
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
|
|
1654
|
+
const metricAttrs = {
|
|
1655
|
+
...modelCallMetricAttrs(evt),
|
|
1656
|
+
"openclaw.errorCategory": errorType,
|
|
1657
|
+
...evt.failureKind ? { "openclaw.failureKind": lowCardinalityAttr(evt.failureKind, "other") } : {}
|
|
1658
|
+
};
|
|
1659
|
+
modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
1660
|
+
recordModelCallSizeTimingMetrics(evt, metricAttrs);
|
|
1661
|
+
genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt, errorType));
|
|
1662
|
+
if (!tracesEnabled) return;
|
|
1663
|
+
const spanAttrs = {
|
|
1664
|
+
"openclaw.provider": evt.provider,
|
|
1665
|
+
"openclaw.model": evt.model,
|
|
1666
|
+
"openclaw.errorCategory": errorType,
|
|
1667
|
+
"error.type": errorType
|
|
1668
|
+
};
|
|
1669
|
+
if (evt.failureKind) spanAttrs["openclaw.failureKind"] = lowCardinalityAttr(evt.failureKind, "other");
|
|
1670
|
+
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1671
|
+
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1672
|
+
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1673
|
+
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
1674
|
+
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1675
|
+
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1676
|
+
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
1677
|
+
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1678
|
+
kind: modelCallSpanKind(),
|
|
1679
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1680
|
+
endTimeMs: evt.ts
|
|
1681
|
+
});
|
|
1682
|
+
setSpanAttrs(span, spanAttrs);
|
|
1683
|
+
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
1684
|
+
span.setStatus({
|
|
1685
|
+
code: SpanStatusCode.ERROR,
|
|
1686
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
1687
|
+
});
|
|
1688
|
+
span.end(evt.ts);
|
|
1689
|
+
};
|
|
1690
|
+
return {
|
|
1691
|
+
recordModelCallSizeTimingMetrics,
|
|
1692
|
+
recordModelCallStarted,
|
|
1693
|
+
recordModelCallCompleted,
|
|
1694
|
+
recordModelCallError
|
|
1695
|
+
};
|
|
1696
|
+
}
|
|
1697
|
+
//#endregion
|
|
1698
|
+
//#region extensions/diagnostics-otel/src/service-recorders-operations.ts
|
|
1699
|
+
function createOperationsRecorders(runtime) {
|
|
1700
|
+
const { durationHistogram, queueDepthHistogram, queueWaitHistogram, laneEnqueueCounter, laneDequeueCounter, sessionStateCounter, sessionTurnCreatedCounter, sessionStuckCounter, sessionStuckAgeHistogram, sessionRecoveryRequestedCounter, sessionRecoveryCompletedCounter, sessionRecoveryAgeHistogram, talkEventCounter, talkEventDurationHistogram, talkAudioBytesHistogram, runAttemptCounter, toolLoopCounter, memoryRssHistogram, memoryHeapUsedHistogram, memoryHeapTotalHistogram, memoryExternalHistogram, memoryArrayBuffersHistogram, memoryPressureCounter, asyncQueueDroppedCounter, tracer, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1701
|
+
const recordLaneEnqueue = (evt) => {
|
|
1702
|
+
const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
|
|
1703
|
+
laneEnqueueCounter.add(1, attrs);
|
|
1704
|
+
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1705
|
+
};
|
|
1706
|
+
const recordLaneDequeue = (evt) => {
|
|
1707
|
+
const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
|
|
1708
|
+
laneDequeueCounter.add(1, attrs);
|
|
1709
|
+
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1710
|
+
if (typeof evt.waitMs === "number") queueWaitHistogram.record(evt.waitMs, attrs);
|
|
1711
|
+
};
|
|
1712
|
+
const recordSessionState = (evt) => {
|
|
1713
|
+
const attrs = { "openclaw.state": evt.state };
|
|
1714
|
+
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1715
|
+
sessionStateCounter.add(1, attrs);
|
|
1716
|
+
};
|
|
1717
|
+
const recordSessionTurnCreated = (evt) => {
|
|
1718
|
+
sessionTurnCreatedCounter.add(1, {
|
|
1719
|
+
"openclaw.agent": lowCardinalityAttr(evt.agentId, "unknown"),
|
|
1720
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel, "unknown"),
|
|
1721
|
+
"openclaw.trigger": evt.trigger
|
|
1722
|
+
});
|
|
1723
|
+
};
|
|
1724
|
+
const recordSessionStuck = (evt) => {
|
|
1725
|
+
const attrs = { "openclaw.state": evt.state };
|
|
1726
|
+
sessionStuckCounter.add(1, attrs);
|
|
1727
|
+
if (typeof evt.ageMs === "number") sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
|
1728
|
+
if (!tracesEnabled) return;
|
|
1729
|
+
const spanAttrs = { ...attrs };
|
|
1730
|
+
spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
|
|
1731
|
+
spanAttrs["openclaw.ageMs"] = evt.ageMs;
|
|
1732
|
+
const span = tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
|
|
1733
|
+
span.setStatus({
|
|
1734
|
+
code: SpanStatusCode.ERROR,
|
|
1735
|
+
message: "session stuck"
|
|
1736
|
+
});
|
|
1737
|
+
span.end();
|
|
1738
|
+
};
|
|
1739
|
+
const sessionRecoveryAttrs = (evt) => {
|
|
1740
|
+
const attrs = { "openclaw.state": evt.state };
|
|
1741
|
+
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1742
|
+
if (evt.activeWorkKind) attrs["openclaw.active_work_kind"] = evt.activeWorkKind;
|
|
1743
|
+
return attrs;
|
|
1744
|
+
};
|
|
1745
|
+
const recordSessionRecoveryRequested = (evt) => {
|
|
1746
|
+
const attrs = sessionRecoveryAttrs(evt);
|
|
1747
|
+
attrs["openclaw.action"] = evt.allowActiveAbort ? "abort" : "recover";
|
|
1748
|
+
sessionRecoveryRequestedCounter.add(1, attrs);
|
|
1749
|
+
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1750
|
+
};
|
|
1751
|
+
const recordSessionRecoveryCompleted = (evt) => {
|
|
1752
|
+
const attrs = sessionRecoveryAttrs(evt);
|
|
1753
|
+
attrs["openclaw.status"] = evt.status;
|
|
1754
|
+
attrs["openclaw.action"] = lowCardinalityAttr(evt.action, "unknown");
|
|
1755
|
+
if (evt.outcomeReason) attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason);
|
|
1756
|
+
sessionRecoveryCompletedCounter.add(1, attrs);
|
|
1757
|
+
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1758
|
+
};
|
|
1759
|
+
const talkEventAttrs = (evt) => ({
|
|
1760
|
+
"openclaw.talk.brain": lowCardinalityAttr(evt.brain),
|
|
1761
|
+
"openclaw.talk.event_type": lowCardinalityAttr(evt.talkEventType),
|
|
1762
|
+
"openclaw.talk.mode": lowCardinalityAttr(evt.mode),
|
|
1763
|
+
"openclaw.talk.provider": lowCardinalityAttr(evt.provider),
|
|
1764
|
+
"openclaw.talk.transport": lowCardinalityAttr(evt.transport)
|
|
1765
|
+
});
|
|
1766
|
+
const recordTalkEvent = (evt, metadata) => {
|
|
1767
|
+
if (!metadata.trusted) return;
|
|
1768
|
+
const attrs = talkEventAttrs(evt);
|
|
1769
|
+
talkEventCounter.add(1, attrs);
|
|
1770
|
+
if (typeof evt.durationMs === "number") talkEventDurationHistogram.record(evt.durationMs, attrs);
|
|
1771
|
+
if (typeof evt.byteLength === "number") talkAudioBytesHistogram.record(evt.byteLength, attrs);
|
|
1772
|
+
};
|
|
1773
|
+
const recordRunAttempt = (evt) => {
|
|
1774
|
+
runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
|
|
1775
|
+
};
|
|
1776
|
+
const toolLoopAttrs = (evt) => ({
|
|
1777
|
+
"openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool"),
|
|
1778
|
+
"openclaw.loop.level": evt.level,
|
|
1779
|
+
"openclaw.loop.action": evt.action,
|
|
1780
|
+
"openclaw.loop.detector": evt.detector,
|
|
1781
|
+
"openclaw.loop.count": evt.count,
|
|
1782
|
+
...evt.pairedToolName ? { "openclaw.loop.paired_tool": lowCardinalityAttr(evt.pairedToolName, "tool") } : {}
|
|
1783
|
+
});
|
|
1784
|
+
const recordToolLoop = (evt) => {
|
|
1785
|
+
const attrs = toolLoopAttrs(evt);
|
|
1786
|
+
toolLoopCounter.add(1, attrs);
|
|
1787
|
+
if (!tracesEnabled) return;
|
|
1788
|
+
const span = spanWithDuration("openclaw.tool.loop", attrs, 0, { endTimeMs: evt.ts });
|
|
1789
|
+
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
1790
|
+
code: SpanStatusCode.ERROR,
|
|
1791
|
+
message: `${evt.detector}:${evt.action}`
|
|
1792
|
+
});
|
|
1793
|
+
span.end(evt.ts);
|
|
1794
|
+
};
|
|
1795
|
+
const recordMemoryUsageMetrics = (evt, attrs = {}) => {
|
|
1796
|
+
memoryRssHistogram.record(evt.memory.rssBytes, attrs);
|
|
1797
|
+
memoryHeapUsedHistogram.record(evt.memory.heapUsedBytes, attrs);
|
|
1798
|
+
memoryHeapTotalHistogram.record(evt.memory.heapTotalBytes, attrs);
|
|
1799
|
+
memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
1800
|
+
memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
1801
|
+
};
|
|
1802
|
+
const recordMemorySample = (evt) => {
|
|
1803
|
+
recordMemoryUsageMetrics(evt);
|
|
1804
|
+
};
|
|
1805
|
+
const recordMemoryPressure = (evt) => {
|
|
1806
|
+
const attrs = {
|
|
1807
|
+
"openclaw.memory.level": evt.level,
|
|
1808
|
+
"openclaw.memory.reason": evt.reason
|
|
1809
|
+
};
|
|
1810
|
+
memoryPressureCounter.add(1, attrs);
|
|
1811
|
+
recordMemoryUsageMetrics(evt, attrs);
|
|
1812
|
+
if (!tracesEnabled) return;
|
|
1813
|
+
const spanAttrs = {
|
|
1814
|
+
...attrs,
|
|
1815
|
+
"openclaw.memory.rss_bytes": evt.memory.rssBytes,
|
|
1816
|
+
"openclaw.memory.heap_used_bytes": evt.memory.heapUsedBytes,
|
|
1817
|
+
"openclaw.memory.heap_total_bytes": evt.memory.heapTotalBytes,
|
|
1818
|
+
"openclaw.memory.external_bytes": evt.memory.externalBytes,
|
|
1819
|
+
"openclaw.memory.array_buffers_bytes": evt.memory.arrayBuffersBytes,
|
|
1820
|
+
...evt.thresholdBytes !== void 0 ? { "openclaw.memory.threshold_bytes": evt.thresholdBytes } : {},
|
|
1821
|
+
...evt.rssGrowthBytes !== void 0 ? { "openclaw.memory.rss_growth_bytes": evt.rssGrowthBytes } : {},
|
|
1822
|
+
...evt.windowMs !== void 0 ? { "openclaw.memory.window_ms": evt.windowMs } : {}
|
|
1823
|
+
};
|
|
1824
|
+
const span = spanWithDuration("openclaw.memory.pressure", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1825
|
+
if (evt.level === "critical") span.setStatus({
|
|
1826
|
+
code: SpanStatusCode.ERROR,
|
|
1827
|
+
message: evt.reason
|
|
1828
|
+
});
|
|
1829
|
+
span.end(evt.ts);
|
|
1830
|
+
};
|
|
1831
|
+
const recordAsyncQueueDropped = (evt) => {
|
|
1832
|
+
asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
|
|
1833
|
+
if (evt.droppedTrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedTrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "trusted" });
|
|
1834
|
+
if (evt.droppedUntrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedUntrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "untrusted" });
|
|
1835
|
+
if (evt.droppedPriorityEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedPriorityEvents, { "openclaw.diagnostic.async_queue.drop_class": "priority" });
|
|
1836
|
+
};
|
|
1837
|
+
const recordRunCompleted = (evt, metadata, privateData) => {
|
|
1838
|
+
const attrs = {
|
|
1839
|
+
"openclaw.outcome": evt.outcome,
|
|
1840
|
+
"openclaw.provider": evt.provider ?? "unknown",
|
|
1841
|
+
"openclaw.model": evt.model ?? "unknown"
|
|
1842
|
+
};
|
|
1843
|
+
if (evt.channel) attrs["openclaw.channel"] = evt.channel;
|
|
1844
|
+
if (evt.blockedBy) attrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
|
|
1845
|
+
durationHistogram.record(evt.durationMs, attrs);
|
|
1846
|
+
if (!tracesEnabled) return;
|
|
1847
|
+
const spanAttrs = { "openclaw.outcome": evt.outcome };
|
|
1848
|
+
addRunAttrs(spanAttrs, evt);
|
|
1849
|
+
if (evt.blockedBy) spanAttrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
|
|
1850
|
+
if (evt.errorCategory) spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other");
|
|
1851
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1852
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1853
|
+
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1854
|
+
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1855
|
+
const span = trackedSpan ?? spanWithDuration("openclaw.run", spanAttrs, evt.durationMs, {
|
|
1856
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1857
|
+
endTimeMs: evt.ts
|
|
1858
|
+
});
|
|
1859
|
+
setSpanAttrs(span, spanAttrs);
|
|
1860
|
+
if (evt.outcome === "error") {
|
|
1861
|
+
const message = redactedError ?? (evt.errorCategory ? redactSensitiveText(evt.errorCategory) : void 0);
|
|
1862
|
+
span.setStatus({
|
|
1863
|
+
code: SpanStatusCode.ERROR,
|
|
1864
|
+
...message ? { message } : {}
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
if (trackedSpan && trustedTrace?.spanId) {
|
|
1868
|
+
completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
span.end(evt.ts);
|
|
1872
|
+
};
|
|
1873
|
+
return {
|
|
1874
|
+
recordLaneEnqueue,
|
|
1875
|
+
recordLaneDequeue,
|
|
1876
|
+
recordSessionState,
|
|
1877
|
+
recordSessionTurnCreated,
|
|
1878
|
+
recordSessionStuck,
|
|
1879
|
+
recordSessionRecoveryRequested,
|
|
1880
|
+
recordSessionRecoveryCompleted,
|
|
1881
|
+
recordTalkEvent,
|
|
1882
|
+
recordRunAttempt,
|
|
1883
|
+
recordToolLoop,
|
|
1884
|
+
recordMemoryUsageMetrics,
|
|
1885
|
+
recordMemorySample,
|
|
1886
|
+
recordMemoryPressure,
|
|
1887
|
+
recordAsyncQueueDropped,
|
|
1888
|
+
recordRunCompleted
|
|
1889
|
+
};
|
|
1890
|
+
}
|
|
1891
|
+
//#endregion
|
|
1892
|
+
//#region extensions/diagnostics-otel/src/service-recorders-tools.ts
|
|
1893
|
+
function createToolAndSystemRecorders(runtime) {
|
|
1894
|
+
const { queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1895
|
+
const toolExecutionBaseAttrs = (evt) => ({
|
|
1896
|
+
"openclaw.toolName": evt.toolName,
|
|
1897
|
+
"openclaw.tool.source": lowCardinalityAttr(evt.toolSource, "core"),
|
|
1898
|
+
"gen_ai.tool.name": evt.toolName,
|
|
1899
|
+
...evt.toolOwner ? { "openclaw.tool.owner": lowCardinalityAttr(evt.toolOwner) } : {},
|
|
1900
|
+
...paramsSummaryAttrs(evt.paramsSummary)
|
|
1901
|
+
});
|
|
1902
|
+
const skillUsedAttrs = (evt) => ({
|
|
1903
|
+
"openclaw.skill.name": lowCardinalityAttr(evt.skillName, "skill"),
|
|
1904
|
+
"openclaw.skill.source": lowCardinalityAttr(evt.skillSource),
|
|
1905
|
+
"openclaw.skill.activation": lowCardinalityAttr(evt.activation),
|
|
1906
|
+
...evt.agentId ? { "openclaw.agent": lowCardinalityAttr(evt.agentId) } : {},
|
|
1907
|
+
...evt.toolName ? { "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool") } : {}
|
|
1908
|
+
});
|
|
1909
|
+
const recordSkillUsed = (evt, metadata) => {
|
|
1910
|
+
if (!metadata.trusted) return;
|
|
1911
|
+
const attrs = skillUsedAttrs(evt);
|
|
1912
|
+
skillUsedCounter.add(1, attrs);
|
|
1913
|
+
if (!tracesEnabled) return;
|
|
1914
|
+
const spanAttrs = { ...attrs };
|
|
1915
|
+
addRunAttrs(spanAttrs, evt);
|
|
1916
|
+
const span = spanWithDuration("openclaw.skill.used", spanAttrs, 0, {
|
|
1917
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1918
|
+
endTimeMs: evt.ts
|
|
1919
|
+
});
|
|
1920
|
+
setSpanAttrs(span, spanAttrs);
|
|
1921
|
+
span.end(evt.ts);
|
|
1922
|
+
};
|
|
1923
|
+
const recordToolExecutionStarted = (evt, metadata) => {
|
|
1924
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1925
|
+
const spanAttrs = toolExecutionBaseAttrs(evt);
|
|
1926
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1927
|
+
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.tool.execution", spanAttrs, void 0, {
|
|
1928
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1929
|
+
startTimeMs: evt.ts
|
|
1930
|
+
}));
|
|
1931
|
+
};
|
|
1932
|
+
const recordToolExecutionCompleted = (evt, metadata, toolContent) => {
|
|
1933
|
+
const attrs = toolExecutionBaseAttrs(evt);
|
|
1934
|
+
toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
1935
|
+
if (!tracesEnabled) return;
|
|
1936
|
+
const spanAttrs = { ...attrs };
|
|
1937
|
+
addRunAttrs(spanAttrs, evt);
|
|
1938
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1939
|
+
assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
|
|
1940
|
+
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
1941
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1942
|
+
endTimeMs: evt.ts
|
|
1943
|
+
});
|
|
1944
|
+
setSpanAttrs(span, spanAttrs);
|
|
1945
|
+
span.end(evt.ts);
|
|
1946
|
+
};
|
|
1947
|
+
const recordToolExecutionError = (evt, metadata, toolContent) => {
|
|
1948
|
+
const attrs = {
|
|
1949
|
+
...toolExecutionBaseAttrs(evt),
|
|
1950
|
+
"openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
|
|
1951
|
+
};
|
|
1952
|
+
toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
1953
|
+
if (!tracesEnabled) return;
|
|
1954
|
+
const spanAttrs = { ...attrs };
|
|
1955
|
+
addRunAttrs(spanAttrs, evt);
|
|
1956
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1957
|
+
if (evt.errorCode) spanAttrs["openclaw.errorCode"] = lowCardinalityAttr(evt.errorCode, "other");
|
|
1958
|
+
assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
|
|
1959
|
+
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
1960
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1961
|
+
endTimeMs: evt.ts
|
|
1962
|
+
});
|
|
1963
|
+
setSpanAttrs(span, spanAttrs);
|
|
1964
|
+
span.setStatus({
|
|
1965
|
+
code: SpanStatusCode.ERROR,
|
|
1966
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
1967
|
+
});
|
|
1968
|
+
span.end(evt.ts);
|
|
1969
|
+
};
|
|
1970
|
+
const recordToolExecutionBlocked = (evt, metadata) => {
|
|
1971
|
+
toolExecutionBlockedCounter.add(1, {
|
|
1972
|
+
...toolExecutionBaseAttrs(evt),
|
|
1973
|
+
"openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
|
|
1974
|
+
});
|
|
1975
|
+
if (!tracesEnabled) return;
|
|
1976
|
+
const spanAttrs = {
|
|
1977
|
+
...toolExecutionBaseAttrs(evt),
|
|
1978
|
+
"openclaw.outcome": "blocked",
|
|
1979
|
+
"openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
|
|
1980
|
+
};
|
|
1981
|
+
addRunAttrs(spanAttrs, evt);
|
|
1982
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1983
|
+
const span = spanWithDuration("openclaw.tool.execution", spanAttrs, 0, {
|
|
1984
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1985
|
+
endTimeMs: evt.ts
|
|
1986
|
+
});
|
|
1987
|
+
setSpanAttrs(span, spanAttrs);
|
|
1988
|
+
span.end(evt.ts);
|
|
1989
|
+
};
|
|
1990
|
+
const recordPayloadLarge = (evt) => {
|
|
1991
|
+
const attrs = {
|
|
1992
|
+
"openclaw.payload.action": evt.action,
|
|
1993
|
+
"openclaw.payload.surface": lowCardinalityAttr(evt.surface, "unknown"),
|
|
1994
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel, "none"),
|
|
1995
|
+
"openclaw.plugin": lowCardinalityAttr(evt.pluginId, "none"),
|
|
1996
|
+
"openclaw.reason": lowCardinalityAttr(evt.reason, "none")
|
|
1997
|
+
};
|
|
1998
|
+
payloadLargeCounter.add(1, attrs);
|
|
1999
|
+
const bytes = positiveFiniteNumber(evt.bytes);
|
|
2000
|
+
if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
|
|
2001
|
+
};
|
|
2002
|
+
const recordExecProcessCompleted = (evt) => {
|
|
2003
|
+
const attrs = {
|
|
2004
|
+
"openclaw.exec.target": evt.target,
|
|
2005
|
+
"openclaw.exec.mode": evt.mode,
|
|
2006
|
+
"openclaw.outcome": evt.outcome
|
|
2007
|
+
};
|
|
2008
|
+
if (evt.failureKind) attrs["openclaw.failureKind"] = evt.failureKind;
|
|
2009
|
+
execProcessDurationHistogram.record(evt.durationMs, attrs);
|
|
2010
|
+
if (!tracesEnabled) return;
|
|
2011
|
+
const spanAttrs = {
|
|
2012
|
+
...attrs,
|
|
2013
|
+
"openclaw.exec.command_length": evt.commandLength
|
|
2014
|
+
};
|
|
2015
|
+
if (typeof evt.exitCode === "number") spanAttrs["openclaw.exec.exit_code"] = evt.exitCode;
|
|
2016
|
+
if (evt.exitSignal) spanAttrs["openclaw.exec.exit_signal"] = lowCardinalityAttr(evt.exitSignal, "other");
|
|
2017
|
+
if (evt.timedOut !== void 0) spanAttrs["openclaw.exec.timed_out"] = evt.timedOut;
|
|
2018
|
+
const span = spanWithDuration("openclaw.exec", spanAttrs, evt.durationMs, { endTimeMs: evt.ts });
|
|
2019
|
+
if (evt.outcome === "failed") span.setStatus({
|
|
2020
|
+
code: SpanStatusCode.ERROR,
|
|
2021
|
+
...evt.failureKind ? { message: evt.failureKind } : {}
|
|
2022
|
+
});
|
|
2023
|
+
span.end(evt.ts);
|
|
2024
|
+
};
|
|
2025
|
+
const recordHeartbeat = (evt) => {
|
|
2026
|
+
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
|
|
2027
|
+
};
|
|
2028
|
+
const recordLivenessWarning = (evt) => {
|
|
2029
|
+
const reason = evt.reasons.join(":");
|
|
2030
|
+
const attrs = { "openclaw.liveness.reason": lowCardinalityAttr(reason, "unknown") };
|
|
2031
|
+
livenessWarningCounter.add(1, attrs);
|
|
2032
|
+
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" });
|
|
2033
|
+
if (evt.eventLoopDelayP99Ms !== void 0) livenessEventLoopDelayP99Histogram.record(evt.eventLoopDelayP99Ms, attrs);
|
|
2034
|
+
if (evt.eventLoopDelayMaxMs !== void 0) livenessEventLoopDelayMaxHistogram.record(evt.eventLoopDelayMaxMs, attrs);
|
|
2035
|
+
if (evt.eventLoopUtilization !== void 0) livenessEventLoopUtilizationHistogram.record(evt.eventLoopUtilization, attrs);
|
|
2036
|
+
if (evt.cpuCoreRatio !== void 0) livenessCpuCoreRatioHistogram.record(evt.cpuCoreRatio, attrs);
|
|
2037
|
+
if (!tracesEnabled) return;
|
|
2038
|
+
const spanAttrs = {
|
|
2039
|
+
...attrs,
|
|
2040
|
+
"openclaw.liveness.active": evt.active,
|
|
2041
|
+
"openclaw.liveness.waiting": evt.waiting,
|
|
2042
|
+
"openclaw.liveness.queued": evt.queued,
|
|
2043
|
+
"openclaw.liveness.interval_ms": evt.intervalMs,
|
|
2044
|
+
...evt.eventLoopDelayP99Ms !== void 0 ? { "openclaw.liveness.event_loop_delay_p99_ms": evt.eventLoopDelayP99Ms } : {},
|
|
2045
|
+
...evt.eventLoopDelayMaxMs !== void 0 ? { "openclaw.liveness.event_loop_delay_max_ms": evt.eventLoopDelayMaxMs } : {},
|
|
2046
|
+
...evt.eventLoopUtilization !== void 0 ? { "openclaw.liveness.event_loop_utilization": evt.eventLoopUtilization } : {},
|
|
2047
|
+
...evt.cpuUserMs !== void 0 ? { "openclaw.liveness.cpu_user_ms": evt.cpuUserMs } : {},
|
|
2048
|
+
...evt.cpuSystemMs !== void 0 ? { "openclaw.liveness.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
2049
|
+
...evt.cpuTotalMs !== void 0 ? { "openclaw.liveness.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
2050
|
+
...evt.cpuCoreRatio !== void 0 ? { "openclaw.liveness.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2051
|
+
};
|
|
2052
|
+
const span = spanWithDuration("openclaw.liveness.warning", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
2053
|
+
span.setStatus({
|
|
2054
|
+
code: SpanStatusCode.ERROR,
|
|
2055
|
+
message: reason
|
|
2056
|
+
});
|
|
2057
|
+
span.end(evt.ts);
|
|
2058
|
+
};
|
|
2059
|
+
const recordDiagnosticPhaseCompleted = (evt) => {
|
|
2060
|
+
if (!tracesEnabled) return;
|
|
2061
|
+
const spanAttrs = {
|
|
2062
|
+
"openclaw.phase": lowCardinalityAttr(evt.name, "unknown"),
|
|
2063
|
+
...evt.cpuUserMs !== void 0 ? { "openclaw.phase.cpu_user_ms": evt.cpuUserMs } : {},
|
|
2064
|
+
...evt.cpuSystemMs !== void 0 ? { "openclaw.phase.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
2065
|
+
...evt.cpuTotalMs !== void 0 ? { "openclaw.phase.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
2066
|
+
...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2067
|
+
};
|
|
2068
|
+
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
2069
|
+
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, { endTimeMs: evt.ts }).end(evt.ts);
|
|
2070
|
+
};
|
|
2071
|
+
const recordTelemetryExporter = (evt, metadata) => {
|
|
2072
|
+
if (!metadata.trusted) return;
|
|
2073
|
+
telemetryExporterCounter.add(1, {
|
|
2074
|
+
"openclaw.exporter": lowCardinalityAttr(evt.exporter, "unknown"),
|
|
2075
|
+
"openclaw.signal": evt.signal,
|
|
2076
|
+
"openclaw.status": evt.status,
|
|
2077
|
+
...evt.reason ? { "openclaw.reason": evt.reason } : {},
|
|
2078
|
+
...evt.errorCategory ? { "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other") } : {}
|
|
2079
|
+
});
|
|
2080
|
+
};
|
|
2081
|
+
return {
|
|
2082
|
+
recordSkillUsed,
|
|
2083
|
+
recordToolExecutionStarted,
|
|
2084
|
+
recordToolExecutionCompleted,
|
|
2085
|
+
recordToolExecutionError,
|
|
2086
|
+
recordToolExecutionBlocked,
|
|
2087
|
+
recordPayloadLarge,
|
|
2088
|
+
recordExecProcessCompleted,
|
|
2089
|
+
recordHeartbeat,
|
|
2090
|
+
recordLivenessWarning,
|
|
2091
|
+
recordDiagnosticPhaseCompleted,
|
|
2092
|
+
recordTelemetryExporter
|
|
2093
|
+
};
|
|
2094
|
+
}
|
|
2095
|
+
//#endregion
|
|
2096
|
+
//#region extensions/diagnostics-otel/src/service-recorders-usage.ts
|
|
2097
|
+
function createUsageRecorders(runtime) {
|
|
2098
|
+
const { tokensCounter, genAiTokenUsageHistogram, costCounter, durationHistogram, contextHistogram, webhookReceivedCounter, webhookErrorCounter, webhookDurationHistogram, messageQueuedCounter, messageReceivedCounter, messageDispatchStartedCounter, messageDispatchCompletedCounter, messageDispatchDurationHistogram, messageProcessedCounter, messageDurationHistogram, messageDeliveryStartedCounter, messageDeliveryDurationHistogram, queueDepthHistogram, tracer, activeTrustedSpans, activeTrustedSpanAliases, trustedSpanAliasKey, spanWithDuration, trustedTraceContext, internalOrTrustedTraceContext, internalOrTrustedExplicitParentContext, activeTrustedParentContext, activeInternalOrTrustedContext, trackTrustedSpan, trackInternalOrTrustedSpan, getTrackedInternalOrTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
2099
|
+
const recordModelUsage = (evt, metadata) => {
|
|
2100
|
+
const attrs = {
|
|
2101
|
+
"openclaw.channel": evt.channel ?? "unknown",
|
|
2102
|
+
"openclaw.agent": lowCardinalityAttr(evt.agentId),
|
|
2103
|
+
"openclaw.provider": evt.provider ?? "unknown",
|
|
2104
|
+
"openclaw.model": evt.model ?? "unknown"
|
|
2105
|
+
};
|
|
2106
|
+
const genAiAttrs = {
|
|
2107
|
+
"gen_ai.operation.name": "chat",
|
|
2108
|
+
"gen_ai.provider.name": lowCardinalityAttr(evt.provider),
|
|
2109
|
+
"gen_ai.request.model": lowCardinalityAttr(evt.model)
|
|
2110
|
+
};
|
|
2111
|
+
const usage = evt.usage;
|
|
2112
|
+
if (usage.input) {
|
|
2113
|
+
tokensCounter.add(usage.input, {
|
|
2114
|
+
...attrs,
|
|
2115
|
+
"openclaw.token": "input"
|
|
2116
|
+
});
|
|
2117
|
+
genAiTokenUsageHistogram.record(usage.input, {
|
|
2118
|
+
...genAiAttrs,
|
|
2119
|
+
"gen_ai.token.type": "input"
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
if (usage.output) {
|
|
2123
|
+
tokensCounter.add(usage.output, {
|
|
2124
|
+
...attrs,
|
|
2125
|
+
"openclaw.token": "output"
|
|
2126
|
+
});
|
|
2127
|
+
genAiTokenUsageHistogram.record(usage.output, {
|
|
2128
|
+
...genAiAttrs,
|
|
2129
|
+
"gen_ai.token.type": "output"
|
|
2130
|
+
});
|
|
2131
|
+
}
|
|
2132
|
+
if (usage.cacheRead) tokensCounter.add(usage.cacheRead, {
|
|
2133
|
+
...attrs,
|
|
2134
|
+
"openclaw.token": "cache_read"
|
|
2135
|
+
});
|
|
2136
|
+
if (usage.cacheWrite) tokensCounter.add(usage.cacheWrite, {
|
|
2137
|
+
...attrs,
|
|
2138
|
+
"openclaw.token": "cache_write"
|
|
2139
|
+
});
|
|
2140
|
+
if (usage.promptTokens) tokensCounter.add(usage.promptTokens, {
|
|
2141
|
+
...attrs,
|
|
2142
|
+
"openclaw.token": "prompt"
|
|
2143
|
+
});
|
|
2144
|
+
if (usage.total) tokensCounter.add(usage.total, {
|
|
2145
|
+
...attrs,
|
|
2146
|
+
"openclaw.token": "total"
|
|
2147
|
+
});
|
|
2148
|
+
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
2149
|
+
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
2150
|
+
if (evt.context?.limit) contextHistogram.record(evt.context.limit, {
|
|
2151
|
+
...attrs,
|
|
2152
|
+
"openclaw.context": "limit"
|
|
2153
|
+
});
|
|
2154
|
+
if (evt.context?.used) contextHistogram.record(evt.context.used, {
|
|
2155
|
+
...attrs,
|
|
2156
|
+
"openclaw.context": "used"
|
|
2157
|
+
});
|
|
2158
|
+
if (!tracesEnabled) return;
|
|
2159
|
+
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
2160
|
+
const spanAttrs = {
|
|
2161
|
+
...attrs,
|
|
2162
|
+
"openclaw.tokens.input": usage.input ?? 0,
|
|
2163
|
+
"openclaw.tokens.output": usage.output ?? 0,
|
|
2164
|
+
"openclaw.tokens.cache_read": usage.cacheRead ?? 0,
|
|
2165
|
+
"openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
|
|
2166
|
+
"openclaw.tokens.total": usage.total ?? 0
|
|
2167
|
+
};
|
|
2168
|
+
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
2169
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
2170
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
2171
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
2172
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite);
|
|
2173
|
+
spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs, {
|
|
2174
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2175
|
+
endTimeMs: evt.ts
|
|
2176
|
+
}).end(evt.ts);
|
|
2177
|
+
};
|
|
2178
|
+
const recordWebhookReceived = (evt) => {
|
|
2179
|
+
const attrs = {
|
|
2180
|
+
"openclaw.channel": evt.channel ?? "unknown",
|
|
2181
|
+
"openclaw.webhook": evt.updateType ?? "unknown"
|
|
2182
|
+
};
|
|
2183
|
+
webhookReceivedCounter.add(1, attrs);
|
|
2184
|
+
};
|
|
2185
|
+
const recordWebhookProcessed = (evt) => {
|
|
2186
|
+
const attrs = {
|
|
2187
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2188
|
+
"openclaw.webhook": lowCardinalityAttr(evt.updateType)
|
|
2189
|
+
};
|
|
2190
|
+
if (typeof evt.durationMs === "number") webhookDurationHistogram.record(evt.durationMs, attrs);
|
|
2191
|
+
if (!tracesEnabled) return;
|
|
2192
|
+
const spanAttrs = { ...attrs };
|
|
2193
|
+
spanWithDuration("openclaw.webhook.processed", spanAttrs, evt.durationMs).end();
|
|
2194
|
+
};
|
|
2195
|
+
const recordWebhookError = (evt) => {
|
|
2196
|
+
const attrs = {
|
|
2197
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2198
|
+
"openclaw.webhook": lowCardinalityAttr(evt.updateType)
|
|
2199
|
+
};
|
|
2200
|
+
webhookErrorCounter.add(1, attrs);
|
|
2201
|
+
if (!tracesEnabled) return;
|
|
2202
|
+
const redactedError = redactSensitiveText(evt.error);
|
|
2203
|
+
const spanAttrs = {
|
|
2204
|
+
...attrs,
|
|
2205
|
+
"openclaw.error": redactedError
|
|
2206
|
+
};
|
|
2207
|
+
const span = tracer.startSpan("openclaw.webhook.error", { attributes: spanAttrs });
|
|
2208
|
+
span.setStatus({
|
|
2209
|
+
code: SpanStatusCode.ERROR,
|
|
2210
|
+
message: redactedError
|
|
2211
|
+
});
|
|
2212
|
+
span.end();
|
|
2213
|
+
};
|
|
2214
|
+
const recordMessageQueued = (evt) => {
|
|
2215
|
+
const attrs = {
|
|
2216
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2217
|
+
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
2218
|
+
};
|
|
2219
|
+
messageQueuedCounter.add(1, attrs);
|
|
2220
|
+
if (typeof evt.queueDepth === "number") queueDepthHistogram.record(evt.queueDepth, attrs);
|
|
2221
|
+
};
|
|
2222
|
+
const recordMessageReceived = (evt) => {
|
|
2223
|
+
messageReceivedCounter.add(1, {
|
|
2224
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2225
|
+
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
2226
|
+
});
|
|
2227
|
+
};
|
|
2228
|
+
const recordMessageDispatchStarted = (evt, metadata) => {
|
|
2229
|
+
const attrs = {
|
|
2230
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2231
|
+
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
2232
|
+
};
|
|
2233
|
+
messageDispatchStartedCounter.add(1, attrs);
|
|
2234
|
+
if (!tracesEnabled) return;
|
|
2235
|
+
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2236
|
+
if (!traceContext?.spanId || activeTrustedSpans.has(traceContext.spanId)) return;
|
|
2237
|
+
trackInternalOrTrustedSpan(evt, metadata, spanWithDuration("openclaw.message.processed", attrs, void 0, {
|
|
2238
|
+
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2239
|
+
startTimeMs: evt.ts
|
|
2240
|
+
}));
|
|
2241
|
+
};
|
|
2242
|
+
const recordMessageDispatchCompleted = (evt) => {
|
|
2243
|
+
const attrs = {
|
|
2244
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2245
|
+
"openclaw.outcome": evt.outcome,
|
|
2246
|
+
"openclaw.reason": lowCardinalityAttr(evt.reason, "none"),
|
|
2247
|
+
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
2248
|
+
};
|
|
2249
|
+
messageDispatchCompletedCounter.add(1, attrs);
|
|
2250
|
+
messageDispatchDurationHistogram.record(evt.durationMs, attrs);
|
|
2251
|
+
};
|
|
2252
|
+
const recordMessageProcessed = (evt, metadata) => {
|
|
2253
|
+
const attrs = {
|
|
2254
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2255
|
+
"openclaw.outcome": evt.outcome ?? "unknown"
|
|
2256
|
+
};
|
|
2257
|
+
messageProcessedCounter.add(1, attrs);
|
|
2258
|
+
if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
|
|
2259
|
+
if (!tracesEnabled) return;
|
|
2260
|
+
const spanAttrs = { ...attrs };
|
|
2261
|
+
if (evt.reason) spanAttrs["openclaw.reason"] = lowCardinalityAttr(evt.reason, "unknown");
|
|
2262
|
+
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
2263
|
+
const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
2264
|
+
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2265
|
+
endTimeMs: evt.ts
|
|
2266
|
+
});
|
|
2267
|
+
setSpanAttrs(span, spanAttrs);
|
|
2268
|
+
if (evt.outcome === "error" && evt.error) span.setStatus({
|
|
2269
|
+
code: SpanStatusCode.ERROR,
|
|
2270
|
+
message: redactSensitiveText(evt.error)
|
|
2271
|
+
});
|
|
2272
|
+
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2273
|
+
if (trackedSpan && traceContext?.spanId) {
|
|
2274
|
+
completeTrackedLifecycleSpan(traceContext.spanId, trackedSpan, evt.ts);
|
|
2275
|
+
return;
|
|
2276
|
+
}
|
|
2277
|
+
span.end(evt.ts);
|
|
2278
|
+
};
|
|
2279
|
+
const messageDeliveryAttrs = (evt) => ({
|
|
2280
|
+
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
2281
|
+
"openclaw.delivery.kind": lowCardinalityAttr(evt.deliveryKind, "other")
|
|
2282
|
+
});
|
|
2283
|
+
const recordMessageDeliveryStarted = (evt) => {
|
|
2284
|
+
messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
|
|
2285
|
+
};
|
|
2286
|
+
const recordMessageDeliveryCompleted = (evt, metadata) => {
|
|
2287
|
+
const attrs = {
|
|
2288
|
+
...messageDeliveryAttrs(evt),
|
|
2289
|
+
"openclaw.outcome": "completed"
|
|
2290
|
+
};
|
|
2291
|
+
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
2292
|
+
if (!tracesEnabled) return;
|
|
2293
|
+
spanWithDuration("openclaw.message.delivery", {
|
|
2294
|
+
...attrs,
|
|
2295
|
+
"openclaw.delivery.result_count": evt.resultCount
|
|
2296
|
+
}, evt.durationMs, {
|
|
2297
|
+
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
2298
|
+
endTimeMs: evt.ts
|
|
2299
|
+
}).end(evt.ts);
|
|
2300
|
+
};
|
|
2301
|
+
const recordMessageDeliveryError = (evt, metadata) => {
|
|
2302
|
+
const attrs = {
|
|
2303
|
+
...messageDeliveryAttrs(evt),
|
|
2304
|
+
"openclaw.outcome": "error",
|
|
2305
|
+
"openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
|
|
2306
|
+
};
|
|
2307
|
+
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
2308
|
+
if (!tracesEnabled) return;
|
|
2309
|
+
const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, {
|
|
2310
|
+
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
2311
|
+
endTimeMs: evt.ts
|
|
2312
|
+
});
|
|
2313
|
+
span.setStatus({
|
|
2314
|
+
code: SpanStatusCode.ERROR,
|
|
2315
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
2316
|
+
});
|
|
2317
|
+
span.end(evt.ts);
|
|
2318
|
+
};
|
|
2319
|
+
const recordRunStarted = (evt, metadata) => {
|
|
2320
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
2321
|
+
const spanAttrs = {};
|
|
2322
|
+
addRunAttrs(spanAttrs, evt);
|
|
2323
|
+
const span = trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.run", spanAttrs, void 0, {
|
|
2324
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2325
|
+
startTimeMs: evt.ts
|
|
2326
|
+
}));
|
|
2327
|
+
const parentSpanId = trustedTraceContext(evt, metadata)?.parentSpanId;
|
|
2328
|
+
if (parentSpanId && !activeTrustedSpans.has(parentSpanId)) {
|
|
2329
|
+
const owner = {
|
|
2330
|
+
kind: "run",
|
|
2331
|
+
id: evt.runId
|
|
2332
|
+
};
|
|
2333
|
+
activeTrustedSpanAliases.set(trustedSpanAliasKey(parentSpanId, owner), {
|
|
2334
|
+
span,
|
|
2335
|
+
spanId: parentSpanId,
|
|
2336
|
+
owner
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2339
|
+
};
|
|
2340
|
+
return {
|
|
2341
|
+
recordModelUsage,
|
|
2342
|
+
recordWebhookReceived,
|
|
2343
|
+
recordWebhookProcessed,
|
|
2344
|
+
recordWebhookError,
|
|
2345
|
+
recordMessageQueued,
|
|
2346
|
+
recordMessageReceived,
|
|
2347
|
+
recordMessageDispatchStarted,
|
|
2348
|
+
recordMessageDispatchCompleted,
|
|
2349
|
+
recordMessageProcessed,
|
|
2350
|
+
recordMessageDeliveryStarted,
|
|
2351
|
+
recordMessageDeliveryCompleted,
|
|
2352
|
+
recordMessageDeliveryError,
|
|
2353
|
+
recordRunStarted
|
|
2354
|
+
};
|
|
2355
|
+
}
|
|
2356
|
+
//#endregion
|
|
2357
|
+
//#region extensions/diagnostics-otel/src/service-traces.ts
|
|
2358
|
+
function createDiagnosticsTraceRuntime(tracer) {
|
|
2359
|
+
const activeTrustedSpans = /* @__PURE__ */ new Map();
|
|
2360
|
+
const activeTrustedSpanAliases = /* @__PURE__ */ new Map();
|
|
2361
|
+
const retainedTrustedSpanContexts = /* @__PURE__ */ new Map();
|
|
2362
|
+
const retainedTrustedSpanContextCleanupTimers = /* @__PURE__ */ new Set();
|
|
2363
|
+
const stopActiveTrustedSpans = () => {
|
|
2364
|
+
const stopAt = Date.now();
|
|
2365
|
+
for (const handle of retainedTrustedSpanContextCleanupTimers) clearTimeout(handle);
|
|
2366
|
+
retainedTrustedSpanContextCleanupTimers.clear();
|
|
2367
|
+
retainedTrustedSpanContexts.clear();
|
|
2368
|
+
for (const span of /* @__PURE__ */ new Set([...activeTrustedSpans.values(), ...Array.from(activeTrustedSpanAliases.values(), (entry) => entry.span)])) span.end(stopAt);
|
|
2369
|
+
activeTrustedSpans.clear();
|
|
2370
|
+
activeTrustedSpanAliases.clear();
|
|
2371
|
+
};
|
|
2372
|
+
const spanWithDuration = (name, attributes, durationMs, options = {}) => {
|
|
2373
|
+
const endTimeMs = options.endTimeMs ?? Date.now();
|
|
2374
|
+
const startTime = typeof options.startTimeMs === "number" ? options.startTimeMs : typeof durationMs === "number" && durationMs >= 0 ? endTimeMs - durationMs : void 0;
|
|
2375
|
+
const parentContext = "parentContext" in options ? options.parentContext ?? void 0 : void 0;
|
|
2376
|
+
return tracer.startSpan(name, {
|
|
2377
|
+
attributes: redactOtelAttributes(attributes),
|
|
2378
|
+
...options.kind !== void 0 ? { kind: options.kind } : {},
|
|
2379
|
+
...startTime !== void 0 ? { startTime } : {}
|
|
2380
|
+
}, parentContext);
|
|
2381
|
+
};
|
|
2382
|
+
const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
|
|
2383
|
+
const internalOrTrustedTraceContext = (evt, metadata) => metadata.trusted || metadata.internal ? normalizeTraceContext(evt.trace) : void 0;
|
|
2384
|
+
const trustedSpanAliasOwner = (evt) => {
|
|
2385
|
+
if ("runId" in evt && evt.runId) return {
|
|
2386
|
+
kind: "run",
|
|
2387
|
+
id: evt.runId
|
|
2388
|
+
};
|
|
2389
|
+
};
|
|
2390
|
+
const sameTrustedSpanAliasOwner = (left, right) => Boolean(left && right && left.kind === right.kind && left.id === right.id);
|
|
2391
|
+
const trustedSpanAliasKey = (spanId, owner) => `${spanId}:${owner.kind}:${owner.id}`;
|
|
2392
|
+
const retainedTrustedSpanContextKey = (traceId, spanId, owner) => `${traceId}:${owner ? trustedSpanAliasKey(spanId, owner) : spanId}`;
|
|
2393
|
+
const retainedTrustedSpanContext = (traceContext, spanId, owner) => {
|
|
2394
|
+
if (!traceContext?.traceId || !spanId) return;
|
|
2395
|
+
const retained = (owner ? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId, owner)) : void 0) ?? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId));
|
|
2396
|
+
if (retained?.spanContext.traceId !== traceContext.traceId) return;
|
|
2397
|
+
if (retained.owner && !sameTrustedSpanAliasOwner(retained.owner, owner)) return;
|
|
2398
|
+
return retained.spanContext;
|
|
2399
|
+
};
|
|
2400
|
+
const activeTrustedSpanAlias = (spanId, owner) => {
|
|
2401
|
+
if (!owner) return;
|
|
2402
|
+
const alias = activeTrustedSpanAliases.get(trustedSpanAliasKey(spanId, owner));
|
|
2403
|
+
if (!alias || !sameTrustedSpanAliasOwner(alias.owner, owner)) return;
|
|
2404
|
+
return alias.span;
|
|
2405
|
+
};
|
|
2406
|
+
const internalOrTrustedParentContext = (evt, metadata) => {
|
|
2407
|
+
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2408
|
+
const parentSpanId = traceContext?.parentSpanId ?? traceContext?.spanId;
|
|
2409
|
+
if (!traceContext || !parentSpanId) return;
|
|
2410
|
+
return contextForTraceContext({
|
|
2411
|
+
...traceContext,
|
|
2412
|
+
spanId: parentSpanId
|
|
2413
|
+
});
|
|
2414
|
+
};
|
|
2415
|
+
const internalOrTrustedExplicitParentContext = (evt, metadata) => {
|
|
2416
|
+
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2417
|
+
if (!traceContext?.parentSpanId) return;
|
|
2418
|
+
return contextForTraceContext({
|
|
2419
|
+
...traceContext,
|
|
2420
|
+
spanId: traceContext.parentSpanId
|
|
2421
|
+
});
|
|
2422
|
+
};
|
|
2423
|
+
const activeTrustedParentContext = (evt, metadata) => {
|
|
2424
|
+
const traceContext = trustedTraceContext(evt, metadata);
|
|
2425
|
+
const parentSpanId = traceContext?.parentSpanId;
|
|
2426
|
+
if (!parentSpanId) return;
|
|
2427
|
+
const owner = trustedSpanAliasOwner(evt);
|
|
2428
|
+
const spanContext = (activeTrustedSpans.get(parentSpanId) ?? activeTrustedSpanAlias(parentSpanId, owner))?.spanContext() ?? retainedTrustedSpanContext(traceContext, parentSpanId, owner);
|
|
2429
|
+
if (!spanContext) return;
|
|
2430
|
+
return trace.setSpanContext(context.active(), spanContext);
|
|
2431
|
+
};
|
|
2432
|
+
const activeInternalOrTrustedContext = (evt, metadata) => {
|
|
2433
|
+
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2434
|
+
if (!traceContext) return;
|
|
2435
|
+
const owner = trustedSpanAliasOwner(evt);
|
|
2436
|
+
const activeSpan = (traceContext.spanId ? activeTrustedSpans.get(traceContext.spanId) ?? activeTrustedSpanAlias(traceContext.spanId, owner) : void 0) ?? (traceContext.parentSpanId ? activeTrustedSpans.get(traceContext.parentSpanId) ?? activeTrustedSpanAlias(traceContext.parentSpanId, owner) : void 0);
|
|
2437
|
+
if (activeSpan) return trace.setSpanContext(context.active(), activeSpan.spanContext());
|
|
2438
|
+
const retainedSpanContext = retainedTrustedSpanContext(traceContext, traceContext.spanId, owner) ?? retainedTrustedSpanContext(traceContext, traceContext.parentSpanId, owner);
|
|
2439
|
+
if (retainedSpanContext) return trace.setSpanContext(context.active(), retainedSpanContext);
|
|
2440
|
+
return internalOrTrustedParentContext(evt, metadata);
|
|
2441
|
+
};
|
|
2442
|
+
const trackTrustedSpan = (evt, metadata, span) => {
|
|
2443
|
+
const spanId = trustedTraceContext(evt, metadata)?.spanId;
|
|
2444
|
+
if (spanId) activeTrustedSpans.set(spanId, span);
|
|
2445
|
+
return span;
|
|
2446
|
+
};
|
|
2447
|
+
const trackInternalOrTrustedSpan = (evt, metadata, span) => {
|
|
2448
|
+
const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
|
|
2449
|
+
if (spanId) activeTrustedSpans.set(spanId, span);
|
|
2450
|
+
return span;
|
|
2451
|
+
};
|
|
2452
|
+
const takeTrackedTrustedSpan = (evt, metadata) => {
|
|
2453
|
+
const spanId = trustedTraceContext(evt, metadata)?.spanId;
|
|
2454
|
+
if (!spanId) return;
|
|
2455
|
+
const span = activeTrustedSpans.get(spanId);
|
|
2456
|
+
if (span) activeTrustedSpans.delete(spanId);
|
|
2457
|
+
return span;
|
|
2458
|
+
};
|
|
2459
|
+
const getTrackedInternalOrTrustedSpan = (evt, metadata) => {
|
|
2460
|
+
const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
|
|
2461
|
+
if (!spanId) return;
|
|
2462
|
+
return activeTrustedSpans.get(spanId);
|
|
2463
|
+
};
|
|
2464
|
+
const setSpanAttrs = (span, attributes) => {
|
|
2465
|
+
span.setAttributes?.(redactOtelAttributes(attributes));
|
|
2466
|
+
};
|
|
2467
|
+
const retainTrustedSpanContext = (traceId, spanId, spanContext, retentionMarker, owner) => {
|
|
2468
|
+
retainedTrustedSpanContexts.set(retainedTrustedSpanContextKey(traceId, spanId, owner), {
|
|
2469
|
+
spanContext,
|
|
2470
|
+
retentionMarker,
|
|
2471
|
+
...owner ? { owner } : {}
|
|
2472
|
+
});
|
|
2473
|
+
while (retainedTrustedSpanContexts.size > MAX_RETAINED_TRUSTED_SPAN_CONTEXTS) {
|
|
2474
|
+
const oldestKey = retainedTrustedSpanContexts.keys().next().value;
|
|
2475
|
+
if (!oldestKey) break;
|
|
2476
|
+
retainedTrustedSpanContexts.delete(oldestKey);
|
|
2477
|
+
}
|
|
2478
|
+
};
|
|
2479
|
+
const scheduleRetainedTrustedSpanContextCleanup = (retentionMarker) => {
|
|
2480
|
+
let drainHandle;
|
|
2481
|
+
let timeoutHandle;
|
|
2482
|
+
const cleanup = () => {
|
|
2483
|
+
if (drainHandle) {
|
|
2484
|
+
clearTimeout(drainHandle);
|
|
2485
|
+
retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
|
|
2486
|
+
drainHandle = void 0;
|
|
2487
|
+
}
|
|
2488
|
+
if (timeoutHandle) {
|
|
2489
|
+
clearTimeout(timeoutHandle);
|
|
2490
|
+
retainedTrustedSpanContextCleanupTimers.delete(timeoutHandle);
|
|
2491
|
+
timeoutHandle = void 0;
|
|
2492
|
+
}
|
|
2493
|
+
for (const [key, retained] of retainedTrustedSpanContexts) if (retained.retentionMarker === retentionMarker) retainedTrustedSpanContexts.delete(key);
|
|
2494
|
+
};
|
|
2495
|
+
drainHandle = setTimeout(() => {
|
|
2496
|
+
if (drainHandle) {
|
|
2497
|
+
retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
|
|
2498
|
+
drainHandle = void 0;
|
|
2499
|
+
}
|
|
2500
|
+
waitForDiagnosticEventsDrained().then(cleanup, cleanup);
|
|
2501
|
+
}, 0);
|
|
2502
|
+
drainHandle.unref?.();
|
|
2503
|
+
retainedTrustedSpanContextCleanupTimers.add(drainHandle);
|
|
2504
|
+
timeoutHandle = setTimeout(cleanup, RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS);
|
|
2505
|
+
timeoutHandle.unref?.();
|
|
2506
|
+
retainedTrustedSpanContextCleanupTimers.add(timeoutHandle);
|
|
2507
|
+
};
|
|
2508
|
+
const completeTrackedLifecycleSpan = (spanId, span, endTimeMs) => {
|
|
2509
|
+
const spanContext = span.spanContext();
|
|
2510
|
+
const retainedKeys = [{ spanId }];
|
|
2511
|
+
const retainedAliasKeys = [];
|
|
2512
|
+
for (const [aliasKey, alias] of activeTrustedSpanAliases) if (alias.span === span) {
|
|
2513
|
+
retainedKeys.push({
|
|
2514
|
+
spanId: alias.spanId,
|
|
2515
|
+
owner: alias.owner
|
|
2516
|
+
});
|
|
2517
|
+
retainedAliasKeys.push(aliasKey);
|
|
2518
|
+
}
|
|
2519
|
+
if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
|
|
2520
|
+
for (const aliasKey of retainedAliasKeys) if (activeTrustedSpanAliases.get(aliasKey)?.span === span) activeTrustedSpanAliases.delete(aliasKey);
|
|
2521
|
+
span.end(endTimeMs);
|
|
2522
|
+
const retentionMarker = Symbol("retainedTrustedSpanContext");
|
|
2523
|
+
for (const retainedKey of retainedKeys) retainTrustedSpanContext(spanContext.traceId, retainedKey.spanId, spanContext, retentionMarker, retainedKey.owner);
|
|
2524
|
+
scheduleRetainedTrustedSpanContextCleanup(retentionMarker);
|
|
2525
|
+
};
|
|
2526
|
+
const addRunAttrs = (spanAttrs, evt) => {
|
|
2527
|
+
if (evt.provider) spanAttrs["openclaw.provider"] = evt.provider;
|
|
2528
|
+
if (evt.model) spanAttrs["openclaw.model"] = evt.model;
|
|
2529
|
+
if (evt.channel) spanAttrs["openclaw.channel"] = evt.channel;
|
|
2530
|
+
if (evt.trigger) spanAttrs["openclaw.trigger"] = evt.trigger;
|
|
2531
|
+
};
|
|
2532
|
+
const paramsSummaryAttrs = (summary) => {
|
|
2533
|
+
if (!summary) return {};
|
|
2534
|
+
return {
|
|
2535
|
+
"openclaw.tool.params.kind": summary.kind,
|
|
2536
|
+
..."length" in summary ? { "openclaw.tool.params.length": summary.length } : {}
|
|
2537
|
+
};
|
|
2538
|
+
};
|
|
2539
|
+
return {
|
|
2540
|
+
tracer,
|
|
2541
|
+
activeTrustedSpans,
|
|
2542
|
+
activeTrustedSpanAliases,
|
|
2543
|
+
trustedSpanAliasKey,
|
|
2544
|
+
trustedSpanAliasOwner,
|
|
2545
|
+
spanWithDuration,
|
|
2546
|
+
trustedTraceContext,
|
|
2547
|
+
internalOrTrustedTraceContext,
|
|
2548
|
+
internalOrTrustedParentContext,
|
|
2549
|
+
internalOrTrustedExplicitParentContext,
|
|
2550
|
+
activeTrustedParentContext,
|
|
2551
|
+
activeInternalOrTrustedContext,
|
|
2552
|
+
trackTrustedSpan,
|
|
2553
|
+
trackInternalOrTrustedSpan,
|
|
2554
|
+
takeTrackedTrustedSpan,
|
|
2555
|
+
getTrackedInternalOrTrustedSpan,
|
|
2556
|
+
setSpanAttrs,
|
|
2557
|
+
completeTrackedLifecycleSpan,
|
|
2558
|
+
addRunAttrs,
|
|
2559
|
+
paramsSummaryAttrs,
|
|
2560
|
+
stopActiveTrustedSpans
|
|
2561
|
+
};
|
|
2562
|
+
}
|
|
2563
|
+
//#endregion
|
|
2564
|
+
//#region extensions/diagnostics-otel/src/service.ts
|
|
2565
|
+
function createDiagnosticsOtelService() {
|
|
2566
|
+
let sdk = null;
|
|
2567
|
+
let logProvider = null;
|
|
2568
|
+
let unsubscribe = null;
|
|
2569
|
+
let stopActiveTrustedSpans = null;
|
|
2570
|
+
let unregisterUnhandledRejectionHandler = null;
|
|
2571
|
+
const stopStarted = async () => {
|
|
2572
|
+
const currentUnsubscribe = unsubscribe;
|
|
2573
|
+
const currentLogProvider = logProvider;
|
|
2574
|
+
const currentSdk = sdk;
|
|
2575
|
+
const currentStopActiveTrustedSpans = stopActiveTrustedSpans;
|
|
2576
|
+
const currentUnregisterUnhandledRejectionHandler = unregisterUnhandledRejectionHandler;
|
|
2577
|
+
unsubscribe = null;
|
|
2578
|
+
logProvider = null;
|
|
2579
|
+
sdk = null;
|
|
2580
|
+
stopActiveTrustedSpans = null;
|
|
2581
|
+
unregisterUnhandledRejectionHandler = null;
|
|
2582
|
+
currentUnregisterUnhandledRejectionHandler?.();
|
|
2583
|
+
currentUnsubscribe?.();
|
|
2584
|
+
currentStopActiveTrustedSpans?.();
|
|
2585
|
+
if (currentLogProvider) await currentLogProvider.shutdown().catch(() => void 0);
|
|
2586
|
+
if (currentSdk) await currentSdk.shutdown().catch(() => void 0);
|
|
2587
|
+
};
|
|
2588
|
+
return {
|
|
2589
|
+
id: "diagnostics-otel",
|
|
2590
|
+
async start(ctx) {
|
|
2591
|
+
await stopStarted();
|
|
2592
|
+
const cfg = ctx.config.diagnostics;
|
|
2593
|
+
const otel = cfg?.otel;
|
|
2594
|
+
if (!cfg || cfg.enabled === false || !otel?.enabled) return;
|
|
2595
|
+
const emitExporterEvent = (event) => {
|
|
2596
|
+
try {
|
|
2597
|
+
ctx.internalDiagnostics?.emit({
|
|
2598
|
+
type: "telemetry.exporter",
|
|
2599
|
+
...event
|
|
2600
|
+
});
|
|
2601
|
+
} catch {}
|
|
2602
|
+
};
|
|
2603
|
+
const emitForSignals = (signals, event) => {
|
|
2604
|
+
for (const signal of signals) emitExporterEvent({
|
|
2605
|
+
signal,
|
|
2606
|
+
...event
|
|
2607
|
+
});
|
|
2608
|
+
};
|
|
2609
|
+
const tracesEnabled = otel.traces !== false;
|
|
2610
|
+
const metricsEnabled = otel.metrics !== false;
|
|
2611
|
+
const logsEnabled = otel.logs === true;
|
|
2612
|
+
const logsExporter = otel.logsExporter ?? "otlp";
|
|
2613
|
+
const logsToOtlp = logsEnabled && (logsExporter === "otlp" || logsExporter === "both");
|
|
2614
|
+
const logsToStdout = logsEnabled && (logsExporter === "stdout" || logsExporter === "both");
|
|
2615
|
+
const otlpSignals = [
|
|
2616
|
+
...tracesEnabled ? ["traces"] : [],
|
|
2617
|
+
...metricsEnabled ? ["metrics"] : [],
|
|
2618
|
+
...logsToOtlp ? ["logs"] : []
|
|
2619
|
+
];
|
|
2620
|
+
const enabledSignals = [
|
|
2621
|
+
...tracesEnabled ? ["traces"] : [],
|
|
2622
|
+
...metricsEnabled ? ["metrics"] : [],
|
|
2623
|
+
...logsEnabled ? ["logs"] : []
|
|
2624
|
+
];
|
|
2625
|
+
if (enabledSignals.length === 0) return;
|
|
2626
|
+
const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
|
|
2627
|
+
if (otlpSignals.length > 0 && protocol !== "http/protobuf") {
|
|
2628
|
+
emitForSignals(otlpSignals, {
|
|
2629
|
+
exporter: "diagnostics-otel",
|
|
2630
|
+
status: "failure",
|
|
2631
|
+
reason: "unsupported_protocol"
|
|
2632
|
+
});
|
|
2633
|
+
ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
const endpoint = normalizeEndpoint(otel.endpoint ?? process.env["OTEL_EXPORTER_OTLP_ENDPOINT"]);
|
|
2637
|
+
const headers = otel.headers ?? void 0;
|
|
2638
|
+
const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || "openclaw";
|
|
2639
|
+
const sampleRate = resolveSampleRate(otel.sampleRate);
|
|
2640
|
+
const contentCapturePolicy = resolveContentCapturePolicy(otel.captureContent);
|
|
2641
|
+
const sdkPreloaded = hasPreloadedOtelSdk();
|
|
2642
|
+
const resource = resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
|
|
2643
|
+
const logUrl = resolveSignalOtelUrl({
|
|
2644
|
+
signalEndpoint: otel.logsEndpoint,
|
|
2645
|
+
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV],
|
|
2646
|
+
endpoint,
|
|
2647
|
+
path: "v1/logs"
|
|
2648
|
+
});
|
|
2649
|
+
if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) {
|
|
2650
|
+
const traceUrl = resolveSignalOtelUrl({
|
|
2651
|
+
signalEndpoint: otel.tracesEndpoint,
|
|
2652
|
+
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV],
|
|
2653
|
+
endpoint,
|
|
2654
|
+
path: "v1/traces"
|
|
2655
|
+
});
|
|
2656
|
+
const metricUrl = resolveSignalOtelUrl({
|
|
2657
|
+
signalEndpoint: otel.metricsEndpoint,
|
|
2658
|
+
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV],
|
|
2659
|
+
endpoint,
|
|
2660
|
+
path: "v1/metrics"
|
|
2661
|
+
});
|
|
2662
|
+
const traceHttpAgentOptions = resolveOtelHttpAgentOptions({
|
|
2663
|
+
url: traceUrl,
|
|
2664
|
+
signalIdentifier: "TRACES",
|
|
2665
|
+
logger: ctx.logger
|
|
2666
|
+
});
|
|
2667
|
+
const metricHttpAgentOptions = resolveOtelHttpAgentOptions({
|
|
2668
|
+
url: metricUrl,
|
|
2669
|
+
signalIdentifier: "METRICS",
|
|
2670
|
+
logger: ctx.logger
|
|
2671
|
+
});
|
|
2672
|
+
const traceExporter = tracesEnabled ? new OTLPTraceExporter({
|
|
2673
|
+
...traceUrl ? { url: traceUrl } : {},
|
|
2674
|
+
...headers ? { headers } : {},
|
|
2675
|
+
...traceHttpAgentOptions ? { httpAgentOptions: traceHttpAgentOptions } : {}
|
|
2676
|
+
}) : void 0;
|
|
2677
|
+
const spanProcessors = traceExporter && typeof otel.flushIntervalMs === "number" ? [new BatchSpanProcessor(traceExporter, { scheduledDelayMillis: Math.max(1e3, otel.flushIntervalMs) })] : void 0;
|
|
2678
|
+
const metricExporter = metricsEnabled ? new OTLPMetricExporter({
|
|
2679
|
+
...metricUrl ? { url: metricUrl } : {},
|
|
2680
|
+
...headers ? { headers } : {},
|
|
2681
|
+
...metricHttpAgentOptions ? { httpAgentOptions: metricHttpAgentOptions } : {}
|
|
2682
|
+
}) : void 0;
|
|
2683
|
+
const metricReader = metricExporter ? new PeriodicExportingMetricReader({
|
|
2684
|
+
exporter: metricExporter,
|
|
2685
|
+
...typeof otel.flushIntervalMs === "number" ? { exportIntervalMillis: Math.max(1e3, otel.flushIntervalMs) } : {}
|
|
2686
|
+
}) : void 0;
|
|
2687
|
+
sdk = new NodeSDK({
|
|
2688
|
+
resource,
|
|
2689
|
+
...spanProcessors ? { spanProcessors } : traceExporter ? { traceExporter } : {},
|
|
2690
|
+
...metricReader ? { metricReader } : {},
|
|
2691
|
+
...sampleRate !== void 0 ? { sampler: new ParentBasedSampler({ root: new TraceIdRatioBasedSampler(sampleRate) }) } : {}
|
|
2692
|
+
});
|
|
2693
|
+
try {
|
|
2694
|
+
sdk.start();
|
|
2695
|
+
} catch (err) {
|
|
2696
|
+
emitForSignals([...tracesEnabled ? ["traces"] : [], ...metricsEnabled ? ["metrics"] : []], {
|
|
2697
|
+
exporter: "diagnostics-otel",
|
|
2698
|
+
status: "failure",
|
|
2699
|
+
reason: "start_failed",
|
|
2700
|
+
errorCategory: errorCategory(err)
|
|
2701
|
+
});
|
|
2702
|
+
await stopStarted();
|
|
2703
|
+
ctx.logger.error(`diagnostics-otel: failed to start SDK: ${formatError(err)}`);
|
|
2704
|
+
throw err;
|
|
2705
|
+
}
|
|
2706
|
+
} else if (sdkPreloaded && (tracesEnabled || metricsEnabled)) ctx.logger.info("diagnostics-otel: using preloaded OpenTelemetry SDK");
|
|
2707
|
+
const meter = metrics.getMeter("openclaw");
|
|
2708
|
+
const diagnosticsTrace = createDiagnosticsTraceRuntime(trace.getTracer("openclaw"));
|
|
2709
|
+
stopActiveTrustedSpans = diagnosticsTrace.stopActiveTrustedSpans;
|
|
2710
|
+
const diagnosticMetrics = createDiagnosticsMetrics(meter);
|
|
2711
|
+
const diagnosticsLogs = createDiagnosticsLogExporter({
|
|
2712
|
+
contentCapturePolicy,
|
|
2713
|
+
emitExporterEvent,
|
|
2714
|
+
flushIntervalMs: otel.flushIntervalMs,
|
|
2715
|
+
headers,
|
|
2716
|
+
logger: ctx.logger,
|
|
2717
|
+
logsEnabled,
|
|
2718
|
+
logsToOtlp,
|
|
2719
|
+
logsToStdout,
|
|
2720
|
+
logUrl,
|
|
2721
|
+
resource,
|
|
2722
|
+
serviceName
|
|
2723
|
+
});
|
|
2724
|
+
logProvider = diagnosticsLogs.logProvider;
|
|
2725
|
+
const { recordLogRecord, recordSecurityEvent } = diagnosticsLogs;
|
|
2726
|
+
const recorderRuntime = createDiagnosticsRecorderRuntime({
|
|
2727
|
+
contentCapturePolicy,
|
|
2728
|
+
metrics: diagnosticMetrics,
|
|
2729
|
+
traces: diagnosticsTrace,
|
|
2730
|
+
tracesEnabled
|
|
2731
|
+
});
|
|
2732
|
+
const recorders = {
|
|
2733
|
+
...createUsageRecorders(recorderRuntime),
|
|
2734
|
+
...createOperationsRecorders(recorderRuntime),
|
|
2735
|
+
...createHarnessRecorders(recorderRuntime),
|
|
2736
|
+
...createModelRecorders(recorderRuntime),
|
|
2737
|
+
...createToolAndSystemRecorders(recorderRuntime)
|
|
2738
|
+
};
|
|
2739
|
+
const subscribe = ctx.internalDiagnostics?.onEvent;
|
|
2740
|
+
if (!subscribe) {
|
|
2741
|
+
ctx.logger.error("diagnostics-otel: internal diagnostics capability unavailable");
|
|
2742
|
+
return;
|
|
2743
|
+
}
|
|
2744
|
+
unsubscribe = subscribe(createDiagnosticsEventHandler({
|
|
2745
|
+
logger: ctx.logger,
|
|
2746
|
+
recorders,
|
|
2747
|
+
recordLogRecord,
|
|
2748
|
+
recordSecurityEvent
|
|
2749
|
+
}));
|
|
2750
|
+
unregisterUnhandledRejectionHandler = registerUnhandledRejectionHandler((reason) => {
|
|
2751
|
+
const otlpError = findOtlpExporterError(reason);
|
|
2752
|
+
if (!otlpError) return false;
|
|
2753
|
+
const code = readErrorCode(otlpError) ?? "unknown";
|
|
2754
|
+
ctx.logger.warn(`diagnostics-otel: suppressed OTLP exporter unhandled rejection (code=${String(code)})`);
|
|
2755
|
+
return true;
|
|
2756
|
+
});
|
|
2757
|
+
emitForSignals(enabledSignals, {
|
|
2758
|
+
exporter: "diagnostics-otel",
|
|
2759
|
+
status: "started",
|
|
2760
|
+
reason: "configured"
|
|
2761
|
+
});
|
|
2762
|
+
if (logsEnabled) {
|
|
2763
|
+
const label = logsExporter === "both" ? "OTLP/Protobuf + stdout JSONL" : logsExporter === "stdout" ? "stdout JSONL" : "OTLP/Protobuf";
|
|
2764
|
+
ctx.logger.info(`diagnostics-otel: logs exporter enabled (${label})`);
|
|
2765
|
+
}
|
|
2766
|
+
},
|
|
2767
|
+
async stop() {
|
|
2768
|
+
await stopStarted();
|
|
2769
|
+
}
|
|
2770
|
+
};
|
|
2771
|
+
}
|
|
2772
|
+
//#endregion
|
|
2773
|
+
export { createDiagnosticsOtelService as t };
|