@openclaw/diagnostics-otel 2026.7.1-beta.2 → 2026.7.1-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1 -2404
- package/dist/runtime-api-BUlvn2ET.js +2589 -0
- package/dist/runtime-api.js +2 -0
- package/node_modules/@types/node/README.md +1 -1
- package/node_modules/@types/node/assert/strict.d.ts +10 -1
- package/node_modules/@types/node/assert.d.ts +144 -17
- package/node_modules/@types/node/async_hooks.d.ts +9 -9
- package/node_modules/@types/node/buffer.d.ts +64 -30
- package/node_modules/@types/node/child_process.d.ts +17 -21
- package/node_modules/@types/node/cluster.d.ts +13 -13
- package/node_modules/@types/node/console.d.ts +1 -1
- package/node_modules/@types/node/crypto.d.ts +471 -575
- package/node_modules/@types/node/diagnostics_channel.d.ts +274 -32
- package/node_modules/@types/node/dns.d.ts +15 -15
- package/node_modules/@types/node/events.d.ts +3 -3
- package/node_modules/@types/node/ffi.d.ts +486 -0
- package/node_modules/@types/node/fs/promises.d.ts +7 -4
- package/node_modules/@types/node/fs.d.ts +24 -18
- package/node_modules/@types/node/globals.d.ts +4 -4
- package/node_modules/@types/node/http.d.ts +30 -5
- package/node_modules/@types/node/http2.d.ts +5 -1
- package/node_modules/@types/node/index.d.ts +1 -3
- package/node_modules/@types/node/inspector.d.ts +1 -1
- package/node_modules/@types/node/module.d.ts +16 -45
- package/node_modules/@types/node/net.d.ts +1 -1
- package/node_modules/@types/node/os.d.ts +2 -2
- package/node_modules/@types/node/package.json +4 -4
- package/node_modules/@types/node/perf_hooks.d.ts +1 -9
- package/node_modules/@types/node/process.d.ts +25 -12
- package/node_modules/@types/node/quic.d.ts +92 -11
- package/node_modules/@types/node/readline.d.ts +12 -11
- package/node_modules/@types/node/repl.d.ts +10 -10
- package/node_modules/@types/node/sqlite.d.ts +56 -1
- package/node_modules/@types/node/stream/iter.d.ts +150 -0
- package/node_modules/@types/node/stream/promises.d.ts +2 -2
- package/node_modules/@types/node/stream/web.d.ts +1 -1
- package/node_modules/@types/node/stream.d.ts +37 -5
- package/node_modules/@types/node/test/reporters.d.ts +1 -1
- package/node_modules/@types/node/test.d.ts +120 -10
- package/node_modules/@types/node/tls.d.ts +1 -1
- package/node_modules/@types/node/ts5.6/index.d.ts +2 -4
- package/node_modules/@types/node/ts5.7/index.d.ts +1 -3
- package/node_modules/@types/node/url.d.ts +3 -3
- package/node_modules/@types/node/util/types.d.ts +1 -1
- package/node_modules/@types/node/util.d.ts +22 -22
- package/node_modules/@types/node/v8.d.ts +85 -3
- package/node_modules/@types/node/vm.d.ts +9 -9
- package/node_modules/@types/node/web-globals/domexception.d.ts +15 -0
- package/node_modules/@types/node/worker_threads.d.ts +10 -12
- package/node_modules/@types/node/zlib.d.ts +3 -3
- package/node_modules/acorn/CHANGELOG.md +14 -0
- package/node_modules/acorn/README.md +3 -0
- package/node_modules/acorn/dist/acorn.d.mts +6 -0
- package/node_modules/acorn/dist/acorn.d.ts +6 -0
- package/node_modules/acorn/dist/acorn.js +87 -49
- package/node_modules/acorn/dist/acorn.mjs +87 -49
- package/node_modules/acorn/package.json +2 -2
- package/node_modules/import-in-the-middle/CHANGELOG.md +21 -0
- package/node_modules/import-in-the-middle/README.md +60 -0
- package/node_modules/import-in-the-middle/create-hook.mjs +375 -146
- package/node_modules/import-in-the-middle/lib/get-exports.mjs +88 -52
- package/node_modules/import-in-the-middle/lib/io.mjs +78 -0
- package/node_modules/import-in-the-middle/package.json +1 -1
- package/node_modules/import-in-the-middle/register-hooks.d.ts +35 -0
- package/node_modules/import-in-the-middle/register-hooks.mjs +63 -0
- package/node_modules/undici-types/agent.d.ts +0 -2
- package/node_modules/undici-types/client.d.ts +25 -19
- package/node_modules/undici-types/dispatcher.d.ts +7 -33
- package/node_modules/undici-types/dispatcher1-wrapper.d.ts +7 -0
- package/node_modules/undici-types/fetch.d.ts +24 -4
- package/node_modules/undici-types/h2c-client.d.ts +6 -6
- package/node_modules/undici-types/handlers.d.ts +1 -2
- package/node_modules/undici-types/header.d.ts +5 -0
- package/node_modules/undici-types/index.d.ts +3 -1
- package/node_modules/undici-types/interceptors.d.ts +1 -1
- package/node_modules/undici-types/package.json +1 -1
- package/node_modules/undici-types/pool.d.ts +0 -2
- package/node_modules/undici-types/proxy-agent.d.ts +2 -2
- package/node_modules/undici-types/round-robin-pool.d.ts +0 -2
- package/node_modules/undici-types/snapshot-agent.d.ts +4 -0
- package/node_modules/undici-types/socks5-proxy-agent.d.ts +2 -2
- package/node_modules/undici-types/webidl.d.ts +0 -1
- package/node_modules/yargs/CHANGELOG.md +330 -0
- package/node_modules/yargs/build/index.cjs +1 -1
- package/node_modules/yargs/package.json +6 -4
- package/node_modules/yargs/yargs +5 -9
- package/node_modules/yargs/yargs.cjs +15 -0
- package/npm-shrinkwrap.json +18 -18
- package/package.json +4 -4
- package/node_modules/@types/node/compatibility/iterators.d.ts +0 -21
- package/node_modules/yargs/browser.d.ts +0 -5
package/dist/index.js
CHANGED
|
@@ -1,2408 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./runtime-api-BUlvn2ET.js";
|
|
2
2
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
3
|
-
import { SpanKind, SpanStatusCode, TraceFlags, context, metrics, trace } from "@opentelemetry/api";
|
|
4
|
-
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
5
|
-
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
|
|
6
|
-
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
7
|
-
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
8
|
-
import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
9
|
-
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
10
|
-
import { NodeSDK } from "@opentelemetry/sdk-node";
|
|
11
|
-
import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
|
12
|
-
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
13
|
-
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_DEFINITIONS } from "@opentelemetry/semantic-conventions/incubating";
|
|
14
|
-
import { waitForDiagnosticEventsDrained } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
15
|
-
import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
|
|
16
|
-
import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
17
|
-
//#region extensions/diagnostics-otel/src/service.ts
|
|
18
|
-
const DEFAULT_SERVICE_NAME = "openclaw";
|
|
19
|
-
const DROPPED_OTEL_ATTRIBUTE_KEYS = new Set([
|
|
20
|
-
"openclaw.callId",
|
|
21
|
-
"openclaw.call_id",
|
|
22
|
-
"openclaw.chatId",
|
|
23
|
-
"openclaw.chat_id",
|
|
24
|
-
"openclaw.messageId",
|
|
25
|
-
"openclaw.message_id",
|
|
26
|
-
"openclaw.parentSpanId",
|
|
27
|
-
"openclaw.parent_span_id",
|
|
28
|
-
"openclaw.runId",
|
|
29
|
-
"openclaw.run_id",
|
|
30
|
-
"openclaw.sessionId",
|
|
31
|
-
"openclaw.session_id",
|
|
32
|
-
"openclaw.sessionKey",
|
|
33
|
-
"openclaw.session_key",
|
|
34
|
-
"openclaw.spanId",
|
|
35
|
-
"openclaw.span_id",
|
|
36
|
-
"openclaw.toolCallId",
|
|
37
|
-
"openclaw.tool_call_id",
|
|
38
|
-
"openclaw.traceId",
|
|
39
|
-
"openclaw.trace_id"
|
|
40
|
-
]);
|
|
41
|
-
const LOW_CARDINALITY_VALUE_RE = /^[A-Za-z0-9_.:-]{1,120}$/u;
|
|
42
|
-
const SECURITY_TARGET_NAME_VALUE_RE = /^[A-Za-z0-9@/_.:-]{1,256}$/u;
|
|
43
|
-
const MAX_OTEL_CONTENT_ATTRIBUTE_CHARS = 128 * 1024;
|
|
44
|
-
const MAX_OTEL_CONTENT_ARRAY_ITEMS = 200;
|
|
45
|
-
const MAX_OTEL_LOG_BODY_CHARS = 4 * 1024;
|
|
46
|
-
const MAX_OTEL_LOG_ATTRIBUTE_COUNT = 64;
|
|
47
|
-
const MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS = 4 * 1024;
|
|
48
|
-
const LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS = 6e4;
|
|
49
|
-
const OTEL_LOG_RAW_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,64}$/u;
|
|
50
|
-
const OTEL_LOG_ATTRIBUTE_KEY_RE = /^[A-Za-z0-9_.:-]{1,96}$/u;
|
|
51
|
-
const BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS = new Set([
|
|
52
|
-
"__proto__",
|
|
53
|
-
"prototype",
|
|
54
|
-
"constructor"
|
|
55
|
-
]);
|
|
56
|
-
const PRELOADED_OTEL_SDK_ENV = "OPENCLAW_OTEL_PRELOADED";
|
|
57
|
-
const OTEL_EXPORTER_OTLP_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_ENDPOINT";
|
|
58
|
-
const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
|
|
59
|
-
const OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT";
|
|
60
|
-
const OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT";
|
|
61
|
-
const OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN";
|
|
62
|
-
const GEN_AI_LATEST_EXPERIMENTAL_OPT_IN = "gen_ai_latest_experimental";
|
|
63
|
-
const GEN_AI_TOKEN_USAGE_BUCKETS = [
|
|
64
|
-
1,
|
|
65
|
-
4,
|
|
66
|
-
16,
|
|
67
|
-
64,
|
|
68
|
-
256,
|
|
69
|
-
1024,
|
|
70
|
-
4096,
|
|
71
|
-
16384,
|
|
72
|
-
65536,
|
|
73
|
-
262144,
|
|
74
|
-
1048576,
|
|
75
|
-
4194304,
|
|
76
|
-
16777216,
|
|
77
|
-
67108864
|
|
78
|
-
];
|
|
79
|
-
const GEN_AI_OPERATION_DURATION_BUCKETS = [
|
|
80
|
-
.01,
|
|
81
|
-
.02,
|
|
82
|
-
.04,
|
|
83
|
-
.08,
|
|
84
|
-
.16,
|
|
85
|
-
.32,
|
|
86
|
-
.64,
|
|
87
|
-
1.28,
|
|
88
|
-
2.56,
|
|
89
|
-
5.12,
|
|
90
|
-
10.24,
|
|
91
|
-
20.48,
|
|
92
|
-
40.96,
|
|
93
|
-
81.92
|
|
94
|
-
];
|
|
95
|
-
const MAX_RETAINED_TRUSTED_SPAN_CONTEXTS = 1024;
|
|
96
|
-
const RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS = 5e3;
|
|
97
|
-
const NO_CONTENT_CAPTURE = {
|
|
98
|
-
inputMessages: false,
|
|
99
|
-
outputMessages: false,
|
|
100
|
-
toolInputs: false,
|
|
101
|
-
toolOutputs: false,
|
|
102
|
-
systemPrompt: false,
|
|
103
|
-
toolDefinitions: false,
|
|
104
|
-
logBodies: false
|
|
105
|
-
};
|
|
106
|
-
function normalizeEndpoint(endpoint) {
|
|
107
|
-
const trimmed = endpoint?.trim();
|
|
108
|
-
return trimmed ? trimmed.replace(/\/+$/, "") : void 0;
|
|
109
|
-
}
|
|
110
|
-
function resolveOtelUrl(endpoint, path) {
|
|
111
|
-
if (!endpoint) return;
|
|
112
|
-
const endpointWithoutQueryOrFragment = endpoint.split(/[?#]/, 1)[0] ?? endpoint;
|
|
113
|
-
if (/\/v1\/(?:traces|metrics|logs)$/i.test(endpointWithoutQueryOrFragment)) return endpoint;
|
|
114
|
-
if (/[?#]/u.test(endpoint)) try {
|
|
115
|
-
const url = new URL(endpoint);
|
|
116
|
-
url.pathname = `${url.pathname.replace(/\/+$/u, "")}/${path}`;
|
|
117
|
-
return url.toString();
|
|
118
|
-
} catch {}
|
|
119
|
-
return `${endpoint}/${path}`;
|
|
120
|
-
}
|
|
121
|
-
function resolveSignalOtelUrl(params) {
|
|
122
|
-
return resolveOtelUrl(normalizeEndpoint(params.signalEndpoint ?? params.signalEnvEndpoint) ?? params.endpoint, params.path);
|
|
123
|
-
}
|
|
124
|
-
function resolveSampleRate(value) {
|
|
125
|
-
if (typeof value !== "number" || !Number.isFinite(value)) return;
|
|
126
|
-
if (value < 0 || value > 1) return;
|
|
127
|
-
return value;
|
|
128
|
-
}
|
|
129
|
-
function formatError(err) {
|
|
130
|
-
if (err instanceof Error) return err.stack ?? err.message;
|
|
131
|
-
if (typeof err === "string") return err;
|
|
132
|
-
try {
|
|
133
|
-
return JSON.stringify(err);
|
|
134
|
-
} catch {
|
|
135
|
-
return String(err);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
function errorCategory(err) {
|
|
139
|
-
try {
|
|
140
|
-
if (err instanceof Error && typeof err.name === "string" && err.name.trim()) return lowCardinalityAttr(err.name, "Error");
|
|
141
|
-
return lowCardinalityAttr(typeof err, "unknown");
|
|
142
|
-
} catch {
|
|
143
|
-
return "unknown";
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
function collectNestedErrorCandidates(err) {
|
|
147
|
-
const queue = [err];
|
|
148
|
-
const seen = /* @__PURE__ */ new Set();
|
|
149
|
-
const candidates = [];
|
|
150
|
-
while (queue.length > 0) {
|
|
151
|
-
const current = queue.shift();
|
|
152
|
-
if (current == null || seen.has(current)) continue;
|
|
153
|
-
seen.add(current);
|
|
154
|
-
candidates.push(current);
|
|
155
|
-
if (Array.isArray(current)) {
|
|
156
|
-
for (const item of current) if (item != null && !seen.has(item)) queue.push(item);
|
|
157
|
-
continue;
|
|
158
|
-
}
|
|
159
|
-
if (typeof current !== "object") continue;
|
|
160
|
-
const record = current;
|
|
161
|
-
for (const nested of [
|
|
162
|
-
record.cause,
|
|
163
|
-
record.reason,
|
|
164
|
-
record.original,
|
|
165
|
-
record.error
|
|
166
|
-
]) if (nested != null && !seen.has(nested)) queue.push(nested);
|
|
167
|
-
if (Array.isArray(record.errors)) {
|
|
168
|
-
for (const nested of record.errors) if (nested != null && !seen.has(nested)) queue.push(nested);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
return candidates;
|
|
172
|
-
}
|
|
173
|
-
function readErrorName(err) {
|
|
174
|
-
if (!err || typeof err !== "object") return;
|
|
175
|
-
const name = err.name;
|
|
176
|
-
return typeof name === "string" && name.trim() ? name : void 0;
|
|
177
|
-
}
|
|
178
|
-
function readErrorCode(err) {
|
|
179
|
-
if (!err || typeof err !== "object") return;
|
|
180
|
-
const code = err.code;
|
|
181
|
-
return typeof code === "string" || typeof code === "number" ? code : void 0;
|
|
182
|
-
}
|
|
183
|
-
function findOtlpExporterError(reason) {
|
|
184
|
-
for (const candidate of collectNestedErrorCandidates(reason)) if (readErrorName(candidate) === "OTLPExporterError" && candidate && typeof candidate === "object") return candidate;
|
|
185
|
-
}
|
|
186
|
-
function redactOtelAttributes(attributes) {
|
|
187
|
-
const redactedAttributes = {};
|
|
188
|
-
for (const [key, value] of Object.entries(attributes)) {
|
|
189
|
-
if (DROPPED_OTEL_ATTRIBUTE_KEYS.has(key)) continue;
|
|
190
|
-
redactedAttributes[key] = typeof value === "string" ? redactSensitiveText(value) : value;
|
|
191
|
-
}
|
|
192
|
-
return redactedAttributes;
|
|
193
|
-
}
|
|
194
|
-
function lowCardinalityAttr(value, fallback = "unknown") {
|
|
195
|
-
if (!value) return fallback;
|
|
196
|
-
const redacted = redactSensitiveText(value.trim());
|
|
197
|
-
const redactedLower = redacted.toLowerCase();
|
|
198
|
-
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
199
|
-
return LOW_CARDINALITY_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
200
|
-
}
|
|
201
|
-
function securityTargetNameAttr(value, fallback = "unknown") {
|
|
202
|
-
if (!value) return fallback;
|
|
203
|
-
const redacted = redactSensitiveText(value.trim());
|
|
204
|
-
const redactedLower = redacted.toLowerCase();
|
|
205
|
-
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
206
|
-
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
207
|
-
}
|
|
208
|
-
function lowCardinalityQueueLaneAttr(value, fallback = "unknown") {
|
|
209
|
-
if (!value) return fallback;
|
|
210
|
-
const redacted = redactSensitiveText(value.trim());
|
|
211
|
-
if (redacted.toLowerCase().startsWith("agent:")) return fallback;
|
|
212
|
-
const scopedLaneIndex = redacted.indexOf(":");
|
|
213
|
-
const lane = scopedLaneIndex >= 0 ? redacted.slice(0, scopedLaneIndex) : redacted;
|
|
214
|
-
return LOW_CARDINALITY_VALUE_RE.test(lane) ? lane : fallback;
|
|
215
|
-
}
|
|
216
|
-
function shouldCaptureOtelLogBody(policy) {
|
|
217
|
-
return policy.logBodies;
|
|
218
|
-
}
|
|
219
|
-
function otelLogTimestampIso(timestamp) {
|
|
220
|
-
if (timestamp instanceof Date) return timestamp.toISOString();
|
|
221
|
-
if (typeof timestamp === "number" && Number.isFinite(timestamp)) return new Date(timestamp).toISOString();
|
|
222
|
-
if (Array.isArray(timestamp)) {
|
|
223
|
-
const [seconds, nanoseconds] = timestamp;
|
|
224
|
-
if (Number.isFinite(seconds) && Number.isFinite(nanoseconds)) return new Date(seconds * 1e3 + Math.trunc(nanoseconds / 1e6)).toISOString();
|
|
225
|
-
}
|
|
226
|
-
return (/* @__PURE__ */ new Date()).toISOString();
|
|
227
|
-
}
|
|
228
|
-
function writeStdoutDiagnosticLogRecord(params) {
|
|
229
|
-
const { logRecord, serviceName, traceContext } = params;
|
|
230
|
-
const line = {
|
|
231
|
-
ts: otelLogTimestampIso(logRecord.timestamp),
|
|
232
|
-
signal: "openclaw.diagnostic.log",
|
|
233
|
-
"service.name": serviceName,
|
|
234
|
-
severityText: logRecord.severityText,
|
|
235
|
-
severityNumber: logRecord.severityNumber,
|
|
236
|
-
body: logRecord.body,
|
|
237
|
-
attributes: logRecord.attributes ?? {},
|
|
238
|
-
...traceContext?.traceId ? { trace_id: traceContext.traceId } : {},
|
|
239
|
-
...traceContext?.spanId ? { span_id: traceContext.spanId } : {},
|
|
240
|
-
...traceContext?.traceFlags ? { trace_flags: traceContext.traceFlags } : {}
|
|
241
|
-
};
|
|
242
|
-
process.stdout.write(`${JSON.stringify(line)}\n`);
|
|
243
|
-
}
|
|
244
|
-
function hasOtelSemconvOptIn(value, optIn) {
|
|
245
|
-
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
246
|
-
}
|
|
247
|
-
function emitLatestGenAiSemconv() {
|
|
248
|
-
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
249
|
-
}
|
|
250
|
-
function genAiOperationName(api) {
|
|
251
|
-
const normalized = api?.trim().toLowerCase();
|
|
252
|
-
if (!normalized) return "chat";
|
|
253
|
-
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
254
|
-
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
255
|
-
return "chat";
|
|
256
|
-
}
|
|
257
|
-
function positiveFiniteNumber(value) {
|
|
258
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : void 0;
|
|
259
|
-
}
|
|
260
|
-
function assignPositiveNumberAttr(attrs, key, value) {
|
|
261
|
-
const normalized = positiveFiniteNumber(value);
|
|
262
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
263
|
-
}
|
|
264
|
-
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
265
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
266
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
267
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
268
|
-
}
|
|
269
|
-
function assignNumberAttr(attrs, key, value) {
|
|
270
|
-
if (typeof value === "number" && Number.isFinite(value)) attrs[key] = value;
|
|
271
|
-
}
|
|
272
|
-
function modelCallPromptTokens(usage) {
|
|
273
|
-
if (typeof usage.promptTokens === "number" && Number.isFinite(usage.promptTokens)) return usage.promptTokens;
|
|
274
|
-
const input = usage.input ?? 0;
|
|
275
|
-
const cacheRead = usage.cacheRead ?? 0;
|
|
276
|
-
const cacheWrite = usage.cacheWrite ?? 0;
|
|
277
|
-
const total = input + cacheRead + cacheWrite;
|
|
278
|
-
return total > 0 ? total : void 0;
|
|
279
|
-
}
|
|
280
|
-
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
281
|
-
const stats = evt.promptStats;
|
|
282
|
-
if (!stats) return;
|
|
283
|
-
for (const [key, value] of [
|
|
284
|
-
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
285
|
-
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
286
|
-
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
287
|
-
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
288
|
-
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
289
|
-
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
290
|
-
]) assignNumberAttr(attrs, key, value);
|
|
291
|
-
}
|
|
292
|
-
function assignModelCallUsageAttrs(attrs, evt) {
|
|
293
|
-
const usage = evt.usage;
|
|
294
|
-
if (!usage) return;
|
|
295
|
-
const promptTokens = modelCallPromptTokens(usage);
|
|
296
|
-
for (const [key, value] of [
|
|
297
|
-
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
298
|
-
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
299
|
-
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
300
|
-
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
301
|
-
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
302
|
-
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
303
|
-
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
304
|
-
["gen_ai.usage.input_tokens", promptTokens],
|
|
305
|
-
["gen_ai.usage.output_tokens", usage.output],
|
|
306
|
-
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
307
|
-
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
308
|
-
]) assignPositiveNumberAttr(attrs, key, value);
|
|
309
|
-
}
|
|
310
|
-
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
311
|
-
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = lowCardinalityAttr(input.provider);
|
|
312
|
-
else attrs["gen_ai.system"] = lowCardinalityAttr(input.provider);
|
|
313
|
-
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
314
|
-
attrs["gen_ai.operation.name"] = genAiOperationName(input.api);
|
|
315
|
-
}
|
|
316
|
-
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
317
|
-
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
318
|
-
}
|
|
319
|
-
function modelCallSpanName(evt) {
|
|
320
|
-
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
321
|
-
return `${genAiOperationName(evt.api)} ${lowCardinalityAttr(evt.model)}`;
|
|
322
|
-
}
|
|
323
|
-
function modelCallSpanKind() {
|
|
324
|
-
return emitLatestGenAiSemconv() ? SpanKind.CLIENT : void 0;
|
|
325
|
-
}
|
|
326
|
-
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
327
|
-
if (!upstreamRequestIdHash) return;
|
|
328
|
-
const boundedHash = lowCardinalityAttr(upstreamRequestIdHash);
|
|
329
|
-
if (boundedHash === "unknown") return;
|
|
330
|
-
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
331
|
-
}
|
|
332
|
-
function clampOtelLogText(value, maxChars) {
|
|
333
|
-
return value.length > maxChars ? `${value.slice(0, maxChars)}...(truncated)` : value;
|
|
334
|
-
}
|
|
335
|
-
function normalizeOtelLogString(value, maxChars) {
|
|
336
|
-
return clampOtelLogText(redactSensitiveText(value), maxChars);
|
|
337
|
-
}
|
|
338
|
-
function resolveContentCapturePolicy(value) {
|
|
339
|
-
if (value === true) return {
|
|
340
|
-
inputMessages: true,
|
|
341
|
-
outputMessages: true,
|
|
342
|
-
toolInputs: true,
|
|
343
|
-
toolOutputs: true,
|
|
344
|
-
systemPrompt: false,
|
|
345
|
-
toolDefinitions: true,
|
|
346
|
-
logBodies: true
|
|
347
|
-
};
|
|
348
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return NO_CONTENT_CAPTURE;
|
|
349
|
-
const config = value;
|
|
350
|
-
if (config.enabled !== true) return NO_CONTENT_CAPTURE;
|
|
351
|
-
return {
|
|
352
|
-
inputMessages: config.inputMessages === true,
|
|
353
|
-
outputMessages: config.outputMessages === true,
|
|
354
|
-
toolInputs: config.toolInputs === true,
|
|
355
|
-
toolOutputs: config.toolOutputs === true,
|
|
356
|
-
systemPrompt: config.systemPrompt === true,
|
|
357
|
-
toolDefinitions: config.toolDefinitions === true,
|
|
358
|
-
logBodies: false
|
|
359
|
-
};
|
|
360
|
-
}
|
|
361
|
-
function hasPreloadedOtelSdk() {
|
|
362
|
-
return process.env[PRELOADED_OTEL_SDK_ENV] === "1";
|
|
363
|
-
}
|
|
364
|
-
function normalizeOtelContentValue(value) {
|
|
365
|
-
if (typeof value === "string") return normalizeOtelLogString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
366
|
-
if (Array.isArray(value)) {
|
|
367
|
-
const items = [];
|
|
368
|
-
for (const item of value.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) if (typeof item === "string") items.push(item);
|
|
369
|
-
if (items.length > 0) return normalizeOtelLogString(items.join("\n"), MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
370
|
-
}
|
|
371
|
-
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
372
|
-
if (json) return json;
|
|
373
|
-
}
|
|
374
|
-
const TRUNCATED_JSON_TEXT_SUFFIX = "...(truncated)";
|
|
375
|
-
const JSON_TRUNCATION_STRING_BUDGETS = [
|
|
376
|
-
8192,
|
|
377
|
-
4096,
|
|
378
|
-
2048,
|
|
379
|
-
1024,
|
|
380
|
-
512,
|
|
381
|
-
256,
|
|
382
|
-
128,
|
|
383
|
-
64,
|
|
384
|
-
32
|
|
385
|
-
];
|
|
386
|
-
const JSON_TRUNCATION_ARRAY_ITEM_BUDGETS = [
|
|
387
|
-
MAX_OTEL_CONTENT_ARRAY_ITEMS,
|
|
388
|
-
100,
|
|
389
|
-
50,
|
|
390
|
-
25,
|
|
391
|
-
10,
|
|
392
|
-
5,
|
|
393
|
-
1
|
|
394
|
-
];
|
|
395
|
-
const JSON_TRUNCATION_MAX_OBJECT_FIELDS = 64;
|
|
396
|
-
const JSON_TRUNCATION_MAX_DEPTH = 8;
|
|
397
|
-
function safeJsonString(value, maxChars) {
|
|
398
|
-
if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
|
|
399
|
-
const exact = stringifyJsonForOtelAttribute(value);
|
|
400
|
-
if (exact && exact.length <= maxChars) return exact;
|
|
401
|
-
for (const maxArrayItems of JSON_TRUNCATION_ARRAY_ITEM_BUDGETS) for (const maxStringChars of JSON_TRUNCATION_STRING_BUDGETS) {
|
|
402
|
-
const json = stringifyJsonForOtelAttribute(truncateJsonValueForOtelAttribute(value, {
|
|
403
|
-
maxArrayItems,
|
|
404
|
-
maxDepth: JSON_TRUNCATION_MAX_DEPTH,
|
|
405
|
-
maxObjectFields: JSON_TRUNCATION_MAX_OBJECT_FIELDS,
|
|
406
|
-
maxStringChars,
|
|
407
|
-
seen: /* @__PURE__ */ new WeakSet()
|
|
408
|
-
}));
|
|
409
|
-
if (json && json.length <= maxChars) return json;
|
|
410
|
-
}
|
|
411
|
-
const summary = stringifyJsonForOtelAttribute({
|
|
412
|
-
truncated: true,
|
|
413
|
-
reason: exact ? "max_attribute_size" : "unserializable_value",
|
|
414
|
-
type: describeJsonValue(value)
|
|
415
|
-
});
|
|
416
|
-
return summary && summary.length <= maxChars ? summary : void 0;
|
|
417
|
-
}
|
|
418
|
-
function stringifyJsonForOtelAttribute(value) {
|
|
419
|
-
try {
|
|
420
|
-
const json = JSON.stringify(value);
|
|
421
|
-
if (!json) return;
|
|
422
|
-
return redactSensitiveText(json);
|
|
423
|
-
} catch {
|
|
424
|
-
return;
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
function truncateJsonValueForOtelAttribute(value, options) {
|
|
428
|
-
if (typeof value === "string") return truncateJsonTextForOtelAttribute(value, options.maxStringChars);
|
|
429
|
-
if (typeof value === "number" || typeof value === "boolean" || value === null) return value;
|
|
430
|
-
if (typeof value === "bigint") return truncateJsonTextForOtelAttribute(String(value), options.maxStringChars);
|
|
431
|
-
if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
|
|
432
|
-
if (options.maxDepth <= 0) return {
|
|
433
|
-
truncated: true,
|
|
434
|
-
reason: "max_depth"
|
|
435
|
-
};
|
|
436
|
-
if (Array.isArray(value)) return truncateJsonArrayForOtelAttribute(value, options);
|
|
437
|
-
if (typeof value === "object") return truncateJsonObjectForOtelAttribute(value, options);
|
|
438
|
-
}
|
|
439
|
-
function truncateJsonArrayForOtelAttribute(value, options) {
|
|
440
|
-
if (options.seen.has(value)) return [{
|
|
441
|
-
truncated: true,
|
|
442
|
-
reason: "circular_reference"
|
|
443
|
-
}];
|
|
444
|
-
options.seen.add(value);
|
|
445
|
-
const nextOptions = {
|
|
446
|
-
...options,
|
|
447
|
-
maxDepth: options.maxDepth - 1
|
|
448
|
-
};
|
|
449
|
-
const items = value.slice(0, options.maxArrayItems).map((item) => truncateJsonValueForOtelAttribute(item, nextOptions));
|
|
450
|
-
if (value.length > items.length) items.push({
|
|
451
|
-
truncated: true,
|
|
452
|
-
omittedItems: value.length - items.length
|
|
453
|
-
});
|
|
454
|
-
options.seen.delete(value);
|
|
455
|
-
return items;
|
|
456
|
-
}
|
|
457
|
-
function truncateJsonObjectForOtelAttribute(value, options) {
|
|
458
|
-
if (options.seen.has(value)) return {
|
|
459
|
-
truncated: true,
|
|
460
|
-
reason: "circular_reference"
|
|
461
|
-
};
|
|
462
|
-
options.seen.add(value);
|
|
463
|
-
const nextOptions = {
|
|
464
|
-
...options,
|
|
465
|
-
maxDepth: options.maxDepth - 1
|
|
466
|
-
};
|
|
467
|
-
const result = {};
|
|
468
|
-
const entries = Object.entries(value).filter(([, field]) => field !== void 0 && typeof field !== "function" && typeof field !== "symbol");
|
|
469
|
-
for (const [key, field] of entries.slice(0, options.maxObjectFields)) result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
|
|
470
|
-
if (entries.length > options.maxObjectFields) {
|
|
471
|
-
result.truncated = true;
|
|
472
|
-
result.omittedFields = entries.length - options.maxObjectFields;
|
|
473
|
-
}
|
|
474
|
-
options.seen.delete(value);
|
|
475
|
-
return result;
|
|
476
|
-
}
|
|
477
|
-
function truncateJsonTextForOtelAttribute(value, maxChars) {
|
|
478
|
-
const redacted = redactSensitiveText(value);
|
|
479
|
-
if (redacted.length <= maxChars) return redacted;
|
|
480
|
-
const suffixBudget = Math.min(14, maxChars);
|
|
481
|
-
const prefixBudget = Math.max(0, maxChars - suffixBudget);
|
|
482
|
-
return `${redacted.slice(0, prefixBudget)}${TRUNCATED_JSON_TEXT_SUFFIX.slice(14 - suffixBudget)}`;
|
|
483
|
-
}
|
|
484
|
-
function describeJsonValue(value) {
|
|
485
|
-
if (Array.isArray(value)) return "array";
|
|
486
|
-
if (value === null) return "null";
|
|
487
|
-
return typeof value;
|
|
488
|
-
}
|
|
489
|
-
function textPart(content) {
|
|
490
|
-
return {
|
|
491
|
-
type: "text",
|
|
492
|
-
content
|
|
493
|
-
};
|
|
494
|
-
}
|
|
495
|
-
function toolCallResponsePart(part) {
|
|
496
|
-
return {
|
|
497
|
-
type: "tool_call_response",
|
|
498
|
-
...typeof part.id === "string" ? { id: part.id } : {},
|
|
499
|
-
result: part.result ?? part.response ?? part.content ?? part.details ?? ""
|
|
500
|
-
};
|
|
501
|
-
}
|
|
502
|
-
function contentParts(value) {
|
|
503
|
-
if (typeof value === "string") return value.length > 0 ? [textPart(value)] : [];
|
|
504
|
-
if (!Array.isArray(value)) {
|
|
505
|
-
if (value === void 0 || value === null) return [];
|
|
506
|
-
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return [textPart(String(value))];
|
|
507
|
-
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
508
|
-
return json ? [textPart(json)] : [];
|
|
509
|
-
}
|
|
510
|
-
const parts = [];
|
|
511
|
-
for (const part of value) {
|
|
512
|
-
if (typeof part === "string") {
|
|
513
|
-
if (part.length > 0) parts.push(textPart(part));
|
|
514
|
-
continue;
|
|
515
|
-
}
|
|
516
|
-
if (!isRecord(part)) continue;
|
|
517
|
-
if (part.type === "text" && typeof part.text === "string") parts.push(textPart(part.text));
|
|
518
|
-
else if (part.type === "text" && typeof part.content === "string") parts.push(textPart(part.content));
|
|
519
|
-
else if (part.type === "thinking" && typeof part.thinking === "string") parts.push({
|
|
520
|
-
type: "reasoning",
|
|
521
|
-
content: part.thinking
|
|
522
|
-
});
|
|
523
|
-
else if (part.type === "toolCall" && typeof part.name === "string") parts.push({
|
|
524
|
-
type: "tool_call",
|
|
525
|
-
name: part.name,
|
|
526
|
-
...typeof part.id === "string" ? { id: part.id } : {},
|
|
527
|
-
...part.arguments !== void 0 ? { arguments: part.arguments } : {}
|
|
528
|
-
});
|
|
529
|
-
else if (part.type === "tool_call" && typeof part.name === "string") parts.push({
|
|
530
|
-
type: "tool_call",
|
|
531
|
-
name: part.name,
|
|
532
|
-
...typeof part.id === "string" ? { id: part.id } : {},
|
|
533
|
-
...part.arguments !== void 0 ? { arguments: part.arguments } : {}
|
|
534
|
-
});
|
|
535
|
-
else if (part.type === "tool_call_response") parts.push(toolCallResponsePart(part));
|
|
536
|
-
else if (part.type === "image") {
|
|
537
|
-
const data = typeof part.data === "string" ? part.data : void 0;
|
|
538
|
-
parts.push({
|
|
539
|
-
type: "blob",
|
|
540
|
-
modality: "image",
|
|
541
|
-
...typeof part.mimeType === "string" ? { mime_type: part.mimeType } : {},
|
|
542
|
-
...typeof part.mime_type === "string" ? { mime_type: part.mime_type } : {},
|
|
543
|
-
...data ? { content: data } : {}
|
|
544
|
-
});
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
return parts;
|
|
548
|
-
}
|
|
549
|
-
function normalizeGenAiMessage(value, fallbackRole = "user") {
|
|
550
|
-
if (typeof value === "string") return {
|
|
551
|
-
role: fallbackRole,
|
|
552
|
-
parts: [textPart(value)]
|
|
553
|
-
};
|
|
554
|
-
if (!isRecord(value)) return;
|
|
555
|
-
const rawRole = typeof value.role === "string" ? value.role : fallbackRole;
|
|
556
|
-
const role = rawRole === "toolResult" ? "tool" : rawRole;
|
|
557
|
-
let parts;
|
|
558
|
-
if (role === "tool") {
|
|
559
|
-
const explicitParts = contentParts(value.parts);
|
|
560
|
-
parts = explicitParts.length > 0 ? explicitParts : [toolCallResponsePart({
|
|
561
|
-
id: value.toolCallId,
|
|
562
|
-
result: value.content ?? value.details ?? ""
|
|
563
|
-
})];
|
|
564
|
-
} else parts = contentParts(value.parts ?? value.content);
|
|
565
|
-
if (parts.length === 0) return;
|
|
566
|
-
return {
|
|
567
|
-
role,
|
|
568
|
-
parts,
|
|
569
|
-
...typeof value.name === "string" ? { name: value.name } : {},
|
|
570
|
-
...typeof value.finish_reason === "string" ? { finish_reason: value.finish_reason } : {},
|
|
571
|
-
...typeof value.stopReason === "string" ? { finish_reason: value.stopReason } : {}
|
|
572
|
-
};
|
|
573
|
-
}
|
|
574
|
-
function normalizeGenAiMessages(value, fallbackRole) {
|
|
575
|
-
const source = Array.isArray(value) ? value : value === void 0 ? [] : [value];
|
|
576
|
-
const messages = [];
|
|
577
|
-
for (const item of source.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
|
|
578
|
-
const message = normalizeGenAiMessage(item, fallbackRole);
|
|
579
|
-
if (message) messages.push(message);
|
|
580
|
-
}
|
|
581
|
-
return messages;
|
|
582
|
-
}
|
|
583
|
-
function normalizeGenAiToolDefinition(value) {
|
|
584
|
-
if (!isRecord(value) || typeof value.name !== "string" || value.name.trim().length === 0) return;
|
|
585
|
-
return {
|
|
586
|
-
type: typeof value.type === "string" ? value.type : "function",
|
|
587
|
-
name: value.name,
|
|
588
|
-
...typeof value.description === "string" ? { description: value.description } : {},
|
|
589
|
-
...value.parameters !== void 0 ? { parameters: value.parameters } : {}
|
|
590
|
-
};
|
|
591
|
-
}
|
|
592
|
-
function normalizeGenAiToolDefinitions(value) {
|
|
593
|
-
if (!Array.isArray(value)) return [];
|
|
594
|
-
const definitions = [];
|
|
595
|
-
for (const item of value.slice(0, MAX_OTEL_CONTENT_ARRAY_ITEMS)) {
|
|
596
|
-
const definition = normalizeGenAiToolDefinition(item);
|
|
597
|
-
if (definition) definitions.push(definition);
|
|
598
|
-
}
|
|
599
|
-
return definitions;
|
|
600
|
-
}
|
|
601
|
-
function assignJsonAttribute(attributes, key, value) {
|
|
602
|
-
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
603
|
-
if (json) attributes[key] = json;
|
|
604
|
-
}
|
|
605
|
-
function assignGenAiModelContentAttributes(attributes, content, policy) {
|
|
606
|
-
if (policy.systemPrompt && typeof content?.systemPrompt === "string") assignJsonAttribute(attributes, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, [textPart(content.systemPrompt)]);
|
|
607
|
-
if (policy.inputMessages) {
|
|
608
|
-
const inputMessages = normalizeGenAiMessages(content?.inputMessages, "user");
|
|
609
|
-
if (inputMessages.length > 0) {
|
|
610
|
-
assignJsonAttribute(attributes, ATTR_GEN_AI_INPUT_MESSAGES, inputMessages);
|
|
611
|
-
assignJsonAttribute(attributes, "input.value", inputMessages);
|
|
612
|
-
attributes["input.mime_type"] = "application/json";
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
if (policy.toolDefinitions) {
|
|
616
|
-
const toolDefinitions = normalizeGenAiToolDefinitions(content?.toolDefinitions);
|
|
617
|
-
if (toolDefinitions.length > 0) assignJsonAttribute(attributes, ATTR_GEN_AI_TOOL_DEFINITIONS, toolDefinitions);
|
|
618
|
-
}
|
|
619
|
-
if (policy.outputMessages) {
|
|
620
|
-
const outputMessages = normalizeGenAiMessages(content?.outputMessages, "assistant");
|
|
621
|
-
if (outputMessages.length > 0) {
|
|
622
|
-
assignJsonAttribute(attributes, ATTR_GEN_AI_OUTPUT_MESSAGES, outputMessages);
|
|
623
|
-
assignJsonAttribute(attributes, "output.value", outputMessages);
|
|
624
|
-
attributes["output.mime_type"] = "application/json";
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
function assignOtelContentAttribute(attributes, key, value) {
|
|
629
|
-
const normalized = normalizeOtelContentValue(value);
|
|
630
|
-
if (normalized) attributes[key] = normalized;
|
|
631
|
-
}
|
|
632
|
-
function assignOtelModelContentAttributes(attributes, content, policy) {
|
|
633
|
-
assignGenAiModelContentAttributes(attributes, content, policy);
|
|
634
|
-
if (policy.inputMessages) assignOtelContentAttribute(attributes, "openclaw.content.input_messages", content?.inputMessages);
|
|
635
|
-
if (policy.toolDefinitions) assignOtelContentAttribute(attributes, "openclaw.content.tool_definitions", content?.toolDefinitions);
|
|
636
|
-
if (policy.outputMessages) assignOtelContentAttribute(attributes, "openclaw.content.output_messages", content?.outputMessages);
|
|
637
|
-
if (policy.systemPrompt) assignOtelContentAttribute(attributes, "openclaw.content.system_prompt", content?.systemPrompt);
|
|
638
|
-
}
|
|
639
|
-
function assignOtelToolContentAttributes(attributes, content, policy) {
|
|
640
|
-
if (policy.toolInputs) assignOtelContentAttribute(attributes, "openclaw.content.tool_input", content?.toolInput);
|
|
641
|
-
if (policy.toolOutputs) assignOtelContentAttribute(attributes, "openclaw.content.tool_output", content?.toolOutput);
|
|
642
|
-
}
|
|
643
|
-
function assignOtelLogAttribute(attributes, key, value) {
|
|
644
|
-
if (Object.keys(attributes).length >= MAX_OTEL_LOG_ATTRIBUTE_COUNT) return;
|
|
645
|
-
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) return;
|
|
646
|
-
if (redactSensitiveText(key) !== key) return;
|
|
647
|
-
if (!OTEL_LOG_ATTRIBUTE_KEY_RE.test(key)) return;
|
|
648
|
-
if (typeof value === "string") {
|
|
649
|
-
attributes[key] = normalizeOtelLogString(value, MAX_OTEL_LOG_ATTRIBUTE_VALUE_CHARS);
|
|
650
|
-
return;
|
|
651
|
-
}
|
|
652
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
653
|
-
attributes[key] = value;
|
|
654
|
-
return;
|
|
655
|
-
}
|
|
656
|
-
if (typeof value === "boolean") attributes[key] = value;
|
|
657
|
-
}
|
|
658
|
-
function normalizeTraceContext(value) {
|
|
659
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
660
|
-
const candidate = value;
|
|
661
|
-
if (!isValidDiagnosticTraceId(candidate.traceId)) return;
|
|
662
|
-
if (candidate.spanId !== void 0 && !isValidDiagnosticSpanId(candidate.spanId)) return;
|
|
663
|
-
if (candidate.parentSpanId !== void 0 && !isValidDiagnosticSpanId(candidate.parentSpanId)) return;
|
|
664
|
-
if (candidate.traceFlags !== void 0 && !isValidDiagnosticTraceFlags(candidate.traceFlags)) return;
|
|
665
|
-
return {
|
|
666
|
-
traceId: candidate.traceId,
|
|
667
|
-
...candidate.spanId ? { spanId: candidate.spanId } : {},
|
|
668
|
-
...candidate.parentSpanId ? { parentSpanId: candidate.parentSpanId } : {},
|
|
669
|
-
...candidate.traceFlags ? { traceFlags: candidate.traceFlags } : {}
|
|
670
|
-
};
|
|
671
|
-
}
|
|
672
|
-
function assignOtelLogEventAttributes(attributes, eventAttributes) {
|
|
673
|
-
if (!eventAttributes) return;
|
|
674
|
-
for (const rawKey in eventAttributes) {
|
|
675
|
-
if (Object.keys(attributes).length >= MAX_OTEL_LOG_ATTRIBUTE_COUNT) break;
|
|
676
|
-
if (!Object.hasOwn(eventAttributes, rawKey)) continue;
|
|
677
|
-
const key = rawKey.trim();
|
|
678
|
-
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
|
|
679
|
-
if (redactSensitiveText(key) !== key) continue;
|
|
680
|
-
if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
|
|
681
|
-
assignOtelLogAttribute(attributes, `openclaw.${key}`, eventAttributes[rawKey]);
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
function assignOtelSecurityEventAttributes(attributes, eventAttributes) {
|
|
685
|
-
if (!eventAttributes) return;
|
|
686
|
-
for (const rawKey in eventAttributes) {
|
|
687
|
-
if (Object.keys(attributes).length >= MAX_OTEL_LOG_ATTRIBUTE_COUNT) break;
|
|
688
|
-
if (!Object.hasOwn(eventAttributes, rawKey)) continue;
|
|
689
|
-
const key = rawKey.trim();
|
|
690
|
-
if (BLOCKED_OTEL_LOG_ATTRIBUTE_KEYS.has(key)) continue;
|
|
691
|
-
if (redactSensitiveText(key) !== key) continue;
|
|
692
|
-
if (!OTEL_LOG_RAW_ATTRIBUTE_KEY_RE.test(key)) continue;
|
|
693
|
-
const value = eventAttributes[rawKey];
|
|
694
|
-
assignOtelLogAttribute(attributes, `openclaw.security.attribute.${key}`, typeof value === "string" ? lowCardinalityAttr(value) : value);
|
|
695
|
-
}
|
|
696
|
-
}
|
|
697
|
-
function securitySeverityText(severity) {
|
|
698
|
-
switch (severity) {
|
|
699
|
-
case "critical": return "FATAL";
|
|
700
|
-
case "high": return "ERROR";
|
|
701
|
-
case "medium": return "WARN";
|
|
702
|
-
case "info":
|
|
703
|
-
case "low": return "INFO";
|
|
704
|
-
}
|
|
705
|
-
return severity;
|
|
706
|
-
}
|
|
707
|
-
function assignOtelSecurityAttributes(attributes, evt) {
|
|
708
|
-
assignOtelLogAttribute(attributes, "openclaw.security.event_id", evt.eventId);
|
|
709
|
-
assignOtelLogAttribute(attributes, "openclaw.security.category", evt.category);
|
|
710
|
-
assignOtelLogAttribute(attributes, "openclaw.security.action", lowCardinalityAttr(evt.action));
|
|
711
|
-
assignOtelLogAttribute(attributes, "openclaw.security.outcome", evt.outcome);
|
|
712
|
-
assignOtelLogAttribute(attributes, "openclaw.security.severity", evt.severity);
|
|
713
|
-
if (evt.reason) assignOtelLogAttribute(attributes, "openclaw.security.reason", lowCardinalityAttr(evt.reason));
|
|
714
|
-
if (evt.actor) {
|
|
715
|
-
assignOtelLogAttribute(attributes, "openclaw.security.actor.kind", evt.actor.kind);
|
|
716
|
-
if (evt.actor.idHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.id_hash", lowCardinalityAttr(evt.actor.idHash));
|
|
717
|
-
if (evt.actor.deviceIdHash) assignOtelLogAttribute(attributes, "openclaw.security.actor.device_id_hash", lowCardinalityAttr(evt.actor.deviceIdHash));
|
|
718
|
-
if (evt.actor.channel) assignOtelLogAttribute(attributes, "openclaw.security.actor.channel", lowCardinalityAttr(evt.actor.channel));
|
|
719
|
-
if (evt.actor.role) assignOtelLogAttribute(attributes, "openclaw.security.actor.role", lowCardinalityAttr(evt.actor.role));
|
|
720
|
-
if (evt.actor.scopes?.length) assignOtelLogAttribute(attributes, "openclaw.security.actor.scopes", evt.actor.scopes.map((scope) => lowCardinalityAttr(scope)).join(","));
|
|
721
|
-
}
|
|
722
|
-
if (evt.target) {
|
|
723
|
-
assignOtelLogAttribute(attributes, "openclaw.security.target.kind", evt.target.kind);
|
|
724
|
-
if (evt.target.idHash) assignOtelLogAttribute(attributes, "openclaw.security.target.id_hash", lowCardinalityAttr(evt.target.idHash));
|
|
725
|
-
if (evt.target.name) assignOtelLogAttribute(attributes, "openclaw.security.target.name", securityTargetNameAttr(evt.target.name));
|
|
726
|
-
if (evt.target.owner) assignOtelLogAttribute(attributes, "openclaw.security.target.owner", lowCardinalityAttr(evt.target.owner));
|
|
727
|
-
}
|
|
728
|
-
if (evt.policy) {
|
|
729
|
-
if (evt.policy.id) assignOtelLogAttribute(attributes, "openclaw.security.policy.id", lowCardinalityAttr(evt.policy.id));
|
|
730
|
-
if (evt.policy.decision) assignOtelLogAttribute(attributes, "openclaw.security.policy.decision", evt.policy.decision);
|
|
731
|
-
if (evt.policy.reason) assignOtelLogAttribute(attributes, "openclaw.security.policy.reason", lowCardinalityAttr(evt.policy.reason));
|
|
732
|
-
}
|
|
733
|
-
if (evt.control) {
|
|
734
|
-
if (evt.control.id) assignOtelLogAttribute(attributes, "openclaw.security.control.id", lowCardinalityAttr(evt.control.id));
|
|
735
|
-
if (evt.control.family) assignOtelLogAttribute(attributes, "openclaw.security.control.family", evt.control.family);
|
|
736
|
-
}
|
|
737
|
-
assignOtelSecurityEventAttributes(attributes, evt.attributes);
|
|
738
|
-
}
|
|
739
|
-
function traceFlagsToOtel(traceFlags) {
|
|
740
|
-
return (Number.parseInt(traceFlags ?? "00", 16) & TraceFlags.SAMPLED) !== 0 ? TraceFlags.SAMPLED : TraceFlags.NONE;
|
|
741
|
-
}
|
|
742
|
-
function contextForTraceContext(traceContext) {
|
|
743
|
-
const normalized = normalizeTraceContext(traceContext);
|
|
744
|
-
if (!normalized?.spanId) return;
|
|
745
|
-
return trace.setSpanContext(context.active(), {
|
|
746
|
-
traceId: normalized.traceId,
|
|
747
|
-
spanId: normalized.spanId,
|
|
748
|
-
traceFlags: traceFlagsToOtel(normalized.traceFlags),
|
|
749
|
-
isRemote: true
|
|
750
|
-
});
|
|
751
|
-
}
|
|
752
|
-
function contextForTrustedTraceContext(evt, metadata) {
|
|
753
|
-
return metadata.trusted || metadata.trustedTraceContext === true ? contextForTraceContext(evt.trace) : void 0;
|
|
754
|
-
}
|
|
755
|
-
function normalizedTrustedTraceContext(evt, metadata) {
|
|
756
|
-
return metadata.trusted || metadata.trustedTraceContext === true ? normalizeTraceContext(evt.trace) : void 0;
|
|
757
|
-
}
|
|
758
|
-
function addTraceAttributes(attributes, traceContext) {
|
|
759
|
-
const normalized = normalizeTraceContext(traceContext);
|
|
760
|
-
if (!normalized) return;
|
|
761
|
-
attributes["openclaw.traceId"] = normalized.traceId;
|
|
762
|
-
if (normalized.spanId) attributes["openclaw.spanId"] = normalized.spanId;
|
|
763
|
-
if (normalized.parentSpanId) attributes["openclaw.parentSpanId"] = normalized.parentSpanId;
|
|
764
|
-
if (normalized.traceFlags) attributes["openclaw.traceFlags"] = normalized.traceFlags;
|
|
765
|
-
}
|
|
766
|
-
function createDiagnosticsOtelService() {
|
|
767
|
-
let sdk = null;
|
|
768
|
-
let logProvider = null;
|
|
769
|
-
let unsubscribe = null;
|
|
770
|
-
let stopActiveTrustedSpans = null;
|
|
771
|
-
let unregisterUnhandledRejectionHandler = null;
|
|
772
|
-
const stopStarted = async () => {
|
|
773
|
-
const currentUnsubscribe = unsubscribe;
|
|
774
|
-
const currentLogProvider = logProvider;
|
|
775
|
-
const currentSdk = sdk;
|
|
776
|
-
const currentStopActiveTrustedSpans = stopActiveTrustedSpans;
|
|
777
|
-
const currentUnregisterUnhandledRejectionHandler = unregisterUnhandledRejectionHandler;
|
|
778
|
-
unsubscribe = null;
|
|
779
|
-
logProvider = null;
|
|
780
|
-
sdk = null;
|
|
781
|
-
stopActiveTrustedSpans = null;
|
|
782
|
-
unregisterUnhandledRejectionHandler = null;
|
|
783
|
-
currentUnregisterUnhandledRejectionHandler?.();
|
|
784
|
-
currentUnsubscribe?.();
|
|
785
|
-
currentStopActiveTrustedSpans?.();
|
|
786
|
-
if (currentLogProvider) await currentLogProvider.shutdown().catch(() => void 0);
|
|
787
|
-
if (currentSdk) await currentSdk.shutdown().catch(() => void 0);
|
|
788
|
-
};
|
|
789
|
-
return {
|
|
790
|
-
id: "diagnostics-otel",
|
|
791
|
-
async start(ctx) {
|
|
792
|
-
await stopStarted();
|
|
793
|
-
const cfg = ctx.config.diagnostics;
|
|
794
|
-
const otel = cfg?.otel;
|
|
795
|
-
if (!cfg || cfg.enabled === false || !otel?.enabled) return;
|
|
796
|
-
const emitExporterEvent = (event) => {
|
|
797
|
-
try {
|
|
798
|
-
ctx.internalDiagnostics?.emit({
|
|
799
|
-
type: "telemetry.exporter",
|
|
800
|
-
...event
|
|
801
|
-
});
|
|
802
|
-
} catch {}
|
|
803
|
-
};
|
|
804
|
-
const emitForSignals = (signals, event) => {
|
|
805
|
-
for (const signal of signals) emitExporterEvent({
|
|
806
|
-
signal,
|
|
807
|
-
...event
|
|
808
|
-
});
|
|
809
|
-
};
|
|
810
|
-
const tracesEnabled = otel.traces !== false;
|
|
811
|
-
const metricsEnabled = otel.metrics !== false;
|
|
812
|
-
const logsEnabled = otel.logs === true;
|
|
813
|
-
const logsExporter = otel.logsExporter ?? "otlp";
|
|
814
|
-
const logsToOtlp = logsEnabled && (logsExporter === "otlp" || logsExporter === "both");
|
|
815
|
-
const logsToStdout = logsEnabled && (logsExporter === "stdout" || logsExporter === "both");
|
|
816
|
-
const otlpSignals = [
|
|
817
|
-
...tracesEnabled ? ["traces"] : [],
|
|
818
|
-
...metricsEnabled ? ["metrics"] : [],
|
|
819
|
-
...logsToOtlp ? ["logs"] : []
|
|
820
|
-
];
|
|
821
|
-
const enabledSignals = [
|
|
822
|
-
...tracesEnabled ? ["traces"] : [],
|
|
823
|
-
...metricsEnabled ? ["metrics"] : [],
|
|
824
|
-
...logsEnabled ? ["logs"] : []
|
|
825
|
-
];
|
|
826
|
-
if (enabledSignals.length === 0) return;
|
|
827
|
-
const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
|
|
828
|
-
if (otlpSignals.length > 0 && protocol !== "http/protobuf") {
|
|
829
|
-
emitForSignals(otlpSignals, {
|
|
830
|
-
exporter: "diagnostics-otel",
|
|
831
|
-
status: "failure",
|
|
832
|
-
reason: "unsupported_protocol"
|
|
833
|
-
});
|
|
834
|
-
ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
|
|
835
|
-
return;
|
|
836
|
-
}
|
|
837
|
-
const endpoint = normalizeEndpoint(otel.endpoint ?? process.env[OTEL_EXPORTER_OTLP_ENDPOINT_ENV]);
|
|
838
|
-
const headers = otel.headers ?? void 0;
|
|
839
|
-
const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME;
|
|
840
|
-
const sampleRate = resolveSampleRate(otel.sampleRate);
|
|
841
|
-
const contentCapturePolicy = resolveContentCapturePolicy(otel.captureContent);
|
|
842
|
-
const sdkPreloaded = hasPreloadedOtelSdk();
|
|
843
|
-
const resource = resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
|
|
844
|
-
const logUrl = resolveSignalOtelUrl({
|
|
845
|
-
signalEndpoint: otel.logsEndpoint,
|
|
846
|
-
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV],
|
|
847
|
-
endpoint,
|
|
848
|
-
path: "v1/logs"
|
|
849
|
-
});
|
|
850
|
-
if (!sdkPreloaded && (tracesEnabled || metricsEnabled)) {
|
|
851
|
-
const traceUrl = resolveSignalOtelUrl({
|
|
852
|
-
signalEndpoint: otel.tracesEndpoint,
|
|
853
|
-
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV],
|
|
854
|
-
endpoint,
|
|
855
|
-
path: "v1/traces"
|
|
856
|
-
});
|
|
857
|
-
const metricUrl = resolveSignalOtelUrl({
|
|
858
|
-
signalEndpoint: otel.metricsEndpoint,
|
|
859
|
-
signalEnvEndpoint: process.env[OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV],
|
|
860
|
-
endpoint,
|
|
861
|
-
path: "v1/metrics"
|
|
862
|
-
});
|
|
863
|
-
const traceExporter = tracesEnabled ? new OTLPTraceExporter({
|
|
864
|
-
...traceUrl ? { url: traceUrl } : {},
|
|
865
|
-
...headers ? { headers } : {}
|
|
866
|
-
}) : void 0;
|
|
867
|
-
const spanProcessors = traceExporter && typeof otel.flushIntervalMs === "number" ? [new BatchSpanProcessor(traceExporter, { scheduledDelayMillis: Math.max(1e3, otel.flushIntervalMs) })] : void 0;
|
|
868
|
-
const metricExporter = metricsEnabled ? new OTLPMetricExporter({
|
|
869
|
-
...metricUrl ? { url: metricUrl } : {},
|
|
870
|
-
...headers ? { headers } : {}
|
|
871
|
-
}) : void 0;
|
|
872
|
-
const metricReader = metricExporter ? new PeriodicExportingMetricReader({
|
|
873
|
-
exporter: metricExporter,
|
|
874
|
-
...typeof otel.flushIntervalMs === "number" ? { exportIntervalMillis: Math.max(1e3, otel.flushIntervalMs) } : {}
|
|
875
|
-
}) : void 0;
|
|
876
|
-
sdk = new NodeSDK({
|
|
877
|
-
resource,
|
|
878
|
-
...spanProcessors ? { spanProcessors } : traceExporter ? { traceExporter } : {},
|
|
879
|
-
...metricReader ? { metricReader } : {},
|
|
880
|
-
...sampleRate !== void 0 ? { sampler: new ParentBasedSampler({ root: new TraceIdRatioBasedSampler(sampleRate) }) } : {}
|
|
881
|
-
});
|
|
882
|
-
try {
|
|
883
|
-
sdk.start();
|
|
884
|
-
} catch (err) {
|
|
885
|
-
emitForSignals([...tracesEnabled ? ["traces"] : [], ...metricsEnabled ? ["metrics"] : []], {
|
|
886
|
-
exporter: "diagnostics-otel",
|
|
887
|
-
status: "failure",
|
|
888
|
-
reason: "start_failed",
|
|
889
|
-
errorCategory: errorCategory(err)
|
|
890
|
-
});
|
|
891
|
-
await stopStarted();
|
|
892
|
-
ctx.logger.error(`diagnostics-otel: failed to start SDK: ${formatError(err)}`);
|
|
893
|
-
throw err;
|
|
894
|
-
}
|
|
895
|
-
} else if (sdkPreloaded && (tracesEnabled || metricsEnabled)) ctx.logger.info("diagnostics-otel: using preloaded OpenTelemetry SDK");
|
|
896
|
-
const logSeverityMap = {
|
|
897
|
-
TRACE: 1,
|
|
898
|
-
DEBUG: 5,
|
|
899
|
-
INFO: 9,
|
|
900
|
-
WARN: 13,
|
|
901
|
-
ERROR: 17,
|
|
902
|
-
FATAL: 21
|
|
903
|
-
};
|
|
904
|
-
const meter = metrics.getMeter("openclaw");
|
|
905
|
-
const tracer = trace.getTracer("openclaw");
|
|
906
|
-
const activeTrustedSpans = /* @__PURE__ */ new Map();
|
|
907
|
-
const activeTrustedSpanAliases = /* @__PURE__ */ new Map();
|
|
908
|
-
const retainedTrustedSpanContexts = /* @__PURE__ */ new Map();
|
|
909
|
-
const retainedTrustedSpanContextCleanupTimers = /* @__PURE__ */ new Set();
|
|
910
|
-
stopActiveTrustedSpans = () => {
|
|
911
|
-
const stopAt = Date.now();
|
|
912
|
-
for (const handle of retainedTrustedSpanContextCleanupTimers) clearTimeout(handle);
|
|
913
|
-
retainedTrustedSpanContextCleanupTimers.clear();
|
|
914
|
-
retainedTrustedSpanContexts.clear();
|
|
915
|
-
for (const span of new Set([...activeTrustedSpans.values(), ...Array.from(activeTrustedSpanAliases.values(), (entry) => entry.span)])) span.end(stopAt);
|
|
916
|
-
activeTrustedSpans.clear();
|
|
917
|
-
activeTrustedSpanAliases.clear();
|
|
918
|
-
};
|
|
919
|
-
const tokensCounter = meter.createCounter("openclaw.tokens", {
|
|
920
|
-
unit: "1",
|
|
921
|
-
description: "Token usage by type"
|
|
922
|
-
});
|
|
923
|
-
const genAiTokenUsageHistogram = meter.createHistogram("gen_ai.client.token.usage", {
|
|
924
|
-
unit: "{token}",
|
|
925
|
-
description: "Number of input and output tokens used by GenAI client operations",
|
|
926
|
-
advice: { explicitBucketBoundaries: GEN_AI_TOKEN_USAGE_BUCKETS }
|
|
927
|
-
});
|
|
928
|
-
const genAiOperationDurationHistogram = meter.createHistogram("gen_ai.client.operation.duration", {
|
|
929
|
-
unit: "s",
|
|
930
|
-
description: "GenAI client operation duration",
|
|
931
|
-
advice: { explicitBucketBoundaries: GEN_AI_OPERATION_DURATION_BUCKETS }
|
|
932
|
-
});
|
|
933
|
-
const costCounter = meter.createCounter("openclaw.cost.usd", {
|
|
934
|
-
unit: "1",
|
|
935
|
-
description: "Estimated model cost (USD)"
|
|
936
|
-
});
|
|
937
|
-
const durationHistogram = meter.createHistogram("openclaw.run.duration_ms", {
|
|
938
|
-
unit: "ms",
|
|
939
|
-
description: "Agent run duration"
|
|
940
|
-
});
|
|
941
|
-
const harnessDurationHistogram = meter.createHistogram("openclaw.harness.duration_ms", {
|
|
942
|
-
unit: "ms",
|
|
943
|
-
description: "Agent harness lifecycle duration"
|
|
944
|
-
});
|
|
945
|
-
const contextHistogram = meter.createHistogram("openclaw.context.tokens", {
|
|
946
|
-
unit: "1",
|
|
947
|
-
description: "Context window size and usage"
|
|
948
|
-
});
|
|
949
|
-
const webhookReceivedCounter = meter.createCounter("openclaw.webhook.received", {
|
|
950
|
-
unit: "1",
|
|
951
|
-
description: "Webhook requests received"
|
|
952
|
-
});
|
|
953
|
-
const webhookErrorCounter = meter.createCounter("openclaw.webhook.error", {
|
|
954
|
-
unit: "1",
|
|
955
|
-
description: "Webhook processing errors"
|
|
956
|
-
});
|
|
957
|
-
const webhookDurationHistogram = meter.createHistogram("openclaw.webhook.duration_ms", {
|
|
958
|
-
unit: "ms",
|
|
959
|
-
description: "Webhook processing duration"
|
|
960
|
-
});
|
|
961
|
-
const messageQueuedCounter = meter.createCounter("openclaw.message.queued", {
|
|
962
|
-
unit: "1",
|
|
963
|
-
description: "Messages queued for processing"
|
|
964
|
-
});
|
|
965
|
-
const messageReceivedCounter = meter.createCounter("openclaw.message.received", {
|
|
966
|
-
unit: "1",
|
|
967
|
-
description: "Inbound messages received"
|
|
968
|
-
});
|
|
969
|
-
const messageDispatchStartedCounter = meter.createCounter("openclaw.message.dispatch.started", {
|
|
970
|
-
unit: "1",
|
|
971
|
-
description: "Inbound message dispatch attempts started"
|
|
972
|
-
});
|
|
973
|
-
const messageDispatchCompletedCounter = meter.createCounter("openclaw.message.dispatch.completed", {
|
|
974
|
-
unit: "1",
|
|
975
|
-
description: "Inbound message dispatch attempts completed"
|
|
976
|
-
});
|
|
977
|
-
const messageDispatchDurationHistogram = meter.createHistogram("openclaw.message.dispatch.duration_ms", {
|
|
978
|
-
unit: "ms",
|
|
979
|
-
description: "Inbound message dispatch duration"
|
|
980
|
-
});
|
|
981
|
-
const messageProcessedCounter = meter.createCounter("openclaw.message.processed", {
|
|
982
|
-
unit: "1",
|
|
983
|
-
description: "Messages processed by outcome"
|
|
984
|
-
});
|
|
985
|
-
const messageDurationHistogram = meter.createHistogram("openclaw.message.duration_ms", {
|
|
986
|
-
unit: "ms",
|
|
987
|
-
description: "Message processing duration"
|
|
988
|
-
});
|
|
989
|
-
const messageDeliveryStartedCounter = meter.createCounter("openclaw.message.delivery.started", {
|
|
990
|
-
unit: "1",
|
|
991
|
-
description: "Outbound message delivery attempts started"
|
|
992
|
-
});
|
|
993
|
-
const messageDeliveryDurationHistogram = meter.createHistogram("openclaw.message.delivery.duration_ms", {
|
|
994
|
-
unit: "ms",
|
|
995
|
-
description: "Outbound message delivery duration"
|
|
996
|
-
});
|
|
997
|
-
const queueDepthHistogram = meter.createHistogram("openclaw.queue.depth", {
|
|
998
|
-
unit: "1",
|
|
999
|
-
description: "Queue depth on enqueue/dequeue"
|
|
1000
|
-
});
|
|
1001
|
-
const queueWaitHistogram = meter.createHistogram("openclaw.queue.wait_ms", {
|
|
1002
|
-
unit: "ms",
|
|
1003
|
-
description: "Queue wait time before execution"
|
|
1004
|
-
});
|
|
1005
|
-
const laneEnqueueCounter = meter.createCounter("openclaw.queue.lane.enqueue", {
|
|
1006
|
-
unit: "1",
|
|
1007
|
-
description: "Command queue lane enqueue events"
|
|
1008
|
-
});
|
|
1009
|
-
const laneDequeueCounter = meter.createCounter("openclaw.queue.lane.dequeue", {
|
|
1010
|
-
unit: "1",
|
|
1011
|
-
description: "Command queue lane dequeue events"
|
|
1012
|
-
});
|
|
1013
|
-
const sessionStateCounter = meter.createCounter("openclaw.session.state", {
|
|
1014
|
-
unit: "1",
|
|
1015
|
-
description: "Session state transitions"
|
|
1016
|
-
});
|
|
1017
|
-
const sessionTurnCreatedCounter = meter.createCounter("openclaw.session.turn.created", {
|
|
1018
|
-
unit: "1",
|
|
1019
|
-
description: "Agent session turns created"
|
|
1020
|
-
});
|
|
1021
|
-
const sessionStuckCounter = meter.createCounter("openclaw.session.stuck", {
|
|
1022
|
-
unit: "1",
|
|
1023
|
-
description: "Sessions stuck in processing"
|
|
1024
|
-
});
|
|
1025
|
-
const sessionStuckAgeHistogram = meter.createHistogram("openclaw.session.stuck_age_ms", {
|
|
1026
|
-
unit: "ms",
|
|
1027
|
-
description: "Age of stuck sessions"
|
|
1028
|
-
});
|
|
1029
|
-
const sessionRecoveryRequestedCounter = meter.createCounter("openclaw.session.recovery.requested", {
|
|
1030
|
-
unit: "1",
|
|
1031
|
-
description: "Session recovery attempts requested"
|
|
1032
|
-
});
|
|
1033
|
-
const sessionRecoveryCompletedCounter = meter.createCounter("openclaw.session.recovery.completed", {
|
|
1034
|
-
unit: "1",
|
|
1035
|
-
description: "Session recovery attempts completed"
|
|
1036
|
-
});
|
|
1037
|
-
const sessionRecoveryAgeHistogram = meter.createHistogram("openclaw.session.recovery.age_ms", {
|
|
1038
|
-
unit: "ms",
|
|
1039
|
-
description: "Age of sessions selected for recovery"
|
|
1040
|
-
});
|
|
1041
|
-
const talkEventCounter = meter.createCounter("openclaw.talk.event", {
|
|
1042
|
-
unit: "1",
|
|
1043
|
-
description: "Talk events emitted by type"
|
|
1044
|
-
});
|
|
1045
|
-
const talkEventDurationHistogram = meter.createHistogram("openclaw.talk.event.duration_ms", {
|
|
1046
|
-
unit: "ms",
|
|
1047
|
-
description: "Talk event duration when reported"
|
|
1048
|
-
});
|
|
1049
|
-
const talkAudioBytesHistogram = meter.createHistogram("openclaw.talk.audio.bytes", {
|
|
1050
|
-
unit: "By",
|
|
1051
|
-
description: "Talk audio frame byte lengths"
|
|
1052
|
-
});
|
|
1053
|
-
const runAttemptCounter = meter.createCounter("openclaw.run.attempt", {
|
|
1054
|
-
unit: "1",
|
|
1055
|
-
description: "Run attempts"
|
|
1056
|
-
});
|
|
1057
|
-
const toolLoopCounter = meter.createCounter("openclaw.tool.loop", {
|
|
1058
|
-
unit: "1",
|
|
1059
|
-
description: "Detected repetitive tool-call loop events"
|
|
1060
|
-
});
|
|
1061
|
-
const skillUsedCounter = meter.createCounter("openclaw.skill.used", {
|
|
1062
|
-
unit: "1",
|
|
1063
|
-
description: "Skills used by agent runs"
|
|
1064
|
-
});
|
|
1065
|
-
const modelCallDurationHistogram = meter.createHistogram("openclaw.model_call.duration_ms", {
|
|
1066
|
-
unit: "ms",
|
|
1067
|
-
description: "Model call duration"
|
|
1068
|
-
});
|
|
1069
|
-
const modelCallRequestBytesHistogram = meter.createHistogram("openclaw.model_call.request_bytes", {
|
|
1070
|
-
unit: "By",
|
|
1071
|
-
description: "UTF-8 byte size of sanitized model request payloads"
|
|
1072
|
-
});
|
|
1073
|
-
const modelCallResponseBytesHistogram = meter.createHistogram("openclaw.model_call.response_bytes", {
|
|
1074
|
-
unit: "By",
|
|
1075
|
-
description: "UTF-8 byte size of bounded streamed model response payloads"
|
|
1076
|
-
});
|
|
1077
|
-
const modelCallTimeToFirstByteHistogram = meter.createHistogram("openclaw.model_call.time_to_first_byte_ms", {
|
|
1078
|
-
unit: "ms",
|
|
1079
|
-
description: "Elapsed time before the first streamed model response event"
|
|
1080
|
-
});
|
|
1081
|
-
const modelFailoverCounter = meter.createCounter("openclaw.model.failover", {
|
|
1082
|
-
unit: "1",
|
|
1083
|
-
description: "Model failovers by source, destination, lane, and reason"
|
|
1084
|
-
});
|
|
1085
|
-
const toolExecutionDurationHistogram = meter.createHistogram("openclaw.tool.execution.duration_ms", {
|
|
1086
|
-
unit: "ms",
|
|
1087
|
-
description: "Tool execution duration"
|
|
1088
|
-
});
|
|
1089
|
-
const toolExecutionBlockedCounter = meter.createCounter("openclaw.tool.execution.blocked", {
|
|
1090
|
-
unit: "1",
|
|
1091
|
-
description: "Tool executions blocked by policy or sandbox diagnostics"
|
|
1092
|
-
});
|
|
1093
|
-
const execProcessDurationHistogram = meter.createHistogram("openclaw.exec.duration_ms", {
|
|
1094
|
-
unit: "ms",
|
|
1095
|
-
description: "Exec process duration"
|
|
1096
|
-
});
|
|
1097
|
-
const memoryRssHistogram = meter.createHistogram("openclaw.memory.rss_bytes", {
|
|
1098
|
-
unit: "By",
|
|
1099
|
-
description: "Resident set size reported by diagnostic memory samples"
|
|
1100
|
-
});
|
|
1101
|
-
const memoryHeapUsedHistogram = meter.createHistogram("openclaw.memory.heap_used_bytes", {
|
|
1102
|
-
unit: "By",
|
|
1103
|
-
description: "Heap used bytes reported by diagnostic memory samples"
|
|
1104
|
-
});
|
|
1105
|
-
const memoryHeapTotalHistogram = meter.createHistogram("openclaw.memory.heap_total_bytes", {
|
|
1106
|
-
unit: "By",
|
|
1107
|
-
description: "Heap total bytes reported by diagnostic memory samples"
|
|
1108
|
-
});
|
|
1109
|
-
const memoryExternalHistogram = meter.createHistogram("openclaw.memory.external_bytes", {
|
|
1110
|
-
unit: "By",
|
|
1111
|
-
description: "External memory bytes reported by diagnostic memory samples"
|
|
1112
|
-
});
|
|
1113
|
-
const memoryArrayBuffersHistogram = meter.createHistogram("openclaw.memory.array_buffers_bytes", {
|
|
1114
|
-
unit: "By",
|
|
1115
|
-
description: "ArrayBuffer bytes reported by diagnostic memory samples"
|
|
1116
|
-
});
|
|
1117
|
-
const memoryPressureCounter = meter.createCounter("openclaw.memory.pressure", {
|
|
1118
|
-
unit: "1",
|
|
1119
|
-
description: "Diagnostic memory pressure events"
|
|
1120
|
-
});
|
|
1121
|
-
const asyncQueueDroppedCounter = meter.createCounter("openclaw.diagnostic.async_queue.dropped", {
|
|
1122
|
-
unit: "1",
|
|
1123
|
-
description: "Async diagnostic queue drops by dropped event class"
|
|
1124
|
-
});
|
|
1125
|
-
const payloadLargeCounter = meter.createCounter("openclaw.payload.large", {
|
|
1126
|
-
unit: "1",
|
|
1127
|
-
description: "Oversized payload diagnostics by surface and action"
|
|
1128
|
-
});
|
|
1129
|
-
const payloadLargeBytesHistogram = meter.createHistogram("openclaw.payload.large_bytes", {
|
|
1130
|
-
unit: "By",
|
|
1131
|
-
description: "Oversized payload byte sizes by surface and action"
|
|
1132
|
-
});
|
|
1133
|
-
const livenessWarningCounter = meter.createCounter("openclaw.liveness.warning", {
|
|
1134
|
-
unit: "1",
|
|
1135
|
-
description: "Diagnostic liveness warning events"
|
|
1136
|
-
});
|
|
1137
|
-
const livenessEventLoopDelayP99Histogram = meter.createHistogram("openclaw.liveness.event_loop_delay_p99_ms", {
|
|
1138
|
-
unit: "ms",
|
|
1139
|
-
description: "P99 event-loop delay reported by diagnostic liveness warnings"
|
|
1140
|
-
});
|
|
1141
|
-
const livenessEventLoopDelayMaxHistogram = meter.createHistogram("openclaw.liveness.event_loop_delay_max_ms", {
|
|
1142
|
-
unit: "ms",
|
|
1143
|
-
description: "Maximum event-loop delay reported by diagnostic liveness warnings"
|
|
1144
|
-
});
|
|
1145
|
-
const livenessEventLoopUtilizationHistogram = meter.createHistogram("openclaw.liveness.event_loop_utilization", {
|
|
1146
|
-
unit: "1",
|
|
1147
|
-
description: "Event-loop utilization reported by diagnostic liveness warnings"
|
|
1148
|
-
});
|
|
1149
|
-
const livenessCpuCoreRatioHistogram = meter.createHistogram("openclaw.liveness.cpu_core_ratio", {
|
|
1150
|
-
unit: "1",
|
|
1151
|
-
description: "CPU core ratio reported by diagnostic liveness warnings"
|
|
1152
|
-
});
|
|
1153
|
-
const telemetryExporterCounter = meter.createCounter("openclaw.telemetry.exporter.events", {
|
|
1154
|
-
unit: "1",
|
|
1155
|
-
description: "Diagnostic telemetry exporter lifecycle and failure events"
|
|
1156
|
-
});
|
|
1157
|
-
let recordLogRecord;
|
|
1158
|
-
let recordSecurityEvent;
|
|
1159
|
-
if (logsEnabled) {
|
|
1160
|
-
let logRecordExportFailureLastReportedAt = Number.NEGATIVE_INFINITY;
|
|
1161
|
-
let otelLogger;
|
|
1162
|
-
if (logsToOtlp) {
|
|
1163
|
-
logProvider = new LoggerProvider({
|
|
1164
|
-
resource,
|
|
1165
|
-
processors: [new BatchLogRecordProcessor(new OTLPLogExporter({
|
|
1166
|
-
...logUrl ? { url: logUrl } : {},
|
|
1167
|
-
...headers ? { headers } : {}
|
|
1168
|
-
}), typeof otel.flushIntervalMs === "number" ? { scheduledDelayMillis: Math.max(1e3, otel.flushIntervalMs) } : {})]
|
|
1169
|
-
});
|
|
1170
|
-
otelLogger = logProvider.getLogger("openclaw");
|
|
1171
|
-
}
|
|
1172
|
-
const reportLogExportFailure = (err, label) => {
|
|
1173
|
-
emitExporterEvent({
|
|
1174
|
-
exporter: "diagnostics-otel",
|
|
1175
|
-
signal: "logs",
|
|
1176
|
-
status: "failure",
|
|
1177
|
-
reason: "emit_failed",
|
|
1178
|
-
errorCategory: errorCategory(err)
|
|
1179
|
-
});
|
|
1180
|
-
const now = Date.now();
|
|
1181
|
-
if (now - logRecordExportFailureLastReportedAt >= LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS) {
|
|
1182
|
-
logRecordExportFailureLastReportedAt = now;
|
|
1183
|
-
ctx.logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`);
|
|
1184
|
-
}
|
|
1185
|
-
};
|
|
1186
|
-
const emitLogRecord = ({ logRecord, traceContext }) => {
|
|
1187
|
-
if (logsToOtlp) otelLogger?.emit(logRecord);
|
|
1188
|
-
if (logsToStdout) writeStdoutDiagnosticLogRecord({
|
|
1189
|
-
logRecord,
|
|
1190
|
-
serviceName,
|
|
1191
|
-
...traceContext ? { traceContext } : {}
|
|
1192
|
-
});
|
|
1193
|
-
};
|
|
1194
|
-
const buildDiagnosticLogRecord = (evt, metadata) => {
|
|
1195
|
-
const logLevelName = evt.level || "INFO";
|
|
1196
|
-
const severityNumber = logSeverityMap[logLevelName] ?? 9;
|
|
1197
|
-
const body = shouldCaptureOtelLogBody(contentCapturePolicy) ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
1198
|
-
const attributes = Object.create(null);
|
|
1199
|
-
assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName);
|
|
1200
|
-
if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
|
|
1201
|
-
if (evt.loggerParents?.length) assignOtelLogAttribute(attributes, "openclaw.logger.parents", evt.loggerParents.join("."));
|
|
1202
|
-
assignOtelLogEventAttributes(attributes, evt.attributes);
|
|
1203
|
-
if (evt.code?.line) assignOtelLogAttribute(attributes, "code.lineno", evt.code.line);
|
|
1204
|
-
if (evt.code?.functionName) assignOtelLogAttribute(attributes, "code.function", evt.code.functionName);
|
|
1205
|
-
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
1206
|
-
addTraceAttributes(attributes, traceContext);
|
|
1207
|
-
const logRecord = {
|
|
1208
|
-
body,
|
|
1209
|
-
severityText: logLevelName,
|
|
1210
|
-
severityNumber,
|
|
1211
|
-
attributes: redactOtelAttributes(attributes),
|
|
1212
|
-
timestamp: evt.ts
|
|
1213
|
-
};
|
|
1214
|
-
const logContext = contextForTrustedTraceContext(evt, metadata);
|
|
1215
|
-
if (logContext) logRecord.context = logContext;
|
|
1216
|
-
return {
|
|
1217
|
-
logRecord,
|
|
1218
|
-
...traceContext ? { traceContext } : {}
|
|
1219
|
-
};
|
|
1220
|
-
};
|
|
1221
|
-
const buildSecurityLogRecord = (evt, metadata) => {
|
|
1222
|
-
const severityText = securitySeverityText(evt.severity);
|
|
1223
|
-
const attributes = Object.create(null);
|
|
1224
|
-
assignOtelSecurityAttributes(attributes, evt);
|
|
1225
|
-
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
1226
|
-
const logRecord = {
|
|
1227
|
-
body: "openclaw.security.event",
|
|
1228
|
-
severityText,
|
|
1229
|
-
severityNumber: logSeverityMap[severityText] ?? 9,
|
|
1230
|
-
attributes: redactOtelAttributes(attributes),
|
|
1231
|
-
timestamp: evt.ts
|
|
1232
|
-
};
|
|
1233
|
-
const logContext = contextForTrustedTraceContext(evt, metadata);
|
|
1234
|
-
if (logContext) logRecord.context = logContext;
|
|
1235
|
-
return {
|
|
1236
|
-
logRecord,
|
|
1237
|
-
...traceContext ? { traceContext } : {}
|
|
1238
|
-
};
|
|
1239
|
-
};
|
|
1240
|
-
recordLogRecord = (evt, metadata) => {
|
|
1241
|
-
try {
|
|
1242
|
-
emitLogRecord(buildDiagnosticLogRecord(evt, metadata));
|
|
1243
|
-
} catch (err) {
|
|
1244
|
-
reportLogExportFailure(err, "log record");
|
|
1245
|
-
}
|
|
1246
|
-
};
|
|
1247
|
-
recordSecurityEvent = (evt, metadata) => {
|
|
1248
|
-
if (!metadata.trusted) return;
|
|
1249
|
-
try {
|
|
1250
|
-
emitLogRecord(buildSecurityLogRecord(evt, metadata));
|
|
1251
|
-
} catch (err) {
|
|
1252
|
-
reportLogExportFailure(err, "security event");
|
|
1253
|
-
}
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
const spanWithDuration = (name, attributes, durationMs, options = {}) => {
|
|
1257
|
-
const endTimeMs = options.endTimeMs ?? Date.now();
|
|
1258
|
-
const startTime = typeof options.startTimeMs === "number" ? options.startTimeMs : typeof durationMs === "number" && durationMs >= 0 ? endTimeMs - durationMs : void 0;
|
|
1259
|
-
const parentContext = "parentContext" in options ? options.parentContext ?? void 0 : void 0;
|
|
1260
|
-
return tracer.startSpan(name, {
|
|
1261
|
-
attributes: redactOtelAttributes(attributes),
|
|
1262
|
-
...options.kind !== void 0 ? { kind: options.kind } : {},
|
|
1263
|
-
...startTime !== void 0 ? { startTime } : {}
|
|
1264
|
-
}, parentContext);
|
|
1265
|
-
};
|
|
1266
|
-
const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
|
|
1267
|
-
const internalOrTrustedTraceContext = (evt, metadata) => metadata.trusted || metadata.internal ? normalizeTraceContext(evt.trace) : void 0;
|
|
1268
|
-
const trustedSpanAliasOwner = (evt) => {
|
|
1269
|
-
if ("runId" in evt && evt.runId) return {
|
|
1270
|
-
kind: "run",
|
|
1271
|
-
id: evt.runId
|
|
1272
|
-
};
|
|
1273
|
-
};
|
|
1274
|
-
const sameTrustedSpanAliasOwner = (left, right) => Boolean(left && right && left.kind === right.kind && left.id === right.id);
|
|
1275
|
-
const trustedSpanAliasKey = (spanId, owner) => `${spanId}:${owner.kind}:${owner.id}`;
|
|
1276
|
-
const retainedTrustedSpanContextKey = (traceId, spanId, owner) => `${traceId}:${owner ? trustedSpanAliasKey(spanId, owner) : spanId}`;
|
|
1277
|
-
const retainedTrustedSpanContext = (traceContext, spanId, owner) => {
|
|
1278
|
-
if (!traceContext?.traceId || !spanId) return;
|
|
1279
|
-
const retained = (owner ? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId, owner)) : void 0) ?? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId));
|
|
1280
|
-
if (retained?.spanContext.traceId !== traceContext.traceId) return;
|
|
1281
|
-
if (retained.owner && !sameTrustedSpanAliasOwner(retained.owner, owner)) return;
|
|
1282
|
-
return retained.spanContext;
|
|
1283
|
-
};
|
|
1284
|
-
const activeTrustedSpanAlias = (spanId, owner) => {
|
|
1285
|
-
if (!owner) return;
|
|
1286
|
-
const alias = activeTrustedSpanAliases.get(trustedSpanAliasKey(spanId, owner));
|
|
1287
|
-
if (!alias || !sameTrustedSpanAliasOwner(alias.owner, owner)) return;
|
|
1288
|
-
return alias.span;
|
|
1289
|
-
};
|
|
1290
|
-
const internalOrTrustedParentContext = (evt, metadata) => {
|
|
1291
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
1292
|
-
const parentSpanId = traceContext?.parentSpanId ?? traceContext?.spanId;
|
|
1293
|
-
if (!traceContext || !parentSpanId) return;
|
|
1294
|
-
return contextForTraceContext({
|
|
1295
|
-
...traceContext,
|
|
1296
|
-
spanId: parentSpanId
|
|
1297
|
-
});
|
|
1298
|
-
};
|
|
1299
|
-
const internalOrTrustedExplicitParentContext = (evt, metadata) => {
|
|
1300
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
1301
|
-
if (!traceContext?.parentSpanId) return;
|
|
1302
|
-
return contextForTraceContext({
|
|
1303
|
-
...traceContext,
|
|
1304
|
-
spanId: traceContext.parentSpanId
|
|
1305
|
-
});
|
|
1306
|
-
};
|
|
1307
|
-
const activeTrustedParentContext = (evt, metadata) => {
|
|
1308
|
-
const traceContext = trustedTraceContext(evt, metadata);
|
|
1309
|
-
const parentSpanId = traceContext?.parentSpanId;
|
|
1310
|
-
if (!parentSpanId) return;
|
|
1311
|
-
const owner = trustedSpanAliasOwner(evt);
|
|
1312
|
-
const spanContext = (activeTrustedSpans.get(parentSpanId) ?? activeTrustedSpanAlias(parentSpanId, owner))?.spanContext() ?? retainedTrustedSpanContext(traceContext, parentSpanId, owner);
|
|
1313
|
-
if (!spanContext) return;
|
|
1314
|
-
return trace.setSpanContext(context.active(), spanContext);
|
|
1315
|
-
};
|
|
1316
|
-
const activeInternalOrTrustedContext = (evt, metadata) => {
|
|
1317
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
1318
|
-
if (!traceContext) return;
|
|
1319
|
-
const owner = trustedSpanAliasOwner(evt);
|
|
1320
|
-
const activeSpan = (traceContext.spanId ? activeTrustedSpans.get(traceContext.spanId) ?? activeTrustedSpanAlias(traceContext.spanId, owner) : void 0) ?? (traceContext.parentSpanId ? activeTrustedSpans.get(traceContext.parentSpanId) ?? activeTrustedSpanAlias(traceContext.parentSpanId, owner) : void 0);
|
|
1321
|
-
if (activeSpan) return trace.setSpanContext(context.active(), activeSpan.spanContext());
|
|
1322
|
-
const retainedSpanContext = retainedTrustedSpanContext(traceContext, traceContext.spanId, owner) ?? retainedTrustedSpanContext(traceContext, traceContext.parentSpanId, owner);
|
|
1323
|
-
if (retainedSpanContext) return trace.setSpanContext(context.active(), retainedSpanContext);
|
|
1324
|
-
return internalOrTrustedParentContext(evt, metadata);
|
|
1325
|
-
};
|
|
1326
|
-
const trackTrustedSpan = (evt, metadata, span) => {
|
|
1327
|
-
const spanId = trustedTraceContext(evt, metadata)?.spanId;
|
|
1328
|
-
if (spanId) activeTrustedSpans.set(spanId, span);
|
|
1329
|
-
return span;
|
|
1330
|
-
};
|
|
1331
|
-
const trackInternalOrTrustedSpan = (evt, metadata, span) => {
|
|
1332
|
-
const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
|
|
1333
|
-
if (spanId) activeTrustedSpans.set(spanId, span);
|
|
1334
|
-
return span;
|
|
1335
|
-
};
|
|
1336
|
-
const takeTrackedTrustedSpan = (evt, metadata) => {
|
|
1337
|
-
const spanId = trustedTraceContext(evt, metadata)?.spanId;
|
|
1338
|
-
if (!spanId) return;
|
|
1339
|
-
const span = activeTrustedSpans.get(spanId);
|
|
1340
|
-
if (span) activeTrustedSpans.delete(spanId);
|
|
1341
|
-
return span;
|
|
1342
|
-
};
|
|
1343
|
-
const getTrackedInternalOrTrustedSpan = (evt, metadata) => {
|
|
1344
|
-
const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
|
|
1345
|
-
if (!spanId) return;
|
|
1346
|
-
return activeTrustedSpans.get(spanId);
|
|
1347
|
-
};
|
|
1348
|
-
const setSpanAttrs = (span, attributes) => {
|
|
1349
|
-
span.setAttributes?.(redactOtelAttributes(attributes));
|
|
1350
|
-
};
|
|
1351
|
-
const retainTrustedSpanContext = (traceId, spanId, spanContext, token, owner) => {
|
|
1352
|
-
retainedTrustedSpanContexts.set(retainedTrustedSpanContextKey(traceId, spanId, owner), {
|
|
1353
|
-
spanContext,
|
|
1354
|
-
token,
|
|
1355
|
-
...owner ? { owner } : {}
|
|
1356
|
-
});
|
|
1357
|
-
while (retainedTrustedSpanContexts.size > MAX_RETAINED_TRUSTED_SPAN_CONTEXTS) {
|
|
1358
|
-
const oldestKey = retainedTrustedSpanContexts.keys().next().value;
|
|
1359
|
-
if (!oldestKey) break;
|
|
1360
|
-
retainedTrustedSpanContexts.delete(oldestKey);
|
|
1361
|
-
}
|
|
1362
|
-
};
|
|
1363
|
-
const scheduleRetainedTrustedSpanContextCleanup = (token) => {
|
|
1364
|
-
let drainHandle;
|
|
1365
|
-
let timeoutHandle;
|
|
1366
|
-
const cleanup = () => {
|
|
1367
|
-
if (drainHandle) {
|
|
1368
|
-
clearTimeout(drainHandle);
|
|
1369
|
-
retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
|
|
1370
|
-
drainHandle = void 0;
|
|
1371
|
-
}
|
|
1372
|
-
if (timeoutHandle) {
|
|
1373
|
-
clearTimeout(timeoutHandle);
|
|
1374
|
-
retainedTrustedSpanContextCleanupTimers.delete(timeoutHandle);
|
|
1375
|
-
timeoutHandle = void 0;
|
|
1376
|
-
}
|
|
1377
|
-
for (const [key, retained] of retainedTrustedSpanContexts) if (retained.token === token) retainedTrustedSpanContexts.delete(key);
|
|
1378
|
-
};
|
|
1379
|
-
drainHandle = setTimeout(() => {
|
|
1380
|
-
if (drainHandle) {
|
|
1381
|
-
retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
|
|
1382
|
-
drainHandle = void 0;
|
|
1383
|
-
}
|
|
1384
|
-
waitForDiagnosticEventsDrained().then(cleanup, cleanup);
|
|
1385
|
-
}, 0);
|
|
1386
|
-
drainHandle.unref?.();
|
|
1387
|
-
retainedTrustedSpanContextCleanupTimers.add(drainHandle);
|
|
1388
|
-
timeoutHandle = setTimeout(cleanup, RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS);
|
|
1389
|
-
timeoutHandle.unref?.();
|
|
1390
|
-
retainedTrustedSpanContextCleanupTimers.add(timeoutHandle);
|
|
1391
|
-
};
|
|
1392
|
-
const completeTrackedLifecycleSpan = (spanId, span, endTimeMs) => {
|
|
1393
|
-
const spanContext = span.spanContext();
|
|
1394
|
-
const retainedKeys = [{ spanId }];
|
|
1395
|
-
const retainedAliasKeys = [];
|
|
1396
|
-
for (const [aliasKey, alias] of activeTrustedSpanAliases) if (alias.span === span) {
|
|
1397
|
-
retainedKeys.push({
|
|
1398
|
-
spanId: alias.spanId,
|
|
1399
|
-
owner: alias.owner
|
|
1400
|
-
});
|
|
1401
|
-
retainedAliasKeys.push(aliasKey);
|
|
1402
|
-
}
|
|
1403
|
-
if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
|
|
1404
|
-
for (const aliasKey of retainedAliasKeys) if (activeTrustedSpanAliases.get(aliasKey)?.span === span) activeTrustedSpanAliases.delete(aliasKey);
|
|
1405
|
-
span.end(endTimeMs);
|
|
1406
|
-
const token = Symbol("retainedTrustedSpanContext");
|
|
1407
|
-
for (const retainedKey of retainedKeys) retainTrustedSpanContext(spanContext.traceId, retainedKey.spanId, spanContext, token, retainedKey.owner);
|
|
1408
|
-
scheduleRetainedTrustedSpanContextCleanup(token);
|
|
1409
|
-
};
|
|
1410
|
-
const addRunAttrs = (spanAttrs, evt) => {
|
|
1411
|
-
if (evt.provider) spanAttrs["openclaw.provider"] = evt.provider;
|
|
1412
|
-
if (evt.model) spanAttrs["openclaw.model"] = evt.model;
|
|
1413
|
-
if (evt.channel) spanAttrs["openclaw.channel"] = evt.channel;
|
|
1414
|
-
if (evt.trigger) spanAttrs["openclaw.trigger"] = evt.trigger;
|
|
1415
|
-
};
|
|
1416
|
-
const paramsSummaryAttrs = (summary) => {
|
|
1417
|
-
if (!summary) return {};
|
|
1418
|
-
return {
|
|
1419
|
-
"openclaw.tool.params.kind": summary.kind,
|
|
1420
|
-
..."length" in summary ? { "openclaw.tool.params.length": summary.length } : {}
|
|
1421
|
-
};
|
|
1422
|
-
};
|
|
1423
|
-
const recordModelUsage = (evt, metadata) => {
|
|
1424
|
-
const attrs = {
|
|
1425
|
-
"openclaw.channel": evt.channel ?? "unknown",
|
|
1426
|
-
"openclaw.agent": lowCardinalityAttr(evt.agentId),
|
|
1427
|
-
"openclaw.provider": evt.provider ?? "unknown",
|
|
1428
|
-
"openclaw.model": evt.model ?? "unknown"
|
|
1429
|
-
};
|
|
1430
|
-
const genAiAttrs = {
|
|
1431
|
-
"gen_ai.operation.name": "chat",
|
|
1432
|
-
"gen_ai.provider.name": lowCardinalityAttr(evt.provider),
|
|
1433
|
-
"gen_ai.request.model": lowCardinalityAttr(evt.model)
|
|
1434
|
-
};
|
|
1435
|
-
const usage = evt.usage;
|
|
1436
|
-
if (usage.input) {
|
|
1437
|
-
tokensCounter.add(usage.input, {
|
|
1438
|
-
...attrs,
|
|
1439
|
-
"openclaw.token": "input"
|
|
1440
|
-
});
|
|
1441
|
-
genAiTokenUsageHistogram.record(usage.input, {
|
|
1442
|
-
...genAiAttrs,
|
|
1443
|
-
"gen_ai.token.type": "input"
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
|
-
if (usage.output) {
|
|
1447
|
-
tokensCounter.add(usage.output, {
|
|
1448
|
-
...attrs,
|
|
1449
|
-
"openclaw.token": "output"
|
|
1450
|
-
});
|
|
1451
|
-
genAiTokenUsageHistogram.record(usage.output, {
|
|
1452
|
-
...genAiAttrs,
|
|
1453
|
-
"gen_ai.token.type": "output"
|
|
1454
|
-
});
|
|
1455
|
-
}
|
|
1456
|
-
if (usage.cacheRead) tokensCounter.add(usage.cacheRead, {
|
|
1457
|
-
...attrs,
|
|
1458
|
-
"openclaw.token": "cache_read"
|
|
1459
|
-
});
|
|
1460
|
-
if (usage.cacheWrite) tokensCounter.add(usage.cacheWrite, {
|
|
1461
|
-
...attrs,
|
|
1462
|
-
"openclaw.token": "cache_write"
|
|
1463
|
-
});
|
|
1464
|
-
if (usage.promptTokens) tokensCounter.add(usage.promptTokens, {
|
|
1465
|
-
...attrs,
|
|
1466
|
-
"openclaw.token": "prompt"
|
|
1467
|
-
});
|
|
1468
|
-
if (usage.total) tokensCounter.add(usage.total, {
|
|
1469
|
-
...attrs,
|
|
1470
|
-
"openclaw.token": "total"
|
|
1471
|
-
});
|
|
1472
|
-
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
1473
|
-
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
1474
|
-
if (evt.context?.limit) contextHistogram.record(evt.context.limit, {
|
|
1475
|
-
...attrs,
|
|
1476
|
-
"openclaw.context": "limit"
|
|
1477
|
-
});
|
|
1478
|
-
if (evt.context?.used) contextHistogram.record(evt.context.used, {
|
|
1479
|
-
...attrs,
|
|
1480
|
-
"openclaw.context": "used"
|
|
1481
|
-
});
|
|
1482
|
-
if (!tracesEnabled) return;
|
|
1483
|
-
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
1484
|
-
const spanAttrs = {
|
|
1485
|
-
...attrs,
|
|
1486
|
-
"openclaw.tokens.input": usage.input ?? 0,
|
|
1487
|
-
"openclaw.tokens.output": usage.output ?? 0,
|
|
1488
|
-
"openclaw.tokens.cache_read": usage.cacheRead ?? 0,
|
|
1489
|
-
"openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
|
|
1490
|
-
"openclaw.tokens.total": usage.total ?? 0
|
|
1491
|
-
};
|
|
1492
|
-
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
1493
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
1494
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
1495
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
1496
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite);
|
|
1497
|
-
spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs, {
|
|
1498
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1499
|
-
endTimeMs: evt.ts
|
|
1500
|
-
}).end(evt.ts);
|
|
1501
|
-
};
|
|
1502
|
-
const recordWebhookReceived = (evt) => {
|
|
1503
|
-
const attrs = {
|
|
1504
|
-
"openclaw.channel": evt.channel ?? "unknown",
|
|
1505
|
-
"openclaw.webhook": evt.updateType ?? "unknown"
|
|
1506
|
-
};
|
|
1507
|
-
webhookReceivedCounter.add(1, attrs);
|
|
1508
|
-
};
|
|
1509
|
-
const recordWebhookProcessed = (evt) => {
|
|
1510
|
-
const attrs = {
|
|
1511
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1512
|
-
"openclaw.webhook": lowCardinalityAttr(evt.updateType)
|
|
1513
|
-
};
|
|
1514
|
-
if (typeof evt.durationMs === "number") webhookDurationHistogram.record(evt.durationMs, attrs);
|
|
1515
|
-
if (!tracesEnabled) return;
|
|
1516
|
-
spanWithDuration("openclaw.webhook.processed", { ...attrs }, evt.durationMs).end();
|
|
1517
|
-
};
|
|
1518
|
-
const recordWebhookError = (evt) => {
|
|
1519
|
-
const attrs = {
|
|
1520
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1521
|
-
"openclaw.webhook": lowCardinalityAttr(evt.updateType)
|
|
1522
|
-
};
|
|
1523
|
-
webhookErrorCounter.add(1, attrs);
|
|
1524
|
-
if (!tracesEnabled) return;
|
|
1525
|
-
const redactedError = redactSensitiveText(evt.error);
|
|
1526
|
-
const spanAttrs = {
|
|
1527
|
-
...attrs,
|
|
1528
|
-
"openclaw.error": redactedError
|
|
1529
|
-
};
|
|
1530
|
-
const span = tracer.startSpan("openclaw.webhook.error", { attributes: spanAttrs });
|
|
1531
|
-
span.setStatus({
|
|
1532
|
-
code: SpanStatusCode.ERROR,
|
|
1533
|
-
message: redactedError
|
|
1534
|
-
});
|
|
1535
|
-
span.end();
|
|
1536
|
-
};
|
|
1537
|
-
const recordMessageQueued = (evt) => {
|
|
1538
|
-
const attrs = {
|
|
1539
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1540
|
-
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
1541
|
-
};
|
|
1542
|
-
messageQueuedCounter.add(1, attrs);
|
|
1543
|
-
if (typeof evt.queueDepth === "number") queueDepthHistogram.record(evt.queueDepth, attrs);
|
|
1544
|
-
};
|
|
1545
|
-
const recordMessageReceived = (evt) => {
|
|
1546
|
-
messageReceivedCounter.add(1, {
|
|
1547
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1548
|
-
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
1549
|
-
});
|
|
1550
|
-
};
|
|
1551
|
-
const recordMessageDispatchStarted = (evt, metadata) => {
|
|
1552
|
-
const attrs = {
|
|
1553
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1554
|
-
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
1555
|
-
};
|
|
1556
|
-
messageDispatchStartedCounter.add(1, attrs);
|
|
1557
|
-
if (!tracesEnabled) return;
|
|
1558
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
1559
|
-
if (!traceContext?.spanId || activeTrustedSpans.has(traceContext.spanId)) return;
|
|
1560
|
-
trackInternalOrTrustedSpan(evt, metadata, spanWithDuration("openclaw.message.processed", attrs, void 0, {
|
|
1561
|
-
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
1562
|
-
startTimeMs: evt.ts
|
|
1563
|
-
}));
|
|
1564
|
-
};
|
|
1565
|
-
const recordMessageDispatchCompleted = (evt) => {
|
|
1566
|
-
const attrs = {
|
|
1567
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1568
|
-
"openclaw.outcome": evt.outcome,
|
|
1569
|
-
"openclaw.reason": lowCardinalityAttr(evt.reason, "none"),
|
|
1570
|
-
"openclaw.source": lowCardinalityAttr(evt.source)
|
|
1571
|
-
};
|
|
1572
|
-
messageDispatchCompletedCounter.add(1, attrs);
|
|
1573
|
-
messageDispatchDurationHistogram.record(evt.durationMs, attrs);
|
|
1574
|
-
};
|
|
1575
|
-
const recordMessageProcessed = (evt, metadata) => {
|
|
1576
|
-
const attrs = {
|
|
1577
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1578
|
-
"openclaw.outcome": evt.outcome ?? "unknown"
|
|
1579
|
-
};
|
|
1580
|
-
messageProcessedCounter.add(1, attrs);
|
|
1581
|
-
if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
|
|
1582
|
-
if (!tracesEnabled) return;
|
|
1583
|
-
const spanAttrs = { ...attrs };
|
|
1584
|
-
if (evt.reason) spanAttrs["openclaw.reason"] = lowCardinalityAttr(evt.reason, "unknown");
|
|
1585
|
-
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1586
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
1587
|
-
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
1588
|
-
endTimeMs: evt.ts
|
|
1589
|
-
});
|
|
1590
|
-
setSpanAttrs(span, spanAttrs);
|
|
1591
|
-
if (evt.outcome === "error" && evt.error) span.setStatus({
|
|
1592
|
-
code: SpanStatusCode.ERROR,
|
|
1593
|
-
message: redactSensitiveText(evt.error)
|
|
1594
|
-
});
|
|
1595
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
1596
|
-
if (trackedSpan && traceContext?.spanId) {
|
|
1597
|
-
completeTrackedLifecycleSpan(traceContext.spanId, trackedSpan, evt.ts);
|
|
1598
|
-
return;
|
|
1599
|
-
}
|
|
1600
|
-
span.end(evt.ts);
|
|
1601
|
-
};
|
|
1602
|
-
const messageDeliveryAttrs = (evt) => ({
|
|
1603
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel),
|
|
1604
|
-
"openclaw.delivery.kind": lowCardinalityAttr(evt.deliveryKind, "other")
|
|
1605
|
-
});
|
|
1606
|
-
const recordMessageDeliveryStarted = (evt) => {
|
|
1607
|
-
messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
|
|
1608
|
-
};
|
|
1609
|
-
const recordMessageDeliveryCompleted = (evt, metadata) => {
|
|
1610
|
-
const attrs = {
|
|
1611
|
-
...messageDeliveryAttrs(evt),
|
|
1612
|
-
"openclaw.outcome": "completed"
|
|
1613
|
-
};
|
|
1614
|
-
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
1615
|
-
if (!tracesEnabled) return;
|
|
1616
|
-
spanWithDuration("openclaw.message.delivery", {
|
|
1617
|
-
...attrs,
|
|
1618
|
-
"openclaw.delivery.result_count": evt.resultCount
|
|
1619
|
-
}, evt.durationMs, {
|
|
1620
|
-
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
1621
|
-
endTimeMs: evt.ts
|
|
1622
|
-
}).end(evt.ts);
|
|
1623
|
-
};
|
|
1624
|
-
const recordMessageDeliveryError = (evt, metadata) => {
|
|
1625
|
-
const attrs = {
|
|
1626
|
-
...messageDeliveryAttrs(evt),
|
|
1627
|
-
"openclaw.outcome": "error",
|
|
1628
|
-
"openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
|
|
1629
|
-
};
|
|
1630
|
-
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
1631
|
-
if (!tracesEnabled) return;
|
|
1632
|
-
const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, {
|
|
1633
|
-
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
1634
|
-
endTimeMs: evt.ts
|
|
1635
|
-
});
|
|
1636
|
-
span.setStatus({
|
|
1637
|
-
code: SpanStatusCode.ERROR,
|
|
1638
|
-
message: redactSensitiveText(evt.errorCategory)
|
|
1639
|
-
});
|
|
1640
|
-
span.end(evt.ts);
|
|
1641
|
-
};
|
|
1642
|
-
const recordRunStarted = (evt, metadata) => {
|
|
1643
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1644
|
-
const spanAttrs = {};
|
|
1645
|
-
addRunAttrs(spanAttrs, evt);
|
|
1646
|
-
const span = trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.run", spanAttrs, void 0, {
|
|
1647
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1648
|
-
startTimeMs: evt.ts
|
|
1649
|
-
}));
|
|
1650
|
-
const parentSpanId = trustedTraceContext(evt, metadata)?.parentSpanId;
|
|
1651
|
-
if (parentSpanId && !activeTrustedSpans.has(parentSpanId)) {
|
|
1652
|
-
const owner = {
|
|
1653
|
-
kind: "run",
|
|
1654
|
-
id: evt.runId
|
|
1655
|
-
};
|
|
1656
|
-
activeTrustedSpanAliases.set(trustedSpanAliasKey(parentSpanId, owner), {
|
|
1657
|
-
span,
|
|
1658
|
-
spanId: parentSpanId,
|
|
1659
|
-
owner
|
|
1660
|
-
});
|
|
1661
|
-
}
|
|
1662
|
-
};
|
|
1663
|
-
const recordLaneEnqueue = (evt) => {
|
|
1664
|
-
const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
|
|
1665
|
-
laneEnqueueCounter.add(1, attrs);
|
|
1666
|
-
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1667
|
-
};
|
|
1668
|
-
const recordLaneDequeue = (evt) => {
|
|
1669
|
-
const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
|
|
1670
|
-
laneDequeueCounter.add(1, attrs);
|
|
1671
|
-
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1672
|
-
if (typeof evt.waitMs === "number") queueWaitHistogram.record(evt.waitMs, attrs);
|
|
1673
|
-
};
|
|
1674
|
-
const recordSessionState = (evt) => {
|
|
1675
|
-
const attrs = { "openclaw.state": evt.state };
|
|
1676
|
-
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1677
|
-
sessionStateCounter.add(1, attrs);
|
|
1678
|
-
};
|
|
1679
|
-
const recordSessionTurnCreated = (evt) => {
|
|
1680
|
-
sessionTurnCreatedCounter.add(1, {
|
|
1681
|
-
"openclaw.agent": lowCardinalityAttr(evt.agentId, "unknown"),
|
|
1682
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel, "unknown"),
|
|
1683
|
-
"openclaw.trigger": evt.trigger
|
|
1684
|
-
});
|
|
1685
|
-
};
|
|
1686
|
-
const recordSessionStuck = (evt) => {
|
|
1687
|
-
const attrs = { "openclaw.state": evt.state };
|
|
1688
|
-
sessionStuckCounter.add(1, attrs);
|
|
1689
|
-
if (typeof evt.ageMs === "number") sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
|
1690
|
-
if (!tracesEnabled) return;
|
|
1691
|
-
const spanAttrs = { ...attrs };
|
|
1692
|
-
spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
|
|
1693
|
-
spanAttrs["openclaw.ageMs"] = evt.ageMs;
|
|
1694
|
-
const span = tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
|
|
1695
|
-
span.setStatus({
|
|
1696
|
-
code: SpanStatusCode.ERROR,
|
|
1697
|
-
message: "session stuck"
|
|
1698
|
-
});
|
|
1699
|
-
span.end();
|
|
1700
|
-
};
|
|
1701
|
-
const sessionRecoveryAttrs = (evt) => {
|
|
1702
|
-
const attrs = { "openclaw.state": evt.state };
|
|
1703
|
-
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1704
|
-
if (evt.activeWorkKind) attrs["openclaw.active_work_kind"] = evt.activeWorkKind;
|
|
1705
|
-
return attrs;
|
|
1706
|
-
};
|
|
1707
|
-
const recordSessionRecoveryRequested = (evt) => {
|
|
1708
|
-
const attrs = sessionRecoveryAttrs(evt);
|
|
1709
|
-
attrs["openclaw.action"] = evt.allowActiveAbort ? "abort" : "recover";
|
|
1710
|
-
sessionRecoveryRequestedCounter.add(1, attrs);
|
|
1711
|
-
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1712
|
-
};
|
|
1713
|
-
const recordSessionRecoveryCompleted = (evt) => {
|
|
1714
|
-
const attrs = sessionRecoveryAttrs(evt);
|
|
1715
|
-
attrs["openclaw.status"] = evt.status;
|
|
1716
|
-
attrs["openclaw.action"] = lowCardinalityAttr(evt.action, "unknown");
|
|
1717
|
-
if (evt.outcomeReason) attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason);
|
|
1718
|
-
sessionRecoveryCompletedCounter.add(1, attrs);
|
|
1719
|
-
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1720
|
-
};
|
|
1721
|
-
const talkEventAttrs = (evt) => ({
|
|
1722
|
-
"openclaw.talk.brain": lowCardinalityAttr(evt.brain),
|
|
1723
|
-
"openclaw.talk.event_type": lowCardinalityAttr(evt.talkEventType),
|
|
1724
|
-
"openclaw.talk.mode": lowCardinalityAttr(evt.mode),
|
|
1725
|
-
"openclaw.talk.provider": lowCardinalityAttr(evt.provider),
|
|
1726
|
-
"openclaw.talk.transport": lowCardinalityAttr(evt.transport)
|
|
1727
|
-
});
|
|
1728
|
-
const recordTalkEvent = (evt, metadata) => {
|
|
1729
|
-
if (!metadata.trusted) return;
|
|
1730
|
-
const attrs = talkEventAttrs(evt);
|
|
1731
|
-
talkEventCounter.add(1, attrs);
|
|
1732
|
-
if (typeof evt.durationMs === "number") talkEventDurationHistogram.record(evt.durationMs, attrs);
|
|
1733
|
-
if (typeof evt.byteLength === "number") talkAudioBytesHistogram.record(evt.byteLength, attrs);
|
|
1734
|
-
};
|
|
1735
|
-
const recordRunAttempt = (evt) => {
|
|
1736
|
-
runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
|
|
1737
|
-
};
|
|
1738
|
-
const toolLoopAttrs = (evt) => ({
|
|
1739
|
-
"openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool"),
|
|
1740
|
-
"openclaw.loop.level": evt.level,
|
|
1741
|
-
"openclaw.loop.action": evt.action,
|
|
1742
|
-
"openclaw.loop.detector": evt.detector,
|
|
1743
|
-
"openclaw.loop.count": evt.count,
|
|
1744
|
-
...evt.pairedToolName ? { "openclaw.loop.paired_tool": lowCardinalityAttr(evt.pairedToolName, "tool") } : {}
|
|
1745
|
-
});
|
|
1746
|
-
const recordToolLoop = (evt) => {
|
|
1747
|
-
const attrs = toolLoopAttrs(evt);
|
|
1748
|
-
toolLoopCounter.add(1, attrs);
|
|
1749
|
-
if (!tracesEnabled) return;
|
|
1750
|
-
const span = spanWithDuration("openclaw.tool.loop", attrs, 0, { endTimeMs: evt.ts });
|
|
1751
|
-
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
1752
|
-
code: SpanStatusCode.ERROR,
|
|
1753
|
-
message: `${evt.detector}:${evt.action}`
|
|
1754
|
-
});
|
|
1755
|
-
span.end(evt.ts);
|
|
1756
|
-
};
|
|
1757
|
-
const recordMemoryUsageMetrics = (evt, attrs = {}) => {
|
|
1758
|
-
memoryRssHistogram.record(evt.memory.rssBytes, attrs);
|
|
1759
|
-
memoryHeapUsedHistogram.record(evt.memory.heapUsedBytes, attrs);
|
|
1760
|
-
memoryHeapTotalHistogram.record(evt.memory.heapTotalBytes, attrs);
|
|
1761
|
-
memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
1762
|
-
memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
1763
|
-
};
|
|
1764
|
-
const recordMemorySample = (evt) => {
|
|
1765
|
-
recordMemoryUsageMetrics(evt);
|
|
1766
|
-
};
|
|
1767
|
-
const recordMemoryPressure = (evt) => {
|
|
1768
|
-
const attrs = {
|
|
1769
|
-
"openclaw.memory.level": evt.level,
|
|
1770
|
-
"openclaw.memory.reason": evt.reason
|
|
1771
|
-
};
|
|
1772
|
-
memoryPressureCounter.add(1, attrs);
|
|
1773
|
-
recordMemoryUsageMetrics(evt, attrs);
|
|
1774
|
-
if (!tracesEnabled) return;
|
|
1775
|
-
const span = spanWithDuration("openclaw.memory.pressure", {
|
|
1776
|
-
...attrs,
|
|
1777
|
-
"openclaw.memory.rss_bytes": evt.memory.rssBytes,
|
|
1778
|
-
"openclaw.memory.heap_used_bytes": evt.memory.heapUsedBytes,
|
|
1779
|
-
"openclaw.memory.heap_total_bytes": evt.memory.heapTotalBytes,
|
|
1780
|
-
"openclaw.memory.external_bytes": evt.memory.externalBytes,
|
|
1781
|
-
"openclaw.memory.array_buffers_bytes": evt.memory.arrayBuffersBytes,
|
|
1782
|
-
...evt.thresholdBytes !== void 0 ? { "openclaw.memory.threshold_bytes": evt.thresholdBytes } : {},
|
|
1783
|
-
...evt.rssGrowthBytes !== void 0 ? { "openclaw.memory.rss_growth_bytes": evt.rssGrowthBytes } : {},
|
|
1784
|
-
...evt.windowMs !== void 0 ? { "openclaw.memory.window_ms": evt.windowMs } : {}
|
|
1785
|
-
}, 0, { endTimeMs: evt.ts });
|
|
1786
|
-
if (evt.level === "critical") span.setStatus({
|
|
1787
|
-
code: SpanStatusCode.ERROR,
|
|
1788
|
-
message: evt.reason
|
|
1789
|
-
});
|
|
1790
|
-
span.end(evt.ts);
|
|
1791
|
-
};
|
|
1792
|
-
const recordAsyncQueueDropped = (evt) => {
|
|
1793
|
-
asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
|
|
1794
|
-
if (evt.droppedTrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedTrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "trusted" });
|
|
1795
|
-
if (evt.droppedUntrustedEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedUntrustedEvents, { "openclaw.diagnostic.async_queue.drop_class": "untrusted" });
|
|
1796
|
-
if (evt.droppedPriorityEvents !== void 0) asyncQueueDroppedCounter.add(evt.droppedPriorityEvents, { "openclaw.diagnostic.async_queue.drop_class": "priority" });
|
|
1797
|
-
};
|
|
1798
|
-
const recordRunCompleted = (evt, metadata) => {
|
|
1799
|
-
const attrs = {
|
|
1800
|
-
"openclaw.outcome": evt.outcome,
|
|
1801
|
-
"openclaw.provider": evt.provider ?? "unknown",
|
|
1802
|
-
"openclaw.model": evt.model ?? "unknown"
|
|
1803
|
-
};
|
|
1804
|
-
if (evt.channel) attrs["openclaw.channel"] = evt.channel;
|
|
1805
|
-
if (evt.blockedBy) attrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
|
|
1806
|
-
durationHistogram.record(evt.durationMs, attrs);
|
|
1807
|
-
if (!tracesEnabled) return;
|
|
1808
|
-
const spanAttrs = { "openclaw.outcome": evt.outcome };
|
|
1809
|
-
addRunAttrs(spanAttrs, evt);
|
|
1810
|
-
if (evt.blockedBy) spanAttrs["openclaw.blocked_by"] = lowCardinalityAttr(evt.blockedBy, "unknown");
|
|
1811
|
-
if (evt.errorCategory) spanAttrs["openclaw.errorCategory"] = lowCardinalityAttr(evt.errorCategory, "other");
|
|
1812
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1813
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1814
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.run", spanAttrs, evt.durationMs, {
|
|
1815
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1816
|
-
endTimeMs: evt.ts
|
|
1817
|
-
});
|
|
1818
|
-
setSpanAttrs(span, spanAttrs);
|
|
1819
|
-
if (evt.outcome === "error") span.setStatus({
|
|
1820
|
-
code: SpanStatusCode.ERROR,
|
|
1821
|
-
...evt.errorCategory ? { message: redactSensitiveText(evt.errorCategory) } : {}
|
|
1822
|
-
});
|
|
1823
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1824
|
-
completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
|
|
1825
|
-
return;
|
|
1826
|
-
}
|
|
1827
|
-
span.end(evt.ts);
|
|
1828
|
-
};
|
|
1829
|
-
const harnessRunMetricAttrs = (evt) => ({
|
|
1830
|
-
"openclaw.harness.id": lowCardinalityAttr(evt.harnessId, "unknown"),
|
|
1831
|
-
"openclaw.harness.plugin": lowCardinalityAttr(evt.pluginId),
|
|
1832
|
-
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1833
|
-
"openclaw.provider": lowCardinalityAttr(evt.provider, "unknown"),
|
|
1834
|
-
"openclaw.model": lowCardinalityAttr(evt.model, "unknown"),
|
|
1835
|
-
...evt.channel ? { "openclaw.channel": lowCardinalityAttr(evt.channel) } : {}
|
|
1836
|
-
});
|
|
1837
|
-
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1838
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1839
|
-
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", harnessRunMetricAttrs(evt), void 0, {
|
|
1840
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1841
|
-
startTimeMs: evt.ts
|
|
1842
|
-
}));
|
|
1843
|
-
};
|
|
1844
|
-
const recordHarnessRunCompleted = (evt, metadata) => {
|
|
1845
|
-
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
|
|
1846
|
-
if (!tracesEnabled) return;
|
|
1847
|
-
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1848
|
-
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = lowCardinalityAttr(evt.resultClassification);
|
|
1849
|
-
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1850
|
-
if (evt.itemLifecycle) {
|
|
1851
|
-
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1852
|
-
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1853
|
-
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1854
|
-
}
|
|
1855
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1856
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1857
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1858
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1859
|
-
endTimeMs: evt.ts
|
|
1860
|
-
});
|
|
1861
|
-
setSpanAttrs(span, spanAttrs);
|
|
1862
|
-
if (evt.outcome === "error") span.setStatus({
|
|
1863
|
-
code: SpanStatusCode.ERROR,
|
|
1864
|
-
message: "error"
|
|
1865
|
-
});
|
|
1866
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1867
|
-
completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
|
|
1868
|
-
return;
|
|
1869
|
-
}
|
|
1870
|
-
span.end(evt.ts);
|
|
1871
|
-
};
|
|
1872
|
-
const recordHarnessRunError = (evt, metadata) => {
|
|
1873
|
-
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
|
|
1874
|
-
const attrs = {
|
|
1875
|
-
...harnessRunMetricAttrs(evt),
|
|
1876
|
-
"openclaw.harness.phase": evt.phase,
|
|
1877
|
-
"openclaw.errorCategory": errorType
|
|
1878
|
-
};
|
|
1879
|
-
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1880
|
-
if (!tracesEnabled) return;
|
|
1881
|
-
const spanAttrs = {
|
|
1882
|
-
...attrs,
|
|
1883
|
-
"error.type": errorType,
|
|
1884
|
-
...evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}
|
|
1885
|
-
};
|
|
1886
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1887
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1888
|
-
endTimeMs: evt.ts
|
|
1889
|
-
});
|
|
1890
|
-
setSpanAttrs(span, spanAttrs);
|
|
1891
|
-
span.setStatus({
|
|
1892
|
-
code: SpanStatusCode.ERROR,
|
|
1893
|
-
message: errorType
|
|
1894
|
-
});
|
|
1895
|
-
span.end(evt.ts);
|
|
1896
|
-
};
|
|
1897
|
-
const recordContextAssembled = (evt, metadata) => {
|
|
1898
|
-
if (!tracesEnabled) return;
|
|
1899
|
-
const spanAttrs = {
|
|
1900
|
-
"openclaw.context.message_count": evt.messageCount,
|
|
1901
|
-
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1902
|
-
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1903
|
-
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1904
|
-
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1905
|
-
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1906
|
-
"openclaw.context.prompt_images": evt.promptImages
|
|
1907
|
-
};
|
|
1908
|
-
addRunAttrs(spanAttrs, evt);
|
|
1909
|
-
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1910
|
-
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1911
|
-
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1912
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1913
|
-
endTimeMs: evt.ts
|
|
1914
|
-
}).end(evt.ts);
|
|
1915
|
-
};
|
|
1916
|
-
const recordModelFailover = (evt, metadata) => {
|
|
1917
|
-
const metricAttrs = {
|
|
1918
|
-
"openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown"),
|
|
1919
|
-
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1920
|
-
"openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane, "unknown"),
|
|
1921
|
-
"openclaw.model": lowCardinalityAttr(evt.fromModel),
|
|
1922
|
-
"openclaw.provider": lowCardinalityAttr(evt.fromProvider),
|
|
1923
|
-
"openclaw.failover.to_model": lowCardinalityAttr(evt.toModel),
|
|
1924
|
-
"openclaw.failover.to_provider": lowCardinalityAttr(evt.toProvider)
|
|
1925
|
-
};
|
|
1926
|
-
modelFailoverCounter.add(1, metricAttrs);
|
|
1927
|
-
if (!tracesEnabled) return;
|
|
1928
|
-
const spanAttrs = { "openclaw.failover.reason": lowCardinalityAttr(evt.reason, "unknown") };
|
|
1929
|
-
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1930
|
-
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1931
|
-
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1932
|
-
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1933
|
-
if (evt.lane) spanAttrs["openclaw.lane"] = lowCardinalityQueueLaneAttr(evt.lane, "unknown");
|
|
1934
|
-
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1935
|
-
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1936
|
-
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1937
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1938
|
-
endTimeMs: evt.ts
|
|
1939
|
-
}).end(evt.ts);
|
|
1940
|
-
};
|
|
1941
|
-
const modelCallMetricAttrs = (evt) => ({
|
|
1942
|
-
"openclaw.provider": evt.provider,
|
|
1943
|
-
"openclaw.model": evt.model,
|
|
1944
|
-
"openclaw.api": lowCardinalityAttr(evt.api),
|
|
1945
|
-
"openclaw.transport": lowCardinalityAttr(evt.transport)
|
|
1946
|
-
});
|
|
1947
|
-
const genAiModelCallMetricAttrs = (evt, errorType) => ({
|
|
1948
|
-
"gen_ai.operation.name": genAiOperationName(evt.api),
|
|
1949
|
-
"gen_ai.provider.name": lowCardinalityAttr(evt.provider),
|
|
1950
|
-
"gen_ai.request.model": lowCardinalityAttr(evt.model),
|
|
1951
|
-
...errorType ? { "error.type": errorType } : {}
|
|
1952
|
-
});
|
|
1953
|
-
const recordModelCallSizeTimingMetrics = (evt, attrs) => {
|
|
1954
|
-
const requestPayloadBytes = positiveFiniteNumber(evt.requestPayloadBytes);
|
|
1955
|
-
if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
|
|
1956
|
-
const responseStreamBytes = positiveFiniteNumber(evt.responseStreamBytes);
|
|
1957
|
-
if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
|
|
1958
|
-
const timeToFirstByteMs = positiveFiniteNumber(evt.timeToFirstByteMs);
|
|
1959
|
-
if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
|
|
1960
|
-
};
|
|
1961
|
-
const recordModelCallStarted = (evt, metadata) => {
|
|
1962
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1963
|
-
const spanAttrs = {
|
|
1964
|
-
"openclaw.provider": evt.provider,
|
|
1965
|
-
"openclaw.model": evt.model
|
|
1966
|
-
};
|
|
1967
|
-
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1968
|
-
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1969
|
-
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1970
|
-
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1971
|
-
trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1972
|
-
kind: modelCallSpanKind(),
|
|
1973
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1974
|
-
startTimeMs: evt.ts
|
|
1975
|
-
}));
|
|
1976
|
-
};
|
|
1977
|
-
const recordModelCallCompleted = (evt, metadata, modelContent) => {
|
|
1978
|
-
const metricAttrs = modelCallMetricAttrs(evt);
|
|
1979
|
-
modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
1980
|
-
recordModelCallSizeTimingMetrics(evt, metricAttrs);
|
|
1981
|
-
genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt));
|
|
1982
|
-
if (!tracesEnabled) return;
|
|
1983
|
-
const spanAttrs = {
|
|
1984
|
-
"openclaw.provider": evt.provider,
|
|
1985
|
-
"openclaw.model": evt.model
|
|
1986
|
-
};
|
|
1987
|
-
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1988
|
-
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1989
|
-
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1990
|
-
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
1991
|
-
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1992
|
-
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1993
|
-
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
1994
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1995
|
-
kind: modelCallSpanKind(),
|
|
1996
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1997
|
-
endTimeMs: evt.ts
|
|
1998
|
-
});
|
|
1999
|
-
setSpanAttrs(span, spanAttrs);
|
|
2000
|
-
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
2001
|
-
span.end(evt.ts);
|
|
2002
|
-
};
|
|
2003
|
-
const recordModelCallError = (evt, metadata, modelContent) => {
|
|
2004
|
-
const errorType = lowCardinalityAttr(evt.errorCategory, "other");
|
|
2005
|
-
const metricAttrs = {
|
|
2006
|
-
...modelCallMetricAttrs(evt),
|
|
2007
|
-
"openclaw.errorCategory": errorType,
|
|
2008
|
-
...evt.failureKind ? { "openclaw.failureKind": lowCardinalityAttr(evt.failureKind, "other") } : {}
|
|
2009
|
-
};
|
|
2010
|
-
modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
2011
|
-
recordModelCallSizeTimingMetrics(evt, metricAttrs);
|
|
2012
|
-
genAiOperationDurationHistogram.record(evt.durationMs / 1e3, genAiModelCallMetricAttrs(evt, errorType));
|
|
2013
|
-
if (!tracesEnabled) return;
|
|
2014
|
-
const spanAttrs = {
|
|
2015
|
-
"openclaw.provider": evt.provider,
|
|
2016
|
-
"openclaw.model": evt.model,
|
|
2017
|
-
"openclaw.errorCategory": errorType,
|
|
2018
|
-
"error.type": errorType
|
|
2019
|
-
};
|
|
2020
|
-
if (evt.failureKind) spanAttrs["openclaw.failureKind"] = lowCardinalityAttr(evt.failureKind, "other");
|
|
2021
|
-
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
2022
|
-
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
2023
|
-
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
2024
|
-
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
2025
|
-
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
2026
|
-
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
2027
|
-
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
2028
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
2029
|
-
kind: modelCallSpanKind(),
|
|
2030
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2031
|
-
endTimeMs: evt.ts
|
|
2032
|
-
});
|
|
2033
|
-
setSpanAttrs(span, spanAttrs);
|
|
2034
|
-
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
2035
|
-
span.setStatus({
|
|
2036
|
-
code: SpanStatusCode.ERROR,
|
|
2037
|
-
message: redactSensitiveText(evt.errorCategory)
|
|
2038
|
-
});
|
|
2039
|
-
span.end(evt.ts);
|
|
2040
|
-
};
|
|
2041
|
-
const toolExecutionBaseAttrs = (evt) => ({
|
|
2042
|
-
"openclaw.toolName": evt.toolName,
|
|
2043
|
-
"openclaw.tool.source": lowCardinalityAttr(evt.toolSource, "core"),
|
|
2044
|
-
"gen_ai.tool.name": evt.toolName,
|
|
2045
|
-
...evt.toolOwner ? { "openclaw.tool.owner": lowCardinalityAttr(evt.toolOwner) } : {},
|
|
2046
|
-
...paramsSummaryAttrs(evt.paramsSummary)
|
|
2047
|
-
});
|
|
2048
|
-
const skillUsedAttrs = (evt) => ({
|
|
2049
|
-
"openclaw.skill.name": lowCardinalityAttr(evt.skillName, "skill"),
|
|
2050
|
-
"openclaw.skill.source": lowCardinalityAttr(evt.skillSource),
|
|
2051
|
-
"openclaw.skill.activation": lowCardinalityAttr(evt.activation),
|
|
2052
|
-
...evt.agentId ? { "openclaw.agent": lowCardinalityAttr(evt.agentId) } : {},
|
|
2053
|
-
...evt.toolName ? { "openclaw.toolName": lowCardinalityAttr(evt.toolName, "tool") } : {}
|
|
2054
|
-
});
|
|
2055
|
-
const recordSkillUsed = (evt, metadata) => {
|
|
2056
|
-
if (!metadata.trusted) return;
|
|
2057
|
-
const attrs = skillUsedAttrs(evt);
|
|
2058
|
-
skillUsedCounter.add(1, attrs);
|
|
2059
|
-
if (!tracesEnabled) return;
|
|
2060
|
-
const spanAttrs = { ...attrs };
|
|
2061
|
-
addRunAttrs(spanAttrs, evt);
|
|
2062
|
-
const span = spanWithDuration("openclaw.skill.used", spanAttrs, 0, {
|
|
2063
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2064
|
-
endTimeMs: evt.ts
|
|
2065
|
-
});
|
|
2066
|
-
setSpanAttrs(span, spanAttrs);
|
|
2067
|
-
span.end(evt.ts);
|
|
2068
|
-
};
|
|
2069
|
-
const recordToolExecutionStarted = (evt, metadata) => {
|
|
2070
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
2071
|
-
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.tool.execution", toolExecutionBaseAttrs(evt), void 0, {
|
|
2072
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2073
|
-
startTimeMs: evt.ts
|
|
2074
|
-
}));
|
|
2075
|
-
};
|
|
2076
|
-
const recordToolExecutionCompleted = (evt, metadata, toolContent) => {
|
|
2077
|
-
const attrs = toolExecutionBaseAttrs(evt);
|
|
2078
|
-
toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
2079
|
-
if (!tracesEnabled) return;
|
|
2080
|
-
const spanAttrs = { ...toolExecutionBaseAttrs(evt) };
|
|
2081
|
-
addRunAttrs(spanAttrs, evt);
|
|
2082
|
-
assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
|
|
2083
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
2084
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2085
|
-
endTimeMs: evt.ts
|
|
2086
|
-
});
|
|
2087
|
-
setSpanAttrs(span, spanAttrs);
|
|
2088
|
-
span.end(evt.ts);
|
|
2089
|
-
};
|
|
2090
|
-
const recordToolExecutionError = (evt, metadata, toolContent) => {
|
|
2091
|
-
const attrs = {
|
|
2092
|
-
...toolExecutionBaseAttrs(evt),
|
|
2093
|
-
"openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
|
|
2094
|
-
};
|
|
2095
|
-
toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
2096
|
-
if (!tracesEnabled) return;
|
|
2097
|
-
const spanAttrs = {
|
|
2098
|
-
...toolExecutionBaseAttrs(evt),
|
|
2099
|
-
"openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other")
|
|
2100
|
-
};
|
|
2101
|
-
addRunAttrs(spanAttrs, evt);
|
|
2102
|
-
if (evt.errorCode) spanAttrs["openclaw.errorCode"] = lowCardinalityAttr(evt.errorCode, "other");
|
|
2103
|
-
assignOtelToolContentAttributes(spanAttrs, toolContent, contentCapturePolicy);
|
|
2104
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
2105
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2106
|
-
endTimeMs: evt.ts
|
|
2107
|
-
});
|
|
2108
|
-
setSpanAttrs(span, spanAttrs);
|
|
2109
|
-
span.setStatus({
|
|
2110
|
-
code: SpanStatusCode.ERROR,
|
|
2111
|
-
message: redactSensitiveText(evt.errorCategory)
|
|
2112
|
-
});
|
|
2113
|
-
span.end(evt.ts);
|
|
2114
|
-
};
|
|
2115
|
-
const recordToolExecutionBlocked = (evt, metadata) => {
|
|
2116
|
-
toolExecutionBlockedCounter.add(1, {
|
|
2117
|
-
...toolExecutionBaseAttrs(evt),
|
|
2118
|
-
"openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
|
|
2119
|
-
});
|
|
2120
|
-
if (!tracesEnabled) return;
|
|
2121
|
-
const spanAttrs = {
|
|
2122
|
-
...toolExecutionBaseAttrs(evt),
|
|
2123
|
-
"openclaw.outcome": "blocked",
|
|
2124
|
-
"openclaw.deniedReason": lowCardinalityAttr(evt.deniedReason, "other")
|
|
2125
|
-
};
|
|
2126
|
-
addRunAttrs(spanAttrs, evt);
|
|
2127
|
-
const span = spanWithDuration("openclaw.tool.execution", spanAttrs, 0, {
|
|
2128
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2129
|
-
endTimeMs: evt.ts
|
|
2130
|
-
});
|
|
2131
|
-
setSpanAttrs(span, spanAttrs);
|
|
2132
|
-
span.end(evt.ts);
|
|
2133
|
-
};
|
|
2134
|
-
const recordPayloadLarge = (evt) => {
|
|
2135
|
-
const attrs = {
|
|
2136
|
-
"openclaw.payload.action": evt.action,
|
|
2137
|
-
"openclaw.payload.surface": lowCardinalityAttr(evt.surface, "unknown"),
|
|
2138
|
-
"openclaw.channel": lowCardinalityAttr(evt.channel, "none"),
|
|
2139
|
-
"openclaw.plugin": lowCardinalityAttr(evt.pluginId, "none"),
|
|
2140
|
-
"openclaw.reason": lowCardinalityAttr(evt.reason, "none")
|
|
2141
|
-
};
|
|
2142
|
-
payloadLargeCounter.add(1, attrs);
|
|
2143
|
-
const bytes = positiveFiniteNumber(evt.bytes);
|
|
2144
|
-
if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
|
|
2145
|
-
};
|
|
2146
|
-
const recordExecProcessCompleted = (evt) => {
|
|
2147
|
-
const attrs = {
|
|
2148
|
-
"openclaw.exec.target": evt.target,
|
|
2149
|
-
"openclaw.exec.mode": evt.mode,
|
|
2150
|
-
"openclaw.outcome": evt.outcome
|
|
2151
|
-
};
|
|
2152
|
-
if (evt.failureKind) attrs["openclaw.failureKind"] = evt.failureKind;
|
|
2153
|
-
execProcessDurationHistogram.record(evt.durationMs, attrs);
|
|
2154
|
-
if (!tracesEnabled) return;
|
|
2155
|
-
const spanAttrs = {
|
|
2156
|
-
...attrs,
|
|
2157
|
-
"openclaw.exec.command_length": evt.commandLength
|
|
2158
|
-
};
|
|
2159
|
-
if (typeof evt.exitCode === "number") spanAttrs["openclaw.exec.exit_code"] = evt.exitCode;
|
|
2160
|
-
if (evt.exitSignal) spanAttrs["openclaw.exec.exit_signal"] = lowCardinalityAttr(evt.exitSignal, "other");
|
|
2161
|
-
if (evt.timedOut !== void 0) spanAttrs["openclaw.exec.timed_out"] = evt.timedOut;
|
|
2162
|
-
const span = spanWithDuration("openclaw.exec", spanAttrs, evt.durationMs, { endTimeMs: evt.ts });
|
|
2163
|
-
if (evt.outcome === "failed") span.setStatus({
|
|
2164
|
-
code: SpanStatusCode.ERROR,
|
|
2165
|
-
...evt.failureKind ? { message: evt.failureKind } : {}
|
|
2166
|
-
});
|
|
2167
|
-
span.end(evt.ts);
|
|
2168
|
-
};
|
|
2169
|
-
const recordHeartbeat = (evt) => {
|
|
2170
|
-
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
|
|
2171
|
-
};
|
|
2172
|
-
const recordLivenessWarning = (evt) => {
|
|
2173
|
-
const reason = evt.reasons.join(":");
|
|
2174
|
-
const attrs = { "openclaw.liveness.reason": lowCardinalityAttr(reason, "unknown") };
|
|
2175
|
-
livenessWarningCounter.add(1, attrs);
|
|
2176
|
-
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" });
|
|
2177
|
-
if (evt.eventLoopDelayP99Ms !== void 0) livenessEventLoopDelayP99Histogram.record(evt.eventLoopDelayP99Ms, attrs);
|
|
2178
|
-
if (evt.eventLoopDelayMaxMs !== void 0) livenessEventLoopDelayMaxHistogram.record(evt.eventLoopDelayMaxMs, attrs);
|
|
2179
|
-
if (evt.eventLoopUtilization !== void 0) livenessEventLoopUtilizationHistogram.record(evt.eventLoopUtilization, attrs);
|
|
2180
|
-
if (evt.cpuCoreRatio !== void 0) livenessCpuCoreRatioHistogram.record(evt.cpuCoreRatio, attrs);
|
|
2181
|
-
if (!tracesEnabled) return;
|
|
2182
|
-
const span = spanWithDuration("openclaw.liveness.warning", {
|
|
2183
|
-
...attrs,
|
|
2184
|
-
"openclaw.liveness.active": evt.active,
|
|
2185
|
-
"openclaw.liveness.waiting": evt.waiting,
|
|
2186
|
-
"openclaw.liveness.queued": evt.queued,
|
|
2187
|
-
"openclaw.liveness.interval_ms": evt.intervalMs,
|
|
2188
|
-
...evt.eventLoopDelayP99Ms !== void 0 ? { "openclaw.liveness.event_loop_delay_p99_ms": evt.eventLoopDelayP99Ms } : {},
|
|
2189
|
-
...evt.eventLoopDelayMaxMs !== void 0 ? { "openclaw.liveness.event_loop_delay_max_ms": evt.eventLoopDelayMaxMs } : {},
|
|
2190
|
-
...evt.eventLoopUtilization !== void 0 ? { "openclaw.liveness.event_loop_utilization": evt.eventLoopUtilization } : {},
|
|
2191
|
-
...evt.cpuUserMs !== void 0 ? { "openclaw.liveness.cpu_user_ms": evt.cpuUserMs } : {},
|
|
2192
|
-
...evt.cpuSystemMs !== void 0 ? { "openclaw.liveness.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
2193
|
-
...evt.cpuTotalMs !== void 0 ? { "openclaw.liveness.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
2194
|
-
...evt.cpuCoreRatio !== void 0 ? { "openclaw.liveness.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2195
|
-
}, 0, { endTimeMs: evt.ts });
|
|
2196
|
-
span.setStatus({
|
|
2197
|
-
code: SpanStatusCode.ERROR,
|
|
2198
|
-
message: reason
|
|
2199
|
-
});
|
|
2200
|
-
span.end(evt.ts);
|
|
2201
|
-
};
|
|
2202
|
-
const recordDiagnosticPhaseCompleted = (evt) => {
|
|
2203
|
-
if (!tracesEnabled) return;
|
|
2204
|
-
const spanAttrs = {
|
|
2205
|
-
"openclaw.phase": lowCardinalityAttr(evt.name, "unknown"),
|
|
2206
|
-
...evt.cpuUserMs !== void 0 ? { "openclaw.phase.cpu_user_ms": evt.cpuUserMs } : {},
|
|
2207
|
-
...evt.cpuSystemMs !== void 0 ? { "openclaw.phase.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
2208
|
-
...evt.cpuTotalMs !== void 0 ? { "openclaw.phase.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
2209
|
-
...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2210
|
-
};
|
|
2211
|
-
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
2212
|
-
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, { endTimeMs: evt.ts }).end(evt.ts);
|
|
2213
|
-
};
|
|
2214
|
-
const recordTelemetryExporter = (evt, metadata) => {
|
|
2215
|
-
if (!metadata.trusted) return;
|
|
2216
|
-
telemetryExporterCounter.add(1, {
|
|
2217
|
-
"openclaw.exporter": lowCardinalityAttr(evt.exporter, "unknown"),
|
|
2218
|
-
"openclaw.signal": evt.signal,
|
|
2219
|
-
"openclaw.status": evt.status,
|
|
2220
|
-
...evt.reason ? { "openclaw.reason": evt.reason } : {},
|
|
2221
|
-
...evt.errorCategory ? { "openclaw.errorCategory": lowCardinalityAttr(evt.errorCategory, "other") } : {}
|
|
2222
|
-
});
|
|
2223
|
-
};
|
|
2224
|
-
const subscribe = ctx.internalDiagnostics?.onEvent;
|
|
2225
|
-
if (!subscribe) {
|
|
2226
|
-
ctx.logger.error("diagnostics-otel: internal diagnostics capability unavailable");
|
|
2227
|
-
return;
|
|
2228
|
-
}
|
|
2229
|
-
unsubscribe = subscribe((evt, metadata, privateData) => {
|
|
2230
|
-
try {
|
|
2231
|
-
switch (evt.type) {
|
|
2232
|
-
case "model.usage":
|
|
2233
|
-
recordModelUsage(evt, metadata);
|
|
2234
|
-
return;
|
|
2235
|
-
case "webhook.received":
|
|
2236
|
-
recordWebhookReceived(evt);
|
|
2237
|
-
return;
|
|
2238
|
-
case "webhook.processed":
|
|
2239
|
-
recordWebhookProcessed(evt);
|
|
2240
|
-
return;
|
|
2241
|
-
case "webhook.error":
|
|
2242
|
-
recordWebhookError(evt);
|
|
2243
|
-
return;
|
|
2244
|
-
case "message.queued":
|
|
2245
|
-
recordMessageQueued(evt);
|
|
2246
|
-
return;
|
|
2247
|
-
case "message.received":
|
|
2248
|
-
recordMessageReceived(evt);
|
|
2249
|
-
return;
|
|
2250
|
-
case "message.dispatch.started":
|
|
2251
|
-
recordMessageDispatchStarted(evt, metadata);
|
|
2252
|
-
return;
|
|
2253
|
-
case "message.dispatch.completed":
|
|
2254
|
-
recordMessageDispatchCompleted(evt);
|
|
2255
|
-
return;
|
|
2256
|
-
case "message.processed":
|
|
2257
|
-
recordMessageProcessed(evt, metadata);
|
|
2258
|
-
return;
|
|
2259
|
-
case "message.delivery.started":
|
|
2260
|
-
recordMessageDeliveryStarted(evt);
|
|
2261
|
-
return;
|
|
2262
|
-
case "message.delivery.completed":
|
|
2263
|
-
recordMessageDeliveryCompleted(evt, metadata);
|
|
2264
|
-
return;
|
|
2265
|
-
case "message.delivery.error":
|
|
2266
|
-
recordMessageDeliveryError(evt, metadata);
|
|
2267
|
-
return;
|
|
2268
|
-
case "talk.event":
|
|
2269
|
-
recordTalkEvent(evt, metadata);
|
|
2270
|
-
return;
|
|
2271
|
-
case "queue.lane.enqueue":
|
|
2272
|
-
recordLaneEnqueue(evt);
|
|
2273
|
-
return;
|
|
2274
|
-
case "queue.lane.dequeue":
|
|
2275
|
-
recordLaneDequeue(evt);
|
|
2276
|
-
return;
|
|
2277
|
-
case "session.state":
|
|
2278
|
-
recordSessionState(evt);
|
|
2279
|
-
return;
|
|
2280
|
-
case "session.long_running":
|
|
2281
|
-
case "session.stalled": return;
|
|
2282
|
-
case "session.turn.created":
|
|
2283
|
-
recordSessionTurnCreated(evt);
|
|
2284
|
-
return;
|
|
2285
|
-
case "session.stuck":
|
|
2286
|
-
recordSessionStuck(evt);
|
|
2287
|
-
return;
|
|
2288
|
-
case "session.recovery.requested":
|
|
2289
|
-
recordSessionRecoveryRequested(evt);
|
|
2290
|
-
return;
|
|
2291
|
-
case "session.recovery.completed":
|
|
2292
|
-
recordSessionRecoveryCompleted(evt);
|
|
2293
|
-
return;
|
|
2294
|
-
case "run.attempt":
|
|
2295
|
-
recordRunAttempt(evt);
|
|
2296
|
-
return;
|
|
2297
|
-
case "run.progress": return;
|
|
2298
|
-
case "diagnostic.heartbeat":
|
|
2299
|
-
recordHeartbeat(evt);
|
|
2300
|
-
return;
|
|
2301
|
-
case "diagnostic.liveness.warning":
|
|
2302
|
-
recordLivenessWarning(evt);
|
|
2303
|
-
return;
|
|
2304
|
-
case "diagnostic.phase.completed":
|
|
2305
|
-
recordDiagnosticPhaseCompleted(evt);
|
|
2306
|
-
return;
|
|
2307
|
-
case "run.started":
|
|
2308
|
-
recordRunStarted(evt, metadata);
|
|
2309
|
-
return;
|
|
2310
|
-
case "run.completed":
|
|
2311
|
-
recordRunCompleted(evt, metadata);
|
|
2312
|
-
return;
|
|
2313
|
-
case "harness.run.started":
|
|
2314
|
-
recordHarnessRunStarted(evt, metadata);
|
|
2315
|
-
return;
|
|
2316
|
-
case "harness.run.completed":
|
|
2317
|
-
recordHarnessRunCompleted(evt, metadata);
|
|
2318
|
-
return;
|
|
2319
|
-
case "harness.run.error":
|
|
2320
|
-
recordHarnessRunError(evt, metadata);
|
|
2321
|
-
return;
|
|
2322
|
-
case "context.assembled":
|
|
2323
|
-
recordContextAssembled(evt, metadata);
|
|
2324
|
-
return;
|
|
2325
|
-
case "model.call.started":
|
|
2326
|
-
recordModelCallStarted(evt, metadata);
|
|
2327
|
-
return;
|
|
2328
|
-
case "model.call.completed":
|
|
2329
|
-
recordModelCallCompleted(evt, metadata, privateData.modelContent);
|
|
2330
|
-
return;
|
|
2331
|
-
case "model.call.error":
|
|
2332
|
-
recordModelCallError(evt, metadata, privateData.modelContent);
|
|
2333
|
-
return;
|
|
2334
|
-
case "tool.execution.started":
|
|
2335
|
-
recordToolExecutionStarted(evt, metadata);
|
|
2336
|
-
return;
|
|
2337
|
-
case "tool.execution.completed":
|
|
2338
|
-
recordToolExecutionCompleted(evt, metadata, privateData.toolContent);
|
|
2339
|
-
return;
|
|
2340
|
-
case "tool.execution.error":
|
|
2341
|
-
recordToolExecutionError(evt, metadata, privateData.toolContent);
|
|
2342
|
-
return;
|
|
2343
|
-
case "tool.execution.blocked":
|
|
2344
|
-
recordToolExecutionBlocked(evt, metadata);
|
|
2345
|
-
return;
|
|
2346
|
-
case "skill.used":
|
|
2347
|
-
recordSkillUsed(evt, metadata);
|
|
2348
|
-
return;
|
|
2349
|
-
case "exec.process.completed":
|
|
2350
|
-
recordExecProcessCompleted(evt);
|
|
2351
|
-
return;
|
|
2352
|
-
case "exec.approval.followup_suppressed": return;
|
|
2353
|
-
case "log.record":
|
|
2354
|
-
recordLogRecord?.(evt, metadata);
|
|
2355
|
-
return;
|
|
2356
|
-
case "security.event":
|
|
2357
|
-
recordSecurityEvent?.(evt, metadata);
|
|
2358
|
-
return;
|
|
2359
|
-
case "tool.loop":
|
|
2360
|
-
recordToolLoop(evt);
|
|
2361
|
-
return;
|
|
2362
|
-
case "diagnostic.memory.sample":
|
|
2363
|
-
recordMemorySample(evt);
|
|
2364
|
-
return;
|
|
2365
|
-
case "diagnostic.memory.pressure":
|
|
2366
|
-
recordMemoryPressure(evt);
|
|
2367
|
-
return;
|
|
2368
|
-
case "diagnostic.async_queue.dropped":
|
|
2369
|
-
recordAsyncQueueDropped(evt);
|
|
2370
|
-
return;
|
|
2371
|
-
case "telemetry.exporter":
|
|
2372
|
-
recordTelemetryExporter(evt, metadata);
|
|
2373
|
-
return;
|
|
2374
|
-
case "payload.large":
|
|
2375
|
-
recordPayloadLarge(evt);
|
|
2376
|
-
return;
|
|
2377
|
-
case "model.failover": recordModelFailover(evt, metadata);
|
|
2378
|
-
}
|
|
2379
|
-
} catch (err) {
|
|
2380
|
-
ctx.logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
|
|
2381
|
-
}
|
|
2382
|
-
});
|
|
2383
|
-
unregisterUnhandledRejectionHandler = registerUnhandledRejectionHandler((reason) => {
|
|
2384
|
-
const otlpError = findOtlpExporterError(reason);
|
|
2385
|
-
if (!otlpError) return false;
|
|
2386
|
-
const code = readErrorCode(otlpError) ?? "unknown";
|
|
2387
|
-
ctx.logger.warn(`diagnostics-otel: suppressed OTLP exporter unhandled rejection (code=${String(code)})`);
|
|
2388
|
-
return true;
|
|
2389
|
-
});
|
|
2390
|
-
emitForSignals(enabledSignals, {
|
|
2391
|
-
exporter: "diagnostics-otel",
|
|
2392
|
-
status: "started",
|
|
2393
|
-
reason: "configured"
|
|
2394
|
-
});
|
|
2395
|
-
if (logsEnabled) {
|
|
2396
|
-
const label = logsExporter === "both" ? "OTLP/Protobuf + stdout JSONL" : logsExporter === "stdout" ? "stdout JSONL" : "OTLP/Protobuf";
|
|
2397
|
-
ctx.logger.info(`diagnostics-otel: logs exporter enabled (${label})`);
|
|
2398
|
-
}
|
|
2399
|
-
},
|
|
2400
|
-
async stop() {
|
|
2401
|
-
await stopStarted();
|
|
2402
|
-
}
|
|
2403
|
-
};
|
|
2404
|
-
}
|
|
2405
|
-
//#endregion
|
|
2406
3
|
//#region extensions/diagnostics-otel/index.ts
|
|
2407
4
|
var diagnostics_otel_default = definePluginEntry({
|
|
2408
5
|
id: "diagnostics-otel",
|