@openclaw/diagnostics-otel 2026.10.2 → 2026.10.5-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -165,8 +165,7 @@ function normalizeOtelContentValue(value) {
|
|
|
165
165
|
const items = value.slice(0, 200).filter((item) => typeof item === "string");
|
|
166
166
|
if (items.length > 0) return normalizeOtelLogString(items.join("\n"), MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
167
167
|
}
|
|
168
|
-
|
|
169
|
-
if (json) return json;
|
|
168
|
+
return safeJsonString(value);
|
|
170
169
|
}
|
|
171
170
|
const TRUNCATED_JSON_TEXT_SUFFIX = "...(truncated)";
|
|
172
171
|
const JSON_TRUNCATION_STRING_BUDGETS = [
|
|
@@ -191,26 +190,20 @@ const JSON_TRUNCATION_ARRAY_ITEM_BUDGETS = [
|
|
|
191
190
|
];
|
|
192
191
|
const JSON_TRUNCATION_MAX_OBJECT_FIELDS = 64;
|
|
193
192
|
const JSON_TRUNCATION_MAX_DEPTH = 8;
|
|
194
|
-
function safeJsonString(value
|
|
193
|
+
function safeJsonString(value) {
|
|
195
194
|
if (value === void 0 || typeof value === "function" || typeof value === "symbol") return;
|
|
196
195
|
const exact = stringifyJsonForOtelAttribute(value);
|
|
197
|
-
if (exact && exact.length <=
|
|
196
|
+
if (exact && exact.length <= MAX_OTEL_CONTENT_ATTRIBUTE_CHARS) return exact;
|
|
198
197
|
for (const maxArrayItems of JSON_TRUNCATION_ARRAY_ITEM_BUDGETS) for (const maxStringChars of JSON_TRUNCATION_STRING_BUDGETS) {
|
|
199
|
-
const json = stringifyJsonForOtelAttribute(truncateJsonValueForOtelAttribute(value,
|
|
200
|
-
|
|
201
|
-
maxDepth: JSON_TRUNCATION_MAX_DEPTH,
|
|
202
|
-
maxObjectFields: JSON_TRUNCATION_MAX_OBJECT_FIELDS,
|
|
203
|
-
maxStringChars,
|
|
204
|
-
seen: /* @__PURE__ */ new WeakSet()
|
|
205
|
-
}));
|
|
206
|
-
if (json && json.length <= maxChars) return json;
|
|
198
|
+
const json = stringifyJsonForOtelAttribute(truncateJsonValueForOtelAttribute(value, maxArrayItems, maxStringChars));
|
|
199
|
+
if (json && json.length <= MAX_OTEL_CONTENT_ATTRIBUTE_CHARS) return json;
|
|
207
200
|
}
|
|
208
201
|
const summary = stringifyJsonForOtelAttribute({
|
|
209
202
|
truncated: true,
|
|
210
203
|
reason: exact ? "max_attribute_size" : "unserializable_value",
|
|
211
204
|
type: describeJsonValue(value)
|
|
212
205
|
});
|
|
213
|
-
return summary && summary.length <=
|
|
206
|
+
return summary && summary.length <= MAX_OTEL_CONTENT_ATTRIBUTE_CHARS ? summary : void 0;
|
|
214
207
|
}
|
|
215
208
|
function stringifyJsonForOtelAttribute(value) {
|
|
216
209
|
try {
|
|
@@ -221,55 +214,46 @@ function stringifyJsonForOtelAttribute(value) {
|
|
|
221
214
|
return;
|
|
222
215
|
}
|
|
223
216
|
}
|
|
224
|
-
function truncateJsonValueForOtelAttribute(
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
maxDepth: options.maxDepth - 1
|
|
263
|
-
};
|
|
264
|
-
const result = {};
|
|
265
|
-
const entries = Object.entries(value).filter(([, field]) => field !== void 0 && typeof field !== "function" && typeof field !== "symbol");
|
|
266
|
-
for (const [key, field] of entries.slice(0, options.maxObjectFields)) result[key] = truncateJsonValueForOtelAttribute(field, nextOptions);
|
|
267
|
-
if (entries.length > options.maxObjectFields) {
|
|
268
|
-
result.truncated = true;
|
|
269
|
-
result.omittedFields = entries.length - options.maxObjectFields;
|
|
217
|
+
function truncateJsonValueForOtelAttribute(input, maxArrayItems, maxStringChars) {
|
|
218
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
219
|
+
function visit(value, depth) {
|
|
220
|
+
if (typeof value === "string" || typeof value === "bigint") return truncateJsonTextForOtelAttribute(String(value), maxStringChars);
|
|
221
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) return value;
|
|
222
|
+
if (typeof value !== "object") return;
|
|
223
|
+
if (depth <= 0) return {
|
|
224
|
+
truncated: true,
|
|
225
|
+
reason: "max_depth"
|
|
226
|
+
};
|
|
227
|
+
if (seen.has(value)) {
|
|
228
|
+
const marker = {
|
|
229
|
+
truncated: true,
|
|
230
|
+
reason: "circular_reference"
|
|
231
|
+
};
|
|
232
|
+
return Array.isArray(value) ? [marker] : marker;
|
|
233
|
+
}
|
|
234
|
+
seen.add(value);
|
|
235
|
+
let result;
|
|
236
|
+
if (Array.isArray(value)) {
|
|
237
|
+
const items = value.slice(0, maxArrayItems).map((item) => visit(item, depth - 1));
|
|
238
|
+
if (value.length > items.length) items.push({
|
|
239
|
+
truncated: true,
|
|
240
|
+
omittedItems: value.length - items.length
|
|
241
|
+
});
|
|
242
|
+
result = items;
|
|
243
|
+
} else {
|
|
244
|
+
const object = {};
|
|
245
|
+
const entries = Object.entries(value).filter(([, field]) => field !== void 0 && typeof field !== "function" && typeof field !== "symbol");
|
|
246
|
+
for (const [key, field] of entries.slice(0, JSON_TRUNCATION_MAX_OBJECT_FIELDS)) object[key] = visit(field, depth - 1);
|
|
247
|
+
if (entries.length > JSON_TRUNCATION_MAX_OBJECT_FIELDS) {
|
|
248
|
+
object.truncated = true;
|
|
249
|
+
object.omittedFields = entries.length - JSON_TRUNCATION_MAX_OBJECT_FIELDS;
|
|
250
|
+
}
|
|
251
|
+
result = object;
|
|
252
|
+
}
|
|
253
|
+
seen.delete(value);
|
|
254
|
+
return result;
|
|
270
255
|
}
|
|
271
|
-
|
|
272
|
-
return result;
|
|
256
|
+
return visit(input, JSON_TRUNCATION_MAX_DEPTH);
|
|
273
257
|
}
|
|
274
258
|
function truncateJsonTextForOtelAttribute(value, maxChars) {
|
|
275
259
|
const redacted = redactSensitiveText(value);
|
|
@@ -404,72 +388,63 @@ function findOtlpExporterError(reason) {
|
|
|
404
388
|
//#region extensions/diagnostics-otel/src/service-events.ts
|
|
405
389
|
function createDiagnosticsEventHandler(params) {
|
|
406
390
|
const { logger, recorders, recordLogEvent } = params;
|
|
391
|
+
const handlers = {
|
|
392
|
+
"diagnostic.gc": recorders.recordGcDuration.bind(recorders),
|
|
393
|
+
"gateway.event_loop.sample": recorders.recordGatewayEventLoopSample.bind(recorders),
|
|
394
|
+
"gateway.rpc": recorders.recordGatewayRpc.bind(recorders),
|
|
395
|
+
"model.usage": (evt, metadata, privateData) => recorders.recordModelUsage(evt, metadata, privateData.hostPluginId),
|
|
396
|
+
"webhook.received": recorders.recordWebhookReceived.bind(recorders),
|
|
397
|
+
"webhook.processed": recorders.recordWebhookProcessed.bind(recorders),
|
|
398
|
+
"webhook.error": recorders.recordWebhookError.bind(recorders),
|
|
399
|
+
"message.queued": recorders.recordMessageQueued.bind(recorders),
|
|
400
|
+
"message.received": recorders.recordMessageReceived.bind(recorders),
|
|
401
|
+
"message.dispatch.started": recorders.recordMessageDispatchStarted.bind(recorders),
|
|
402
|
+
"message.dispatch.completed": recorders.recordMessageDispatchCompleted.bind(recorders),
|
|
403
|
+
"message.processed": recorders.recordMessageProcessed.bind(recorders),
|
|
404
|
+
"message.delivery.started": recorders.recordMessageDeliveryStarted.bind(recorders),
|
|
405
|
+
"message.delivery.completed": recorders.recordMessageDeliveryFinished.bind(recorders),
|
|
406
|
+
"message.delivery.error": recorders.recordMessageDeliveryFinished.bind(recorders),
|
|
407
|
+
"talk.event": recorders.recordTalkEvent.bind(recorders),
|
|
408
|
+
"queue.lane.enqueue": recorders.recordLaneEnqueue.bind(recorders),
|
|
409
|
+
"queue.lane.dequeue": recorders.recordLaneDequeue.bind(recorders),
|
|
410
|
+
"session.state": recorders.recordSessionState.bind(recorders),
|
|
411
|
+
"session.turn.created": recorders.recordSessionTurnCreated.bind(recorders),
|
|
412
|
+
"session.stuck": recorders.recordSessionStuck.bind(recorders),
|
|
413
|
+
"session.recovery.requested": recorders.recordSessionRecoveryRequested.bind(recorders),
|
|
414
|
+
"session.recovery.completed": recorders.recordSessionRecoveryCompleted.bind(recorders),
|
|
415
|
+
"run.attempt": recorders.recordRunAttempt.bind(recorders),
|
|
416
|
+
"diagnostic.heartbeat": recorders.recordHeartbeat.bind(recorders),
|
|
417
|
+
"diagnostic.liveness.warning": recorders.recordLivenessWarning.bind(recorders),
|
|
418
|
+
"diagnostic.phase.completed": recorders.recordDiagnosticPhaseCompleted.bind(recorders),
|
|
419
|
+
"run.started": recorders.recordRunStarted.bind(recorders),
|
|
420
|
+
"run.completed": recorders.recordRunCompleted.bind(recorders),
|
|
421
|
+
"harness.run.started": recorders.recordHarnessRunStarted.bind(recorders),
|
|
422
|
+
"agent.commentary": recorders.recordAgentCommentary.bind(recorders),
|
|
423
|
+
"harness.run.completed": recorders.recordHarnessRunFinished.bind(recorders),
|
|
424
|
+
"harness.run.error": recorders.recordHarnessRunFinished.bind(recorders),
|
|
425
|
+
"context.assembled": recorders.recordContextAssembled.bind(recorders),
|
|
426
|
+
"model.call.started": recorders.recordModelCallStarted.bind(recorders),
|
|
427
|
+
"model.call.completed": (evt, metadata, privateData) => recorders.recordModelCallFinished(evt, metadata, privateData.modelContent),
|
|
428
|
+
"model.call.error": (evt, metadata, privateData) => recorders.recordModelCallFinished(evt, metadata, privateData.modelContent),
|
|
429
|
+
"tool.execution.started": recorders.recordToolExecutionStarted.bind(recorders),
|
|
430
|
+
"tool.execution.completed": (evt, metadata, privateData) => recorders.recordToolExecutionFinished(evt, metadata, privateData.toolContent),
|
|
431
|
+
"tool.execution.error": (evt, metadata, privateData) => recorders.recordToolExecutionFinished(evt, metadata, privateData.toolContent),
|
|
432
|
+
"tool.execution.blocked": recorders.recordToolExecutionBlocked.bind(recorders),
|
|
433
|
+
"skill.used": recorders.recordSkillUsed.bind(recorders),
|
|
434
|
+
"exec.process.completed": recorders.recordExecProcessCompleted.bind(recorders),
|
|
435
|
+
"log.record": (evt, metadata) => recordLogEvent?.(evt, metadata),
|
|
436
|
+
"security.event": (evt, metadata) => recordLogEvent?.(evt, metadata),
|
|
437
|
+
"tool.loop": recorders.recordToolLoop.bind(recorders),
|
|
438
|
+
"diagnostic.memory.sample": (evt) => recorders.recordMemoryUsageMetrics(evt),
|
|
439
|
+
"diagnostic.memory.pressure": recorders.recordMemoryPressure.bind(recorders),
|
|
440
|
+
"diagnostic.async_queue.dropped": recorders.recordAsyncQueueDropped.bind(recorders),
|
|
441
|
+
"telemetry.exporter": recorders.recordTelemetryExporter.bind(recorders),
|
|
442
|
+
"payload.large": recorders.recordPayloadLarge.bind(recorders),
|
|
443
|
+
"model.failover": recorders.recordModelFailover.bind(recorders)
|
|
444
|
+
};
|
|
407
445
|
return (evt, metadata, privateData) => {
|
|
408
446
|
try {
|
|
409
|
-
|
|
410
|
-
case "diagnostic.child_process.spawn": return;
|
|
411
|
-
case "diagnostic.gc": return recorders.recordGcDuration(evt, metadata);
|
|
412
|
-
case "gateway.event_loop.sample": return recorders.recordGatewayEventLoopSample(evt, metadata);
|
|
413
|
-
case "gateway.rpc": return recorders.recordGatewayRpc(evt, metadata);
|
|
414
|
-
case "model.usage": return recorders.recordModelUsage(evt, metadata, privateData.hostPluginId);
|
|
415
|
-
case "webhook.received": return recorders.recordWebhookReceived(evt);
|
|
416
|
-
case "webhook.processed": return recorders.recordWebhookProcessed(evt);
|
|
417
|
-
case "webhook.error": return recorders.recordWebhookError(evt);
|
|
418
|
-
case "message.queued": return recorders.recordMessageQueued(evt);
|
|
419
|
-
case "message.received": return recorders.recordMessageReceived(evt);
|
|
420
|
-
case "message.dispatch.started": return recorders.recordMessageDispatchStarted(evt, metadata);
|
|
421
|
-
case "message.dispatch.completed": return recorders.recordMessageDispatchCompleted(evt);
|
|
422
|
-
case "message.processed": return recorders.recordMessageProcessed(evt, metadata);
|
|
423
|
-
case "message.delivery.started": return recorders.recordMessageDeliveryStarted(evt);
|
|
424
|
-
case "message.delivery.completed":
|
|
425
|
-
case "message.delivery.error": return recorders.recordMessageDeliveryFinished(evt, metadata);
|
|
426
|
-
case "talk.event": return recorders.recordTalkEvent(evt, metadata);
|
|
427
|
-
case "queue.lane.enqueue": return recorders.recordLaneEnqueue(evt);
|
|
428
|
-
case "queue.lane.dequeue": return recorders.recordLaneDequeue(evt);
|
|
429
|
-
case "session.state": return recorders.recordSessionState(evt);
|
|
430
|
-
case "session.long_running":
|
|
431
|
-
case "session.stalled": break;
|
|
432
|
-
case "session.turn.created": return recorders.recordSessionTurnCreated(evt);
|
|
433
|
-
case "session.stuck": return recorders.recordSessionStuck(evt);
|
|
434
|
-
case "session.recovery.requested": return recorders.recordSessionRecoveryRequested(evt);
|
|
435
|
-
case "session.recovery.completed": return recorders.recordSessionRecoveryCompleted(evt);
|
|
436
|
-
case "run.attempt": return recorders.recordRunAttempt(evt);
|
|
437
|
-
case "run.progress": break;
|
|
438
|
-
case "run.execution_phase": break;
|
|
439
|
-
case "diagnostic.heartbeat": return recorders.recordHeartbeat(evt);
|
|
440
|
-
case "diagnostic.liveness.warning": return recorders.recordLivenessWarning(evt);
|
|
441
|
-
case "diagnostic.phase.completed": return recorders.recordDiagnosticPhaseCompleted(evt, metadata);
|
|
442
|
-
case "run.started": return recorders.recordRunStarted(evt, metadata);
|
|
443
|
-
case "run.completed": return recorders.recordRunCompleted(evt, metadata, privateData);
|
|
444
|
-
case "harness.run.started": return recorders.recordHarnessRunStarted(evt, metadata);
|
|
445
|
-
case "agent.commentary": return recorders.recordAgentCommentary(evt, metadata, privateData);
|
|
446
|
-
case "harness.run.completed":
|
|
447
|
-
case "harness.run.error": return recorders.recordHarnessRunFinished(evt, metadata, privateData);
|
|
448
|
-
case "context.assembled": return recorders.recordContextAssembled(evt, metadata);
|
|
449
|
-
case "model.call.started":
|
|
450
|
-
recorders.recordModelCallStarted(evt, metadata);
|
|
451
|
-
return;
|
|
452
|
-
case "model.call.completed":
|
|
453
|
-
case "model.call.error": return recorders.recordModelCallFinished(evt, metadata, privateData.modelContent);
|
|
454
|
-
case "tool.execution.started":
|
|
455
|
-
recorders.recordToolExecutionStarted(evt, metadata);
|
|
456
|
-
return;
|
|
457
|
-
case "tool.execution.completed":
|
|
458
|
-
case "tool.execution.error": return recorders.recordToolExecutionFinished(evt, metadata, privateData.toolContent);
|
|
459
|
-
case "tool.execution.blocked": return recorders.recordToolExecutionBlocked(evt, metadata);
|
|
460
|
-
case "skill.used": return recorders.recordSkillUsed(evt, metadata);
|
|
461
|
-
case "exec.process.completed": return recorders.recordExecProcessCompleted(evt, metadata);
|
|
462
|
-
case "exec.approval.followup_suppressed": break;
|
|
463
|
-
case "log.record":
|
|
464
|
-
case "security.event": return recordLogEvent?.(evt, metadata);
|
|
465
|
-
case "tool.loop": return recorders.recordToolLoop(evt);
|
|
466
|
-
case "diagnostic.memory.sample": return recorders.recordMemoryUsageMetrics(evt);
|
|
467
|
-
case "diagnostic.memory.pressure": return recorders.recordMemoryPressure(evt);
|
|
468
|
-
case "diagnostic.async_queue.dropped": return recorders.recordAsyncQueueDropped(evt);
|
|
469
|
-
case "telemetry.exporter": return recorders.recordTelemetryExporter(evt, metadata);
|
|
470
|
-
case "payload.large": return recorders.recordPayloadLarge(evt);
|
|
471
|
-
case "model.failover": return recorders.recordModelFailover(evt, metadata);
|
|
472
|
-
}
|
|
447
|
+
if (Object.hasOwn(handlers, evt.type)) handlers[evt.type]?.(evt, metadata, privateData);
|
|
473
448
|
} catch (err) {
|
|
474
449
|
logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
|
|
475
450
|
}
|
|
@@ -571,9 +546,8 @@ function createPublicExporterHealthEventEmitter(publish) {
|
|
|
571
546
|
* transport has exhausted dependency-owned retries.
|
|
572
547
|
*/
|
|
573
548
|
function observeOtlpExporterHealth(exporter, params) {
|
|
574
|
-
const
|
|
575
|
-
const
|
|
576
|
-
const shutdown = observed.shutdown.bind(observed);
|
|
549
|
+
const exportItems = exporter.export.bind(exporter);
|
|
550
|
+
const shutdown = exporter.shutdown.bind(exporter);
|
|
577
551
|
const emit = (status, reason, error) => {
|
|
578
552
|
params.emitExporterEvent({
|
|
579
553
|
exporter: "diagnostics-otel",
|
|
@@ -584,7 +558,7 @@ function observeOtlpExporterHealth(exporter, params) {
|
|
|
584
558
|
...error ? { errorCategory: errorCategory(error) } : {}
|
|
585
559
|
});
|
|
586
560
|
};
|
|
587
|
-
|
|
561
|
+
exporter.export = (items, resultCallback) => {
|
|
588
562
|
let dependencyCallbackInvoked = false;
|
|
589
563
|
try {
|
|
590
564
|
exportItems(items, (result) => {
|
|
@@ -598,7 +572,7 @@ function observeOtlpExporterHealth(exporter, params) {
|
|
|
598
572
|
throw error;
|
|
599
573
|
}
|
|
600
574
|
};
|
|
601
|
-
|
|
575
|
+
exporter.shutdown = async () => {
|
|
602
576
|
try {
|
|
603
577
|
await shutdown();
|
|
604
578
|
} catch (error) {
|
|
@@ -610,6 +584,9 @@ function observeOtlpExporterHealth(exporter, params) {
|
|
|
610
584
|
}
|
|
611
585
|
//#endregion
|
|
612
586
|
//#region extensions/diagnostics-otel/src/service-attributes.ts
|
|
587
|
+
function assignOptionalNumberAttrs(attributes, prefix, values) {
|
|
588
|
+
for (const [key, value] of Object.entries(values)) if (value !== void 0) attributes[`${prefix}${key}`] = value;
|
|
589
|
+
}
|
|
613
590
|
function redactOtelAttributes(attributes) {
|
|
614
591
|
const redactedAttributes = {};
|
|
615
592
|
for (const [key, value] of Object.entries(attributes)) {
|
|
@@ -618,12 +595,11 @@ function redactOtelAttributes(attributes) {
|
|
|
618
595
|
}
|
|
619
596
|
return redactedAttributes;
|
|
620
597
|
}
|
|
621
|
-
function securityTargetNameAttr(value
|
|
622
|
-
if (!value) return fallback;
|
|
598
|
+
function securityTargetNameAttr(value) {
|
|
623
599
|
const redacted = redactSensitiveText(value.trim());
|
|
624
600
|
const redactedLower = redacted.toLowerCase();
|
|
625
|
-
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return
|
|
626
|
-
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted :
|
|
601
|
+
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return "unknown";
|
|
602
|
+
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : "unknown";
|
|
627
603
|
}
|
|
628
604
|
function otelLogTimestampIso(timestamp) {
|
|
629
605
|
if (timestamp instanceof Date) return timestamp.toISOString();
|
|
@@ -1077,7 +1053,7 @@ function contentParts(value) {
|
|
|
1077
1053
|
if (!Array.isArray(value)) {
|
|
1078
1054
|
if (value === void 0 || value === null) return [];
|
|
1079
1055
|
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return [textPart(String(value))];
|
|
1080
|
-
const json = safeJsonString(value
|
|
1056
|
+
const json = safeJsonString(value);
|
|
1081
1057
|
return json ? [textPart(json)] : [];
|
|
1082
1058
|
}
|
|
1083
1059
|
const parts = [];
|
|
@@ -1146,7 +1122,7 @@ function redactInternalReasoningFromMessage(value) {
|
|
|
1146
1122
|
function redactInternalReasoningFromMessages(value) {
|
|
1147
1123
|
return Array.isArray(value) ? value.map((message) => redactInternalReasoningFromMessage(message)) : redactInternalReasoningFromMessage(value);
|
|
1148
1124
|
}
|
|
1149
|
-
function normalizeGenAiMessage(value, fallbackRole
|
|
1125
|
+
function normalizeGenAiMessage(value, fallbackRole) {
|
|
1150
1126
|
if (typeof value === "string") return {
|
|
1151
1127
|
role: fallbackRole,
|
|
1152
1128
|
parts: [textPart(value)]
|
|
@@ -1199,7 +1175,7 @@ function normalizeGenAiToolDefinitions(value) {
|
|
|
1199
1175
|
return definitions;
|
|
1200
1176
|
}
|
|
1201
1177
|
function assignJsonAttribute(attributes, key, value) {
|
|
1202
|
-
const json = safeJsonString(value
|
|
1178
|
+
const json = safeJsonString(value);
|
|
1203
1179
|
if (json) attributes[key] = json;
|
|
1204
1180
|
}
|
|
1205
1181
|
function assignOtelContentAttribute(attributes, key, value) {
|
|
@@ -1249,20 +1225,6 @@ function assignOtelToolContentAttributes(attributes, content, captureContent) {
|
|
|
1249
1225
|
//#endregion
|
|
1250
1226
|
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1251
1227
|
function createHarnessRecorders(runtime) {
|
|
1252
|
-
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled, getTrackedInternalOrTrustedSpan, captureContent } = runtime;
|
|
1253
|
-
const recordAgentCommentary = (evt, metadata, privateData) => {
|
|
1254
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1255
|
-
const span = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1256
|
-
if (!span) return;
|
|
1257
|
-
const attrs = {
|
|
1258
|
-
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1259
|
-
"openclaw.commentary.sequence": evt.sourceSequence,
|
|
1260
|
-
"openclaw.commentary.text_length": evt.textLength,
|
|
1261
|
-
"openclaw.commentary.content_truncated": evt.contentTruncated
|
|
1262
|
-
};
|
|
1263
|
-
assignOtelModelContentAttributes(attrs, privateData.modelContent, captureContent);
|
|
1264
|
-
span.addEvent("openclaw.agent.commentary", redactOtelAttributes(attrs), evt.sourceTimestampMs);
|
|
1265
|
-
};
|
|
1266
1228
|
const harnessRunMetricAttrs = (evt) => ({
|
|
1267
1229
|
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1268
1230
|
"openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId),
|
|
@@ -1271,105 +1233,113 @@ function createHarnessRecorders(runtime) {
|
|
|
1271
1233
|
"openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"),
|
|
1272
1234
|
...evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}
|
|
1273
1235
|
});
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1236
|
+
return {
|
|
1237
|
+
recordAgentCommentary(evt, metadata, privateData) {
|
|
1238
|
+
if (!runtime.tracesEnabled || !metadata.trusted) return;
|
|
1239
|
+
const span = runtime.getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1240
|
+
if (!span) return;
|
|
1241
|
+
const attrs = {
|
|
1242
|
+
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1243
|
+
"openclaw.commentary.sequence": evt.sourceSequence,
|
|
1244
|
+
"openclaw.commentary.text_length": evt.textLength,
|
|
1245
|
+
"openclaw.commentary.content_truncated": evt.contentTruncated
|
|
1246
|
+
};
|
|
1247
|
+
assignOtelModelContentAttributes(attrs, privateData.modelContent, runtime.captureContent);
|
|
1248
|
+
span.addEvent("openclaw.agent.commentary", redactOtelAttributes(attrs), evt.sourceTimestampMs);
|
|
1249
|
+
},
|
|
1250
|
+
recordHarnessRunStarted(evt, metadata) {
|
|
1251
|
+
if (!runtime.tracesEnabled || !metadata.trusted) return;
|
|
1252
|
+
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1253
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1254
|
+
runtime.trackTrustedSpan(evt, metadata, runtime.spanWithDuration("openclaw.harness.run", spanAttrs, void 0, {
|
|
1255
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1256
|
+
startTimeMs: evt.ts
|
|
1257
|
+
}));
|
|
1258
|
+
},
|
|
1259
|
+
recordHarnessRunFinished(evt, metadata, privateData) {
|
|
1260
|
+
const errorType = evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "error";
|
|
1261
|
+
const attrs = {
|
|
1262
|
+
...harnessRunMetricAttrs(evt),
|
|
1263
|
+
...evt.type === "harness.run.error" ? {
|
|
1264
|
+
"openclaw.harness.phase": evt.phase,
|
|
1265
|
+
"openclaw.errorCategory": errorType
|
|
1266
|
+
} : {}
|
|
1267
|
+
};
|
|
1268
|
+
runtime.harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1269
|
+
if (!runtime.tracesEnabled) return;
|
|
1270
|
+
const spanAttrs = { ...attrs };
|
|
1271
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1272
|
+
if (evt.type === "harness.run.completed") {
|
|
1273
|
+
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue(evt.resultClassification);
|
|
1274
|
+
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1275
|
+
if (evt.itemLifecycle) {
|
|
1276
|
+
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1277
|
+
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1278
|
+
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1279
|
+
}
|
|
1280
|
+
} else {
|
|
1281
|
+
spanAttrs["error.type"] = errorType;
|
|
1282
|
+
if (evt.cleanupFailed) spanAttrs["openclaw.harness.cleanup_failed"] = true;
|
|
1303
1283
|
}
|
|
1304
|
-
|
|
1305
|
-
spanAttrs["error
|
|
1306
|
-
|
|
1284
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1285
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1286
|
+
const trustedTrace = runtime.trustedTraceContext(evt, metadata);
|
|
1287
|
+
const trackedSpan = trustedTrace?.spanId ? runtime.activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1288
|
+
const span = trackedSpan ?? runtime.spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1289
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1290
|
+
endTimeMs: evt.ts
|
|
1291
|
+
});
|
|
1292
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1293
|
+
if (evt.type === "harness.run.error" || evt.outcome === "error") span.setStatus({
|
|
1294
|
+
code: SpanStatusCode.ERROR,
|
|
1295
|
+
message: redactedError ?? errorType
|
|
1296
|
+
});
|
|
1297
|
+
runtime.completeTrackedLifecycleSpan(trackedSpan ? trustedTrace : void 0, span, evt.ts);
|
|
1298
|
+
},
|
|
1299
|
+
recordContextAssembled(evt, metadata) {
|
|
1300
|
+
if (!runtime.tracesEnabled) return;
|
|
1301
|
+
const spanAttrs = {
|
|
1302
|
+
"openclaw.context.message_count": evt.messageCount,
|
|
1303
|
+
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1304
|
+
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1305
|
+
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1306
|
+
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1307
|
+
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1308
|
+
"openclaw.context.prompt_images": evt.promptImages
|
|
1309
|
+
};
|
|
1310
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1311
|
+
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1312
|
+
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1313
|
+
runtime.spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1314
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1315
|
+
endTimeMs: evt.ts
|
|
1316
|
+
}).end(evt.ts);
|
|
1317
|
+
},
|
|
1318
|
+
recordModelFailover(evt, metadata) {
|
|
1319
|
+
const metricAttrs = {
|
|
1320
|
+
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1321
|
+
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1322
|
+
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1323
|
+
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1324
|
+
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1325
|
+
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1326
|
+
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1327
|
+
};
|
|
1328
|
+
runtime.modelFailoverCounter.add(1, metricAttrs);
|
|
1329
|
+
if (!runtime.tracesEnabled) return;
|
|
1330
|
+
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1331
|
+
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1332
|
+
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1333
|
+
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1334
|
+
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1335
|
+
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1336
|
+
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1337
|
+
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1338
|
+
runtime.spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1339
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1340
|
+
endTimeMs: evt.ts
|
|
1341
|
+
}).end(evt.ts);
|
|
1307
1342
|
}
|
|
1308
|
-
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1309
|
-
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1310
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1311
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1312
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1313
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1314
|
-
endTimeMs: evt.ts
|
|
1315
|
-
});
|
|
1316
|
-
setSpanAttrs(span, spanAttrs);
|
|
1317
|
-
if (evt.type === "harness.run.error" || evt.outcome === "error") span.setStatus({
|
|
1318
|
-
code: SpanStatusCode.ERROR,
|
|
1319
|
-
message: redactedError ?? errorType
|
|
1320
|
-
});
|
|
1321
|
-
completeTrackedLifecycleSpan(trackedSpan ? trustedTrace : void 0, span, evt.ts);
|
|
1322
|
-
};
|
|
1323
|
-
const recordContextAssembled = (evt, metadata) => {
|
|
1324
|
-
if (!tracesEnabled) return;
|
|
1325
|
-
const spanAttrs = {
|
|
1326
|
-
"openclaw.context.message_count": evt.messageCount,
|
|
1327
|
-
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1328
|
-
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1329
|
-
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1330
|
-
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1331
|
-
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1332
|
-
"openclaw.context.prompt_images": evt.promptImages
|
|
1333
|
-
};
|
|
1334
|
-
addRunAttrs(spanAttrs, evt);
|
|
1335
|
-
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1336
|
-
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1337
|
-
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1338
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1339
|
-
endTimeMs: evt.ts
|
|
1340
|
-
}).end(evt.ts);
|
|
1341
|
-
};
|
|
1342
|
-
const recordModelFailover = (evt, metadata) => {
|
|
1343
|
-
const metricAttrs = {
|
|
1344
|
-
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1345
|
-
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1346
|
-
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1347
|
-
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1348
|
-
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1349
|
-
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1350
|
-
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1351
|
-
};
|
|
1352
|
-
modelFailoverCounter.add(1, metricAttrs);
|
|
1353
|
-
if (!tracesEnabled) return;
|
|
1354
|
-
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1355
|
-
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1356
|
-
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1357
|
-
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1358
|
-
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1359
|
-
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1360
|
-
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1361
|
-
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1362
|
-
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1363
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1364
|
-
endTimeMs: evt.ts
|
|
1365
|
-
}).end(evt.ts);
|
|
1366
|
-
};
|
|
1367
|
-
return {
|
|
1368
|
-
recordAgentCommentary,
|
|
1369
|
-
recordHarnessRunStarted,
|
|
1370
|
-
recordHarnessRunFinished,
|
|
1371
|
-
recordContextAssembled,
|
|
1372
|
-
recordModelFailover
|
|
1373
1343
|
};
|
|
1374
1344
|
}
|
|
1375
1345
|
//#endregion
|
|
@@ -1389,15 +1359,6 @@ function assignPositiveNumberAttr(attrs, key, value) {
|
|
|
1389
1359
|
const normalized = asPositiveFiniteNumber(value);
|
|
1390
1360
|
if (normalized !== void 0) attrs[key] = normalized;
|
|
1391
1361
|
}
|
|
1392
|
-
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1393
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1394
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1395
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1396
|
-
}
|
|
1397
|
-
function assignNumberAttr(attrs, key, value) {
|
|
1398
|
-
const normalized = asFiniteNumber(value);
|
|
1399
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1400
|
-
}
|
|
1401
1362
|
function modelCallPromptTokens(usage) {
|
|
1402
1363
|
const promptTokens = asNonNegativeFiniteNumber(usage.promptTokens);
|
|
1403
1364
|
if (promptTokens !== void 0) return promptTokens;
|
|
@@ -1417,7 +1378,10 @@ function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
|
1417
1378
|
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1418
1379
|
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1419
1380
|
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1420
|
-
])
|
|
1381
|
+
]) {
|
|
1382
|
+
const normalized = asFiniteNumber(value);
|
|
1383
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1384
|
+
}
|
|
1421
1385
|
}
|
|
1422
1386
|
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1423
1387
|
const usage = evt.usage;
|
|
@@ -1448,10 +1412,7 @@ function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
|
1448
1412
|
}
|
|
1449
1413
|
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1450
1414
|
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1451
|
-
attrs["openclaw.model_call.observation_unit"] =
|
|
1452
|
-
}
|
|
1453
|
-
function modelCallObservationUnit(evt) {
|
|
1454
|
-
return evt.observationUnit ?? "request";
|
|
1415
|
+
attrs["openclaw.model_call.observation_unit"] = evt.observationUnit ?? "request";
|
|
1455
1416
|
}
|
|
1456
1417
|
function modelCallSpanName(evt) {
|
|
1457
1418
|
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
@@ -1467,752 +1428,711 @@ function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
|
1467
1428
|
//#endregion
|
|
1468
1429
|
//#region extensions/diagnostics-otel/src/service-recorders-model.ts
|
|
1469
1430
|
function createModelRecorders(runtime) {
|
|
1470
|
-
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, captureContent, tracesEnabled } = runtime;
|
|
1471
|
-
const modelCallMetricAttrs = (evt) => ({
|
|
1472
|
-
"openclaw.provider": evt.provider,
|
|
1473
|
-
"openclaw.model": evt.model,
|
|
1474
|
-
"openclaw.api": normalizeDiagnosticValue(evt.api),
|
|
1475
|
-
"openclaw.transport": normalizeDiagnosticValue(evt.transport),
|
|
1476
|
-
"openclaw.model_call.observation_unit": modelCallObservationUnit(evt)
|
|
1477
|
-
});
|
|
1478
|
-
const recordModelCallSizeTimingMetrics = (evt, attrs) => {
|
|
1479
|
-
const requestPayloadBytes = asPositiveFiniteNumber(evt.requestPayloadBytes);
|
|
1480
|
-
if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
|
|
1481
|
-
const responseStreamBytes = asPositiveFiniteNumber(evt.responseStreamBytes);
|
|
1482
|
-
if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
|
|
1483
|
-
const timeToFirstByteMs = asPositiveFiniteNumber(evt.timeToFirstByteMs);
|
|
1484
|
-
if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
|
|
1485
|
-
};
|
|
1486
|
-
const recordModelCallStarted = (evt, metadata) => {
|
|
1487
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1488
|
-
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1489
|
-
if (trackedSpan) return trackedSpan.spanContext();
|
|
1490
|
-
const spanAttrs = {
|
|
1491
|
-
"openclaw.provider": evt.provider,
|
|
1492
|
-
"openclaw.model": evt.model
|
|
1493
|
-
};
|
|
1494
|
-
addRunAttrs(spanAttrs, evt);
|
|
1495
|
-
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1496
|
-
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1497
|
-
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1498
|
-
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1499
|
-
return trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1500
|
-
kind: SpanKind.CLIENT,
|
|
1501
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1502
|
-
startTimeMs: evt.ts
|
|
1503
|
-
})).spanContext();
|
|
1504
|
-
};
|
|
1505
|
-
const recordModelCallFinished = (evt, metadata, modelContent) => {
|
|
1506
|
-
const errorType = evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : void 0;
|
|
1507
|
-
const metricAttrs = {
|
|
1508
|
-
...modelCallMetricAttrs(evt),
|
|
1509
|
-
...errorType !== void 0 ? { "openclaw.errorCategory": errorType } : {},
|
|
1510
|
-
...evt.type === "model.call.error" && evt.failureKind ? { "openclaw.failureKind": normalizeDiagnosticValue(evt.failureKind, "other") } : {}
|
|
1511
|
-
};
|
|
1512
|
-
modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
1513
|
-
recordModelCallSizeTimingMetrics(evt, metricAttrs);
|
|
1514
|
-
genAiOperationDurationHistogram.record(evt.durationMs / 1e3, {
|
|
1515
|
-
"gen_ai.operation.name": genAiOperationName(evt.api, evt.observationUnit),
|
|
1516
|
-
"gen_ai.provider.name": normalizeDiagnosticValue(evt.provider),
|
|
1517
|
-
"gen_ai.request.model": normalizeDiagnosticValue(evt.model),
|
|
1518
|
-
...errorType ? { "error.type": errorType } : {}
|
|
1519
|
-
});
|
|
1520
|
-
if (!tracesEnabled) return;
|
|
1521
|
-
const spanAttrs = {
|
|
1522
|
-
"openclaw.provider": evt.provider,
|
|
1523
|
-
"openclaw.model": evt.model,
|
|
1524
|
-
...errorType !== void 0 ? {
|
|
1525
|
-
"openclaw.errorCategory": errorType,
|
|
1526
|
-
"error.type": errorType
|
|
1527
|
-
} : {}
|
|
1528
|
-
};
|
|
1529
|
-
addRunAttrs(spanAttrs, evt);
|
|
1530
|
-
if (evt.type === "model.call.error" && evt.failureKind) spanAttrs["openclaw.failureKind"] = normalizeDiagnosticValue(evt.failureKind, "other");
|
|
1531
|
-
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1532
|
-
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1533
|
-
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1534
|
-
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
1535
|
-
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1536
|
-
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1537
|
-
assignOtelModelContentAttributes(spanAttrs, modelContent, captureContent);
|
|
1538
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1539
|
-
kind: SpanKind.CLIENT,
|
|
1540
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1541
|
-
endTimeMs: evt.ts
|
|
1542
|
-
});
|
|
1543
|
-
setSpanAttrs(span, spanAttrs);
|
|
1544
|
-
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
1545
|
-
if (evt.type === "model.call.error") span.setStatus({
|
|
1546
|
-
code: SpanStatusCode.ERROR,
|
|
1547
|
-
message: redactSensitiveText(evt.errorCategory)
|
|
1548
|
-
});
|
|
1549
|
-
span.end(evt.ts);
|
|
1550
|
-
};
|
|
1551
1431
|
return {
|
|
1552
|
-
recordModelCallStarted,
|
|
1553
|
-
|
|
1432
|
+
recordModelCallStarted(evt, metadata) {
|
|
1433
|
+
if (!runtime.tracesEnabled || !metadata.trusted) return;
|
|
1434
|
+
const trackedSpan = runtime.getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1435
|
+
if (trackedSpan) return trackedSpan.spanContext();
|
|
1436
|
+
const spanAttrs = {
|
|
1437
|
+
"openclaw.provider": evt.provider,
|
|
1438
|
+
"openclaw.model": evt.model
|
|
1439
|
+
};
|
|
1440
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1441
|
+
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1442
|
+
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1443
|
+
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1444
|
+
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1445
|
+
return runtime.trackTrustedSpan(evt, metadata, runtime.spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1446
|
+
kind: SpanKind.CLIENT,
|
|
1447
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1448
|
+
startTimeMs: evt.ts
|
|
1449
|
+
})).spanContext();
|
|
1450
|
+
},
|
|
1451
|
+
recordModelCallFinished(evt, metadata, modelContent) {
|
|
1452
|
+
const errorType = evt.type === "model.call.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : void 0;
|
|
1453
|
+
const metricAttrs = {
|
|
1454
|
+
"openclaw.provider": evt.provider,
|
|
1455
|
+
"openclaw.model": evt.model,
|
|
1456
|
+
"openclaw.api": normalizeDiagnosticValue(evt.api),
|
|
1457
|
+
"openclaw.transport": normalizeDiagnosticValue(evt.transport),
|
|
1458
|
+
"openclaw.model_call.observation_unit": evt.observationUnit ?? "request",
|
|
1459
|
+
...errorType !== void 0 ? { "openclaw.errorCategory": errorType } : {},
|
|
1460
|
+
...evt.type === "model.call.error" && evt.failureKind ? { "openclaw.failureKind": normalizeDiagnosticValue(evt.failureKind, "other") } : {}
|
|
1461
|
+
};
|
|
1462
|
+
runtime.modelCallDurationHistogram.record(evt.durationMs, metricAttrs);
|
|
1463
|
+
for (const [histogram, value] of [
|
|
1464
|
+
[runtime.modelCallRequestBytesHistogram, evt.requestPayloadBytes],
|
|
1465
|
+
[runtime.modelCallResponseBytesHistogram, evt.responseStreamBytes],
|
|
1466
|
+
[runtime.modelCallTimeToFirstByteHistogram, evt.timeToFirstByteMs]
|
|
1467
|
+
]) {
|
|
1468
|
+
const normalized = asPositiveFiniteNumber(value);
|
|
1469
|
+
if (normalized !== void 0) histogram.record(normalized, metricAttrs);
|
|
1470
|
+
}
|
|
1471
|
+
runtime.genAiOperationDurationHistogram.record(evt.durationMs / 1e3, {
|
|
1472
|
+
"gen_ai.operation.name": genAiOperationName(evt.api, evt.observationUnit),
|
|
1473
|
+
"gen_ai.provider.name": normalizeDiagnosticValue(evt.provider),
|
|
1474
|
+
"gen_ai.request.model": normalizeDiagnosticValue(evt.model),
|
|
1475
|
+
...errorType ? { "error.type": errorType } : {}
|
|
1476
|
+
});
|
|
1477
|
+
if (!runtime.tracesEnabled) return;
|
|
1478
|
+
const spanAttrs = {
|
|
1479
|
+
"openclaw.provider": evt.provider,
|
|
1480
|
+
"openclaw.model": evt.model,
|
|
1481
|
+
...errorType !== void 0 ? {
|
|
1482
|
+
"openclaw.errorCategory": errorType,
|
|
1483
|
+
"error.type": errorType
|
|
1484
|
+
} : {}
|
|
1485
|
+
};
|
|
1486
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1487
|
+
if (evt.type === "model.call.error" && evt.failureKind) spanAttrs["openclaw.failureKind"] = normalizeDiagnosticValue(evt.failureKind, "other");
|
|
1488
|
+
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1489
|
+
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1490
|
+
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1491
|
+
for (const [key, value] of [
|
|
1492
|
+
["openclaw.model_call.request_bytes", evt.requestPayloadBytes],
|
|
1493
|
+
["openclaw.model_call.response_bytes", evt.responseStreamBytes],
|
|
1494
|
+
["openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs]
|
|
1495
|
+
]) assignPositiveNumberAttr(spanAttrs, key, value);
|
|
1496
|
+
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1497
|
+
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1498
|
+
assignOtelModelContentAttributes(spanAttrs, modelContent, runtime.captureContent);
|
|
1499
|
+
const span = runtime.takeTrackedTrustedSpan(evt, metadata) ?? runtime.spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1500
|
+
kind: SpanKind.CLIENT,
|
|
1501
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1502
|
+
endTimeMs: evt.ts
|
|
1503
|
+
});
|
|
1504
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1505
|
+
addUpstreamRequestIdSpanEvent(span, evt.upstreamRequestIdHash);
|
|
1506
|
+
if (evt.type === "model.call.error") span.setStatus({
|
|
1507
|
+
code: SpanStatusCode.ERROR,
|
|
1508
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
1509
|
+
});
|
|
1510
|
+
span.end(evt.ts);
|
|
1511
|
+
}
|
|
1554
1512
|
};
|
|
1555
1513
|
}
|
|
1556
1514
|
//#endregion
|
|
1557
1515
|
//#region extensions/diagnostics-otel/src/service-recorders-operations.ts
|
|
1558
1516
|
function createOperationsRecorders(runtime) {
|
|
1559
|
-
const { durationHistogram, gatewayRpcRequestsCounter, gatewayRpcOutcomesCounter, gatewayRpcFirstResponseHistogram, gatewayRpcHandlerHistogram, gatewayRpcAdmissionHistogram, gatewayRpcQueueWaitHistogram, queueDepthHistogram, queueWaitHistogram, laneEnqueueCounter, laneDequeueCounter, sessionStateCounter, sessionTurnCreatedCounter, sessionStuckCounter, sessionStuckAgeHistogram, sessionRecoveryRequestedCounter, sessionRecoveryCompletedCounter, sessionRecoveryAgeHistogram, talkEventCounter, talkEventDurationHistogram, talkAudioBytesHistogram, runAttemptCounter, toolLoopCounter, memoryRssHistogram, memoryHeapUsedHistogram, memoryHeapTotalHistogram, memoryExternalHistogram, memoryArrayBuffersHistogram, memoryPressureCounter, asyncQueueDroppedCounter, tracer, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, internalOrTrustedExplicitParentContext, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1560
|
-
const recordGatewayRpc = (evt, metadata) => {
|
|
1561
|
-
if (!metadata.trusted || evt.phase === "response" && evt.firstResponse === false) return;
|
|
1562
|
-
const attrs = { "openclaw.gateway.rpc.method": evt.method };
|
|
1563
|
-
if (evt.phase === "received") {
|
|
1564
|
-
gatewayRpcRequestsCounter.add(1, attrs);
|
|
1565
|
-
return;
|
|
1566
|
-
}
|
|
1567
|
-
const outcomeAttrs = {
|
|
1568
|
-
"openclaw.gateway.rpc.phase": evt.phase,
|
|
1569
|
-
"openclaw.gateway.rpc.outcome": evt.outcome
|
|
1570
|
-
};
|
|
1571
|
-
gatewayRpcOutcomesCounter.add(1, outcomeAttrs);
|
|
1572
|
-
switch (evt.phase) {
|
|
1573
|
-
case "response":
|
|
1574
|
-
if (evt.outcome === "ok" || evt.outcome === "error") gatewayRpcFirstResponseHistogram.record(evt.durationMs, attrs);
|
|
1575
|
-
break;
|
|
1576
|
-
case "handler":
|
|
1577
|
-
gatewayRpcHandlerHistogram.record(evt.durationMs, attrs);
|
|
1578
|
-
gatewayRpcAdmissionHistogram.record(evt.admissionMs, attrs);
|
|
1579
|
-
break;
|
|
1580
|
-
case "dispatch": if (evt.queueWaitMs !== void 0) gatewayRpcQueueWaitHistogram.record(evt.queueWaitMs, attrs);
|
|
1581
|
-
}
|
|
1582
|
-
if (!tracesEnabled) return;
|
|
1583
|
-
const span = spanWithDuration(`openclaw.gateway.rpc.${evt.phase}`, {
|
|
1584
|
-
...attrs,
|
|
1585
|
-
...outcomeAttrs,
|
|
1586
|
-
...evt.phase === "handler" ? { "openclaw.gateway.rpc.admission_ms": evt.admissionMs } : {},
|
|
1587
|
-
...evt.phase === "dispatch" ? { "openclaw.gateway.rpc.response": evt.response } : {}
|
|
1588
|
-
}, evt.durationMs, {
|
|
1589
|
-
endTimeMs: evt.ts,
|
|
1590
|
-
parentContext: internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT
|
|
1591
|
-
});
|
|
1592
|
-
if (evt.outcome === "error" || evt.outcome === "threw") span.setStatus({ code: SpanStatusCode.ERROR });
|
|
1593
|
-
span.end(evt.ts);
|
|
1594
|
-
};
|
|
1595
|
-
const recordLaneEnqueue = (evt) => {
|
|
1596
|
-
const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) };
|
|
1597
|
-
laneEnqueueCounter.add(1, attrs);
|
|
1598
|
-
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1599
|
-
};
|
|
1600
|
-
const recordLaneDequeue = (evt) => {
|
|
1601
|
-
const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) };
|
|
1602
|
-
laneDequeueCounter.add(1, attrs);
|
|
1603
|
-
queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1604
|
-
if (typeof evt.waitMs === "number") queueWaitHistogram.record(evt.waitMs, attrs);
|
|
1605
|
-
};
|
|
1606
|
-
const recordSessionState = (evt) => {
|
|
1607
|
-
const attrs = { "openclaw.state": evt.state };
|
|
1608
|
-
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1609
|
-
sessionStateCounter.add(1, attrs);
|
|
1610
|
-
};
|
|
1611
|
-
const recordSessionTurnCreated = (evt) => {
|
|
1612
|
-
sessionTurnCreatedCounter.add(1, {
|
|
1613
|
-
"openclaw.agent": normalizeDiagnosticValue(evt.agentId, "unknown"),
|
|
1614
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel, "unknown"),
|
|
1615
|
-
"openclaw.trigger": evt.trigger
|
|
1616
|
-
});
|
|
1617
|
-
};
|
|
1618
|
-
const recordSessionStuck = (evt) => {
|
|
1619
|
-
const attrs = { "openclaw.state": evt.state };
|
|
1620
|
-
sessionStuckCounter.add(1, attrs);
|
|
1621
|
-
if (typeof evt.ageMs === "number") sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
|
1622
|
-
if (!tracesEnabled) return;
|
|
1623
|
-
const spanAttrs = { ...attrs };
|
|
1624
|
-
spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
|
|
1625
|
-
spanAttrs["openclaw.ageMs"] = evt.ageMs;
|
|
1626
|
-
const span = tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
|
|
1627
|
-
span.setStatus({
|
|
1628
|
-
code: SpanStatusCode.ERROR,
|
|
1629
|
-
message: "session stuck"
|
|
1630
|
-
});
|
|
1631
|
-
span.end();
|
|
1632
|
-
};
|
|
1633
1517
|
const sessionRecoveryAttrs = (evt) => {
|
|
1634
1518
|
const attrs = { "openclaw.state": evt.state };
|
|
1635
1519
|
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1636
1520
|
if (evt.activeWorkKind) attrs["openclaw.active_work_kind"] = evt.activeWorkKind;
|
|
1637
1521
|
return attrs;
|
|
1638
1522
|
};
|
|
1639
|
-
const recordSessionRecoveryRequested = (evt) => {
|
|
1640
|
-
const attrs = sessionRecoveryAttrs(evt);
|
|
1641
|
-
attrs["openclaw.action"] = evt.allowActiveAbort ? "abort" : "recover";
|
|
1642
|
-
sessionRecoveryRequestedCounter.add(1, attrs);
|
|
1643
|
-
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1644
|
-
};
|
|
1645
|
-
const recordSessionRecoveryCompleted = (evt) => {
|
|
1646
|
-
const attrs = sessionRecoveryAttrs(evt);
|
|
1647
|
-
attrs["openclaw.status"] = evt.status;
|
|
1648
|
-
attrs["openclaw.action"] = normalizeDiagnosticValue(evt.action, "unknown");
|
|
1649
|
-
if (evt.outcomeReason) attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason);
|
|
1650
|
-
sessionRecoveryCompletedCounter.add(1, attrs);
|
|
1651
|
-
sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1652
|
-
};
|
|
1653
|
-
const talkEventAttrs = (evt) => ({
|
|
1654
|
-
"openclaw.talk.brain": normalizeDiagnosticValue(evt.brain),
|
|
1655
|
-
"openclaw.talk.event_type": normalizeDiagnosticValue(evt.talkEventType),
|
|
1656
|
-
"openclaw.talk.mode": normalizeDiagnosticValue(evt.mode),
|
|
1657
|
-
"openclaw.talk.provider": normalizeDiagnosticValue(evt.provider),
|
|
1658
|
-
"openclaw.talk.transport": normalizeDiagnosticValue(evt.transport)
|
|
1659
|
-
});
|
|
1660
|
-
const recordTalkEvent = (evt, metadata) => {
|
|
1661
|
-
if (!metadata.trusted) return;
|
|
1662
|
-
const attrs = talkEventAttrs(evt);
|
|
1663
|
-
talkEventCounter.add(1, attrs);
|
|
1664
|
-
if (typeof evt.durationMs === "number") talkEventDurationHistogram.record(evt.durationMs, attrs);
|
|
1665
|
-
if (typeof evt.byteLength === "number") talkAudioBytesHistogram.record(evt.byteLength, attrs);
|
|
1666
|
-
};
|
|
1667
|
-
const recordRunAttempt = (evt) => {
|
|
1668
|
-
runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
|
|
1669
|
-
};
|
|
1670
|
-
const toolLoopAttrs = (evt) => ({
|
|
1671
|
-
"openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
1672
|
-
"openclaw.loop.level": evt.level,
|
|
1673
|
-
"openclaw.loop.action": evt.action,
|
|
1674
|
-
"openclaw.loop.detector": evt.detector,
|
|
1675
|
-
"openclaw.loop.count": evt.count,
|
|
1676
|
-
...evt.pairedToolName ? { "openclaw.loop.paired_tool": normalizeDiagnosticValue(evt.pairedToolName, "tool") } : {}
|
|
1677
|
-
});
|
|
1678
|
-
const recordToolLoop = (evt) => {
|
|
1679
|
-
const attrs = toolLoopAttrs(evt);
|
|
1680
|
-
toolLoopCounter.add(1, attrs);
|
|
1681
|
-
if (!tracesEnabled) return;
|
|
1682
|
-
const spanAttrs = { ...attrs };
|
|
1683
|
-
addRunAttrs(spanAttrs, evt);
|
|
1684
|
-
const span = spanWithDuration("openclaw.tool.loop", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1685
|
-
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
1686
|
-
code: SpanStatusCode.ERROR,
|
|
1687
|
-
message: `${evt.detector}:${evt.action}`
|
|
1688
|
-
});
|
|
1689
|
-
span.end(evt.ts);
|
|
1690
|
-
};
|
|
1691
1523
|
const recordMemoryUsageMetrics = (evt, attrs = {}) => {
|
|
1692
|
-
memoryRssHistogram.record(evt.memory.rssBytes, attrs);
|
|
1693
|
-
memoryHeapUsedHistogram.record(evt.memory.heapUsedBytes, attrs);
|
|
1694
|
-
memoryHeapTotalHistogram.record(evt.memory.heapTotalBytes, attrs);
|
|
1695
|
-
memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
1696
|
-
memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
1697
|
-
};
|
|
1698
|
-
const recordMemoryPressure = (evt) => {
|
|
1699
|
-
const attrs = {
|
|
1700
|
-
"openclaw.memory.level": evt.level,
|
|
1701
|
-
"openclaw.memory.reason": evt.reason
|
|
1702
|
-
};
|
|
1703
|
-
memoryPressureCounter.add(1, attrs);
|
|
1704
|
-
recordMemoryUsageMetrics(evt, attrs);
|
|
1705
|
-
if (!tracesEnabled) return;
|
|
1706
|
-
const spanAttrs = {
|
|
1707
|
-
...attrs,
|
|
1708
|
-
"openclaw.memory.rss_bytes": evt.memory.rssBytes,
|
|
1709
|
-
"openclaw.memory.heap_used_bytes": evt.memory.heapUsedBytes,
|
|
1710
|
-
"openclaw.memory.heap_total_bytes": evt.memory.heapTotalBytes,
|
|
1711
|
-
"openclaw.memory.external_bytes": evt.memory.externalBytes,
|
|
1712
|
-
"openclaw.memory.array_buffers_bytes": evt.memory.arrayBuffersBytes,
|
|
1713
|
-
...evt.thresholdBytes !== void 0 ? { "openclaw.memory.threshold_bytes": evt.thresholdBytes } : {},
|
|
1714
|
-
...evt.rssGrowthBytes !== void 0 ? { "openclaw.memory.rss_growth_bytes": evt.rssGrowthBytes } : {},
|
|
1715
|
-
...evt.windowMs !== void 0 ? { "openclaw.memory.window_ms": evt.windowMs } : {}
|
|
1716
|
-
};
|
|
1717
|
-
const span = spanWithDuration("openclaw.memory.pressure", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1718
|
-
if (evt.level === "critical") span.setStatus({
|
|
1719
|
-
code: SpanStatusCode.ERROR,
|
|
1720
|
-
message: evt.reason
|
|
1721
|
-
});
|
|
1722
|
-
span.end(evt.ts);
|
|
1524
|
+
runtime.memoryRssHistogram.record(evt.memory.rssBytes, attrs);
|
|
1525
|
+
runtime.memoryHeapUsedHistogram.record(evt.memory.heapUsedBytes, attrs);
|
|
1526
|
+
runtime.memoryHeapTotalHistogram.record(evt.memory.heapTotalBytes, attrs);
|
|
1527
|
+
runtime.memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
1528
|
+
runtime.memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
1723
1529
|
};
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1530
|
+
return {
|
|
1531
|
+
recordGatewayRpc(evt, metadata) {
|
|
1532
|
+
if (!metadata.trusted || evt.phase === "response" && evt.firstResponse === false) return;
|
|
1533
|
+
const attrs = { "openclaw.gateway.rpc.method": evt.method };
|
|
1534
|
+
if (evt.phase === "received") {
|
|
1535
|
+
runtime.gatewayRpcRequestsCounter.add(1, attrs);
|
|
1536
|
+
return;
|
|
1537
|
+
}
|
|
1538
|
+
const outcomeAttrs = {
|
|
1539
|
+
"openclaw.gateway.rpc.phase": evt.phase,
|
|
1540
|
+
"openclaw.gateway.rpc.outcome": evt.outcome
|
|
1541
|
+
};
|
|
1542
|
+
runtime.gatewayRpcOutcomesCounter.add(1, outcomeAttrs);
|
|
1543
|
+
switch (evt.phase) {
|
|
1544
|
+
case "response":
|
|
1545
|
+
if (evt.outcome === "ok" || evt.outcome === "error") runtime.gatewayRpcFirstResponseHistogram.record(evt.durationMs, attrs);
|
|
1546
|
+
break;
|
|
1547
|
+
case "handler":
|
|
1548
|
+
runtime.gatewayRpcHandlerHistogram.record(evt.durationMs, attrs);
|
|
1549
|
+
runtime.gatewayRpcAdmissionHistogram.record(evt.admissionMs, attrs);
|
|
1550
|
+
break;
|
|
1551
|
+
case "dispatch": if (evt.queueWaitMs !== void 0) runtime.gatewayRpcQueueWaitHistogram.record(evt.queueWaitMs, attrs);
|
|
1552
|
+
}
|
|
1553
|
+
if (!runtime.tracesEnabled) return;
|
|
1554
|
+
const span = runtime.spanWithDuration(`openclaw.gateway.rpc.${evt.phase}`, {
|
|
1555
|
+
...attrs,
|
|
1556
|
+
...outcomeAttrs,
|
|
1557
|
+
...evt.phase === "handler" ? { "openclaw.gateway.rpc.admission_ms": evt.admissionMs } : {},
|
|
1558
|
+
...evt.phase === "dispatch" ? { "openclaw.gateway.rpc.response": evt.response } : {}
|
|
1559
|
+
}, evt.durationMs, {
|
|
1560
|
+
endTimeMs: evt.ts,
|
|
1561
|
+
parentContext: runtime.internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT
|
|
1562
|
+
});
|
|
1563
|
+
if (evt.outcome === "error" || evt.outcome === "threw") span.setStatus({ code: SpanStatusCode.ERROR });
|
|
1564
|
+
span.end(evt.ts);
|
|
1565
|
+
},
|
|
1566
|
+
recordLaneEnqueue(evt) {
|
|
1567
|
+
const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) };
|
|
1568
|
+
runtime.laneEnqueueCounter.add(1, attrs);
|
|
1569
|
+
runtime.queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1570
|
+
},
|
|
1571
|
+
recordLaneDequeue(evt) {
|
|
1572
|
+
const attrs = { "openclaw.lane": normalizeDiagnosticLane(evt.lane) };
|
|
1573
|
+
runtime.laneDequeueCounter.add(1, attrs);
|
|
1574
|
+
runtime.queueDepthHistogram.record(evt.queueSize, attrs);
|
|
1575
|
+
if (typeof evt.waitMs === "number") runtime.queueWaitHistogram.record(evt.waitMs, attrs);
|
|
1576
|
+
},
|
|
1577
|
+
recordSessionState(evt) {
|
|
1578
|
+
const attrs = { "openclaw.state": evt.state };
|
|
1579
|
+
if (evt.reason) attrs["openclaw.reason"] = redactSensitiveText(evt.reason);
|
|
1580
|
+
runtime.sessionStateCounter.add(1, attrs);
|
|
1581
|
+
},
|
|
1582
|
+
recordSessionTurnCreated(evt) {
|
|
1583
|
+
runtime.sessionTurnCreatedCounter.add(1, {
|
|
1584
|
+
"openclaw.agent": normalizeDiagnosticValue(evt.agentId, "unknown"),
|
|
1585
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel, "unknown"),
|
|
1586
|
+
"openclaw.trigger": evt.trigger
|
|
1587
|
+
});
|
|
1588
|
+
},
|
|
1589
|
+
recordSessionStuck(evt) {
|
|
1590
|
+
const attrs = { "openclaw.state": evt.state };
|
|
1591
|
+
runtime.sessionStuckCounter.add(1, attrs);
|
|
1592
|
+
if (typeof evt.ageMs === "number") runtime.sessionStuckAgeHistogram.record(evt.ageMs, attrs);
|
|
1593
|
+
if (!runtime.tracesEnabled) return;
|
|
1594
|
+
const spanAttrs = { ...attrs };
|
|
1595
|
+
spanAttrs["openclaw.queueDepth"] = evt.queueDepth ?? 0;
|
|
1596
|
+
spanAttrs["openclaw.ageMs"] = evt.ageMs;
|
|
1597
|
+
const span = runtime.tracer.startSpan("openclaw.session.stuck", { attributes: spanAttrs });
|
|
1757
1598
|
span.setStatus({
|
|
1758
1599
|
code: SpanStatusCode.ERROR,
|
|
1759
|
-
|
|
1600
|
+
message: "session stuck"
|
|
1760
1601
|
});
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1602
|
+
span.end();
|
|
1603
|
+
},
|
|
1604
|
+
recordSessionRecoveryRequested(evt) {
|
|
1605
|
+
const attrs = sessionRecoveryAttrs(evt);
|
|
1606
|
+
attrs["openclaw.action"] = evt.allowActiveAbort ? "abort" : "recover";
|
|
1607
|
+
runtime.sessionRecoveryRequestedCounter.add(1, attrs);
|
|
1608
|
+
runtime.sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1609
|
+
},
|
|
1610
|
+
recordSessionRecoveryCompleted(evt) {
|
|
1611
|
+
const attrs = sessionRecoveryAttrs(evt);
|
|
1612
|
+
attrs["openclaw.status"] = evt.status;
|
|
1613
|
+
attrs["openclaw.action"] = normalizeDiagnosticValue(evt.action, "unknown");
|
|
1614
|
+
if (evt.outcomeReason) attrs["openclaw.reason"] = redactSensitiveText(evt.outcomeReason);
|
|
1615
|
+
runtime.sessionRecoveryCompletedCounter.add(1, attrs);
|
|
1616
|
+
runtime.sessionRecoveryAgeHistogram.record(evt.ageMs, attrs);
|
|
1617
|
+
},
|
|
1618
|
+
recordTalkEvent(evt, metadata) {
|
|
1619
|
+
if (!metadata.trusted) return;
|
|
1620
|
+
const attrs = {
|
|
1621
|
+
"openclaw.talk.brain": normalizeDiagnosticValue(evt.brain),
|
|
1622
|
+
"openclaw.talk.event_type": normalizeDiagnosticValue(evt.talkEventType),
|
|
1623
|
+
"openclaw.talk.mode": normalizeDiagnosticValue(evt.mode),
|
|
1624
|
+
"openclaw.talk.provider": normalizeDiagnosticValue(evt.provider),
|
|
1625
|
+
"openclaw.talk.transport": normalizeDiagnosticValue(evt.transport)
|
|
1626
|
+
};
|
|
1627
|
+
runtime.talkEventCounter.add(1, attrs);
|
|
1628
|
+
if (typeof evt.durationMs === "number") runtime.talkEventDurationHistogram.record(evt.durationMs, attrs);
|
|
1629
|
+
if (typeof evt.byteLength === "number") runtime.talkAudioBytesHistogram.record(evt.byteLength, attrs);
|
|
1630
|
+
},
|
|
1631
|
+
recordRunAttempt(evt) {
|
|
1632
|
+
runtime.runAttemptCounter.add(1, { "openclaw.attempt": evt.attempt });
|
|
1633
|
+
},
|
|
1634
|
+
recordToolLoop(evt) {
|
|
1635
|
+
const attrs = {
|
|
1636
|
+
"openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool"),
|
|
1637
|
+
"openclaw.loop.level": evt.level,
|
|
1638
|
+
"openclaw.loop.action": evt.action,
|
|
1639
|
+
"openclaw.loop.detector": evt.detector,
|
|
1640
|
+
"openclaw.loop.count": evt.count,
|
|
1641
|
+
...evt.pairedToolName ? { "openclaw.loop.paired_tool": normalizeDiagnosticValue(evt.pairedToolName, "tool") } : {}
|
|
1642
|
+
};
|
|
1643
|
+
runtime.toolLoopCounter.add(1, attrs);
|
|
1644
|
+
if (!runtime.tracesEnabled) return;
|
|
1645
|
+
const spanAttrs = { ...attrs };
|
|
1646
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1647
|
+
const span = runtime.spanWithDuration("openclaw.tool.loop", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1648
|
+
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
1649
|
+
code: SpanStatusCode.ERROR,
|
|
1650
|
+
message: `${evt.detector}:${evt.action}`
|
|
1651
|
+
});
|
|
1652
|
+
span.end(evt.ts);
|
|
1653
|
+
},
|
|
1776
1654
|
recordMemoryUsageMetrics,
|
|
1777
|
-
recordMemoryPressure
|
|
1778
|
-
|
|
1779
|
-
|
|
1655
|
+
recordMemoryPressure(evt) {
|
|
1656
|
+
const attrs = {
|
|
1657
|
+
"openclaw.memory.level": evt.level,
|
|
1658
|
+
"openclaw.memory.reason": evt.reason
|
|
1659
|
+
};
|
|
1660
|
+
runtime.memoryPressureCounter.add(1, attrs);
|
|
1661
|
+
recordMemoryUsageMetrics(evt, attrs);
|
|
1662
|
+
if (!runtime.tracesEnabled) return;
|
|
1663
|
+
const spanAttrs = {
|
|
1664
|
+
...attrs,
|
|
1665
|
+
"openclaw.memory.rss_bytes": evt.memory.rssBytes,
|
|
1666
|
+
"openclaw.memory.heap_used_bytes": evt.memory.heapUsedBytes,
|
|
1667
|
+
"openclaw.memory.heap_total_bytes": evt.memory.heapTotalBytes,
|
|
1668
|
+
"openclaw.memory.external_bytes": evt.memory.externalBytes,
|
|
1669
|
+
"openclaw.memory.array_buffers_bytes": evt.memory.arrayBuffersBytes
|
|
1670
|
+
};
|
|
1671
|
+
assignOptionalNumberAttrs(spanAttrs, "openclaw.memory.", {
|
|
1672
|
+
threshold_bytes: evt.thresholdBytes,
|
|
1673
|
+
rss_growth_bytes: evt.rssGrowthBytes,
|
|
1674
|
+
window_ms: evt.windowMs
|
|
1675
|
+
});
|
|
1676
|
+
const span = runtime.spanWithDuration("openclaw.memory.pressure", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1677
|
+
if (evt.level === "critical") span.setStatus({
|
|
1678
|
+
code: SpanStatusCode.ERROR,
|
|
1679
|
+
message: evt.reason
|
|
1680
|
+
});
|
|
1681
|
+
span.end(evt.ts);
|
|
1682
|
+
},
|
|
1683
|
+
recordAsyncQueueDropped(evt) {
|
|
1684
|
+
runtime.asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
|
|
1685
|
+
for (const [dropClass, field] of [
|
|
1686
|
+
["trusted", "droppedTrustedEvents"],
|
|
1687
|
+
["untrusted", "droppedUntrustedEvents"],
|
|
1688
|
+
["priority", "droppedPriorityEvents"]
|
|
1689
|
+
]) if (evt[field] !== void 0) runtime.asyncQueueDroppedCounter.add(evt[field], { "openclaw.diagnostic.async_queue.drop_class": dropClass });
|
|
1690
|
+
},
|
|
1691
|
+
recordRunCompleted(evt, metadata, privateData) {
|
|
1692
|
+
const attrs = {
|
|
1693
|
+
"openclaw.outcome": evt.outcome,
|
|
1694
|
+
"openclaw.provider": evt.provider ?? "unknown",
|
|
1695
|
+
"openclaw.model": evt.model ?? "unknown"
|
|
1696
|
+
};
|
|
1697
|
+
if (evt.channel) attrs["openclaw.channel"] = evt.channel;
|
|
1698
|
+
if (evt.blockedBy) attrs["openclaw.blocked_by"] = normalizeDiagnosticValue(evt.blockedBy, "unknown");
|
|
1699
|
+
runtime.durationHistogram.record(evt.durationMs, attrs);
|
|
1700
|
+
if (!runtime.tracesEnabled) return;
|
|
1701
|
+
const spanAttrs = { "openclaw.outcome": evt.outcome };
|
|
1702
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1703
|
+
if (evt.blockedBy) spanAttrs["openclaw.blocked_by"] = normalizeDiagnosticValue(evt.blockedBy, "unknown");
|
|
1704
|
+
if (evt.errorCategory) spanAttrs["openclaw.errorCategory"] = normalizeDiagnosticValue(evt.errorCategory, "other");
|
|
1705
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1706
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1707
|
+
const trustedTrace = runtime.trustedTraceContext(evt, metadata);
|
|
1708
|
+
const trackedSpan = trustedTrace?.spanId ? runtime.activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1709
|
+
const span = trackedSpan ?? runtime.spanWithDuration("openclaw.run", spanAttrs, evt.durationMs, {
|
|
1710
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1711
|
+
endTimeMs: evt.ts
|
|
1712
|
+
});
|
|
1713
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1714
|
+
if (evt.outcome === "error") {
|
|
1715
|
+
const message = redactedError ?? (evt.errorCategory ? redactSensitiveText(evt.errorCategory) : void 0);
|
|
1716
|
+
span.setStatus({
|
|
1717
|
+
code: SpanStatusCode.ERROR,
|
|
1718
|
+
...message ? { message } : {}
|
|
1719
|
+
});
|
|
1720
|
+
}
|
|
1721
|
+
runtime.completeTrackedLifecycleSpan(trackedSpan ? trustedTrace : void 0, span, evt.ts);
|
|
1722
|
+
}
|
|
1780
1723
|
};
|
|
1781
1724
|
}
|
|
1782
1725
|
//#endregion
|
|
1783
1726
|
//#region extensions/diagnostics-otel/src/service-recorders-tools.ts
|
|
1784
1727
|
function createToolAndSystemRecorders(runtime) {
|
|
1785
|
-
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, internalOrTrustedExplicitParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, captureContent, tracesEnabled } = runtime;
|
|
1786
1728
|
const toolExecutionBaseAttrs = (evt) => ({
|
|
1787
1729
|
"openclaw.toolName": evt.toolName,
|
|
1788
1730
|
"openclaw.tool.source": normalizeDiagnosticValue(evt.toolSource, "core"),
|
|
1789
1731
|
"gen_ai.tool.name": evt.toolName,
|
|
1790
1732
|
...evt.toolOwner ? { "openclaw.tool.owner": normalizeDiagnosticValue(evt.toolOwner) } : {},
|
|
1791
|
-
...paramsSummaryAttrs(evt.paramsSummary)
|
|
1733
|
+
...runtime.paramsSummaryAttrs(evt.paramsSummary)
|
|
1792
1734
|
});
|
|
1793
1735
|
const toolTimestampMs = (evt) => evt.sourceTimestampMs ?? evt.ts;
|
|
1794
|
-
const skillUsedAttrs = (evt) => ({
|
|
1795
|
-
"openclaw.skill.name": normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
1796
|
-
"openclaw.skill.source": normalizeDiagnosticValue(evt.skillSource),
|
|
1797
|
-
"openclaw.skill.activation": normalizeDiagnosticValue(evt.activation),
|
|
1798
|
-
...evt.agentId ? { "openclaw.agent": normalizeDiagnosticValue(evt.agentId) } : {},
|
|
1799
|
-
...evt.toolName ? { "openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool") } : {}
|
|
1800
|
-
});
|
|
1801
|
-
const recordSkillUsed = (evt, metadata) => {
|
|
1802
|
-
if (!metadata.trusted) return;
|
|
1803
|
-
const attrs = skillUsedAttrs(evt);
|
|
1804
|
-
skillUsedCounter.add(1, attrs);
|
|
1805
|
-
if (!tracesEnabled) return;
|
|
1806
|
-
const spanAttrs = { ...attrs };
|
|
1807
|
-
addRunAttrs(spanAttrs, evt);
|
|
1808
|
-
const span = spanWithDuration("openclaw.skill.used", spanAttrs, 0, {
|
|
1809
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1810
|
-
endTimeMs: evt.ts
|
|
1811
|
-
});
|
|
1812
|
-
setSpanAttrs(span, spanAttrs);
|
|
1813
|
-
span.end(evt.ts);
|
|
1814
|
-
};
|
|
1815
|
-
const recordToolExecutionStarted = (evt, metadata) => {
|
|
1816
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1817
|
-
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1818
|
-
if (trackedSpan) return trackedSpan.spanContext();
|
|
1819
|
-
const spanAttrs = toolExecutionBaseAttrs(evt);
|
|
1820
|
-
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1821
|
-
return trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.tool.execution", spanAttrs, void 0, {
|
|
1822
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1823
|
-
startTimeMs: toolTimestampMs(evt)
|
|
1824
|
-
})).spanContext();
|
|
1825
|
-
};
|
|
1826
|
-
const recordToolExecutionFinished = (evt, metadata, toolContent) => {
|
|
1827
|
-
const attrs = toolExecutionBaseAttrs(evt);
|
|
1828
|
-
if (evt.type === "tool.execution.error") attrs["openclaw.errorCategory"] = normalizeDiagnosticValue(evt.errorCategory, "other");
|
|
1829
|
-
toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
1830
|
-
if (!tracesEnabled) return;
|
|
1831
|
-
const spanAttrs = { ...attrs };
|
|
1832
|
-
addRunAttrs(spanAttrs, evt);
|
|
1833
|
-
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1834
|
-
if (evt.type === "tool.execution.error" && evt.errorCode) spanAttrs["openclaw.errorCode"] = normalizeDiagnosticValue(evt.errorCode, "other");
|
|
1835
|
-
assignOtelToolContentAttributes(spanAttrs, toolContent, captureContent);
|
|
1836
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
1837
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1838
|
-
endTimeMs: toolTimestampMs(evt)
|
|
1839
|
-
});
|
|
1840
|
-
setSpanAttrs(span, spanAttrs);
|
|
1841
|
-
if (evt.type === "tool.execution.error") span.setStatus({
|
|
1842
|
-
code: SpanStatusCode.ERROR,
|
|
1843
|
-
message: redactSensitiveText(evt.errorCategory)
|
|
1844
|
-
});
|
|
1845
|
-
span.end(toolTimestampMs(evt));
|
|
1846
|
-
};
|
|
1847
|
-
const recordToolExecutionBlocked = (evt, metadata) => {
|
|
1848
|
-
toolExecutionBlockedCounter.add(1, {
|
|
1849
|
-
...toolExecutionBaseAttrs(evt),
|
|
1850
|
-
"openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other")
|
|
1851
|
-
});
|
|
1852
|
-
if (!tracesEnabled) return;
|
|
1853
|
-
const spanAttrs = {
|
|
1854
|
-
...toolExecutionBaseAttrs(evt),
|
|
1855
|
-
"openclaw.outcome": "blocked",
|
|
1856
|
-
"openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other")
|
|
1857
|
-
};
|
|
1858
|
-
addRunAttrs(spanAttrs, evt);
|
|
1859
|
-
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1860
|
-
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, 0, {
|
|
1861
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1862
|
-
endTimeMs: toolTimestampMs(evt)
|
|
1863
|
-
});
|
|
1864
|
-
setSpanAttrs(span, spanAttrs);
|
|
1865
|
-
span.end(toolTimestampMs(evt));
|
|
1866
|
-
};
|
|
1867
|
-
const recordPayloadLarge = (evt) => {
|
|
1868
|
-
const attrs = {
|
|
1869
|
-
"openclaw.payload.action": evt.action,
|
|
1870
|
-
"openclaw.payload.surface": normalizeDiagnosticValue(evt.surface, "unknown"),
|
|
1871
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel, "none"),
|
|
1872
|
-
"openclaw.plugin": normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
1873
|
-
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none")
|
|
1874
|
-
};
|
|
1875
|
-
payloadLargeCounter.add(1, attrs);
|
|
1876
|
-
const bytes = asPositiveFiniteNumber(evt.bytes);
|
|
1877
|
-
if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
|
|
1878
|
-
};
|
|
1879
|
-
const recordExecProcessCompleted = (evt, metadata) => {
|
|
1880
|
-
const attrs = {
|
|
1881
|
-
"openclaw.exec.target": evt.target,
|
|
1882
|
-
"openclaw.exec.mode": evt.mode,
|
|
1883
|
-
"openclaw.outcome": evt.outcome
|
|
1884
|
-
};
|
|
1885
|
-
if (evt.failureKind) attrs["openclaw.failureKind"] = evt.failureKind;
|
|
1886
|
-
execProcessDurationHistogram.record(evt.durationMs, attrs);
|
|
1887
|
-
if (!tracesEnabled) return;
|
|
1888
|
-
const spanAttrs = {
|
|
1889
|
-
...attrs,
|
|
1890
|
-
"openclaw.exec.command_length": evt.commandLength
|
|
1891
|
-
};
|
|
1892
|
-
if (typeof evt.exitCode === "number") spanAttrs["openclaw.exec.exit_code"] = evt.exitCode;
|
|
1893
|
-
if (evt.exitSignal) spanAttrs["openclaw.exec.exit_signal"] = normalizeDiagnosticValue(evt.exitSignal, "other");
|
|
1894
|
-
if (evt.timedOut !== void 0) spanAttrs["openclaw.exec.timed_out"] = evt.timedOut;
|
|
1895
|
-
const span = spanWithDuration("openclaw.exec", spanAttrs, evt.durationMs, {
|
|
1896
|
-
parentContext: exportedInternalOrTrustedContext(evt, metadata),
|
|
1897
|
-
endTimeMs: evt.ts
|
|
1898
|
-
});
|
|
1899
|
-
if (evt.outcome === "failed") span.setStatus({
|
|
1900
|
-
code: SpanStatusCode.ERROR,
|
|
1901
|
-
...evt.failureKind ? { message: evt.failureKind } : {}
|
|
1902
|
-
});
|
|
1903
|
-
span.end(evt.ts);
|
|
1904
|
-
};
|
|
1905
|
-
const recordGcDuration = (evt, metadata) => {
|
|
1906
|
-
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
1907
|
-
gcDurationHistogram.record(evt.durationMs, void 0, ROOT_CONTEXT);
|
|
1908
|
-
};
|
|
1909
|
-
const recordGatewayEventLoopSample = (evt, metadata) => {
|
|
1910
|
-
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
1911
|
-
gatewayEventLoopDelayMaxHistogram.record(evt.delayMaxMs, void 0, ROOT_CONTEXT);
|
|
1912
|
-
gatewayEventLoopObservedCounter.add(evt.intervalMs, void 0, ROOT_CONTEXT);
|
|
1913
|
-
};
|
|
1914
|
-
const recordHeartbeat = (evt) => {
|
|
1915
|
-
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
|
|
1916
|
-
};
|
|
1917
|
-
const recordLivenessWarning = (evt) => {
|
|
1918
|
-
const reason = evt.reasons.join(":");
|
|
1919
|
-
const attrs = { "openclaw.liveness.reason": normalizeDiagnosticValue(reason, "unknown") };
|
|
1920
|
-
livenessWarningCounter.add(1, attrs);
|
|
1921
|
-
queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" });
|
|
1922
|
-
if (evt.eventLoopDelayP99Ms !== void 0) livenessEventLoopDelayP99Histogram.record(evt.eventLoopDelayP99Ms, attrs);
|
|
1923
|
-
if (evt.eventLoopDelayMaxMs !== void 0) livenessEventLoopDelayMaxHistogram.record(evt.eventLoopDelayMaxMs, attrs);
|
|
1924
|
-
if (evt.eventLoopUtilization !== void 0) livenessEventLoopUtilizationHistogram.record(evt.eventLoopUtilization, attrs);
|
|
1925
|
-
if (evt.cpuCoreRatio !== void 0) livenessCpuCoreRatioHistogram.record(evt.cpuCoreRatio, attrs);
|
|
1926
|
-
if (!tracesEnabled) return;
|
|
1927
|
-
const spanAttrs = {
|
|
1928
|
-
...attrs,
|
|
1929
|
-
"openclaw.liveness.active": evt.active,
|
|
1930
|
-
"openclaw.liveness.waiting": evt.waiting,
|
|
1931
|
-
"openclaw.liveness.queued": evt.queued,
|
|
1932
|
-
"openclaw.liveness.interval_ms": evt.intervalMs,
|
|
1933
|
-
...evt.eventLoopDelayP99Ms !== void 0 ? { "openclaw.liveness.event_loop_delay_p99_ms": evt.eventLoopDelayP99Ms } : {},
|
|
1934
|
-
...evt.eventLoopDelayMaxMs !== void 0 ? { "openclaw.liveness.event_loop_delay_max_ms": evt.eventLoopDelayMaxMs } : {},
|
|
1935
|
-
...evt.eventLoopUtilization !== void 0 ? { "openclaw.liveness.event_loop_utilization": evt.eventLoopUtilization } : {},
|
|
1936
|
-
...evt.cpuUserMs !== void 0 ? { "openclaw.liveness.cpu_user_ms": evt.cpuUserMs } : {},
|
|
1937
|
-
...evt.cpuSystemMs !== void 0 ? { "openclaw.liveness.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
1938
|
-
...evt.cpuTotalMs !== void 0 ? { "openclaw.liveness.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
1939
|
-
...evt.cpuCoreRatio !== void 0 ? { "openclaw.liveness.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
1940
|
-
};
|
|
1941
|
-
const span = spanWithDuration("openclaw.liveness.warning", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1942
|
-
span.setStatus({
|
|
1943
|
-
code: SpanStatusCode.ERROR,
|
|
1944
|
-
message: reason
|
|
1945
|
-
});
|
|
1946
|
-
span.end(evt.ts);
|
|
1947
|
-
};
|
|
1948
|
-
const recordDiagnosticPhaseCompleted = (evt, metadata) => {
|
|
1949
|
-
if (!tracesEnabled) return;
|
|
1950
|
-
const spanAttrs = {
|
|
1951
|
-
"openclaw.phase": normalizeDiagnosticValue(evt.name, "unknown"),
|
|
1952
|
-
...evt.cpuUserMs !== void 0 ? { "openclaw.phase.cpu_user_ms": evt.cpuUserMs } : {},
|
|
1953
|
-
...evt.cpuSystemMs !== void 0 ? { "openclaw.phase.cpu_system_ms": evt.cpuSystemMs } : {},
|
|
1954
|
-
...evt.cpuTotalMs !== void 0 ? { "openclaw.phase.cpu_total_ms": evt.cpuTotalMs } : {},
|
|
1955
|
-
...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
1956
|
-
};
|
|
1957
|
-
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
1958
|
-
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
1959
|
-
endTimeMs: evt.ts,
|
|
1960
|
-
...metadata.trusted ? { parentContext: internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT } : {}
|
|
1961
|
-
}).end(evt.ts);
|
|
1962
|
-
};
|
|
1963
|
-
const recordTelemetryExporter = (evt, metadata) => {
|
|
1964
|
-
if (!metadata.trusted) return;
|
|
1965
|
-
telemetryExporterCounter.add(1, {
|
|
1966
|
-
"openclaw.exporter": normalizeDiagnosticValue(evt.exporter, "unknown"),
|
|
1967
|
-
"openclaw.signal": evt.signal,
|
|
1968
|
-
"openclaw.status": evt.status,
|
|
1969
|
-
...evt.reason ? { "openclaw.reason": evt.reason } : {},
|
|
1970
|
-
...evt.errorCategory ? { "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other") } : {}
|
|
1971
|
-
});
|
|
1972
|
-
};
|
|
1973
1736
|
return {
|
|
1974
|
-
recordGcDuration,
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
"openclaw.
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
1737
|
+
recordGcDuration(evt, metadata) {
|
|
1738
|
+
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
1739
|
+
runtime.gcDurationHistogram.record(evt.durationMs, void 0, ROOT_CONTEXT);
|
|
1740
|
+
},
|
|
1741
|
+
recordGatewayEventLoopSample(evt, metadata) {
|
|
1742
|
+
if (!metadata.trusted && !isInternalDiagnosticEventMetadata(metadata)) return;
|
|
1743
|
+
runtime.gatewayEventLoopDelayMaxHistogram.record(evt.delayMaxMs, void 0, ROOT_CONTEXT);
|
|
1744
|
+
runtime.gatewayEventLoopObservedCounter.add(evt.intervalMs, void 0, ROOT_CONTEXT);
|
|
1745
|
+
},
|
|
1746
|
+
recordSkillUsed(evt, metadata) {
|
|
1747
|
+
if (!metadata.trusted) return;
|
|
1748
|
+
const attrs = {
|
|
1749
|
+
"openclaw.skill.name": normalizeDiagnosticValue(evt.skillName, "skill"),
|
|
1750
|
+
"openclaw.skill.source": normalizeDiagnosticValue(evt.skillSource),
|
|
1751
|
+
"openclaw.skill.activation": normalizeDiagnosticValue(evt.activation),
|
|
1752
|
+
...evt.agentId ? { "openclaw.agent": normalizeDiagnosticValue(evt.agentId) } : {},
|
|
1753
|
+
...evt.toolName ? { "openclaw.toolName": normalizeDiagnosticValue(evt.toolName, "tool") } : {}
|
|
1754
|
+
};
|
|
1755
|
+
runtime.skillUsedCounter.add(1, attrs);
|
|
1756
|
+
if (!runtime.tracesEnabled) return;
|
|
1757
|
+
const spanAttrs = { ...attrs };
|
|
1758
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1759
|
+
const span = runtime.spanWithDuration("openclaw.skill.used", spanAttrs, 0, {
|
|
1760
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1761
|
+
endTimeMs: evt.ts
|
|
1762
|
+
});
|
|
1763
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1764
|
+
span.end(evt.ts);
|
|
1765
|
+
},
|
|
1766
|
+
recordToolExecutionStarted(evt, metadata) {
|
|
1767
|
+
if (!runtime.tracesEnabled || !metadata.trusted) return;
|
|
1768
|
+
const trackedSpan = runtime.getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1769
|
+
if (trackedSpan) return trackedSpan.spanContext();
|
|
1770
|
+
const spanAttrs = toolExecutionBaseAttrs(evt);
|
|
1771
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1772
|
+
return runtime.trackTrustedSpan(evt, metadata, runtime.spanWithDuration("openclaw.tool.execution", spanAttrs, void 0, {
|
|
1773
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1774
|
+
startTimeMs: toolTimestampMs(evt)
|
|
1775
|
+
})).spanContext();
|
|
1776
|
+
},
|
|
1777
|
+
recordToolExecutionFinished(evt, metadata, toolContent) {
|
|
1778
|
+
const attrs = toolExecutionBaseAttrs(evt);
|
|
1779
|
+
if (evt.type === "tool.execution.error") attrs["openclaw.errorCategory"] = normalizeDiagnosticValue(evt.errorCategory, "other");
|
|
1780
|
+
runtime.toolExecutionDurationHistogram.record(evt.durationMs, attrs);
|
|
1781
|
+
if (!runtime.tracesEnabled) return;
|
|
1782
|
+
const spanAttrs = { ...attrs };
|
|
1783
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1784
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1785
|
+
if (evt.type === "tool.execution.error" && evt.errorCode) spanAttrs["openclaw.errorCode"] = normalizeDiagnosticValue(evt.errorCode, "other");
|
|
1786
|
+
assignOtelToolContentAttributes(spanAttrs, toolContent, runtime.captureContent);
|
|
1787
|
+
const span = runtime.takeTrackedTrustedSpan(evt, metadata) ?? runtime.spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
1788
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1789
|
+
endTimeMs: toolTimestampMs(evt)
|
|
1790
|
+
});
|
|
1791
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1792
|
+
if (evt.type === "tool.execution.error") span.setStatus({
|
|
1793
|
+
code: SpanStatusCode.ERROR,
|
|
1794
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
1795
|
+
});
|
|
1796
|
+
span.end(toolTimestampMs(evt));
|
|
1797
|
+
},
|
|
1798
|
+
recordToolExecutionBlocked(evt, metadata) {
|
|
1799
|
+
runtime.toolExecutionBlockedCounter.add(1, {
|
|
1800
|
+
...toolExecutionBaseAttrs(evt),
|
|
1801
|
+
"openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other")
|
|
1802
|
+
});
|
|
1803
|
+
if (!runtime.tracesEnabled) return;
|
|
1804
|
+
const spanAttrs = {
|
|
1805
|
+
...toolExecutionBaseAttrs(evt),
|
|
1806
|
+
"openclaw.outcome": "blocked",
|
|
1807
|
+
"openclaw.deniedReason": normalizeDiagnosticValue(evt.deniedReason, "other")
|
|
1808
|
+
};
|
|
1809
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1810
|
+
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
1811
|
+
const span = runtime.takeTrackedTrustedSpan(evt, metadata) ?? runtime.spanWithDuration("openclaw.tool.execution", spanAttrs, 0, {
|
|
1812
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1813
|
+
endTimeMs: toolTimestampMs(evt)
|
|
1814
|
+
});
|
|
1815
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
1816
|
+
span.end(toolTimestampMs(evt));
|
|
1817
|
+
},
|
|
1818
|
+
recordPayloadLarge(evt) {
|
|
1819
|
+
const attrs = {
|
|
1820
|
+
"openclaw.payload.action": evt.action,
|
|
1821
|
+
"openclaw.payload.surface": normalizeDiagnosticValue(evt.surface, "unknown"),
|
|
1822
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel, "none"),
|
|
1823
|
+
"openclaw.plugin": normalizeDiagnosticValue(evt.pluginId, "none"),
|
|
1824
|
+
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none")
|
|
1825
|
+
};
|
|
1826
|
+
runtime.payloadLargeCounter.add(1, attrs);
|
|
1827
|
+
const bytes = asPositiveFiniteNumber(evt.bytes);
|
|
1828
|
+
if (bytes !== void 0) runtime.payloadLargeBytesHistogram.record(bytes, attrs);
|
|
1829
|
+
},
|
|
1830
|
+
recordExecProcessCompleted(evt, metadata) {
|
|
1831
|
+
const attrs = {
|
|
1832
|
+
"openclaw.exec.target": evt.target,
|
|
1833
|
+
"openclaw.exec.mode": evt.mode,
|
|
1834
|
+
"openclaw.outcome": evt.outcome
|
|
1835
|
+
};
|
|
1836
|
+
if (evt.failureKind) attrs["openclaw.failureKind"] = evt.failureKind;
|
|
1837
|
+
runtime.execProcessDurationHistogram.record(evt.durationMs, attrs);
|
|
1838
|
+
if (!runtime.tracesEnabled) return;
|
|
1839
|
+
const spanAttrs = {
|
|
2016
1840
|
...attrs,
|
|
2017
|
-
"openclaw.
|
|
1841
|
+
"openclaw.exec.command_length": evt.commandLength
|
|
1842
|
+
};
|
|
1843
|
+
if (typeof evt.exitCode === "number") spanAttrs["openclaw.exec.exit_code"] = evt.exitCode;
|
|
1844
|
+
if (evt.exitSignal) spanAttrs["openclaw.exec.exit_signal"] = normalizeDiagnosticValue(evt.exitSignal, "other");
|
|
1845
|
+
if (evt.timedOut !== void 0) spanAttrs["openclaw.exec.timed_out"] = evt.timedOut;
|
|
1846
|
+
const span = runtime.spanWithDuration("openclaw.exec", spanAttrs, evt.durationMs, {
|
|
1847
|
+
parentContext: runtime.exportedInternalOrTrustedContext(evt, metadata),
|
|
1848
|
+
endTimeMs: evt.ts
|
|
2018
1849
|
});
|
|
2019
|
-
if (
|
|
2020
|
-
|
|
2021
|
-
|
|
1850
|
+
if (evt.outcome === "failed") span.setStatus({
|
|
1851
|
+
code: SpanStatusCode.ERROR,
|
|
1852
|
+
...evt.failureKind ? { message: evt.failureKind } : {}
|
|
2022
1853
|
});
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
1854
|
+
span.end(evt.ts);
|
|
1855
|
+
},
|
|
1856
|
+
recordHeartbeat(evt) {
|
|
1857
|
+
runtime.queueDepthHistogram.record(evt.queued, { "openclaw.channel": "heartbeat" });
|
|
1858
|
+
},
|
|
1859
|
+
recordLivenessWarning(evt) {
|
|
1860
|
+
const reason = evt.reasons.join(":");
|
|
1861
|
+
const attrs = { "openclaw.liveness.reason": normalizeDiagnosticValue(reason, "unknown") };
|
|
1862
|
+
runtime.livenessWarningCounter.add(1, attrs);
|
|
1863
|
+
runtime.queueDepthHistogram.record(evt.queued, { "openclaw.channel": "liveness" });
|
|
1864
|
+
for (const [histogram, value] of [
|
|
1865
|
+
[runtime.livenessEventLoopDelayP99Histogram, evt.eventLoopDelayP99Ms],
|
|
1866
|
+
[runtime.livenessEventLoopDelayMaxHistogram, evt.eventLoopDelayMaxMs],
|
|
1867
|
+
[runtime.livenessEventLoopUtilizationHistogram, evt.eventLoopUtilization],
|
|
1868
|
+
[runtime.livenessCpuCoreRatioHistogram, evt.cpuCoreRatio]
|
|
1869
|
+
]) if (value !== void 0) histogram.record(value, attrs);
|
|
1870
|
+
if (!runtime.tracesEnabled) return;
|
|
1871
|
+
const spanAttrs = {
|
|
2029
1872
|
...attrs,
|
|
2030
|
-
"openclaw.
|
|
1873
|
+
"openclaw.liveness.active": evt.active,
|
|
1874
|
+
"openclaw.liveness.waiting": evt.waiting,
|
|
1875
|
+
"openclaw.liveness.queued": evt.queued,
|
|
1876
|
+
"openclaw.liveness.interval_ms": evt.intervalMs
|
|
1877
|
+
};
|
|
1878
|
+
assignOptionalNumberAttrs(spanAttrs, "openclaw.liveness.", {
|
|
1879
|
+
event_loop_delay_p99_ms: evt.eventLoopDelayP99Ms,
|
|
1880
|
+
event_loop_delay_max_ms: evt.eventLoopDelayMaxMs,
|
|
1881
|
+
event_loop_utilization: evt.eventLoopUtilization,
|
|
1882
|
+
cpu_user_ms: evt.cpuUserMs,
|
|
1883
|
+
cpu_system_ms: evt.cpuSystemMs,
|
|
1884
|
+
cpu_total_ms: evt.cpuTotalMs,
|
|
1885
|
+
cpu_core_ratio: evt.cpuCoreRatio
|
|
1886
|
+
});
|
|
1887
|
+
const span = runtime.spanWithDuration("openclaw.liveness.warning", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
1888
|
+
span.setStatus({
|
|
1889
|
+
code: SpanStatusCode.ERROR,
|
|
1890
|
+
message: reason
|
|
1891
|
+
});
|
|
1892
|
+
span.end(evt.ts);
|
|
1893
|
+
},
|
|
1894
|
+
recordDiagnosticPhaseCompleted(evt, metadata) {
|
|
1895
|
+
if (!runtime.tracesEnabled) return;
|
|
1896
|
+
const spanAttrs = { "openclaw.phase": normalizeDiagnosticValue(evt.name, "unknown") };
|
|
1897
|
+
assignOptionalNumberAttrs(spanAttrs, "openclaw.phase.", {
|
|
1898
|
+
cpu_user_ms: evt.cpuUserMs,
|
|
1899
|
+
cpu_system_ms: evt.cpuSystemMs,
|
|
1900
|
+
cpu_total_ms: evt.cpuTotalMs,
|
|
1901
|
+
cpu_core_ratio: evt.cpuCoreRatio
|
|
1902
|
+
});
|
|
1903
|
+
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
1904
|
+
runtime.spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
1905
|
+
endTimeMs: evt.ts,
|
|
1906
|
+
...metadata.trusted ? { parentContext: runtime.internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT } : {}
|
|
1907
|
+
}).end(evt.ts);
|
|
1908
|
+
},
|
|
1909
|
+
recordTelemetryExporter(evt, metadata) {
|
|
1910
|
+
if (!metadata.trusted) return;
|
|
1911
|
+
runtime.telemetryExporterCounter.add(1, {
|
|
1912
|
+
"openclaw.exporter": normalizeDiagnosticValue(evt.exporter, "unknown"),
|
|
1913
|
+
"openclaw.signal": evt.signal,
|
|
1914
|
+
"openclaw.status": evt.status,
|
|
1915
|
+
...evt.reason ? { "openclaw.reason": evt.reason } : {},
|
|
1916
|
+
...evt.errorCategory ? { "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other") } : {}
|
|
2031
1917
|
});
|
|
2032
1918
|
}
|
|
2033
|
-
if (!tracesEnabled) return;
|
|
2034
|
-
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
2035
|
-
const spanAttrs = {
|
|
2036
|
-
...attrs,
|
|
2037
|
-
"openclaw.tokens.input": usage.input ?? 0,
|
|
2038
|
-
"openclaw.tokens.output": usage.output ?? 0,
|
|
2039
|
-
"openclaw.tokens.cache_read": usage.cacheRead ?? 0,
|
|
2040
|
-
"openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
|
|
2041
|
-
"openclaw.tokens.total": usage.total ?? 0
|
|
2042
|
-
};
|
|
2043
|
-
if (metadata.trusted && metadata.internal && hostPluginId) spanAttrs["openclaw.plugin"] = normalizeDiagnosticValue(hostPluginId);
|
|
2044
|
-
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
2045
|
-
addRunAttrs(spanAttrs, evt);
|
|
2046
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
2047
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
2048
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
2049
|
-
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite);
|
|
2050
|
-
spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs, {
|
|
2051
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2052
|
-
endTimeMs: evt.ts
|
|
2053
|
-
}).end(evt.ts);
|
|
2054
|
-
};
|
|
2055
|
-
const recordWebhookReceived = (evt) => {
|
|
2056
|
-
const attrs = {
|
|
2057
|
-
"openclaw.channel": evt.channel ?? "unknown",
|
|
2058
|
-
"openclaw.webhook": evt.updateType ?? "unknown"
|
|
2059
|
-
};
|
|
2060
|
-
webhookReceivedCounter.add(1, attrs);
|
|
2061
|
-
};
|
|
2062
|
-
const recordWebhookProcessed = (evt) => {
|
|
2063
|
-
const attrs = {
|
|
2064
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2065
|
-
"openclaw.webhook": normalizeDiagnosticValue(evt.updateType)
|
|
2066
|
-
};
|
|
2067
|
-
if (typeof evt.durationMs === "number") webhookDurationHistogram.record(evt.durationMs, attrs);
|
|
2068
|
-
if (!tracesEnabled) return;
|
|
2069
|
-
const spanAttrs = { ...attrs };
|
|
2070
|
-
spanWithDuration("openclaw.webhook.processed", spanAttrs, evt.durationMs).end();
|
|
2071
|
-
};
|
|
2072
|
-
const recordWebhookError = (evt) => {
|
|
2073
|
-
const attrs = {
|
|
2074
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2075
|
-
"openclaw.webhook": normalizeDiagnosticValue(evt.updateType)
|
|
2076
|
-
};
|
|
2077
|
-
webhookErrorCounter.add(1, attrs);
|
|
2078
|
-
if (!tracesEnabled) return;
|
|
2079
|
-
const redactedError = redactSensitiveText(evt.error);
|
|
2080
|
-
const spanAttrs = {
|
|
2081
|
-
...attrs,
|
|
2082
|
-
"openclaw.error": redactedError
|
|
2083
|
-
};
|
|
2084
|
-
const span = tracer.startSpan("openclaw.webhook.error", { attributes: spanAttrs });
|
|
2085
|
-
span.setStatus({
|
|
2086
|
-
code: SpanStatusCode.ERROR,
|
|
2087
|
-
message: redactedError
|
|
2088
|
-
});
|
|
2089
|
-
span.end();
|
|
2090
|
-
};
|
|
2091
|
-
const recordMessageQueued = (evt) => {
|
|
2092
|
-
const attrs = {
|
|
2093
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2094
|
-
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2095
|
-
};
|
|
2096
|
-
messageQueuedCounter.add(1, attrs);
|
|
2097
|
-
if (typeof evt.queueDepth === "number") queueDepthHistogram.record(evt.queueDepth, attrs);
|
|
2098
|
-
};
|
|
2099
|
-
const recordMessageReceived = (evt) => {
|
|
2100
|
-
messageReceivedCounter.add(1, {
|
|
2101
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2102
|
-
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2103
|
-
});
|
|
2104
|
-
};
|
|
2105
|
-
const recordMessageDispatchStarted = (evt, metadata) => {
|
|
2106
|
-
const attrs = {
|
|
2107
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2108
|
-
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2109
|
-
};
|
|
2110
|
-
messageDispatchStartedCounter.add(1, attrs);
|
|
2111
|
-
if (!tracesEnabled) return;
|
|
2112
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2113
|
-
if (!traceContext?.spanId || activeTrustedSpans.has(traceContext.spanId)) return;
|
|
2114
|
-
trackInternalOrTrustedSpan(evt, metadata, spanWithDuration("openclaw.message.processed", attrs, void 0, {
|
|
2115
|
-
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2116
|
-
startTimeMs: evt.ts
|
|
2117
|
-
}));
|
|
2118
|
-
};
|
|
2119
|
-
const recordMessageDispatchCompleted = (evt) => {
|
|
2120
|
-
const attrs = {
|
|
2121
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2122
|
-
"openclaw.outcome": evt.outcome,
|
|
2123
|
-
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none"),
|
|
2124
|
-
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2125
|
-
};
|
|
2126
|
-
messageDispatchCompletedCounter.add(1, attrs);
|
|
2127
|
-
messageDispatchDurationHistogram.record(evt.durationMs, attrs);
|
|
2128
|
-
};
|
|
2129
|
-
const recordMessageProcessed = (evt, metadata) => {
|
|
2130
|
-
const attrs = {
|
|
2131
|
-
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2132
|
-
"openclaw.outcome": evt.outcome ?? "unknown"
|
|
2133
|
-
};
|
|
2134
|
-
messageProcessedCounter.add(1, attrs);
|
|
2135
|
-
if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
|
|
2136
|
-
if (!tracesEnabled) return;
|
|
2137
|
-
const spanAttrs = { ...attrs };
|
|
2138
|
-
addRunAttrs(spanAttrs, evt);
|
|
2139
|
-
if (evt.reason) spanAttrs["openclaw.reason"] = normalizeDiagnosticValue(evt.reason, "unknown");
|
|
2140
|
-
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
2141
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
2142
|
-
parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2143
|
-
endTimeMs: evt.ts
|
|
2144
|
-
});
|
|
2145
|
-
setSpanAttrs(span, spanAttrs);
|
|
2146
|
-
if (evt.outcome === "error" && evt.error) span.setStatus({
|
|
2147
|
-
code: SpanStatusCode.ERROR,
|
|
2148
|
-
message: redactSensitiveText(evt.error)
|
|
2149
|
-
});
|
|
2150
|
-
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2151
|
-
completeTrackedLifecycleSpan(trackedSpan ? traceContext : void 0, span, evt.ts);
|
|
2152
1919
|
};
|
|
1920
|
+
}
|
|
1921
|
+
//#endregion
|
|
1922
|
+
//#region extensions/diagnostics-otel/src/service-recorders-usage.ts
|
|
1923
|
+
function createUsageRecorders(runtime) {
|
|
2153
1924
|
const messageDeliveryAttrs = (evt) => ({
|
|
2154
1925
|
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2155
1926
|
"openclaw.delivery.kind": normalizeDiagnosticValue(evt.deliveryKind, "other")
|
|
2156
1927
|
});
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
1928
|
+
return {
|
|
1929
|
+
recordModelUsage(evt, metadata, hostPluginId) {
|
|
1930
|
+
const attrs = {
|
|
1931
|
+
"openclaw.channel": evt.channel ?? "unknown",
|
|
1932
|
+
"openclaw.agent": normalizeDiagnosticValue(evt.agentId),
|
|
1933
|
+
"openclaw.provider": evt.provider ?? "unknown",
|
|
1934
|
+
"openclaw.model": evt.model ?? "unknown"
|
|
1935
|
+
};
|
|
1936
|
+
const genAiAttrs = {
|
|
1937
|
+
"gen_ai.operation.name": "chat",
|
|
1938
|
+
"gen_ai.provider.name": normalizeDiagnosticValue(evt.provider),
|
|
1939
|
+
"gen_ai.request.model": normalizeDiagnosticValue(evt.model)
|
|
1940
|
+
};
|
|
1941
|
+
const usage = evt.usage;
|
|
1942
|
+
for (const [tokenType, field] of [
|
|
1943
|
+
["input", "input"],
|
|
1944
|
+
["output", "output"],
|
|
1945
|
+
["cache_read", "cacheRead"],
|
|
1946
|
+
["cache_write", "cacheWrite"],
|
|
1947
|
+
["prompt", "promptTokens"],
|
|
1948
|
+
["total", "total"]
|
|
1949
|
+
]) {
|
|
1950
|
+
const amount = usage[field];
|
|
1951
|
+
if (!amount) continue;
|
|
1952
|
+
runtime.tokensCounter.add(amount, {
|
|
1953
|
+
...attrs,
|
|
1954
|
+
"openclaw.token": tokenType
|
|
1955
|
+
});
|
|
1956
|
+
if (tokenType === "input" || tokenType === "output") runtime.genAiTokenUsageHistogram.record(amount, {
|
|
1957
|
+
...genAiAttrs,
|
|
1958
|
+
"gen_ai.token.type": tokenType
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
if (evt.costUsd) runtime.costCounter.add(evt.costUsd, attrs);
|
|
1962
|
+
if (evt.durationMs) runtime.durationHistogram.record(evt.durationMs, attrs);
|
|
1963
|
+
for (const kind of ["limit", "used"]) {
|
|
1964
|
+
const amount = evt.context?.[kind];
|
|
1965
|
+
if (amount) runtime.contextHistogram.record(amount, {
|
|
1966
|
+
...attrs,
|
|
1967
|
+
"openclaw.context": kind
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
if (!runtime.tracesEnabled) return;
|
|
1971
|
+
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
1972
|
+
const spanAttrs = {
|
|
1973
|
+
...attrs,
|
|
1974
|
+
"openclaw.tokens.input": usage.input ?? 0,
|
|
1975
|
+
"openclaw.tokens.output": usage.output ?? 0,
|
|
1976
|
+
"openclaw.tokens.cache_read": usage.cacheRead ?? 0,
|
|
1977
|
+
"openclaw.tokens.cache_write": usage.cacheWrite ?? 0,
|
|
1978
|
+
"openclaw.tokens.total": usage.total ?? 0
|
|
2194
1979
|
};
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
1980
|
+
if (metadata.trusted && metadata.internal && hostPluginId) spanAttrs["openclaw.plugin"] = normalizeDiagnosticValue(hostPluginId);
|
|
1981
|
+
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
1982
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
1983
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
1984
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
1985
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
1986
|
+
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite);
|
|
1987
|
+
runtime.spanWithDuration("openclaw.model.usage", spanAttrs, evt.durationMs, {
|
|
1988
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
1989
|
+
endTimeMs: evt.ts
|
|
1990
|
+
}).end(evt.ts);
|
|
1991
|
+
},
|
|
1992
|
+
recordWebhookReceived(evt) {
|
|
1993
|
+
const attrs = {
|
|
1994
|
+
"openclaw.channel": evt.channel ?? "unknown",
|
|
1995
|
+
"openclaw.webhook": evt.updateType ?? "unknown"
|
|
1996
|
+
};
|
|
1997
|
+
runtime.webhookReceivedCounter.add(1, attrs);
|
|
1998
|
+
},
|
|
1999
|
+
recordWebhookProcessed(evt) {
|
|
2000
|
+
const attrs = {
|
|
2001
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2002
|
+
"openclaw.webhook": normalizeDiagnosticValue(evt.updateType)
|
|
2003
|
+
};
|
|
2004
|
+
if (typeof evt.durationMs === "number") runtime.webhookDurationHistogram.record(evt.durationMs, attrs);
|
|
2005
|
+
if (!runtime.tracesEnabled) return;
|
|
2006
|
+
const spanAttrs = { ...attrs };
|
|
2007
|
+
runtime.spanWithDuration("openclaw.webhook.processed", spanAttrs, evt.durationMs).end();
|
|
2008
|
+
},
|
|
2009
|
+
recordWebhookError(evt) {
|
|
2010
|
+
const attrs = {
|
|
2011
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2012
|
+
"openclaw.webhook": normalizeDiagnosticValue(evt.updateType)
|
|
2013
|
+
};
|
|
2014
|
+
runtime.webhookErrorCounter.add(1, attrs);
|
|
2015
|
+
if (!runtime.tracesEnabled) return;
|
|
2016
|
+
const redactedError = redactSensitiveText(evt.error);
|
|
2017
|
+
const spanAttrs = {
|
|
2018
|
+
...attrs,
|
|
2019
|
+
"openclaw.error": redactedError
|
|
2020
|
+
};
|
|
2021
|
+
const span = runtime.tracer.startSpan("openclaw.webhook.error", { attributes: spanAttrs });
|
|
2022
|
+
span.setStatus({
|
|
2023
|
+
code: SpanStatusCode.ERROR,
|
|
2024
|
+
message: redactedError
|
|
2025
|
+
});
|
|
2026
|
+
span.end();
|
|
2027
|
+
},
|
|
2028
|
+
recordMessageQueued(evt) {
|
|
2029
|
+
const attrs = {
|
|
2030
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2031
|
+
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2032
|
+
};
|
|
2033
|
+
runtime.messageQueuedCounter.add(1, attrs);
|
|
2034
|
+
if (typeof evt.queueDepth === "number") runtime.queueDepthHistogram.record(evt.queueDepth, attrs);
|
|
2035
|
+
},
|
|
2036
|
+
recordMessageReceived(evt) {
|
|
2037
|
+
runtime.messageReceivedCounter.add(1, {
|
|
2038
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2039
|
+
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2199
2040
|
});
|
|
2041
|
+
},
|
|
2042
|
+
recordMessageDispatchStarted(evt, metadata) {
|
|
2043
|
+
const attrs = {
|
|
2044
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2045
|
+
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2046
|
+
};
|
|
2047
|
+
runtime.messageDispatchStartedCounter.add(1, attrs);
|
|
2048
|
+
if (!runtime.tracesEnabled) return;
|
|
2049
|
+
const traceContext = runtime.internalOrTrustedTraceContext(evt, metadata);
|
|
2050
|
+
if (!traceContext?.spanId || runtime.activeTrustedSpans.has(traceContext.spanId)) return;
|
|
2051
|
+
runtime.trackInternalOrTrustedSpan(evt, metadata, runtime.spanWithDuration("openclaw.message.processed", attrs, void 0, {
|
|
2052
|
+
parentContext: runtime.internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2053
|
+
startTimeMs: evt.ts
|
|
2054
|
+
}));
|
|
2055
|
+
},
|
|
2056
|
+
recordMessageDispatchCompleted(evt) {
|
|
2057
|
+
const attrs = {
|
|
2058
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2059
|
+
"openclaw.outcome": evt.outcome,
|
|
2060
|
+
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none"),
|
|
2061
|
+
"openclaw.source": normalizeDiagnosticValue(evt.source)
|
|
2062
|
+
};
|
|
2063
|
+
runtime.messageDispatchCompletedCounter.add(1, attrs);
|
|
2064
|
+
runtime.messageDispatchDurationHistogram.record(evt.durationMs, attrs);
|
|
2065
|
+
},
|
|
2066
|
+
recordMessageProcessed(evt, metadata) {
|
|
2067
|
+
const attrs = {
|
|
2068
|
+
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
2069
|
+
"openclaw.outcome": evt.outcome ?? "unknown"
|
|
2070
|
+
};
|
|
2071
|
+
runtime.messageProcessedCounter.add(1, attrs);
|
|
2072
|
+
if (typeof evt.durationMs === "number") runtime.messageDurationHistogram.record(evt.durationMs, attrs);
|
|
2073
|
+
if (!runtime.tracesEnabled) return;
|
|
2074
|
+
const spanAttrs = { ...attrs };
|
|
2075
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
2076
|
+
if (evt.reason) spanAttrs["openclaw.reason"] = normalizeDiagnosticValue(evt.reason, "unknown");
|
|
2077
|
+
const trackedSpan = runtime.getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
2078
|
+
const span = trackedSpan ?? runtime.spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
2079
|
+
parentContext: runtime.internalOrTrustedExplicitParentContext(evt, metadata),
|
|
2080
|
+
endTimeMs: evt.ts
|
|
2081
|
+
});
|
|
2082
|
+
runtime.setSpanAttrs(span, spanAttrs);
|
|
2083
|
+
if (evt.outcome === "error" && evt.error) span.setStatus({
|
|
2084
|
+
code: SpanStatusCode.ERROR,
|
|
2085
|
+
message: redactSensitiveText(evt.error)
|
|
2086
|
+
});
|
|
2087
|
+
const traceContext = runtime.internalOrTrustedTraceContext(evt, metadata);
|
|
2088
|
+
runtime.completeTrackedLifecycleSpan(trackedSpan ? traceContext : void 0, span, evt.ts);
|
|
2089
|
+
},
|
|
2090
|
+
recordMessageDeliveryStarted(evt) {
|
|
2091
|
+
runtime.messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
|
|
2092
|
+
},
|
|
2093
|
+
recordMessageDeliveryFinished(evt, metadata) {
|
|
2094
|
+
const attrs = {
|
|
2095
|
+
...messageDeliveryAttrs(evt),
|
|
2096
|
+
"openclaw.outcome": evt.type === "message.delivery.error" ? "error" : "completed",
|
|
2097
|
+
...evt.type === "message.delivery.error" ? { "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other") } : {}
|
|
2098
|
+
};
|
|
2099
|
+
runtime.messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
2100
|
+
if (!runtime.tracesEnabled) return;
|
|
2101
|
+
const span = runtime.spanWithDuration("openclaw.message.delivery", {
|
|
2102
|
+
...attrs,
|
|
2103
|
+
...evt.type === "message.delivery.completed" ? { "openclaw.delivery.result_count": evt.resultCount } : {}
|
|
2104
|
+
}, evt.durationMs, {
|
|
2105
|
+
parentContext: runtime.activeInternalOrTrustedContext(evt, metadata),
|
|
2106
|
+
endTimeMs: evt.ts
|
|
2107
|
+
});
|
|
2108
|
+
if (evt.type === "message.delivery.error") span.setStatus({
|
|
2109
|
+
code: SpanStatusCode.ERROR,
|
|
2110
|
+
message: redactSensitiveText(evt.errorCategory)
|
|
2111
|
+
});
|
|
2112
|
+
span.end(evt.ts);
|
|
2113
|
+
},
|
|
2114
|
+
recordRunStarted(evt, metadata) {
|
|
2115
|
+
if (!runtime.tracesEnabled || !metadata.trusted) return;
|
|
2116
|
+
const spanAttrs = {};
|
|
2117
|
+
runtime.addRunAttrs(spanAttrs, evt);
|
|
2118
|
+
const span = runtime.trackTrustedSpan(evt, metadata, runtime.spanWithDuration("openclaw.run", spanAttrs, void 0, {
|
|
2119
|
+
parentContext: runtime.activeTrustedParentContext(evt, metadata),
|
|
2120
|
+
startTimeMs: evt.ts
|
|
2121
|
+
}));
|
|
2122
|
+
const parentSpanId = runtime.trustedTraceContext(evt, metadata)?.parentSpanId;
|
|
2123
|
+
if (parentSpanId && !runtime.activeTrustedSpans.has(parentSpanId)) {
|
|
2124
|
+
const owner = {
|
|
2125
|
+
kind: "run",
|
|
2126
|
+
id: evt.runId
|
|
2127
|
+
};
|
|
2128
|
+
runtime.activeTrustedSpanAliases.set(runtime.trustedSpanAliasKey(parentSpanId, owner), {
|
|
2129
|
+
span,
|
|
2130
|
+
spanId: parentSpanId,
|
|
2131
|
+
owner
|
|
2132
|
+
});
|
|
2133
|
+
}
|
|
2200
2134
|
}
|
|
2201
2135
|
};
|
|
2202
|
-
return {
|
|
2203
|
-
recordModelUsage,
|
|
2204
|
-
recordWebhookReceived,
|
|
2205
|
-
recordWebhookProcessed,
|
|
2206
|
-
recordWebhookError,
|
|
2207
|
-
recordMessageQueued,
|
|
2208
|
-
recordMessageReceived,
|
|
2209
|
-
recordMessageDispatchStarted,
|
|
2210
|
-
recordMessageDispatchCompleted,
|
|
2211
|
-
recordMessageProcessed,
|
|
2212
|
-
recordMessageDeliveryStarted,
|
|
2213
|
-
recordMessageDeliveryFinished,
|
|
2214
|
-
recordRunStarted
|
|
2215
|
-
};
|
|
2216
2136
|
}
|
|
2217
2137
|
//#endregion
|
|
2218
2138
|
//#region extensions/diagnostics-otel/src/service-traces.ts
|