@openclaw/diagnostics-otel 2026.9.6 → 2026.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/assets/icon.png
CHANGED
|
Binary file
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, redactSensitiveText } from "../api.js";
|
|
2
1
|
import { ROOT_CONTEXT, SpanKind, SpanStatusCode, TraceFlags, context, createContextKey, createNoopMeter, diag, isSpanContextValid, metrics, propagation, trace } from "@opentelemetry/api";
|
|
3
2
|
import * as otelCore from "@opentelemetry/core";
|
|
4
3
|
import { CompositePropagator, ExportResultCode, W3CBaggagePropagator, W3CTraceContextPropagator, getStringListFromEnv } from "@opentelemetry/core";
|
|
@@ -8,11 +7,14 @@ import * as resources from "@opentelemetry/resources";
|
|
|
8
7
|
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
9
8
|
import { BasicTracerProvider, BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
|
10
9
|
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
10
|
+
import { asFiniteNumberInRange, asNonNegativeFiniteNumber, asPositiveFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
|
|
11
11
|
import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
|
|
12
|
+
import { asFiniteNumber, isRecord, normalizeOptionalString, readNonBlankString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
13
|
+
import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime";
|
|
12
14
|
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
13
|
-
import { isInternalDiagnosticEventMetadata, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
14
15
|
import { readFileSync } from "node:fs";
|
|
15
16
|
import nodePath from "node:path";
|
|
17
|
+
import { isInternalDiagnosticEventMetadata, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
16
18
|
import { collectErrorGraphCandidates } from "openclaw/plugin-sdk/error-runtime";
|
|
17
19
|
import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime";
|
|
18
20
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
@@ -21,7 +23,6 @@ import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-ho
|
|
|
21
23
|
import { B3InjectEncoding, B3Propagator } from "@opentelemetry/propagator-b3";
|
|
22
24
|
import { JaegerPropagator } from "@opentelemetry/propagator-jaeger";
|
|
23
25
|
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_TOOL_DEFINITIONS, GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL, GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT } from "@opentelemetry/semantic-conventions/incubating";
|
|
24
|
-
import { asFiniteNumber, asFiniteNumberInRange, isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
25
26
|
const DROPPED_OTEL_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
|
|
26
27
|
"openclaw.callId",
|
|
27
28
|
"openclaw.call_id",
|
|
@@ -308,9 +309,6 @@ function describeJsonValue(value) {
|
|
|
308
309
|
}
|
|
309
310
|
//#endregion
|
|
310
311
|
//#region extensions/diagnostics-otel/src/service-exporter.ts
|
|
311
|
-
function normalizeEndpoint(endpoint) {
|
|
312
|
-
return endpoint?.trim() || void 0;
|
|
313
|
-
}
|
|
314
312
|
const SIGNAL_QUALIFIED_OTLP_PATH_PATTERN = /\/v1\/(traces|metrics|logs)$/iu;
|
|
315
313
|
function appendOrReplaceSignalPath(value, path) {
|
|
316
314
|
const base = value.replace(/\/+$/u, "");
|
|
@@ -330,10 +328,10 @@ function resolveSharedOtelUrl(endpoint, path) {
|
|
|
330
328
|
return appendOrReplaceSignalPath(endpoint, path);
|
|
331
329
|
}
|
|
332
330
|
function resolveSignalOtelUrl(params) {
|
|
333
|
-
const signalEndpoint =
|
|
331
|
+
const signalEndpoint = normalizeOptionalString(params.signalEndpoint ?? params.signalEnvEndpoint);
|
|
334
332
|
const endpoint = signalEndpoint ?? params.endpoint;
|
|
335
|
-
const signalEnvEndpoint = params.signalEnvEndpoint
|
|
336
|
-
const sharedEnvEndpoint = params.sharedEnvEndpoint
|
|
333
|
+
const signalEnvEndpoint = readNonBlankString(params.signalEnvEndpoint);
|
|
334
|
+
const sharedEnvEndpoint = readNonBlankString(params.sharedEnvEndpoint);
|
|
337
335
|
const consumedSharedEnvEndpoint = signalEnvEndpoint ? void 0 : sharedEnvEndpoint;
|
|
338
336
|
const appendedSharedEnvEndpoint = consumedSharedEnvEndpoint ? `${consumedSharedEnvEndpoint}${consumedSharedEnvEndpoint.endsWith("/") ? "" : "/"}${params.path}` : void 0;
|
|
339
337
|
const resolvedEndpoint = endpoint && URL.canParse(endpoint) && !signalEndpoint ? resolveSharedOtelUrl(endpoint, params.path) : endpoint;
|
|
@@ -347,7 +345,7 @@ function resolveSignalOtelUrl(params) {
|
|
|
347
345
|
}
|
|
348
346
|
function readOtelEnvFile(params) {
|
|
349
347
|
const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`;
|
|
350
|
-
const filePath =
|
|
348
|
+
const filePath = readNonBlankString(process.env[signalEnvName]) ?? readNonBlankString(process.env[params.sharedEnvName]);
|
|
351
349
|
if (!filePath) return;
|
|
352
350
|
try {
|
|
353
351
|
const material = readFileSync(nodePath.resolve(process.cwd(), filePath));
|
|
@@ -355,9 +353,6 @@ function readOtelEnvFile(params) {
|
|
|
355
353
|
} catch {}
|
|
356
354
|
throw new Error(`Configured OpenTelemetry ${params.label} file is missing, empty, or unreadable; refusing insecure export`);
|
|
357
355
|
}
|
|
358
|
-
function normalizeOtelEnvValue(value) {
|
|
359
|
-
return value?.trim() ? value : void 0;
|
|
360
|
-
}
|
|
361
356
|
function resolveOtelHttpAgentOptions(params) {
|
|
362
357
|
const { url, signalIdentifier } = params;
|
|
363
358
|
const ca = readOtelEnvFile({
|
|
@@ -398,11 +393,6 @@ function resolveOtelHttpAgentOptions(params) {
|
|
|
398
393
|
}
|
|
399
394
|
return (ca || cert || key) && new URL(url).protocol === "https:" ? agentOptions : void 0;
|
|
400
395
|
}
|
|
401
|
-
function resolveSampleRate(value) {
|
|
402
|
-
if (typeof value !== "number" || !Number.isFinite(value)) return;
|
|
403
|
-
if (value < 0 || value > 1) return;
|
|
404
|
-
return value;
|
|
405
|
-
}
|
|
406
396
|
function formatError(err) {
|
|
407
397
|
if (err instanceof Error) return err.stack ?? err.message;
|
|
408
398
|
if (typeof err === "string") return err;
|
|
@@ -443,163 +433,71 @@ function findOtlpExporterError(reason) {
|
|
|
443
433
|
//#region extensions/diagnostics-otel/src/service-events.ts
|
|
444
434
|
function createDiagnosticsEventHandler(params) {
|
|
445
435
|
const { logger, recorders, recordLogRecord, recordSecurityEvent } = params;
|
|
446
|
-
const { recordGcDuration, recordGatewayEventLoopSample, recordGatewayRpc, recordModelUsage, recordWebhookReceived, recordWebhookProcessed, recordWebhookError, recordMessageQueued, recordMessageReceived, recordMessageDispatchStarted, recordMessageDispatchCompleted, recordMessageProcessed, recordMessageDeliveryStarted, recordMessageDeliveryCompleted, recordMessageDeliveryError, recordTalkEvent, recordLaneEnqueue, recordLaneDequeue, recordSessionState, recordSessionTurnCreated, recordSessionStuck, recordSessionRecoveryRequested, recordSessionRecoveryCompleted, recordRunAttempt, recordHeartbeat, recordLivenessWarning, recordDiagnosticPhaseCompleted, recordRunStarted, recordRunCompleted, recordHarnessRunStarted, recordHarnessRunCompleted, recordHarnessRunError, recordContextAssembled, recordModelCallStarted, recordModelCallFinished, recordToolExecutionStarted, recordToolExecutionFinished, recordToolExecutionBlocked, recordSkillUsed, recordExecProcessCompleted, recordToolLoop, recordMemorySample, recordMemoryPressure, recordAsyncQueueDropped, recordTelemetryExporter, recordPayloadLarge, recordModelFailover } = recorders;
|
|
447
436
|
return (evt, metadata, privateData) => {
|
|
448
437
|
try {
|
|
449
438
|
switch (evt.type) {
|
|
450
439
|
case "diagnostic.child_process.spawn": return;
|
|
451
|
-
case "diagnostic.gc":
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
case "
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
case "
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
case "
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
case "
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
case "
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
case "
|
|
470
|
-
recordWebhookError(evt);
|
|
471
|
-
return;
|
|
472
|
-
case "message.queued":
|
|
473
|
-
recordMessageQueued(evt);
|
|
474
|
-
return;
|
|
475
|
-
case "message.received":
|
|
476
|
-
recordMessageReceived(evt);
|
|
477
|
-
return;
|
|
478
|
-
case "message.dispatch.started":
|
|
479
|
-
recordMessageDispatchStarted(evt, metadata);
|
|
480
|
-
return;
|
|
481
|
-
case "message.dispatch.completed":
|
|
482
|
-
recordMessageDispatchCompleted(evt);
|
|
483
|
-
return;
|
|
484
|
-
case "message.processed":
|
|
485
|
-
recordMessageProcessed(evt, metadata);
|
|
486
|
-
return;
|
|
487
|
-
case "message.delivery.started":
|
|
488
|
-
recordMessageDeliveryStarted(evt);
|
|
489
|
-
return;
|
|
490
|
-
case "message.delivery.completed":
|
|
491
|
-
recordMessageDeliveryCompleted(evt, metadata);
|
|
492
|
-
return;
|
|
493
|
-
case "message.delivery.error":
|
|
494
|
-
recordMessageDeliveryError(evt, metadata);
|
|
495
|
-
return;
|
|
496
|
-
case "talk.event":
|
|
497
|
-
recordTalkEvent(evt, metadata);
|
|
498
|
-
return;
|
|
499
|
-
case "queue.lane.enqueue":
|
|
500
|
-
recordLaneEnqueue(evt);
|
|
501
|
-
return;
|
|
502
|
-
case "queue.lane.dequeue":
|
|
503
|
-
recordLaneDequeue(evt);
|
|
504
|
-
return;
|
|
505
|
-
case "session.state":
|
|
506
|
-
recordSessionState(evt);
|
|
507
|
-
break;
|
|
440
|
+
case "diagnostic.gc": return recorders.recordGcDuration(evt, metadata);
|
|
441
|
+
case "gateway.event_loop.sample": return recorders.recordGatewayEventLoopSample(evt, metadata);
|
|
442
|
+
case "gateway.rpc": return recorders.recordGatewayRpc(evt, metadata);
|
|
443
|
+
case "model.usage": return recorders.recordModelUsage(evt, metadata, privateData.hostPluginId);
|
|
444
|
+
case "webhook.received": return recorders.recordWebhookReceived(evt);
|
|
445
|
+
case "webhook.processed": return recorders.recordWebhookProcessed(evt);
|
|
446
|
+
case "webhook.error": return recorders.recordWebhookError(evt);
|
|
447
|
+
case "message.queued": return recorders.recordMessageQueued(evt);
|
|
448
|
+
case "message.received": return recorders.recordMessageReceived(evt);
|
|
449
|
+
case "message.dispatch.started": return recorders.recordMessageDispatchStarted(evt, metadata);
|
|
450
|
+
case "message.dispatch.completed": return recorders.recordMessageDispatchCompleted(evt);
|
|
451
|
+
case "message.processed": return recorders.recordMessageProcessed(evt, metadata);
|
|
452
|
+
case "message.delivery.started": return recorders.recordMessageDeliveryStarted(evt);
|
|
453
|
+
case "message.delivery.completed": return recorders.recordMessageDeliveryCompleted(evt, metadata);
|
|
454
|
+
case "message.delivery.error": return recorders.recordMessageDeliveryError(evt, metadata);
|
|
455
|
+
case "talk.event": return recorders.recordTalkEvent(evt, metadata);
|
|
456
|
+
case "queue.lane.enqueue": return recorders.recordLaneEnqueue(evt);
|
|
457
|
+
case "queue.lane.dequeue": return recorders.recordLaneDequeue(evt);
|
|
458
|
+
case "session.state": return recorders.recordSessionState(evt);
|
|
508
459
|
case "session.long_running":
|
|
509
460
|
case "session.stalled": break;
|
|
510
|
-
case "session.turn.created":
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
case "session.
|
|
514
|
-
|
|
515
|
-
return;
|
|
516
|
-
case "session.recovery.requested":
|
|
517
|
-
recordSessionRecoveryRequested(evt);
|
|
518
|
-
return;
|
|
519
|
-
case "session.recovery.completed":
|
|
520
|
-
recordSessionRecoveryCompleted(evt);
|
|
521
|
-
return;
|
|
522
|
-
case "run.attempt":
|
|
523
|
-
recordRunAttempt(evt);
|
|
524
|
-
break;
|
|
461
|
+
case "session.turn.created": return recorders.recordSessionTurnCreated(evt);
|
|
462
|
+
case "session.stuck": return recorders.recordSessionStuck(evt);
|
|
463
|
+
case "session.recovery.requested": return recorders.recordSessionRecoveryRequested(evt);
|
|
464
|
+
case "session.recovery.completed": return recorders.recordSessionRecoveryCompleted(evt);
|
|
465
|
+
case "run.attempt": return recorders.recordRunAttempt(evt);
|
|
525
466
|
case "run.progress": break;
|
|
526
467
|
case "run.execution_phase": break;
|
|
527
|
-
case "diagnostic.heartbeat":
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
case "
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
case "
|
|
534
|
-
recordDiagnosticPhaseCompleted(evt);
|
|
535
|
-
return;
|
|
536
|
-
case "run.started":
|
|
537
|
-
recordRunStarted(evt, metadata);
|
|
538
|
-
return;
|
|
539
|
-
case "run.completed":
|
|
540
|
-
recordRunCompleted(evt, metadata, privateData);
|
|
541
|
-
return;
|
|
542
|
-
case "harness.run.started":
|
|
543
|
-
recordHarnessRunStarted(evt, metadata);
|
|
544
|
-
return;
|
|
468
|
+
case "diagnostic.heartbeat": return recorders.recordHeartbeat(evt);
|
|
469
|
+
case "diagnostic.liveness.warning": return recorders.recordLivenessWarning(evt);
|
|
470
|
+
case "diagnostic.phase.completed": return recorders.recordDiagnosticPhaseCompleted(evt, metadata);
|
|
471
|
+
case "run.started": return recorders.recordRunStarted(evt, metadata);
|
|
472
|
+
case "run.completed": return recorders.recordRunCompleted(evt, metadata, privateData);
|
|
473
|
+
case "harness.run.started": return recorders.recordHarnessRunStarted(evt, metadata);
|
|
474
|
+
case "agent.commentary": return recorders.recordAgentCommentary(evt, metadata, privateData);
|
|
545
475
|
case "harness.run.completed":
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
case "harness.run.error":
|
|
549
|
-
recordHarnessRunError(evt, metadata, privateData);
|
|
550
|
-
return;
|
|
551
|
-
case "context.assembled":
|
|
552
|
-
recordContextAssembled(evt, metadata);
|
|
553
|
-
return;
|
|
476
|
+
case "harness.run.error": return recorders.recordHarnessRunFinished(evt, metadata, privateData);
|
|
477
|
+
case "context.assembled": return recorders.recordContextAssembled(evt, metadata);
|
|
554
478
|
case "model.call.started":
|
|
555
|
-
recordModelCallStarted(evt, metadata);
|
|
479
|
+
recorders.recordModelCallStarted(evt, metadata);
|
|
556
480
|
return;
|
|
557
481
|
case "model.call.completed":
|
|
558
|
-
case "model.call.error":
|
|
559
|
-
recordModelCallFinished(evt, metadata, privateData.modelContent);
|
|
560
|
-
return;
|
|
482
|
+
case "model.call.error": return recorders.recordModelCallFinished(evt, metadata, privateData.modelContent);
|
|
561
483
|
case "tool.execution.started":
|
|
562
|
-
recordToolExecutionStarted(evt, metadata);
|
|
484
|
+
recorders.recordToolExecutionStarted(evt, metadata);
|
|
563
485
|
return;
|
|
564
486
|
case "tool.execution.completed":
|
|
565
|
-
case "tool.execution.error":
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
case "
|
|
569
|
-
recordToolExecutionBlocked(evt, metadata);
|
|
570
|
-
return;
|
|
571
|
-
case "skill.used":
|
|
572
|
-
recordSkillUsed(evt, metadata);
|
|
573
|
-
return;
|
|
574
|
-
case "exec.process.completed":
|
|
575
|
-
recordExecProcessCompleted(evt, metadata);
|
|
576
|
-
break;
|
|
487
|
+
case "tool.execution.error": return recorders.recordToolExecutionFinished(evt, metadata, privateData.toolContent);
|
|
488
|
+
case "tool.execution.blocked": return recorders.recordToolExecutionBlocked(evt, metadata);
|
|
489
|
+
case "skill.used": return recorders.recordSkillUsed(evt, metadata);
|
|
490
|
+
case "exec.process.completed": return recorders.recordExecProcessCompleted(evt, metadata);
|
|
577
491
|
case "exec.approval.followup_suppressed": break;
|
|
578
|
-
case "log.record":
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
case "
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
case "
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
case "diagnostic.memory.sample":
|
|
588
|
-
recordMemorySample(evt);
|
|
589
|
-
return;
|
|
590
|
-
case "diagnostic.memory.pressure":
|
|
591
|
-
recordMemoryPressure(evt);
|
|
592
|
-
return;
|
|
593
|
-
case "diagnostic.async_queue.dropped":
|
|
594
|
-
recordAsyncQueueDropped(evt);
|
|
595
|
-
return;
|
|
596
|
-
case "telemetry.exporter":
|
|
597
|
-
recordTelemetryExporter(evt, metadata);
|
|
598
|
-
return;
|
|
599
|
-
case "payload.large":
|
|
600
|
-
recordPayloadLarge(evt);
|
|
601
|
-
return;
|
|
602
|
-
case "model.failover": recordModelFailover(evt, metadata);
|
|
492
|
+
case "log.record": return recordLogRecord?.(evt, metadata);
|
|
493
|
+
case "security.event": return recordSecurityEvent?.(evt, metadata);
|
|
494
|
+
case "tool.loop": return recorders.recordToolLoop(evt);
|
|
495
|
+
case "diagnostic.memory.sample": return recorders.recordMemoryUsageMetrics(evt);
|
|
496
|
+
case "diagnostic.memory.pressure": return recorders.recordMemoryPressure(evt);
|
|
497
|
+
case "diagnostic.async_queue.dropped": return recorders.recordAsyncQueueDropped(evt);
|
|
498
|
+
case "telemetry.exporter": return recorders.recordTelemetryExporter(evt, metadata);
|
|
499
|
+
case "payload.large": return recorders.recordPayloadLarge(evt);
|
|
500
|
+
case "model.failover": return recorders.recordModelFailover(evt, metadata);
|
|
603
501
|
}
|
|
604
502
|
} catch (err) {
|
|
605
503
|
logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
|
|
@@ -756,9 +654,6 @@ function securityTargetNameAttr(value, fallback = "unknown") {
|
|
|
756
654
|
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
757
655
|
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
758
656
|
}
|
|
759
|
-
function shouldCaptureOtelLogBody(policy) {
|
|
760
|
-
return policy.logBodies;
|
|
761
|
-
}
|
|
762
657
|
function otelLogTimestampIso(timestamp) {
|
|
763
658
|
if (timestamp instanceof Date) return timestamp.toISOString();
|
|
764
659
|
if (typeof timestamp === "number" && Number.isFinite(timestamp)) return new Date(timestamp).toISOString();
|
|
@@ -913,7 +808,6 @@ const LOG_SEVERITY_MAP = {
|
|
|
913
808
|
function createDiagnosticsLogExporter(params) {
|
|
914
809
|
const { contentCapturePolicy, emitExporterEvent, flushIntervalMs, headers, logger, logsEnabled, logsToOtlp, logsToStdout, logHttpAgentOptions, logUrl, resource, serviceName } = params;
|
|
915
810
|
let logProvider = null;
|
|
916
|
-
const logSeverityMap = LOG_SEVERITY_MAP;
|
|
917
811
|
let recordLogRecord;
|
|
918
812
|
let recordSecurityEvent;
|
|
919
813
|
if (logsEnabled) {
|
|
@@ -986,8 +880,8 @@ function createDiagnosticsLogExporter(params) {
|
|
|
986
880
|
};
|
|
987
881
|
const buildDiagnosticLogRecord = (evt, metadata) => {
|
|
988
882
|
const logLevelName = evt.level || "INFO";
|
|
989
|
-
const severityNumber =
|
|
990
|
-
const body =
|
|
883
|
+
const severityNumber = LOG_SEVERITY_MAP[logLevelName] ?? 9;
|
|
884
|
+
const body = contentCapturePolicy.logBodies ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
991
885
|
const attributes = Object.create(null);
|
|
992
886
|
assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName);
|
|
993
887
|
if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
|
|
@@ -1019,7 +913,7 @@ function createDiagnosticsLogExporter(params) {
|
|
|
1019
913
|
const logRecord = {
|
|
1020
914
|
body: "openclaw.security.event",
|
|
1021
915
|
severityText,
|
|
1022
|
-
severityNumber:
|
|
916
|
+
severityNumber: LOG_SEVERITY_MAP[severityText] ?? 9,
|
|
1023
917
|
attributes: redactOtelAttributes(attributes),
|
|
1024
918
|
timestamp: evt.ts
|
|
1025
919
|
};
|
|
@@ -1438,247 +1332,6 @@ function createDiagnosticsRecorderRuntime(params) {
|
|
|
1438
1332
|
};
|
|
1439
1333
|
}
|
|
1440
1334
|
//#endregion
|
|
1441
|
-
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1442
|
-
function createHarnessRecorders(runtime) {
|
|
1443
|
-
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1444
|
-
const harnessRunMetricAttrs = (evt) => ({
|
|
1445
|
-
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1446
|
-
"openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId),
|
|
1447
|
-
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1448
|
-
"openclaw.provider": normalizeDiagnosticValue(evt.provider, "unknown"),
|
|
1449
|
-
"openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"),
|
|
1450
|
-
...evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}
|
|
1451
|
-
});
|
|
1452
|
-
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1453
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1454
|
-
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", harnessRunMetricAttrs(evt), void 0, {
|
|
1455
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1456
|
-
startTimeMs: evt.ts
|
|
1457
|
-
}));
|
|
1458
|
-
};
|
|
1459
|
-
const recordHarnessRunCompleted = (evt, metadata, privateData) => {
|
|
1460
|
-
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
|
|
1461
|
-
if (!tracesEnabled) return;
|
|
1462
|
-
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1463
|
-
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue(evt.resultClassification);
|
|
1464
|
-
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1465
|
-
if (evt.itemLifecycle) {
|
|
1466
|
-
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1467
|
-
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1468
|
-
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1469
|
-
}
|
|
1470
|
-
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1471
|
-
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1472
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1473
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1474
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1475
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1476
|
-
endTimeMs: evt.ts
|
|
1477
|
-
});
|
|
1478
|
-
setSpanAttrs(span, spanAttrs);
|
|
1479
|
-
if (evt.outcome === "error") span.setStatus({
|
|
1480
|
-
code: SpanStatusCode.ERROR,
|
|
1481
|
-
message: redactedError ?? "error"
|
|
1482
|
-
});
|
|
1483
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1484
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1485
|
-
return;
|
|
1486
|
-
}
|
|
1487
|
-
span.end(evt.ts);
|
|
1488
|
-
};
|
|
1489
|
-
const recordHarnessRunError = (evt, metadata, privateData) => {
|
|
1490
|
-
const errorType = normalizeDiagnosticValue(evt.errorCategory, "other");
|
|
1491
|
-
const attrs = {
|
|
1492
|
-
...harnessRunMetricAttrs(evt),
|
|
1493
|
-
"openclaw.harness.phase": evt.phase,
|
|
1494
|
-
"openclaw.errorCategory": errorType
|
|
1495
|
-
};
|
|
1496
|
-
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1497
|
-
if (!tracesEnabled) return;
|
|
1498
|
-
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1499
|
-
const spanAttrs = {
|
|
1500
|
-
...attrs,
|
|
1501
|
-
"error.type": errorType,
|
|
1502
|
-
...redactedError ? { "openclaw.error": redactedError } : {},
|
|
1503
|
-
...evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}
|
|
1504
|
-
};
|
|
1505
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1506
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1507
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1508
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1509
|
-
endTimeMs: evt.ts
|
|
1510
|
-
});
|
|
1511
|
-
setSpanAttrs(span, spanAttrs);
|
|
1512
|
-
span.setStatus({
|
|
1513
|
-
code: SpanStatusCode.ERROR,
|
|
1514
|
-
message: redactedError ?? errorType
|
|
1515
|
-
});
|
|
1516
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1517
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1518
|
-
return;
|
|
1519
|
-
}
|
|
1520
|
-
span.end(evt.ts);
|
|
1521
|
-
};
|
|
1522
|
-
const recordContextAssembled = (evt, metadata) => {
|
|
1523
|
-
if (!tracesEnabled) return;
|
|
1524
|
-
const spanAttrs = {
|
|
1525
|
-
"openclaw.context.message_count": evt.messageCount,
|
|
1526
|
-
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1527
|
-
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1528
|
-
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1529
|
-
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1530
|
-
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1531
|
-
"openclaw.context.prompt_images": evt.promptImages
|
|
1532
|
-
};
|
|
1533
|
-
addRunAttrs(spanAttrs, evt);
|
|
1534
|
-
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1535
|
-
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1536
|
-
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1537
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1538
|
-
endTimeMs: evt.ts
|
|
1539
|
-
}).end(evt.ts);
|
|
1540
|
-
};
|
|
1541
|
-
const recordModelFailover = (evt, metadata) => {
|
|
1542
|
-
const metricAttrs = {
|
|
1543
|
-
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1544
|
-
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1545
|
-
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1546
|
-
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1547
|
-
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1548
|
-
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1549
|
-
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1550
|
-
};
|
|
1551
|
-
modelFailoverCounter.add(1, metricAttrs);
|
|
1552
|
-
if (!tracesEnabled) return;
|
|
1553
|
-
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1554
|
-
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1555
|
-
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1556
|
-
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1557
|
-
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1558
|
-
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1559
|
-
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1560
|
-
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1561
|
-
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1562
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1563
|
-
endTimeMs: evt.ts
|
|
1564
|
-
}).end(evt.ts);
|
|
1565
|
-
};
|
|
1566
|
-
return {
|
|
1567
|
-
recordHarnessRunStarted,
|
|
1568
|
-
recordHarnessRunCompleted,
|
|
1569
|
-
recordHarnessRunError,
|
|
1570
|
-
recordContextAssembled,
|
|
1571
|
-
recordModelFailover
|
|
1572
|
-
};
|
|
1573
|
-
}
|
|
1574
|
-
//#endregion
|
|
1575
|
-
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1576
|
-
function hasOtelSemconvOptIn(value, optIn) {
|
|
1577
|
-
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1578
|
-
}
|
|
1579
|
-
function emitLatestGenAiSemconv() {
|
|
1580
|
-
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
1581
|
-
}
|
|
1582
|
-
function genAiOperationName(api, observationUnit) {
|
|
1583
|
-
if (observationUnit === "turn") return GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT;
|
|
1584
|
-
const normalized = api?.trim().toLowerCase();
|
|
1585
|
-
if (!normalized) return "chat";
|
|
1586
|
-
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
1587
|
-
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
1588
|
-
return "chat";
|
|
1589
|
-
}
|
|
1590
|
-
function positiveFiniteNumber(value) {
|
|
1591
|
-
return asFiniteNumberInRange(value, {
|
|
1592
|
-
min: 0,
|
|
1593
|
-
minExclusive: true
|
|
1594
|
-
});
|
|
1595
|
-
}
|
|
1596
|
-
function nonNegativeFiniteNumber(value) {
|
|
1597
|
-
return asFiniteNumberInRange(value, { min: 0 });
|
|
1598
|
-
}
|
|
1599
|
-
function assignPositiveNumberAttr(attrs, key, value) {
|
|
1600
|
-
const normalized = positiveFiniteNumber(value);
|
|
1601
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1602
|
-
}
|
|
1603
|
-
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1604
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1605
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1606
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1607
|
-
}
|
|
1608
|
-
function assignNumberAttr(attrs, key, value) {
|
|
1609
|
-
const normalized = asFiniteNumber(value);
|
|
1610
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1611
|
-
}
|
|
1612
|
-
function modelCallPromptTokens(usage) {
|
|
1613
|
-
const promptTokens = nonNegativeFiniteNumber(usage.promptTokens);
|
|
1614
|
-
if (promptTokens !== void 0) return promptTokens;
|
|
1615
|
-
const input = nonNegativeFiniteNumber(usage.input);
|
|
1616
|
-
const cacheRead = nonNegativeFiniteNumber(usage.cacheRead);
|
|
1617
|
-
const cacheWrite = nonNegativeFiniteNumber(usage.cacheWrite);
|
|
1618
|
-
if (input === void 0 && cacheRead === void 0 && cacheWrite === void 0) return;
|
|
1619
|
-
return (input ?? 0) + (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
1620
|
-
}
|
|
1621
|
-
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
1622
|
-
const stats = evt.promptStats;
|
|
1623
|
-
if (!stats) return;
|
|
1624
|
-
for (const [key, value] of [
|
|
1625
|
-
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
1626
|
-
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
1627
|
-
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
1628
|
-
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1629
|
-
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1630
|
-
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1631
|
-
]) assignNumberAttr(attrs, key, value);
|
|
1632
|
-
}
|
|
1633
|
-
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1634
|
-
const usage = evt.usage;
|
|
1635
|
-
if (!usage) return;
|
|
1636
|
-
const promptTokens = modelCallPromptTokens(usage);
|
|
1637
|
-
for (const [key, value] of [
|
|
1638
|
-
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
1639
|
-
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
1640
|
-
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
1641
|
-
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
1642
|
-
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
1643
|
-
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
1644
|
-
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
1645
|
-
["gen_ai.usage.input_tokens", promptTokens],
|
|
1646
|
-
["gen_ai.usage.output_tokens", usage.output],
|
|
1647
|
-
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
1648
|
-
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
1649
|
-
]) {
|
|
1650
|
-
const normalized = nonNegativeFiniteNumber(value);
|
|
1651
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1652
|
-
}
|
|
1653
|
-
}
|
|
1654
|
-
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
1655
|
-
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = normalizeDiagnosticValue(input.provider);
|
|
1656
|
-
else attrs["gen_ai.system"] = normalizeDiagnosticValue(input.provider);
|
|
1657
|
-
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
1658
|
-
attrs["gen_ai.operation.name"] = genAiOperationName(input.api, input.observationUnit);
|
|
1659
|
-
}
|
|
1660
|
-
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1661
|
-
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1662
|
-
attrs["openclaw.model_call.observation_unit"] = modelCallObservationUnit(evt);
|
|
1663
|
-
}
|
|
1664
|
-
function modelCallObservationUnit(evt) {
|
|
1665
|
-
return evt.observationUnit ?? "request";
|
|
1666
|
-
}
|
|
1667
|
-
function modelCallSpanName(evt) {
|
|
1668
|
-
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
1669
|
-
const operationName = genAiOperationName(evt.api, evt.observationUnit);
|
|
1670
|
-
return operationName === GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT ? operationName : `${operationName} ${normalizeDiagnosticValue(evt.model)}`;
|
|
1671
|
-
}
|
|
1672
|
-
function modelCallSpanKind() {
|
|
1673
|
-
return SpanKind.CLIENT;
|
|
1674
|
-
}
|
|
1675
|
-
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
1676
|
-
if (!upstreamRequestIdHash) return;
|
|
1677
|
-
const boundedHash = normalizeDiagnosticValue(upstreamRequestIdHash);
|
|
1678
|
-
if (boundedHash === "unknown") return;
|
|
1679
|
-
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
1680
|
-
}
|
|
1681
|
-
//#endregion
|
|
1682
1335
|
//#region extensions/diagnostics-otel/src/service-genai-content.ts
|
|
1683
1336
|
function textPart(content) {
|
|
1684
1337
|
return {
|
|
@@ -1906,9 +1559,234 @@ function assignOtelToolContentAttributes(attributes, content, policy) {
|
|
|
1906
1559
|
}
|
|
1907
1560
|
}
|
|
1908
1561
|
//#endregion
|
|
1562
|
+
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1563
|
+
function createHarnessRecorders(runtime) {
|
|
1564
|
+
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled, getTrackedInternalOrTrustedSpan, contentCapturePolicy } = runtime;
|
|
1565
|
+
const recordAgentCommentary = (evt, metadata, privateData) => {
|
|
1566
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1567
|
+
const span = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1568
|
+
if (!span) return;
|
|
1569
|
+
const attrs = {
|
|
1570
|
+
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1571
|
+
"openclaw.commentary.sequence": evt.sourceSequence,
|
|
1572
|
+
"openclaw.commentary.text_length": evt.textLength,
|
|
1573
|
+
"openclaw.commentary.content_truncated": evt.contentTruncated
|
|
1574
|
+
};
|
|
1575
|
+
assignOtelModelContentAttributes(attrs, privateData.modelContent, contentCapturePolicy);
|
|
1576
|
+
span.addEvent("openclaw.agent.commentary", redactOtelAttributes(attrs), evt.sourceTimestampMs);
|
|
1577
|
+
};
|
|
1578
|
+
const harnessRunMetricAttrs = (evt) => ({
|
|
1579
|
+
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1580
|
+
"openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId),
|
|
1581
|
+
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1582
|
+
"openclaw.provider": normalizeDiagnosticValue(evt.provider, "unknown"),
|
|
1583
|
+
"openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"),
|
|
1584
|
+
...evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}
|
|
1585
|
+
});
|
|
1586
|
+
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1587
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1588
|
+
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1589
|
+
addRunAttrs(spanAttrs, evt);
|
|
1590
|
+
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", spanAttrs, void 0, {
|
|
1591
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1592
|
+
startTimeMs: evt.ts
|
|
1593
|
+
}));
|
|
1594
|
+
};
|
|
1595
|
+
const recordHarnessRunFinished = (evt, metadata, privateData) => {
|
|
1596
|
+
const errorType = evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "error";
|
|
1597
|
+
const attrs = {
|
|
1598
|
+
...harnessRunMetricAttrs(evt),
|
|
1599
|
+
...evt.type === "harness.run.error" ? {
|
|
1600
|
+
"openclaw.harness.phase": evt.phase,
|
|
1601
|
+
"openclaw.errorCategory": errorType
|
|
1602
|
+
} : {}
|
|
1603
|
+
};
|
|
1604
|
+
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1605
|
+
if (!tracesEnabled) return;
|
|
1606
|
+
const spanAttrs = { ...attrs };
|
|
1607
|
+
addRunAttrs(spanAttrs, evt);
|
|
1608
|
+
if (evt.type === "harness.run.completed") {
|
|
1609
|
+
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue(evt.resultClassification);
|
|
1610
|
+
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1611
|
+
if (evt.itemLifecycle) {
|
|
1612
|
+
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1613
|
+
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1614
|
+
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1615
|
+
}
|
|
1616
|
+
} else {
|
|
1617
|
+
spanAttrs["error.type"] = errorType;
|
|
1618
|
+
if (evt.cleanupFailed) spanAttrs["openclaw.harness.cleanup_failed"] = true;
|
|
1619
|
+
}
|
|
1620
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1621
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1622
|
+
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1623
|
+
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1624
|
+
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1625
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1626
|
+
endTimeMs: evt.ts
|
|
1627
|
+
});
|
|
1628
|
+
setSpanAttrs(span, spanAttrs);
|
|
1629
|
+
if (evt.type === "harness.run.error" || evt.outcome === "error") span.setStatus({
|
|
1630
|
+
code: SpanStatusCode.ERROR,
|
|
1631
|
+
message: redactedError ?? errorType
|
|
1632
|
+
});
|
|
1633
|
+
if (trackedSpan && trustedTrace?.spanId) {
|
|
1634
|
+
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1635
|
+
return;
|
|
1636
|
+
}
|
|
1637
|
+
span.end(evt.ts);
|
|
1638
|
+
};
|
|
1639
|
+
const recordContextAssembled = (evt, metadata) => {
|
|
1640
|
+
if (!tracesEnabled) return;
|
|
1641
|
+
const spanAttrs = {
|
|
1642
|
+
"openclaw.context.message_count": evt.messageCount,
|
|
1643
|
+
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1644
|
+
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1645
|
+
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1646
|
+
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1647
|
+
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1648
|
+
"openclaw.context.prompt_images": evt.promptImages
|
|
1649
|
+
};
|
|
1650
|
+
addRunAttrs(spanAttrs, evt);
|
|
1651
|
+
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1652
|
+
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1653
|
+
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1654
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1655
|
+
endTimeMs: evt.ts
|
|
1656
|
+
}).end(evt.ts);
|
|
1657
|
+
};
|
|
1658
|
+
const recordModelFailover = (evt, metadata) => {
|
|
1659
|
+
const metricAttrs = {
|
|
1660
|
+
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1661
|
+
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1662
|
+
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1663
|
+
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1664
|
+
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1665
|
+
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1666
|
+
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1667
|
+
};
|
|
1668
|
+
modelFailoverCounter.add(1, metricAttrs);
|
|
1669
|
+
if (!tracesEnabled) return;
|
|
1670
|
+
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1671
|
+
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1672
|
+
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1673
|
+
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1674
|
+
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1675
|
+
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1676
|
+
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1677
|
+
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1678
|
+
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1679
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1680
|
+
endTimeMs: evt.ts
|
|
1681
|
+
}).end(evt.ts);
|
|
1682
|
+
};
|
|
1683
|
+
return {
|
|
1684
|
+
recordAgentCommentary,
|
|
1685
|
+
recordHarnessRunStarted,
|
|
1686
|
+
recordHarnessRunFinished,
|
|
1687
|
+
recordContextAssembled,
|
|
1688
|
+
recordModelFailover
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1691
|
+
//#endregion
|
|
1692
|
+
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1693
|
+
function hasOtelSemconvOptIn(value, optIn) {
|
|
1694
|
+
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1695
|
+
}
|
|
1696
|
+
function emitLatestGenAiSemconv() {
|
|
1697
|
+
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
1698
|
+
}
|
|
1699
|
+
function genAiOperationName(api, observationUnit) {
|
|
1700
|
+
if (observationUnit === "turn") return GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT;
|
|
1701
|
+
const normalized = api?.trim().toLowerCase();
|
|
1702
|
+
if (!normalized) return "chat";
|
|
1703
|
+
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
1704
|
+
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
1705
|
+
return "chat";
|
|
1706
|
+
}
|
|
1707
|
+
function assignPositiveNumberAttr(attrs, key, value) {
|
|
1708
|
+
const normalized = asPositiveFiniteNumber(value);
|
|
1709
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1710
|
+
}
|
|
1711
|
+
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1712
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1713
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1714
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1715
|
+
}
|
|
1716
|
+
function assignNumberAttr(attrs, key, value) {
|
|
1717
|
+
const normalized = asFiniteNumber(value);
|
|
1718
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1719
|
+
}
|
|
1720
|
+
function modelCallPromptTokens(usage) {
|
|
1721
|
+
const promptTokens = asNonNegativeFiniteNumber(usage.promptTokens);
|
|
1722
|
+
if (promptTokens !== void 0) return promptTokens;
|
|
1723
|
+
const input = asNonNegativeFiniteNumber(usage.input);
|
|
1724
|
+
const cacheRead = asNonNegativeFiniteNumber(usage.cacheRead);
|
|
1725
|
+
const cacheWrite = asNonNegativeFiniteNumber(usage.cacheWrite);
|
|
1726
|
+
if (input === void 0 && cacheRead === void 0 && cacheWrite === void 0) return;
|
|
1727
|
+
return (input ?? 0) + (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
1728
|
+
}
|
|
1729
|
+
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
1730
|
+
const stats = evt.promptStats;
|
|
1731
|
+
if (!stats) return;
|
|
1732
|
+
for (const [key, value] of [
|
|
1733
|
+
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
1734
|
+
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
1735
|
+
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
1736
|
+
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1737
|
+
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1738
|
+
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1739
|
+
]) assignNumberAttr(attrs, key, value);
|
|
1740
|
+
}
|
|
1741
|
+
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1742
|
+
const usage = evt.usage;
|
|
1743
|
+
if (!usage) return;
|
|
1744
|
+
const promptTokens = modelCallPromptTokens(usage);
|
|
1745
|
+
for (const [key, value] of [
|
|
1746
|
+
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
1747
|
+
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
1748
|
+
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
1749
|
+
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
1750
|
+
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
1751
|
+
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
1752
|
+
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
1753
|
+
["gen_ai.usage.input_tokens", promptTokens],
|
|
1754
|
+
["gen_ai.usage.output_tokens", usage.output],
|
|
1755
|
+
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
1756
|
+
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
1757
|
+
]) {
|
|
1758
|
+
const normalized = asNonNegativeFiniteNumber(value);
|
|
1759
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
1763
|
+
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = normalizeDiagnosticValue(input.provider);
|
|
1764
|
+
else attrs["gen_ai.system"] = normalizeDiagnosticValue(input.provider);
|
|
1765
|
+
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
1766
|
+
attrs["gen_ai.operation.name"] = genAiOperationName(input.api, input.observationUnit);
|
|
1767
|
+
}
|
|
1768
|
+
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1769
|
+
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1770
|
+
attrs["openclaw.model_call.observation_unit"] = modelCallObservationUnit(evt);
|
|
1771
|
+
}
|
|
1772
|
+
function modelCallObservationUnit(evt) {
|
|
1773
|
+
return evt.observationUnit ?? "request";
|
|
1774
|
+
}
|
|
1775
|
+
function modelCallSpanName(evt) {
|
|
1776
|
+
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
1777
|
+
const operationName = genAiOperationName(evt.api, evt.observationUnit);
|
|
1778
|
+
return operationName === GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT ? operationName : `${operationName} ${normalizeDiagnosticValue(evt.model)}`;
|
|
1779
|
+
}
|
|
1780
|
+
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
1781
|
+
if (!upstreamRequestIdHash) return;
|
|
1782
|
+
const boundedHash = normalizeDiagnosticValue(upstreamRequestIdHash);
|
|
1783
|
+
if (boundedHash === "unknown") return;
|
|
1784
|
+
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
1785
|
+
}
|
|
1786
|
+
//#endregion
|
|
1909
1787
|
//#region extensions/diagnostics-otel/src/service-recorders-model.ts
|
|
1910
1788
|
function createModelRecorders(runtime) {
|
|
1911
|
-
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1789
|
+
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1912
1790
|
const modelCallMetricAttrs = (evt) => ({
|
|
1913
1791
|
"openclaw.provider": evt.provider,
|
|
1914
1792
|
"openclaw.model": evt.model,
|
|
@@ -1917,11 +1795,11 @@ function createModelRecorders(runtime) {
|
|
|
1917
1795
|
"openclaw.model_call.observation_unit": modelCallObservationUnit(evt)
|
|
1918
1796
|
});
|
|
1919
1797
|
const recordModelCallSizeTimingMetrics = (evt, attrs) => {
|
|
1920
|
-
const requestPayloadBytes =
|
|
1798
|
+
const requestPayloadBytes = asPositiveFiniteNumber(evt.requestPayloadBytes);
|
|
1921
1799
|
if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
|
|
1922
|
-
const responseStreamBytes =
|
|
1800
|
+
const responseStreamBytes = asPositiveFiniteNumber(evt.responseStreamBytes);
|
|
1923
1801
|
if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
|
|
1924
|
-
const timeToFirstByteMs =
|
|
1802
|
+
const timeToFirstByteMs = asPositiveFiniteNumber(evt.timeToFirstByteMs);
|
|
1925
1803
|
if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
|
|
1926
1804
|
};
|
|
1927
1805
|
const recordModelCallStarted = (evt, metadata) => {
|
|
@@ -1932,12 +1810,13 @@ function createModelRecorders(runtime) {
|
|
|
1932
1810
|
"openclaw.provider": evt.provider,
|
|
1933
1811
|
"openclaw.model": evt.model
|
|
1934
1812
|
};
|
|
1813
|
+
addRunAttrs(spanAttrs, evt);
|
|
1935
1814
|
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1936
1815
|
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1937
1816
|
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1938
1817
|
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1939
1818
|
return trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1940
|
-
kind:
|
|
1819
|
+
kind: SpanKind.CLIENT,
|
|
1941
1820
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1942
1821
|
startTimeMs: evt.ts
|
|
1943
1822
|
})).spanContext();
|
|
@@ -1966,6 +1845,7 @@ function createModelRecorders(runtime) {
|
|
|
1966
1845
|
"error.type": errorType
|
|
1967
1846
|
} : {}
|
|
1968
1847
|
};
|
|
1848
|
+
addRunAttrs(spanAttrs, evt);
|
|
1969
1849
|
if (evt.type === "model.call.error" && evt.failureKind) spanAttrs["openclaw.failureKind"] = normalizeDiagnosticValue(evt.failureKind, "other");
|
|
1970
1850
|
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1971
1851
|
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
@@ -1975,7 +1855,7 @@ function createModelRecorders(runtime) {
|
|
|
1975
1855
|
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1976
1856
|
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
1977
1857
|
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1978
|
-
kind:
|
|
1858
|
+
kind: SpanKind.CLIENT,
|
|
1979
1859
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1980
1860
|
endTimeMs: evt.ts
|
|
1981
1861
|
});
|
|
@@ -2118,7 +1998,9 @@ function createOperationsRecorders(runtime) {
|
|
|
2118
1998
|
const attrs = toolLoopAttrs(evt);
|
|
2119
1999
|
toolLoopCounter.add(1, attrs);
|
|
2120
2000
|
if (!tracesEnabled) return;
|
|
2121
|
-
const
|
|
2001
|
+
const spanAttrs = { ...attrs };
|
|
2002
|
+
addRunAttrs(spanAttrs, evt);
|
|
2003
|
+
const span = spanWithDuration("openclaw.tool.loop", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
2122
2004
|
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
2123
2005
|
code: SpanStatusCode.ERROR,
|
|
2124
2006
|
message: `${evt.detector}:${evt.action}`
|
|
@@ -2132,9 +2014,6 @@ function createOperationsRecorders(runtime) {
|
|
|
2132
2014
|
memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
2133
2015
|
memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
2134
2016
|
};
|
|
2135
|
-
const recordMemorySample = (evt) => {
|
|
2136
|
-
recordMemoryUsageMetrics(evt);
|
|
2137
|
-
};
|
|
2138
2017
|
const recordMemoryPressure = (evt) => {
|
|
2139
2018
|
const attrs = {
|
|
2140
2019
|
"openclaw.memory.level": evt.level,
|
|
@@ -2163,9 +2042,11 @@ function createOperationsRecorders(runtime) {
|
|
|
2163
2042
|
};
|
|
2164
2043
|
const recordAsyncQueueDropped = (evt) => {
|
|
2165
2044
|
asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2045
|
+
for (const [dropClass, field] of [
|
|
2046
|
+
["trusted", "droppedTrustedEvents"],
|
|
2047
|
+
["untrusted", "droppedUntrustedEvents"],
|
|
2048
|
+
["priority", "droppedPriorityEvents"]
|
|
2049
|
+
]) if (evt[field] !== void 0) asyncQueueDroppedCounter.add(evt[field], { "openclaw.diagnostic.async_queue.drop_class": dropClass });
|
|
2169
2050
|
};
|
|
2170
2051
|
const recordRunCompleted = (evt, metadata, privateData) => {
|
|
2171
2052
|
const attrs = {
|
|
@@ -2216,7 +2097,6 @@ function createOperationsRecorders(runtime) {
|
|
|
2216
2097
|
recordRunAttempt,
|
|
2217
2098
|
recordToolLoop,
|
|
2218
2099
|
recordMemoryUsageMetrics,
|
|
2219
|
-
recordMemorySample,
|
|
2220
2100
|
recordMemoryPressure,
|
|
2221
2101
|
recordAsyncQueueDropped,
|
|
2222
2102
|
recordRunCompleted
|
|
@@ -2225,7 +2105,7 @@ function createOperationsRecorders(runtime) {
|
|
|
2225
2105
|
//#endregion
|
|
2226
2106
|
//#region extensions/diagnostics-otel/src/service-recorders-tools.ts
|
|
2227
2107
|
function createToolAndSystemRecorders(runtime) {
|
|
2228
|
-
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
2108
|
+
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, internalOrTrustedExplicitParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
2229
2109
|
const toolExecutionBaseAttrs = (evt) => ({
|
|
2230
2110
|
"openclaw.toolName": evt.toolName,
|
|
2231
2111
|
"openclaw.tool.source": normalizeDiagnosticValue(evt.toolSource, "core"),
|
|
@@ -2316,7 +2196,7 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2316
2196
|
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none")
|
|
2317
2197
|
};
|
|
2318
2198
|
payloadLargeCounter.add(1, attrs);
|
|
2319
|
-
const bytes =
|
|
2199
|
+
const bytes = asPositiveFiniteNumber(evt.bytes);
|
|
2320
2200
|
if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
|
|
2321
2201
|
};
|
|
2322
2202
|
const recordExecProcessCompleted = (evt, metadata) => {
|
|
@@ -2388,7 +2268,7 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2388
2268
|
});
|
|
2389
2269
|
span.end(evt.ts);
|
|
2390
2270
|
};
|
|
2391
|
-
const recordDiagnosticPhaseCompleted = (evt) => {
|
|
2271
|
+
const recordDiagnosticPhaseCompleted = (evt, metadata) => {
|
|
2392
2272
|
if (!tracesEnabled) return;
|
|
2393
2273
|
const spanAttrs = {
|
|
2394
2274
|
"openclaw.phase": normalizeDiagnosticValue(evt.name, "unknown"),
|
|
@@ -2398,7 +2278,10 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2398
2278
|
...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2399
2279
|
};
|
|
2400
2280
|
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
2401
|
-
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
2281
|
+
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
2282
|
+
endTimeMs: evt.ts,
|
|
2283
|
+
...metadata.trusted ? { parentContext: internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT } : {}
|
|
2284
|
+
}).end(evt.ts);
|
|
2402
2285
|
};
|
|
2403
2286
|
const recordTelemetryExporter = (evt, metadata) => {
|
|
2404
2287
|
if (!metadata.trusted) return;
|
|
@@ -2442,52 +2325,34 @@ function createUsageRecorders(runtime) {
|
|
|
2442
2325
|
"gen_ai.request.model": normalizeDiagnosticValue(evt.model)
|
|
2443
2326
|
};
|
|
2444
2327
|
const usage = evt.usage;
|
|
2445
|
-
|
|
2446
|
-
|
|
2328
|
+
for (const [tokenType, field] of [
|
|
2329
|
+
["input", "input"],
|
|
2330
|
+
["output", "output"],
|
|
2331
|
+
["cache_read", "cacheRead"],
|
|
2332
|
+
["cache_write", "cacheWrite"],
|
|
2333
|
+
["prompt", "promptTokens"],
|
|
2334
|
+
["total", "total"]
|
|
2335
|
+
]) {
|
|
2336
|
+
const amount = usage[field];
|
|
2337
|
+
if (!amount) continue;
|
|
2338
|
+
tokensCounter.add(amount, {
|
|
2447
2339
|
...attrs,
|
|
2448
|
-
"openclaw.token":
|
|
2340
|
+
"openclaw.token": tokenType
|
|
2449
2341
|
});
|
|
2450
|
-
genAiTokenUsageHistogram.record(
|
|
2342
|
+
if (tokenType === "input" || tokenType === "output") genAiTokenUsageHistogram.record(amount, {
|
|
2451
2343
|
...genAiAttrs,
|
|
2452
|
-
"gen_ai.token.type":
|
|
2344
|
+
"gen_ai.token.type": tokenType
|
|
2453
2345
|
});
|
|
2454
2346
|
}
|
|
2455
|
-
if (
|
|
2456
|
-
|
|
2347
|
+
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
2348
|
+
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
2349
|
+
for (const kind of ["limit", "used"]) {
|
|
2350
|
+
const amount = evt.context?.[kind];
|
|
2351
|
+
if (amount) contextHistogram.record(amount, {
|
|
2457
2352
|
...attrs,
|
|
2458
|
-
"openclaw.
|
|
2459
|
-
});
|
|
2460
|
-
genAiTokenUsageHistogram.record(usage.output, {
|
|
2461
|
-
...genAiAttrs,
|
|
2462
|
-
"gen_ai.token.type": "output"
|
|
2353
|
+
"openclaw.context": kind
|
|
2463
2354
|
});
|
|
2464
2355
|
}
|
|
2465
|
-
if (usage.cacheRead) tokensCounter.add(usage.cacheRead, {
|
|
2466
|
-
...attrs,
|
|
2467
|
-
"openclaw.token": "cache_read"
|
|
2468
|
-
});
|
|
2469
|
-
if (usage.cacheWrite) tokensCounter.add(usage.cacheWrite, {
|
|
2470
|
-
...attrs,
|
|
2471
|
-
"openclaw.token": "cache_write"
|
|
2472
|
-
});
|
|
2473
|
-
if (usage.promptTokens) tokensCounter.add(usage.promptTokens, {
|
|
2474
|
-
...attrs,
|
|
2475
|
-
"openclaw.token": "prompt"
|
|
2476
|
-
});
|
|
2477
|
-
if (usage.total) tokensCounter.add(usage.total, {
|
|
2478
|
-
...attrs,
|
|
2479
|
-
"openclaw.token": "total"
|
|
2480
|
-
});
|
|
2481
|
-
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
2482
|
-
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
2483
|
-
if (evt.context?.limit) contextHistogram.record(evt.context.limit, {
|
|
2484
|
-
...attrs,
|
|
2485
|
-
"openclaw.context": "limit"
|
|
2486
|
-
});
|
|
2487
|
-
if (evt.context?.used) contextHistogram.record(evt.context.used, {
|
|
2488
|
-
...attrs,
|
|
2489
|
-
"openclaw.context": "used"
|
|
2490
|
-
});
|
|
2491
2356
|
if (!tracesEnabled) return;
|
|
2492
2357
|
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
2493
2358
|
const spanAttrs = {
|
|
@@ -2500,6 +2365,7 @@ function createUsageRecorders(runtime) {
|
|
|
2500
2365
|
};
|
|
2501
2366
|
if (metadata.trusted && metadata.internal && hostPluginId) spanAttrs["openclaw.plugin"] = normalizeDiagnosticValue(hostPluginId);
|
|
2502
2367
|
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
2368
|
+
addRunAttrs(spanAttrs, evt);
|
|
2503
2369
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
2504
2370
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
2505
2371
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
@@ -2592,6 +2458,7 @@ function createUsageRecorders(runtime) {
|
|
|
2592
2458
|
if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
|
|
2593
2459
|
if (!tracesEnabled) return;
|
|
2594
2460
|
const spanAttrs = { ...attrs };
|
|
2461
|
+
addRunAttrs(spanAttrs, evt);
|
|
2595
2462
|
if (evt.reason) spanAttrs["openclaw.reason"] = normalizeDiagnosticValue(evt.reason, "unknown");
|
|
2596
2463
|
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
2597
2464
|
const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
@@ -2703,12 +2570,11 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2703
2570
|
const spanWithDuration = (name, attributes, durationMs, options = {}) => {
|
|
2704
2571
|
const endTimeMs = options.endTimeMs ?? Date.now();
|
|
2705
2572
|
const startTime = typeof options.startTimeMs === "number" ? options.startTimeMs : typeof durationMs === "number" && durationMs >= 0 ? endTimeMs - durationMs : void 0;
|
|
2706
|
-
const parentContext = "parentContext" in options ? options.parentContext ?? void 0 : void 0;
|
|
2707
2573
|
return tracer.startSpan(name, {
|
|
2708
2574
|
attributes: redactOtelAttributes(attributes),
|
|
2709
2575
|
...options.kind !== void 0 ? { kind: options.kind } : {},
|
|
2710
2576
|
...startTime !== void 0 ? { startTime } : {}
|
|
2711
|
-
}, parentContext);
|
|
2577
|
+
}, options.parentContext ?? void 0);
|
|
2712
2578
|
};
|
|
2713
2579
|
const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
|
|
2714
2580
|
const internalOrTrustedTraceContext = (evt, metadata) => metadata.internal ? normalizeTraceContext(evt.trace) : normalizedTrustedTraceContext(evt, metadata);
|
|
@@ -2837,6 +2703,7 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2837
2703
|
if (evt.model) spanAttrs["openclaw.model"] = evt.model;
|
|
2838
2704
|
if (evt.channel) spanAttrs["openclaw.channel"] = evt.channel;
|
|
2839
2705
|
if (evt.trigger) spanAttrs["openclaw.trigger"] = evt.trigger;
|
|
2706
|
+
if (evt.agentId) spanAttrs["openclaw.agent"] = normalizeDiagnosticValue(evt.agentId);
|
|
2840
2707
|
};
|
|
2841
2708
|
const paramsSummaryAttrs = (summary) => {
|
|
2842
2709
|
if (!summary) return {};
|
|
@@ -2893,10 +2760,6 @@ function isOtelSdkDisabled(logger) {
|
|
|
2893
2760
|
logger.warn("diagnostics-otel: invalid OTEL_SDK_DISABLED value; expected true or false, using false");
|
|
2894
2761
|
return false;
|
|
2895
2762
|
}
|
|
2896
|
-
function readNonblankOtelEnv(name) {
|
|
2897
|
-
const value = process.env[name];
|
|
2898
|
-
return value?.trim() ? value : void 0;
|
|
2899
|
-
}
|
|
2900
2763
|
function readPositiveOtelNumber(name, fallback) {
|
|
2901
2764
|
const value = otelCore.getNumberFromEnv(name);
|
|
2902
2765
|
if (value !== void 0 && value <= 0) {
|
|
@@ -2921,7 +2784,7 @@ function resolveResourceDetectors() {
|
|
|
2921
2784
|
});
|
|
2922
2785
|
}
|
|
2923
2786
|
function resolveSignalProtocol(signal, configuredProtocol) {
|
|
2924
|
-
return configuredProtocol ??
|
|
2787
|
+
return configuredProtocol ?? readNonBlankString(process.env[OTEL_SIGNAL_PROTOCOL_ENV[signal]]) ?? readNonBlankString(process.env["OTEL_EXPORTER_OTLP_PROTOCOL"]) ?? OTLP_HTTP_PROTOBUF_PROTOCOL;
|
|
2925
2788
|
}
|
|
2926
2789
|
function createStartupRollbackError(startupError, cleanupError) {
|
|
2927
2790
|
return new AggregateError([startupError, cleanupError], "diagnostics-otel startup failed and rollback cleanup failed", { cause: startupError });
|
|
@@ -3070,10 +2933,13 @@ function createDiagnosticsOtelService() {
|
|
|
3070
2933
|
if (!tracesActive && !metricsActive && !logsActive) return;
|
|
3071
2934
|
const hasOwnedOtlpSignal = ownedOtlpSignals.length > 0;
|
|
3072
2935
|
const sharedEnvEndpoint = hasOwnedOtlpSignal ? process.env[OTEL_EXPORTER_OTLP_ENDPOINT_ENV] : void 0;
|
|
3073
|
-
const endpoint = hasOwnedOtlpSignal ?
|
|
2936
|
+
const endpoint = hasOwnedOtlpSignal ? normalizeOptionalString(otel.endpoint ?? sharedEnvEndpoint) : void 0;
|
|
3074
2937
|
const headers = otel.headers ?? void 0;
|
|
3075
2938
|
const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || "openclaw";
|
|
3076
|
-
const sampleRate =
|
|
2939
|
+
const sampleRate = asFiniteNumberInRange(otel.sampleRate, {
|
|
2940
|
+
min: 0,
|
|
2941
|
+
max: 1
|
|
2942
|
+
});
|
|
3077
2943
|
const contentCapturePolicy = resolveContentCapturePolicy(otel.captureContent);
|
|
3078
2944
|
const resource = resources.resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
|
|
3079
2945
|
const logUrl = logsToOtlp ? resolveSignalOtelUrl({
|
|
@@ -3219,7 +3085,7 @@ function createDiagnosticsOtelService() {
|
|
|
3219
3085
|
recorders,
|
|
3220
3086
|
recordLogRecord,
|
|
3221
3087
|
recordSecurityEvent
|
|
3222
|
-
}), metricsActive ? void 0 : tracesActive ? { exclude: ["gateway.event_loop.sample", "diagnostic.gc"] } : { include: ["log.record", "security.event"] });
|
|
3088
|
+
}), metricsActive ? tracesActive ? void 0 : { exclude: ["diagnostic.phase.completed"] } : tracesActive ? { exclude: ["gateway.event_loop.sample", "diagnostic.gc"] } : { include: ["log.record", "security.event"] });
|
|
3223
3089
|
if (tracesActive) active.unregisterTracePropagationBridge = ctx.internalDiagnostics?.registerTracePropagationBridge?.({
|
|
3224
3090
|
shouldPrepareEvent(event) {
|
|
3225
3091
|
return event.type === "model.call.started" || event.type === "tool.execution.started";
|
package/dist/api.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { emptyPluginConfigSchema } from "openclaw/plugin-sdk/plugin-entry";
|
|
2
|
-
import { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
3
2
|
import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime";
|
|
3
|
+
import { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
4
4
|
export { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, emptyPluginConfigSchema, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent, redactSensitiveText };
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-ButGOVne.mjs";
|
|
2
2
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
3
3
|
//#region extensions/diagnostics-otel/index.ts
|
|
4
4
|
var diagnostics_otel_default = definePluginEntry({
|
package/dist/runtime-api.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-ButGOVne.mjs";
|
|
2
2
|
export { createDiagnosticsOtelService };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-otel",
|
|
3
|
-
"version": "2026.9.
|
|
3
|
+
"version": "2026.9.8",
|
|
4
4
|
"description": "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -34,10 +34,10 @@
|
|
|
34
34
|
"minHostVersion": ">=2026.4.25"
|
|
35
35
|
},
|
|
36
36
|
"compat": {
|
|
37
|
-
"pluginApi": ">=2026.9.
|
|
37
|
+
"pluginApi": ">=2026.9.8"
|
|
38
38
|
},
|
|
39
39
|
"build": {
|
|
40
|
-
"openclawVersion": "2026.9.
|
|
40
|
+
"openclawVersion": "2026.9.8"
|
|
41
41
|
},
|
|
42
42
|
"release": {
|
|
43
43
|
"publishToClawHub": true,
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"assets/activity.svg"
|
|
56
56
|
],
|
|
57
57
|
"peerDependencies": {
|
|
58
|
-
"openclaw": ">=2026.9.
|
|
58
|
+
"openclaw": ">=2026.9.8"
|
|
59
59
|
},
|
|
60
60
|
"peerDependenciesMeta": {
|
|
61
61
|
"openclaw": {
|