@openclaw/diagnostics-otel 2026.9.5 → 2026.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/assets/icon.png
CHANGED
|
Binary file
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, redactSensitiveText } from "../api.js";
|
|
2
1
|
import { ROOT_CONTEXT, SpanKind, SpanStatusCode, TraceFlags, context, createContextKey, createNoopMeter, diag, isSpanContextValid, metrics, propagation, trace } from "@opentelemetry/api";
|
|
3
2
|
import * as otelCore from "@opentelemetry/core";
|
|
4
3
|
import { CompositePropagator, ExportResultCode, W3CBaggagePropagator, W3CTraceContextPropagator, getStringListFromEnv } from "@opentelemetry/core";
|
|
@@ -8,11 +7,14 @@ import * as resources from "@opentelemetry/resources";
|
|
|
8
7
|
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
9
8
|
import { BasicTracerProvider, BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
|
|
10
9
|
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
10
|
+
import { asFiniteNumberInRange, asNonNegativeFiniteNumber, asPositiveFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
|
|
11
11
|
import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
|
|
12
|
+
import { asFiniteNumber, isRecord, normalizeOptionalString, readNonBlankString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
13
|
+
import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime";
|
|
12
14
|
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
13
|
-
import { isInternalDiagnosticEventMetadata, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
14
15
|
import { readFileSync } from "node:fs";
|
|
15
16
|
import nodePath from "node:path";
|
|
17
|
+
import { isInternalDiagnosticEventMetadata, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
16
18
|
import { collectErrorGraphCandidates } from "openclaw/plugin-sdk/error-runtime";
|
|
17
19
|
import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime";
|
|
18
20
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
@@ -21,7 +23,6 @@ import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-ho
|
|
|
21
23
|
import { B3InjectEncoding, B3Propagator } from "@opentelemetry/propagator-b3";
|
|
22
24
|
import { JaegerPropagator } from "@opentelemetry/propagator-jaeger";
|
|
23
25
|
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_TOOL_DEFINITIONS, GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL, GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT } from "@opentelemetry/semantic-conventions/incubating";
|
|
24
|
-
import { asFiniteNumber, asFiniteNumberInRange, isRecord } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
25
26
|
const DROPPED_OTEL_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
|
|
26
27
|
"openclaw.callId",
|
|
27
28
|
"openclaw.call_id",
|
|
@@ -308,9 +309,6 @@ function describeJsonValue(value) {
|
|
|
308
309
|
}
|
|
309
310
|
//#endregion
|
|
310
311
|
//#region extensions/diagnostics-otel/src/service-exporter.ts
|
|
311
|
-
function normalizeEndpoint(endpoint) {
|
|
312
|
-
return endpoint?.trim() || void 0;
|
|
313
|
-
}
|
|
314
312
|
const SIGNAL_QUALIFIED_OTLP_PATH_PATTERN = /\/v1\/(traces|metrics|logs)$/iu;
|
|
315
313
|
function appendOrReplaceSignalPath(value, path) {
|
|
316
314
|
const base = value.replace(/\/+$/u, "");
|
|
@@ -330,10 +328,10 @@ function resolveSharedOtelUrl(endpoint, path) {
|
|
|
330
328
|
return appendOrReplaceSignalPath(endpoint, path);
|
|
331
329
|
}
|
|
332
330
|
function resolveSignalOtelUrl(params) {
|
|
333
|
-
const signalEndpoint =
|
|
331
|
+
const signalEndpoint = normalizeOptionalString(params.signalEndpoint ?? params.signalEnvEndpoint);
|
|
334
332
|
const endpoint = signalEndpoint ?? params.endpoint;
|
|
335
|
-
const signalEnvEndpoint = params.signalEnvEndpoint
|
|
336
|
-
const sharedEnvEndpoint = params.sharedEnvEndpoint
|
|
333
|
+
const signalEnvEndpoint = readNonBlankString(params.signalEnvEndpoint);
|
|
334
|
+
const sharedEnvEndpoint = readNonBlankString(params.sharedEnvEndpoint);
|
|
337
335
|
const consumedSharedEnvEndpoint = signalEnvEndpoint ? void 0 : sharedEnvEndpoint;
|
|
338
336
|
const appendedSharedEnvEndpoint = consumedSharedEnvEndpoint ? `${consumedSharedEnvEndpoint}${consumedSharedEnvEndpoint.endsWith("/") ? "" : "/"}${params.path}` : void 0;
|
|
339
337
|
const resolvedEndpoint = endpoint && URL.canParse(endpoint) && !signalEndpoint ? resolveSharedOtelUrl(endpoint, params.path) : endpoint;
|
|
@@ -347,7 +345,7 @@ function resolveSignalOtelUrl(params) {
|
|
|
347
345
|
}
|
|
348
346
|
function readOtelEnvFile(params) {
|
|
349
347
|
const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`;
|
|
350
|
-
const filePath =
|
|
348
|
+
const filePath = readNonBlankString(process.env[signalEnvName]) ?? readNonBlankString(process.env[params.sharedEnvName]);
|
|
351
349
|
if (!filePath) return;
|
|
352
350
|
try {
|
|
353
351
|
const material = readFileSync(nodePath.resolve(process.cwd(), filePath));
|
|
@@ -355,9 +353,6 @@ function readOtelEnvFile(params) {
|
|
|
355
353
|
} catch {}
|
|
356
354
|
throw new Error(`Configured OpenTelemetry ${params.label} file is missing, empty, or unreadable; refusing insecure export`);
|
|
357
355
|
}
|
|
358
|
-
function normalizeOtelEnvValue(value) {
|
|
359
|
-
return value?.trim() ? value : void 0;
|
|
360
|
-
}
|
|
361
356
|
function resolveOtelHttpAgentOptions(params) {
|
|
362
357
|
const { url, signalIdentifier } = params;
|
|
363
358
|
const ca = readOtelEnvFile({
|
|
@@ -398,11 +393,6 @@ function resolveOtelHttpAgentOptions(params) {
|
|
|
398
393
|
}
|
|
399
394
|
return (ca || cert || key) && new URL(url).protocol === "https:" ? agentOptions : void 0;
|
|
400
395
|
}
|
|
401
|
-
function resolveSampleRate(value) {
|
|
402
|
-
if (typeof value !== "number" || !Number.isFinite(value)) return;
|
|
403
|
-
if (value < 0 || value > 1) return;
|
|
404
|
-
return value;
|
|
405
|
-
}
|
|
406
396
|
function formatError(err) {
|
|
407
397
|
if (err instanceof Error) return err.stack ?? err.message;
|
|
408
398
|
if (typeof err === "string") return err;
|
|
@@ -443,162 +433,71 @@ function findOtlpExporterError(reason) {
|
|
|
443
433
|
//#region extensions/diagnostics-otel/src/service-events.ts
|
|
444
434
|
function createDiagnosticsEventHandler(params) {
|
|
445
435
|
const { logger, recorders, recordLogRecord, recordSecurityEvent } = params;
|
|
446
|
-
const { recordGcDuration, recordGatewayEventLoopSample, recordGatewayRpc, recordModelUsage, recordWebhookReceived, recordWebhookProcessed, recordWebhookError, recordMessageQueued, recordMessageReceived, recordMessageDispatchStarted, recordMessageDispatchCompleted, recordMessageProcessed, recordMessageDeliveryStarted, recordMessageDeliveryCompleted, recordMessageDeliveryError, recordTalkEvent, recordLaneEnqueue, recordLaneDequeue, recordSessionState, recordSessionTurnCreated, recordSessionStuck, recordSessionRecoveryRequested, recordSessionRecoveryCompleted, recordRunAttempt, recordHeartbeat, recordLivenessWarning, recordDiagnosticPhaseCompleted, recordRunStarted, recordRunCompleted, recordHarnessRunStarted, recordHarnessRunCompleted, recordHarnessRunError, recordContextAssembled, recordModelCallStarted, recordModelCallFinished, recordToolExecutionStarted, recordToolExecutionFinished, recordToolExecutionBlocked, recordSkillUsed, recordExecProcessCompleted, recordToolLoop, recordMemorySample, recordMemoryPressure, recordAsyncQueueDropped, recordTelemetryExporter, recordPayloadLarge, recordModelFailover } = recorders;
|
|
447
436
|
return (evt, metadata, privateData) => {
|
|
448
437
|
try {
|
|
449
438
|
switch (evt.type) {
|
|
450
|
-
case "diagnostic.
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
case "gateway.
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
case "
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
case "
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
case "
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
case "
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
case "
|
|
469
|
-
|
|
470
|
-
return;
|
|
471
|
-
case "message.queued":
|
|
472
|
-
recordMessageQueued(evt);
|
|
473
|
-
return;
|
|
474
|
-
case "message.received":
|
|
475
|
-
recordMessageReceived(evt);
|
|
476
|
-
return;
|
|
477
|
-
case "message.dispatch.started":
|
|
478
|
-
recordMessageDispatchStarted(evt, metadata);
|
|
479
|
-
return;
|
|
480
|
-
case "message.dispatch.completed":
|
|
481
|
-
recordMessageDispatchCompleted(evt);
|
|
482
|
-
return;
|
|
483
|
-
case "message.processed":
|
|
484
|
-
recordMessageProcessed(evt, metadata);
|
|
485
|
-
return;
|
|
486
|
-
case "message.delivery.started":
|
|
487
|
-
recordMessageDeliveryStarted(evt);
|
|
488
|
-
return;
|
|
489
|
-
case "message.delivery.completed":
|
|
490
|
-
recordMessageDeliveryCompleted(evt, metadata);
|
|
491
|
-
return;
|
|
492
|
-
case "message.delivery.error":
|
|
493
|
-
recordMessageDeliveryError(evt, metadata);
|
|
494
|
-
return;
|
|
495
|
-
case "talk.event":
|
|
496
|
-
recordTalkEvent(evt, metadata);
|
|
497
|
-
return;
|
|
498
|
-
case "queue.lane.enqueue":
|
|
499
|
-
recordLaneEnqueue(evt);
|
|
500
|
-
return;
|
|
501
|
-
case "queue.lane.dequeue":
|
|
502
|
-
recordLaneDequeue(evt);
|
|
503
|
-
return;
|
|
504
|
-
case "session.state":
|
|
505
|
-
recordSessionState(evt);
|
|
506
|
-
break;
|
|
439
|
+
case "diagnostic.child_process.spawn": return;
|
|
440
|
+
case "diagnostic.gc": return recorders.recordGcDuration(evt, metadata);
|
|
441
|
+
case "gateway.event_loop.sample": return recorders.recordGatewayEventLoopSample(evt, metadata);
|
|
442
|
+
case "gateway.rpc": return recorders.recordGatewayRpc(evt, metadata);
|
|
443
|
+
case "model.usage": return recorders.recordModelUsage(evt, metadata, privateData.hostPluginId);
|
|
444
|
+
case "webhook.received": return recorders.recordWebhookReceived(evt);
|
|
445
|
+
case "webhook.processed": return recorders.recordWebhookProcessed(evt);
|
|
446
|
+
case "webhook.error": return recorders.recordWebhookError(evt);
|
|
447
|
+
case "message.queued": return recorders.recordMessageQueued(evt);
|
|
448
|
+
case "message.received": return recorders.recordMessageReceived(evt);
|
|
449
|
+
case "message.dispatch.started": return recorders.recordMessageDispatchStarted(evt, metadata);
|
|
450
|
+
case "message.dispatch.completed": return recorders.recordMessageDispatchCompleted(evt);
|
|
451
|
+
case "message.processed": return recorders.recordMessageProcessed(evt, metadata);
|
|
452
|
+
case "message.delivery.started": return recorders.recordMessageDeliveryStarted(evt);
|
|
453
|
+
case "message.delivery.completed": return recorders.recordMessageDeliveryCompleted(evt, metadata);
|
|
454
|
+
case "message.delivery.error": return recorders.recordMessageDeliveryError(evt, metadata);
|
|
455
|
+
case "talk.event": return recorders.recordTalkEvent(evt, metadata);
|
|
456
|
+
case "queue.lane.enqueue": return recorders.recordLaneEnqueue(evt);
|
|
457
|
+
case "queue.lane.dequeue": return recorders.recordLaneDequeue(evt);
|
|
458
|
+
case "session.state": return recorders.recordSessionState(evt);
|
|
507
459
|
case "session.long_running":
|
|
508
460
|
case "session.stalled": break;
|
|
509
|
-
case "session.turn.created":
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
case "session.
|
|
513
|
-
|
|
514
|
-
return;
|
|
515
|
-
case "session.recovery.requested":
|
|
516
|
-
recordSessionRecoveryRequested(evt);
|
|
517
|
-
return;
|
|
518
|
-
case "session.recovery.completed":
|
|
519
|
-
recordSessionRecoveryCompleted(evt);
|
|
520
|
-
return;
|
|
521
|
-
case "run.attempt":
|
|
522
|
-
recordRunAttempt(evt);
|
|
523
|
-
break;
|
|
461
|
+
case "session.turn.created": return recorders.recordSessionTurnCreated(evt);
|
|
462
|
+
case "session.stuck": return recorders.recordSessionStuck(evt);
|
|
463
|
+
case "session.recovery.requested": return recorders.recordSessionRecoveryRequested(evt);
|
|
464
|
+
case "session.recovery.completed": return recorders.recordSessionRecoveryCompleted(evt);
|
|
465
|
+
case "run.attempt": return recorders.recordRunAttempt(evt);
|
|
524
466
|
case "run.progress": break;
|
|
525
467
|
case "run.execution_phase": break;
|
|
526
|
-
case "diagnostic.heartbeat":
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
case "
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
case "
|
|
533
|
-
recordDiagnosticPhaseCompleted(evt);
|
|
534
|
-
return;
|
|
535
|
-
case "run.started":
|
|
536
|
-
recordRunStarted(evt, metadata);
|
|
537
|
-
return;
|
|
538
|
-
case "run.completed":
|
|
539
|
-
recordRunCompleted(evt, metadata, privateData);
|
|
540
|
-
return;
|
|
541
|
-
case "harness.run.started":
|
|
542
|
-
recordHarnessRunStarted(evt, metadata);
|
|
543
|
-
return;
|
|
468
|
+
case "diagnostic.heartbeat": return recorders.recordHeartbeat(evt);
|
|
469
|
+
case "diagnostic.liveness.warning": return recorders.recordLivenessWarning(evt);
|
|
470
|
+
case "diagnostic.phase.completed": return recorders.recordDiagnosticPhaseCompleted(evt, metadata);
|
|
471
|
+
case "run.started": return recorders.recordRunStarted(evt, metadata);
|
|
472
|
+
case "run.completed": return recorders.recordRunCompleted(evt, metadata, privateData);
|
|
473
|
+
case "harness.run.started": return recorders.recordHarnessRunStarted(evt, metadata);
|
|
474
|
+
case "agent.commentary": return recorders.recordAgentCommentary(evt, metadata, privateData);
|
|
544
475
|
case "harness.run.completed":
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
case "harness.run.error":
|
|
548
|
-
recordHarnessRunError(evt, metadata, privateData);
|
|
549
|
-
return;
|
|
550
|
-
case "context.assembled":
|
|
551
|
-
recordContextAssembled(evt, metadata);
|
|
552
|
-
return;
|
|
476
|
+
case "harness.run.error": return recorders.recordHarnessRunFinished(evt, metadata, privateData);
|
|
477
|
+
case "context.assembled": return recorders.recordContextAssembled(evt, metadata);
|
|
553
478
|
case "model.call.started":
|
|
554
|
-
recordModelCallStarted(evt, metadata);
|
|
479
|
+
recorders.recordModelCallStarted(evt, metadata);
|
|
555
480
|
return;
|
|
556
481
|
case "model.call.completed":
|
|
557
|
-
case "model.call.error":
|
|
558
|
-
recordModelCallFinished(evt, metadata, privateData.modelContent);
|
|
559
|
-
return;
|
|
482
|
+
case "model.call.error": return recorders.recordModelCallFinished(evt, metadata, privateData.modelContent);
|
|
560
483
|
case "tool.execution.started":
|
|
561
|
-
recordToolExecutionStarted(evt, metadata);
|
|
484
|
+
recorders.recordToolExecutionStarted(evt, metadata);
|
|
562
485
|
return;
|
|
563
486
|
case "tool.execution.completed":
|
|
564
|
-
case "tool.execution.error":
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
case "
|
|
568
|
-
recordToolExecutionBlocked(evt, metadata);
|
|
569
|
-
return;
|
|
570
|
-
case "skill.used":
|
|
571
|
-
recordSkillUsed(evt, metadata);
|
|
572
|
-
return;
|
|
573
|
-
case "exec.process.completed":
|
|
574
|
-
recordExecProcessCompleted(evt, metadata);
|
|
575
|
-
break;
|
|
487
|
+
case "tool.execution.error": return recorders.recordToolExecutionFinished(evt, metadata, privateData.toolContent);
|
|
488
|
+
case "tool.execution.blocked": return recorders.recordToolExecutionBlocked(evt, metadata);
|
|
489
|
+
case "skill.used": return recorders.recordSkillUsed(evt, metadata);
|
|
490
|
+
case "exec.process.completed": return recorders.recordExecProcessCompleted(evt, metadata);
|
|
576
491
|
case "exec.approval.followup_suppressed": break;
|
|
577
|
-
case "log.record":
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
case "
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
case "
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
case "diagnostic.memory.sample":
|
|
587
|
-
recordMemorySample(evt);
|
|
588
|
-
return;
|
|
589
|
-
case "diagnostic.memory.pressure":
|
|
590
|
-
recordMemoryPressure(evt);
|
|
591
|
-
return;
|
|
592
|
-
case "diagnostic.async_queue.dropped":
|
|
593
|
-
recordAsyncQueueDropped(evt);
|
|
594
|
-
return;
|
|
595
|
-
case "telemetry.exporter":
|
|
596
|
-
recordTelemetryExporter(evt, metadata);
|
|
597
|
-
return;
|
|
598
|
-
case "payload.large":
|
|
599
|
-
recordPayloadLarge(evt);
|
|
600
|
-
return;
|
|
601
|
-
case "model.failover": recordModelFailover(evt, metadata);
|
|
492
|
+
case "log.record": return recordLogRecord?.(evt, metadata);
|
|
493
|
+
case "security.event": return recordSecurityEvent?.(evt, metadata);
|
|
494
|
+
case "tool.loop": return recorders.recordToolLoop(evt);
|
|
495
|
+
case "diagnostic.memory.sample": return recorders.recordMemoryUsageMetrics(evt);
|
|
496
|
+
case "diagnostic.memory.pressure": return recorders.recordMemoryPressure(evt);
|
|
497
|
+
case "diagnostic.async_queue.dropped": return recorders.recordAsyncQueueDropped(evt);
|
|
498
|
+
case "telemetry.exporter": return recorders.recordTelemetryExporter(evt, metadata);
|
|
499
|
+
case "payload.large": return recorders.recordPayloadLarge(evt);
|
|
500
|
+
case "model.failover": return recorders.recordModelFailover(evt, metadata);
|
|
602
501
|
}
|
|
603
502
|
} catch (err) {
|
|
604
503
|
logger.error(`diagnostics-otel: event handler failed (${evt.type}): ${formatError(err)}`);
|
|
@@ -755,9 +654,6 @@ function securityTargetNameAttr(value, fallback = "unknown") {
|
|
|
755
654
|
if (redactedLower.startsWith("agent:") || redactedLower.includes(":agent:")) return fallback;
|
|
756
655
|
return SECURITY_TARGET_NAME_VALUE_RE.test(redacted) ? redacted : fallback;
|
|
757
656
|
}
|
|
758
|
-
function shouldCaptureOtelLogBody(policy) {
|
|
759
|
-
return policy.logBodies;
|
|
760
|
-
}
|
|
761
657
|
function otelLogTimestampIso(timestamp) {
|
|
762
658
|
if (timestamp instanceof Date) return timestamp.toISOString();
|
|
763
659
|
if (typeof timestamp === "number" && Number.isFinite(timestamp)) return new Date(timestamp).toISOString();
|
|
@@ -912,7 +808,6 @@ const LOG_SEVERITY_MAP = {
|
|
|
912
808
|
function createDiagnosticsLogExporter(params) {
|
|
913
809
|
const { contentCapturePolicy, emitExporterEvent, flushIntervalMs, headers, logger, logsEnabled, logsToOtlp, logsToStdout, logHttpAgentOptions, logUrl, resource, serviceName } = params;
|
|
914
810
|
let logProvider = null;
|
|
915
|
-
const logSeverityMap = LOG_SEVERITY_MAP;
|
|
916
811
|
let recordLogRecord;
|
|
917
812
|
let recordSecurityEvent;
|
|
918
813
|
if (logsEnabled) {
|
|
@@ -985,8 +880,8 @@ function createDiagnosticsLogExporter(params) {
|
|
|
985
880
|
};
|
|
986
881
|
const buildDiagnosticLogRecord = (evt, metadata) => {
|
|
987
882
|
const logLevelName = evt.level || "INFO";
|
|
988
|
-
const severityNumber =
|
|
989
|
-
const body =
|
|
883
|
+
const severityNumber = LOG_SEVERITY_MAP[logLevelName] ?? 9;
|
|
884
|
+
const body = contentCapturePolicy.logBodies ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
990
885
|
const attributes = Object.create(null);
|
|
991
886
|
assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName);
|
|
992
887
|
if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
|
|
@@ -1018,7 +913,7 @@ function createDiagnosticsLogExporter(params) {
|
|
|
1018
913
|
const logRecord = {
|
|
1019
914
|
body: "openclaw.security.event",
|
|
1020
915
|
severityText,
|
|
1021
|
-
severityNumber:
|
|
916
|
+
severityNumber: LOG_SEVERITY_MAP[severityText] ?? 9,
|
|
1022
917
|
attributes: redactOtelAttributes(attributes),
|
|
1023
918
|
timestamp: evt.ts
|
|
1024
919
|
};
|
|
@@ -1437,247 +1332,6 @@ function createDiagnosticsRecorderRuntime(params) {
|
|
|
1437
1332
|
};
|
|
1438
1333
|
}
|
|
1439
1334
|
//#endregion
|
|
1440
|
-
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1441
|
-
function createHarnessRecorders(runtime) {
|
|
1442
|
-
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1443
|
-
const harnessRunMetricAttrs = (evt) => ({
|
|
1444
|
-
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1445
|
-
"openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId),
|
|
1446
|
-
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1447
|
-
"openclaw.provider": normalizeDiagnosticValue(evt.provider, "unknown"),
|
|
1448
|
-
"openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"),
|
|
1449
|
-
...evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}
|
|
1450
|
-
});
|
|
1451
|
-
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1452
|
-
if (!tracesEnabled || !metadata.trusted) return;
|
|
1453
|
-
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", harnessRunMetricAttrs(evt), void 0, {
|
|
1454
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1455
|
-
startTimeMs: evt.ts
|
|
1456
|
-
}));
|
|
1457
|
-
};
|
|
1458
|
-
const recordHarnessRunCompleted = (evt, metadata, privateData) => {
|
|
1459
|
-
harnessDurationHistogram.record(evt.durationMs, harnessRunMetricAttrs(evt));
|
|
1460
|
-
if (!tracesEnabled) return;
|
|
1461
|
-
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1462
|
-
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue(evt.resultClassification);
|
|
1463
|
-
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1464
|
-
if (evt.itemLifecycle) {
|
|
1465
|
-
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1466
|
-
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1467
|
-
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1468
|
-
}
|
|
1469
|
-
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1470
|
-
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1471
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1472
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1473
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1474
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1475
|
-
endTimeMs: evt.ts
|
|
1476
|
-
});
|
|
1477
|
-
setSpanAttrs(span, spanAttrs);
|
|
1478
|
-
if (evt.outcome === "error") span.setStatus({
|
|
1479
|
-
code: SpanStatusCode.ERROR,
|
|
1480
|
-
message: redactedError ?? "error"
|
|
1481
|
-
});
|
|
1482
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1483
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1484
|
-
return;
|
|
1485
|
-
}
|
|
1486
|
-
span.end(evt.ts);
|
|
1487
|
-
};
|
|
1488
|
-
const recordHarnessRunError = (evt, metadata, privateData) => {
|
|
1489
|
-
const errorType = normalizeDiagnosticValue(evt.errorCategory, "other");
|
|
1490
|
-
const attrs = {
|
|
1491
|
-
...harnessRunMetricAttrs(evt),
|
|
1492
|
-
"openclaw.harness.phase": evt.phase,
|
|
1493
|
-
"openclaw.errorCategory": errorType
|
|
1494
|
-
};
|
|
1495
|
-
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1496
|
-
if (!tracesEnabled) return;
|
|
1497
|
-
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1498
|
-
const spanAttrs = {
|
|
1499
|
-
...attrs,
|
|
1500
|
-
"error.type": errorType,
|
|
1501
|
-
...redactedError ? { "openclaw.error": redactedError } : {},
|
|
1502
|
-
...evt.cleanupFailed ? { "openclaw.harness.cleanup_failed": true } : {}
|
|
1503
|
-
};
|
|
1504
|
-
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1505
|
-
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1506
|
-
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1507
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1508
|
-
endTimeMs: evt.ts
|
|
1509
|
-
});
|
|
1510
|
-
setSpanAttrs(span, spanAttrs);
|
|
1511
|
-
span.setStatus({
|
|
1512
|
-
code: SpanStatusCode.ERROR,
|
|
1513
|
-
message: redactedError ?? errorType
|
|
1514
|
-
});
|
|
1515
|
-
if (trackedSpan && trustedTrace?.spanId) {
|
|
1516
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1517
|
-
return;
|
|
1518
|
-
}
|
|
1519
|
-
span.end(evt.ts);
|
|
1520
|
-
};
|
|
1521
|
-
const recordContextAssembled = (evt, metadata) => {
|
|
1522
|
-
if (!tracesEnabled) return;
|
|
1523
|
-
const spanAttrs = {
|
|
1524
|
-
"openclaw.context.message_count": evt.messageCount,
|
|
1525
|
-
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1526
|
-
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1527
|
-
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1528
|
-
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1529
|
-
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1530
|
-
"openclaw.context.prompt_images": evt.promptImages
|
|
1531
|
-
};
|
|
1532
|
-
addRunAttrs(spanAttrs, evt);
|
|
1533
|
-
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1534
|
-
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1535
|
-
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1536
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1537
|
-
endTimeMs: evt.ts
|
|
1538
|
-
}).end(evt.ts);
|
|
1539
|
-
};
|
|
1540
|
-
const recordModelFailover = (evt, metadata) => {
|
|
1541
|
-
const metricAttrs = {
|
|
1542
|
-
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1543
|
-
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1544
|
-
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1545
|
-
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1546
|
-
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1547
|
-
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1548
|
-
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1549
|
-
};
|
|
1550
|
-
modelFailoverCounter.add(1, metricAttrs);
|
|
1551
|
-
if (!tracesEnabled) return;
|
|
1552
|
-
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1553
|
-
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1554
|
-
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1555
|
-
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1556
|
-
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1557
|
-
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1558
|
-
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1559
|
-
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1560
|
-
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1561
|
-
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1562
|
-
endTimeMs: evt.ts
|
|
1563
|
-
}).end(evt.ts);
|
|
1564
|
-
};
|
|
1565
|
-
return {
|
|
1566
|
-
recordHarnessRunStarted,
|
|
1567
|
-
recordHarnessRunCompleted,
|
|
1568
|
-
recordHarnessRunError,
|
|
1569
|
-
recordContextAssembled,
|
|
1570
|
-
recordModelFailover
|
|
1571
|
-
};
|
|
1572
|
-
}
|
|
1573
|
-
//#endregion
|
|
1574
|
-
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1575
|
-
function hasOtelSemconvOptIn(value, optIn) {
|
|
1576
|
-
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1577
|
-
}
|
|
1578
|
-
function emitLatestGenAiSemconv() {
|
|
1579
|
-
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
1580
|
-
}
|
|
1581
|
-
function genAiOperationName(api, observationUnit) {
|
|
1582
|
-
if (observationUnit === "turn") return GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT;
|
|
1583
|
-
const normalized = api?.trim().toLowerCase();
|
|
1584
|
-
if (!normalized) return "chat";
|
|
1585
|
-
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
1586
|
-
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
1587
|
-
return "chat";
|
|
1588
|
-
}
|
|
1589
|
-
function positiveFiniteNumber(value) {
|
|
1590
|
-
return asFiniteNumberInRange(value, {
|
|
1591
|
-
min: 0,
|
|
1592
|
-
minExclusive: true
|
|
1593
|
-
});
|
|
1594
|
-
}
|
|
1595
|
-
function nonNegativeFiniteNumber(value) {
|
|
1596
|
-
return asFiniteNumberInRange(value, { min: 0 });
|
|
1597
|
-
}
|
|
1598
|
-
function assignPositiveNumberAttr(attrs, key, value) {
|
|
1599
|
-
const normalized = positiveFiniteNumber(value);
|
|
1600
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1601
|
-
}
|
|
1602
|
-
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1603
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1604
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1605
|
-
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1606
|
-
}
|
|
1607
|
-
function assignNumberAttr(attrs, key, value) {
|
|
1608
|
-
const normalized = asFiniteNumber(value);
|
|
1609
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1610
|
-
}
|
|
1611
|
-
function modelCallPromptTokens(usage) {
|
|
1612
|
-
const promptTokens = nonNegativeFiniteNumber(usage.promptTokens);
|
|
1613
|
-
if (promptTokens !== void 0) return promptTokens;
|
|
1614
|
-
const input = nonNegativeFiniteNumber(usage.input);
|
|
1615
|
-
const cacheRead = nonNegativeFiniteNumber(usage.cacheRead);
|
|
1616
|
-
const cacheWrite = nonNegativeFiniteNumber(usage.cacheWrite);
|
|
1617
|
-
if (input === void 0 && cacheRead === void 0 && cacheWrite === void 0) return;
|
|
1618
|
-
return (input ?? 0) + (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
1619
|
-
}
|
|
1620
|
-
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
1621
|
-
const stats = evt.promptStats;
|
|
1622
|
-
if (!stats) return;
|
|
1623
|
-
for (const [key, value] of [
|
|
1624
|
-
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
1625
|
-
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
1626
|
-
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
1627
|
-
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1628
|
-
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1629
|
-
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1630
|
-
]) assignNumberAttr(attrs, key, value);
|
|
1631
|
-
}
|
|
1632
|
-
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1633
|
-
const usage = evt.usage;
|
|
1634
|
-
if (!usage) return;
|
|
1635
|
-
const promptTokens = modelCallPromptTokens(usage);
|
|
1636
|
-
for (const [key, value] of [
|
|
1637
|
-
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
1638
|
-
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
1639
|
-
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
1640
|
-
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
1641
|
-
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
1642
|
-
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
1643
|
-
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
1644
|
-
["gen_ai.usage.input_tokens", promptTokens],
|
|
1645
|
-
["gen_ai.usage.output_tokens", usage.output],
|
|
1646
|
-
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
1647
|
-
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
1648
|
-
]) {
|
|
1649
|
-
const normalized = nonNegativeFiniteNumber(value);
|
|
1650
|
-
if (normalized !== void 0) attrs[key] = normalized;
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
1654
|
-
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = normalizeDiagnosticValue(input.provider);
|
|
1655
|
-
else attrs["gen_ai.system"] = normalizeDiagnosticValue(input.provider);
|
|
1656
|
-
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
1657
|
-
attrs["gen_ai.operation.name"] = genAiOperationName(input.api, input.observationUnit);
|
|
1658
|
-
}
|
|
1659
|
-
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1660
|
-
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1661
|
-
attrs["openclaw.model_call.observation_unit"] = modelCallObservationUnit(evt);
|
|
1662
|
-
}
|
|
1663
|
-
function modelCallObservationUnit(evt) {
|
|
1664
|
-
return evt.observationUnit ?? "request";
|
|
1665
|
-
}
|
|
1666
|
-
function modelCallSpanName(evt) {
|
|
1667
|
-
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
1668
|
-
const operationName = genAiOperationName(evt.api, evt.observationUnit);
|
|
1669
|
-
return operationName === GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT ? operationName : `${operationName} ${normalizeDiagnosticValue(evt.model)}`;
|
|
1670
|
-
}
|
|
1671
|
-
function modelCallSpanKind() {
|
|
1672
|
-
return SpanKind.CLIENT;
|
|
1673
|
-
}
|
|
1674
|
-
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
1675
|
-
if (!upstreamRequestIdHash) return;
|
|
1676
|
-
const boundedHash = normalizeDiagnosticValue(upstreamRequestIdHash);
|
|
1677
|
-
if (boundedHash === "unknown") return;
|
|
1678
|
-
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
1679
|
-
}
|
|
1680
|
-
//#endregion
|
|
1681
1335
|
//#region extensions/diagnostics-otel/src/service-genai-content.ts
|
|
1682
1336
|
function textPart(content) {
|
|
1683
1337
|
return {
|
|
@@ -1905,9 +1559,234 @@ function assignOtelToolContentAttributes(attributes, content, policy) {
|
|
|
1905
1559
|
}
|
|
1906
1560
|
}
|
|
1907
1561
|
//#endregion
|
|
1562
|
+
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1563
|
+
function createHarnessRecorders(runtime) {
|
|
1564
|
+
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled, getTrackedInternalOrTrustedSpan, contentCapturePolicy } = runtime;
|
|
1565
|
+
const recordAgentCommentary = (evt, metadata, privateData) => {
|
|
1566
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1567
|
+
const span = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
1568
|
+
if (!span) return;
|
|
1569
|
+
const attrs = {
|
|
1570
|
+
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1571
|
+
"openclaw.commentary.sequence": evt.sourceSequence,
|
|
1572
|
+
"openclaw.commentary.text_length": evt.textLength,
|
|
1573
|
+
"openclaw.commentary.content_truncated": evt.contentTruncated
|
|
1574
|
+
};
|
|
1575
|
+
assignOtelModelContentAttributes(attrs, privateData.modelContent, contentCapturePolicy);
|
|
1576
|
+
span.addEvent("openclaw.agent.commentary", redactOtelAttributes(attrs), evt.sourceTimestampMs);
|
|
1577
|
+
};
|
|
1578
|
+
const harnessRunMetricAttrs = (evt) => ({
|
|
1579
|
+
"openclaw.harness.id": normalizeDiagnosticValue(evt.harnessId, "unknown"),
|
|
1580
|
+
"openclaw.harness.plugin": normalizeDiagnosticValue(evt.pluginId),
|
|
1581
|
+
...evt.type === "harness.run.started" ? {} : { "openclaw.outcome": evt.type === "harness.run.error" ? "error" : evt.outcome },
|
|
1582
|
+
"openclaw.provider": normalizeDiagnosticValue(evt.provider, "unknown"),
|
|
1583
|
+
"openclaw.model": normalizeDiagnosticValue(evt.model, "unknown"),
|
|
1584
|
+
...evt.channel ? { "openclaw.channel": normalizeDiagnosticValue(evt.channel) } : {}
|
|
1585
|
+
});
|
|
1586
|
+
const recordHarnessRunStarted = (evt, metadata) => {
|
|
1587
|
+
if (!tracesEnabled || !metadata.trusted) return;
|
|
1588
|
+
const spanAttrs = { ...harnessRunMetricAttrs(evt) };
|
|
1589
|
+
addRunAttrs(spanAttrs, evt);
|
|
1590
|
+
trackTrustedSpan(evt, metadata, spanWithDuration("openclaw.harness.run", spanAttrs, void 0, {
|
|
1591
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1592
|
+
startTimeMs: evt.ts
|
|
1593
|
+
}));
|
|
1594
|
+
};
|
|
1595
|
+
const recordHarnessRunFinished = (evt, metadata, privateData) => {
|
|
1596
|
+
const errorType = evt.type === "harness.run.error" ? normalizeDiagnosticValue(evt.errorCategory, "other") : "error";
|
|
1597
|
+
const attrs = {
|
|
1598
|
+
...harnessRunMetricAttrs(evt),
|
|
1599
|
+
...evt.type === "harness.run.error" ? {
|
|
1600
|
+
"openclaw.harness.phase": evt.phase,
|
|
1601
|
+
"openclaw.errorCategory": errorType
|
|
1602
|
+
} : {}
|
|
1603
|
+
};
|
|
1604
|
+
harnessDurationHistogram.record(evt.durationMs, attrs);
|
|
1605
|
+
if (!tracesEnabled) return;
|
|
1606
|
+
const spanAttrs = { ...attrs };
|
|
1607
|
+
addRunAttrs(spanAttrs, evt);
|
|
1608
|
+
if (evt.type === "harness.run.completed") {
|
|
1609
|
+
if (evt.resultClassification) spanAttrs["openclaw.harness.result_classification"] = normalizeDiagnosticValue(evt.resultClassification);
|
|
1610
|
+
if (typeof evt.yieldDetected === "boolean") spanAttrs["openclaw.harness.yield_detected"] = evt.yieldDetected;
|
|
1611
|
+
if (evt.itemLifecycle) {
|
|
1612
|
+
spanAttrs["openclaw.harness.items.started"] = evt.itemLifecycle.startedCount;
|
|
1613
|
+
spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
|
|
1614
|
+
spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
|
|
1615
|
+
}
|
|
1616
|
+
} else {
|
|
1617
|
+
spanAttrs["error.type"] = errorType;
|
|
1618
|
+
if (evt.cleanupFailed) spanAttrs["openclaw.harness.cleanup_failed"] = true;
|
|
1619
|
+
}
|
|
1620
|
+
const redactedError = normalizeOtelErrorMessage(privateData.errorMessage);
|
|
1621
|
+
if (redactedError) spanAttrs["openclaw.error"] = redactedError;
|
|
1622
|
+
const trustedTrace = trustedTraceContext(evt, metadata);
|
|
1623
|
+
const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
|
|
1624
|
+
const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
|
|
1625
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1626
|
+
endTimeMs: evt.ts
|
|
1627
|
+
});
|
|
1628
|
+
setSpanAttrs(span, spanAttrs);
|
|
1629
|
+
if (evt.type === "harness.run.error" || evt.outcome === "error") span.setStatus({
|
|
1630
|
+
code: SpanStatusCode.ERROR,
|
|
1631
|
+
message: redactedError ?? errorType
|
|
1632
|
+
});
|
|
1633
|
+
if (trackedSpan && trustedTrace?.spanId) {
|
|
1634
|
+
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1635
|
+
return;
|
|
1636
|
+
}
|
|
1637
|
+
span.end(evt.ts);
|
|
1638
|
+
};
|
|
1639
|
+
const recordContextAssembled = (evt, metadata) => {
|
|
1640
|
+
if (!tracesEnabled) return;
|
|
1641
|
+
const spanAttrs = {
|
|
1642
|
+
"openclaw.context.message_count": evt.messageCount,
|
|
1643
|
+
"openclaw.context.history_text_chars": evt.historyTextChars,
|
|
1644
|
+
"openclaw.context.history_image_blocks": evt.historyImageBlocks,
|
|
1645
|
+
"openclaw.context.max_message_text_chars": evt.maxMessageTextChars,
|
|
1646
|
+
"openclaw.context.system_prompt_chars": evt.systemPromptChars,
|
|
1647
|
+
"openclaw.context.prompt_chars": evt.promptChars,
|
|
1648
|
+
"openclaw.context.prompt_images": evt.promptImages
|
|
1649
|
+
};
|
|
1650
|
+
addRunAttrs(spanAttrs, evt);
|
|
1651
|
+
if (evt.contextTokenBudget !== void 0) spanAttrs["openclaw.context.token_budget"] = evt.contextTokenBudget;
|
|
1652
|
+
if (evt.reserveTokens !== void 0) spanAttrs["openclaw.context.reserve_tokens"] = evt.reserveTokens;
|
|
1653
|
+
spanWithDuration("openclaw.context.assembled", spanAttrs, 0, {
|
|
1654
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1655
|
+
endTimeMs: evt.ts
|
|
1656
|
+
}).end(evt.ts);
|
|
1657
|
+
};
|
|
1658
|
+
const recordModelFailover = (evt, metadata) => {
|
|
1659
|
+
const metricAttrs = {
|
|
1660
|
+
"openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown"),
|
|
1661
|
+
"openclaw.failover.suspended": evt.suspended === void 0 ? "unknown" : String(evt.suspended),
|
|
1662
|
+
"openclaw.lane": normalizeDiagnosticLane(evt.lane, "unknown"),
|
|
1663
|
+
"openclaw.model": normalizeDiagnosticValue(evt.fromModel),
|
|
1664
|
+
"openclaw.provider": normalizeDiagnosticValue(evt.fromProvider),
|
|
1665
|
+
"openclaw.failover.to_model": normalizeDiagnosticValue(evt.toModel),
|
|
1666
|
+
"openclaw.failover.to_provider": normalizeDiagnosticValue(evt.toProvider)
|
|
1667
|
+
};
|
|
1668
|
+
modelFailoverCounter.add(1, metricAttrs);
|
|
1669
|
+
if (!tracesEnabled) return;
|
|
1670
|
+
const spanAttrs = { "openclaw.failover.reason": normalizeDiagnosticValue(evt.reason, "unknown") };
|
|
1671
|
+
if (evt.fromProvider) spanAttrs["openclaw.provider"] = evt.fromProvider;
|
|
1672
|
+
if (evt.fromModel) spanAttrs["openclaw.model"] = evt.fromModel;
|
|
1673
|
+
if (evt.toProvider) spanAttrs["openclaw.failover.to_provider"] = evt.toProvider;
|
|
1674
|
+
if (evt.toModel) spanAttrs["openclaw.failover.to_model"] = evt.toModel;
|
|
1675
|
+
if (evt.lane) spanAttrs["openclaw.lane"] = normalizeDiagnosticLane(evt.lane, "unknown");
|
|
1676
|
+
if (evt.suspended !== void 0) spanAttrs["openclaw.failover.suspended"] = evt.suspended;
|
|
1677
|
+
if (evt.cascadeDepth !== void 0) spanAttrs["openclaw.failover.cascade_depth"] = evt.cascadeDepth;
|
|
1678
|
+
spanWithDuration("openclaw.model.failover", spanAttrs, 0, {
|
|
1679
|
+
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1680
|
+
endTimeMs: evt.ts
|
|
1681
|
+
}).end(evt.ts);
|
|
1682
|
+
};
|
|
1683
|
+
return {
|
|
1684
|
+
recordAgentCommentary,
|
|
1685
|
+
recordHarnessRunStarted,
|
|
1686
|
+
recordHarnessRunFinished,
|
|
1687
|
+
recordContextAssembled,
|
|
1688
|
+
recordModelFailover
|
|
1689
|
+
};
|
|
1690
|
+
}
|
|
1691
|
+
//#endregion
|
|
1692
|
+
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1693
|
+
function hasOtelSemconvOptIn(value, optIn) {
|
|
1694
|
+
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1695
|
+
}
|
|
1696
|
+
function emitLatestGenAiSemconv() {
|
|
1697
|
+
return hasOtelSemconvOptIn(process.env[OTEL_SEMCONV_STABILITY_OPT_IN_ENV], GEN_AI_LATEST_EXPERIMENTAL_OPT_IN);
|
|
1698
|
+
}
|
|
1699
|
+
function genAiOperationName(api, observationUnit) {
|
|
1700
|
+
if (observationUnit === "turn") return GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT;
|
|
1701
|
+
const normalized = api?.trim().toLowerCase();
|
|
1702
|
+
if (!normalized) return "chat";
|
|
1703
|
+
if (normalized === "completions" || normalized.endsWith("-completions")) return "text_completion";
|
|
1704
|
+
if (normalized === "generate_content" || normalized.includes("generative-ai")) return "generate_content";
|
|
1705
|
+
return "chat";
|
|
1706
|
+
}
|
|
1707
|
+
function assignPositiveNumberAttr(attrs, key, value) {
|
|
1708
|
+
const normalized = asPositiveFiniteNumber(value);
|
|
1709
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1710
|
+
}
|
|
1711
|
+
function assignModelCallSizeTimingAttrs(attrs, evt) {
|
|
1712
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.request_bytes", evt.requestPayloadBytes);
|
|
1713
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.response_bytes", evt.responseStreamBytes);
|
|
1714
|
+
assignPositiveNumberAttr(attrs, "openclaw.model_call.time_to_first_byte_ms", evt.timeToFirstByteMs);
|
|
1715
|
+
}
|
|
1716
|
+
function assignNumberAttr(attrs, key, value) {
|
|
1717
|
+
const normalized = asFiniteNumber(value);
|
|
1718
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1719
|
+
}
|
|
1720
|
+
function modelCallPromptTokens(usage) {
|
|
1721
|
+
const promptTokens = asNonNegativeFiniteNumber(usage.promptTokens);
|
|
1722
|
+
if (promptTokens !== void 0) return promptTokens;
|
|
1723
|
+
const input = asNonNegativeFiniteNumber(usage.input);
|
|
1724
|
+
const cacheRead = asNonNegativeFiniteNumber(usage.cacheRead);
|
|
1725
|
+
const cacheWrite = asNonNegativeFiniteNumber(usage.cacheWrite);
|
|
1726
|
+
if (input === void 0 && cacheRead === void 0 && cacheWrite === void 0) return;
|
|
1727
|
+
return (input ?? 0) + (cacheRead ?? 0) + (cacheWrite ?? 0);
|
|
1728
|
+
}
|
|
1729
|
+
function assignModelCallPromptStatsAttrs(attrs, evt) {
|
|
1730
|
+
const stats = evt.promptStats;
|
|
1731
|
+
if (!stats) return;
|
|
1732
|
+
for (const [key, value] of [
|
|
1733
|
+
["openclaw.model_call.prompt.input_messages_count", stats.inputMessagesCount],
|
|
1734
|
+
["openclaw.model_call.prompt.input_messages_chars", stats.inputMessagesChars],
|
|
1735
|
+
["openclaw.model_call.prompt.system_prompt_chars", stats.systemPromptChars],
|
|
1736
|
+
["openclaw.model_call.prompt.tool_definitions_count", stats.toolDefinitionsCount],
|
|
1737
|
+
["openclaw.model_call.prompt.tool_definitions_chars", stats.toolDefinitionsChars],
|
|
1738
|
+
["openclaw.model_call.prompt.total_chars", stats.totalChars]
|
|
1739
|
+
]) assignNumberAttr(attrs, key, value);
|
|
1740
|
+
}
|
|
1741
|
+
function assignModelCallUsageAttrs(attrs, evt) {
|
|
1742
|
+
const usage = evt.usage;
|
|
1743
|
+
if (!usage) return;
|
|
1744
|
+
const promptTokens = modelCallPromptTokens(usage);
|
|
1745
|
+
for (const [key, value] of [
|
|
1746
|
+
["openclaw.model_call.usage.input_tokens", usage.input],
|
|
1747
|
+
["openclaw.model_call.usage.output_tokens", usage.output],
|
|
1748
|
+
["openclaw.model_call.usage.cache_read_input_tokens", usage.cacheRead],
|
|
1749
|
+
["openclaw.model_call.usage.cache_creation_input_tokens", usage.cacheWrite],
|
|
1750
|
+
["openclaw.model_call.usage.reasoning_output_tokens", usage.reasoningTokens],
|
|
1751
|
+
["openclaw.model_call.usage.prompt_tokens", promptTokens],
|
|
1752
|
+
["openclaw.model_call.usage.total_tokens", usage.total],
|
|
1753
|
+
["gen_ai.usage.input_tokens", promptTokens],
|
|
1754
|
+
["gen_ai.usage.output_tokens", usage.output],
|
|
1755
|
+
["gen_ai.usage.cache_read.input_tokens", usage.cacheRead],
|
|
1756
|
+
["gen_ai.usage.cache_creation.input_tokens", usage.cacheWrite]
|
|
1757
|
+
]) {
|
|
1758
|
+
const normalized = asNonNegativeFiniteNumber(value);
|
|
1759
|
+
if (normalized !== void 0) attrs[key] = normalized;
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
function assignGenAiSpanIdentityAttrs(attrs, input) {
|
|
1763
|
+
if (emitLatestGenAiSemconv()) attrs["gen_ai.provider.name"] = normalizeDiagnosticValue(input.provider);
|
|
1764
|
+
else attrs["gen_ai.system"] = normalizeDiagnosticValue(input.provider);
|
|
1765
|
+
if (input.model) attrs["gen_ai.request.model"] = redactSensitiveText(input.model.trim());
|
|
1766
|
+
attrs["gen_ai.operation.name"] = genAiOperationName(input.api, input.observationUnit);
|
|
1767
|
+
}
|
|
1768
|
+
function assignGenAiModelCallAttrs(attrs, evt) {
|
|
1769
|
+
assignGenAiSpanIdentityAttrs(attrs, evt);
|
|
1770
|
+
attrs["openclaw.model_call.observation_unit"] = modelCallObservationUnit(evt);
|
|
1771
|
+
}
|
|
1772
|
+
function modelCallObservationUnit(evt) {
|
|
1773
|
+
return evt.observationUnit ?? "request";
|
|
1774
|
+
}
|
|
1775
|
+
function modelCallSpanName(evt) {
|
|
1776
|
+
if (!emitLatestGenAiSemconv()) return "openclaw.model.call";
|
|
1777
|
+
const operationName = genAiOperationName(evt.api, evt.observationUnit);
|
|
1778
|
+
return operationName === GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT ? operationName : `${operationName} ${normalizeDiagnosticValue(evt.model)}`;
|
|
1779
|
+
}
|
|
1780
|
+
function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
1781
|
+
if (!upstreamRequestIdHash) return;
|
|
1782
|
+
const boundedHash = normalizeDiagnosticValue(upstreamRequestIdHash);
|
|
1783
|
+
if (boundedHash === "unknown") return;
|
|
1784
|
+
span.addEvent?.("openclaw.provider.request", { "openclaw.upstreamRequestIdHash": boundedHash });
|
|
1785
|
+
}
|
|
1786
|
+
//#endregion
|
|
1908
1787
|
//#region extensions/diagnostics-otel/src/service-recorders-model.ts
|
|
1909
1788
|
function createModelRecorders(runtime) {
|
|
1910
|
-
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1789
|
+
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
1911
1790
|
const modelCallMetricAttrs = (evt) => ({
|
|
1912
1791
|
"openclaw.provider": evt.provider,
|
|
1913
1792
|
"openclaw.model": evt.model,
|
|
@@ -1916,11 +1795,11 @@ function createModelRecorders(runtime) {
|
|
|
1916
1795
|
"openclaw.model_call.observation_unit": modelCallObservationUnit(evt)
|
|
1917
1796
|
});
|
|
1918
1797
|
const recordModelCallSizeTimingMetrics = (evt, attrs) => {
|
|
1919
|
-
const requestPayloadBytes =
|
|
1798
|
+
const requestPayloadBytes = asPositiveFiniteNumber(evt.requestPayloadBytes);
|
|
1920
1799
|
if (requestPayloadBytes !== void 0) modelCallRequestBytesHistogram.record(requestPayloadBytes, attrs);
|
|
1921
|
-
const responseStreamBytes =
|
|
1800
|
+
const responseStreamBytes = asPositiveFiniteNumber(evt.responseStreamBytes);
|
|
1922
1801
|
if (responseStreamBytes !== void 0) modelCallResponseBytesHistogram.record(responseStreamBytes, attrs);
|
|
1923
|
-
const timeToFirstByteMs =
|
|
1802
|
+
const timeToFirstByteMs = asPositiveFiniteNumber(evt.timeToFirstByteMs);
|
|
1924
1803
|
if (timeToFirstByteMs !== void 0) modelCallTimeToFirstByteHistogram.record(timeToFirstByteMs, attrs);
|
|
1925
1804
|
};
|
|
1926
1805
|
const recordModelCallStarted = (evt, metadata) => {
|
|
@@ -1931,12 +1810,13 @@ function createModelRecorders(runtime) {
|
|
|
1931
1810
|
"openclaw.provider": evt.provider,
|
|
1932
1811
|
"openclaw.model": evt.model
|
|
1933
1812
|
};
|
|
1813
|
+
addRunAttrs(spanAttrs, evt);
|
|
1934
1814
|
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1935
1815
|
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
1936
1816
|
if (evt.transport) spanAttrs["openclaw.transport"] = evt.transport;
|
|
1937
1817
|
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1938
1818
|
return trackTrustedSpan(evt, metadata, spanWithDuration(modelCallSpanName(evt), spanAttrs, void 0, {
|
|
1939
|
-
kind:
|
|
1819
|
+
kind: SpanKind.CLIENT,
|
|
1940
1820
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1941
1821
|
startTimeMs: evt.ts
|
|
1942
1822
|
})).spanContext();
|
|
@@ -1965,6 +1845,7 @@ function createModelRecorders(runtime) {
|
|
|
1965
1845
|
"error.type": errorType
|
|
1966
1846
|
} : {}
|
|
1967
1847
|
};
|
|
1848
|
+
addRunAttrs(spanAttrs, evt);
|
|
1968
1849
|
if (evt.type === "model.call.error" && evt.failureKind) spanAttrs["openclaw.failureKind"] = normalizeDiagnosticValue(evt.failureKind, "other");
|
|
1969
1850
|
assignGenAiModelCallAttrs(spanAttrs, evt);
|
|
1970
1851
|
if (evt.api) spanAttrs["openclaw.api"] = evt.api;
|
|
@@ -1974,7 +1855,7 @@ function createModelRecorders(runtime) {
|
|
|
1974
1855
|
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1975
1856
|
assignOtelModelContentAttributes(spanAttrs, modelContent, contentCapturePolicy);
|
|
1976
1857
|
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1977
|
-
kind:
|
|
1858
|
+
kind: SpanKind.CLIENT,
|
|
1978
1859
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
1979
1860
|
endTimeMs: evt.ts
|
|
1980
1861
|
});
|
|
@@ -2117,7 +1998,9 @@ function createOperationsRecorders(runtime) {
|
|
|
2117
1998
|
const attrs = toolLoopAttrs(evt);
|
|
2118
1999
|
toolLoopCounter.add(1, attrs);
|
|
2119
2000
|
if (!tracesEnabled) return;
|
|
2120
|
-
const
|
|
2001
|
+
const spanAttrs = { ...attrs };
|
|
2002
|
+
addRunAttrs(spanAttrs, evt);
|
|
2003
|
+
const span = spanWithDuration("openclaw.tool.loop", spanAttrs, 0, { endTimeMs: evt.ts });
|
|
2121
2004
|
if (evt.level === "critical" || evt.action === "block") span.setStatus({
|
|
2122
2005
|
code: SpanStatusCode.ERROR,
|
|
2123
2006
|
message: `${evt.detector}:${evt.action}`
|
|
@@ -2131,9 +2014,6 @@ function createOperationsRecorders(runtime) {
|
|
|
2131
2014
|
memoryExternalHistogram.record(evt.memory.externalBytes, attrs);
|
|
2132
2015
|
memoryArrayBuffersHistogram.record(evt.memory.arrayBuffersBytes, attrs);
|
|
2133
2016
|
};
|
|
2134
|
-
const recordMemorySample = (evt) => {
|
|
2135
|
-
recordMemoryUsageMetrics(evt);
|
|
2136
|
-
};
|
|
2137
2017
|
const recordMemoryPressure = (evt) => {
|
|
2138
2018
|
const attrs = {
|
|
2139
2019
|
"openclaw.memory.level": evt.level,
|
|
@@ -2162,9 +2042,11 @@ function createOperationsRecorders(runtime) {
|
|
|
2162
2042
|
};
|
|
2163
2043
|
const recordAsyncQueueDropped = (evt) => {
|
|
2164
2044
|
asyncQueueDroppedCounter.add(evt.droppedEvents, { "openclaw.diagnostic.async_queue.drop_class": "total" });
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2045
|
+
for (const [dropClass, field] of [
|
|
2046
|
+
["trusted", "droppedTrustedEvents"],
|
|
2047
|
+
["untrusted", "droppedUntrustedEvents"],
|
|
2048
|
+
["priority", "droppedPriorityEvents"]
|
|
2049
|
+
]) if (evt[field] !== void 0) asyncQueueDroppedCounter.add(evt[field], { "openclaw.diagnostic.async_queue.drop_class": dropClass });
|
|
2168
2050
|
};
|
|
2169
2051
|
const recordRunCompleted = (evt, metadata, privateData) => {
|
|
2170
2052
|
const attrs = {
|
|
@@ -2215,7 +2097,6 @@ function createOperationsRecorders(runtime) {
|
|
|
2215
2097
|
recordRunAttempt,
|
|
2216
2098
|
recordToolLoop,
|
|
2217
2099
|
recordMemoryUsageMetrics,
|
|
2218
|
-
recordMemorySample,
|
|
2219
2100
|
recordMemoryPressure,
|
|
2220
2101
|
recordAsyncQueueDropped,
|
|
2221
2102
|
recordRunCompleted
|
|
@@ -2224,7 +2105,7 @@ function createOperationsRecorders(runtime) {
|
|
|
2224
2105
|
//#endregion
|
|
2225
2106
|
//#region extensions/diagnostics-otel/src/service-recorders-tools.ts
|
|
2226
2107
|
function createToolAndSystemRecorders(runtime) {
|
|
2227
|
-
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
2108
|
+
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, internalOrTrustedExplicitParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, contentCapturePolicy, tracesEnabled } = runtime;
|
|
2228
2109
|
const toolExecutionBaseAttrs = (evt) => ({
|
|
2229
2110
|
"openclaw.toolName": evt.toolName,
|
|
2230
2111
|
"openclaw.tool.source": normalizeDiagnosticValue(evt.toolSource, "core"),
|
|
@@ -2315,7 +2196,7 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2315
2196
|
"openclaw.reason": normalizeDiagnosticValue(evt.reason, "none")
|
|
2316
2197
|
};
|
|
2317
2198
|
payloadLargeCounter.add(1, attrs);
|
|
2318
|
-
const bytes =
|
|
2199
|
+
const bytes = asPositiveFiniteNumber(evt.bytes);
|
|
2319
2200
|
if (bytes !== void 0) payloadLargeBytesHistogram.record(bytes, attrs);
|
|
2320
2201
|
};
|
|
2321
2202
|
const recordExecProcessCompleted = (evt, metadata) => {
|
|
@@ -2387,7 +2268,7 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2387
2268
|
});
|
|
2388
2269
|
span.end(evt.ts);
|
|
2389
2270
|
};
|
|
2390
|
-
const recordDiagnosticPhaseCompleted = (evt) => {
|
|
2271
|
+
const recordDiagnosticPhaseCompleted = (evt, metadata) => {
|
|
2391
2272
|
if (!tracesEnabled) return;
|
|
2392
2273
|
const spanAttrs = {
|
|
2393
2274
|
"openclaw.phase": normalizeDiagnosticValue(evt.name, "unknown"),
|
|
@@ -2397,7 +2278,10 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2397
2278
|
...evt.cpuCoreRatio !== void 0 ? { "openclaw.phase.cpu_core_ratio": evt.cpuCoreRatio } : {}
|
|
2398
2279
|
};
|
|
2399
2280
|
for (const [key, value] of Object.entries(evt.details ?? {})) spanAttrs[`openclaw.phase.detail.${key}`] = typeof value === "boolean" ? String(value) : value;
|
|
2400
|
-
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
2281
|
+
spanWithDuration("openclaw.diagnostic.phase", spanAttrs, evt.durationMs, {
|
|
2282
|
+
endTimeMs: evt.ts,
|
|
2283
|
+
...metadata.trusted ? { parentContext: internalOrTrustedExplicitParentContext(evt, metadata) ?? ROOT_CONTEXT } : {}
|
|
2284
|
+
}).end(evt.ts);
|
|
2401
2285
|
};
|
|
2402
2286
|
const recordTelemetryExporter = (evt, metadata) => {
|
|
2403
2287
|
if (!metadata.trusted) return;
|
|
@@ -2441,52 +2325,34 @@ function createUsageRecorders(runtime) {
|
|
|
2441
2325
|
"gen_ai.request.model": normalizeDiagnosticValue(evt.model)
|
|
2442
2326
|
};
|
|
2443
2327
|
const usage = evt.usage;
|
|
2444
|
-
|
|
2445
|
-
|
|
2328
|
+
for (const [tokenType, field] of [
|
|
2329
|
+
["input", "input"],
|
|
2330
|
+
["output", "output"],
|
|
2331
|
+
["cache_read", "cacheRead"],
|
|
2332
|
+
["cache_write", "cacheWrite"],
|
|
2333
|
+
["prompt", "promptTokens"],
|
|
2334
|
+
["total", "total"]
|
|
2335
|
+
]) {
|
|
2336
|
+
const amount = usage[field];
|
|
2337
|
+
if (!amount) continue;
|
|
2338
|
+
tokensCounter.add(amount, {
|
|
2446
2339
|
...attrs,
|
|
2447
|
-
"openclaw.token":
|
|
2340
|
+
"openclaw.token": tokenType
|
|
2448
2341
|
});
|
|
2449
|
-
genAiTokenUsageHistogram.record(
|
|
2342
|
+
if (tokenType === "input" || tokenType === "output") genAiTokenUsageHistogram.record(amount, {
|
|
2450
2343
|
...genAiAttrs,
|
|
2451
|
-
"gen_ai.token.type":
|
|
2344
|
+
"gen_ai.token.type": tokenType
|
|
2452
2345
|
});
|
|
2453
2346
|
}
|
|
2454
|
-
if (
|
|
2455
|
-
|
|
2347
|
+
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
2348
|
+
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
2349
|
+
for (const kind of ["limit", "used"]) {
|
|
2350
|
+
const amount = evt.context?.[kind];
|
|
2351
|
+
if (amount) contextHistogram.record(amount, {
|
|
2456
2352
|
...attrs,
|
|
2457
|
-
"openclaw.
|
|
2458
|
-
});
|
|
2459
|
-
genAiTokenUsageHistogram.record(usage.output, {
|
|
2460
|
-
...genAiAttrs,
|
|
2461
|
-
"gen_ai.token.type": "output"
|
|
2353
|
+
"openclaw.context": kind
|
|
2462
2354
|
});
|
|
2463
2355
|
}
|
|
2464
|
-
if (usage.cacheRead) tokensCounter.add(usage.cacheRead, {
|
|
2465
|
-
...attrs,
|
|
2466
|
-
"openclaw.token": "cache_read"
|
|
2467
|
-
});
|
|
2468
|
-
if (usage.cacheWrite) tokensCounter.add(usage.cacheWrite, {
|
|
2469
|
-
...attrs,
|
|
2470
|
-
"openclaw.token": "cache_write"
|
|
2471
|
-
});
|
|
2472
|
-
if (usage.promptTokens) tokensCounter.add(usage.promptTokens, {
|
|
2473
|
-
...attrs,
|
|
2474
|
-
"openclaw.token": "prompt"
|
|
2475
|
-
});
|
|
2476
|
-
if (usage.total) tokensCounter.add(usage.total, {
|
|
2477
|
-
...attrs,
|
|
2478
|
-
"openclaw.token": "total"
|
|
2479
|
-
});
|
|
2480
|
-
if (evt.costUsd) costCounter.add(evt.costUsd, attrs);
|
|
2481
|
-
if (evt.durationMs) durationHistogram.record(evt.durationMs, attrs);
|
|
2482
|
-
if (evt.context?.limit) contextHistogram.record(evt.context.limit, {
|
|
2483
|
-
...attrs,
|
|
2484
|
-
"openclaw.context": "limit"
|
|
2485
|
-
});
|
|
2486
|
-
if (evt.context?.used) contextHistogram.record(evt.context.used, {
|
|
2487
|
-
...attrs,
|
|
2488
|
-
"openclaw.context": "used"
|
|
2489
|
-
});
|
|
2490
2356
|
if (!tracesEnabled) return;
|
|
2491
2357
|
const genAiInputTokens = usage.promptTokens ?? (usage.input ?? 0) + (usage.cacheRead ?? 0) + (usage.cacheWrite ?? 0);
|
|
2492
2358
|
const spanAttrs = {
|
|
@@ -2499,6 +2365,7 @@ function createUsageRecorders(runtime) {
|
|
|
2499
2365
|
};
|
|
2500
2366
|
if (metadata.trusted && metadata.internal && hostPluginId) spanAttrs["openclaw.plugin"] = normalizeDiagnosticValue(hostPluginId);
|
|
2501
2367
|
assignGenAiSpanIdentityAttrs(spanAttrs, evt);
|
|
2368
|
+
addRunAttrs(spanAttrs, evt);
|
|
2502
2369
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.input_tokens", genAiInputTokens);
|
|
2503
2370
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.output_tokens", usage.output);
|
|
2504
2371
|
assignPositiveNumberAttr(spanAttrs, "gen_ai.usage.cache_read.input_tokens", usage.cacheRead);
|
|
@@ -2591,6 +2458,7 @@ function createUsageRecorders(runtime) {
|
|
|
2591
2458
|
if (typeof evt.durationMs === "number") messageDurationHistogram.record(evt.durationMs, attrs);
|
|
2592
2459
|
if (!tracesEnabled) return;
|
|
2593
2460
|
const spanAttrs = { ...attrs };
|
|
2461
|
+
addRunAttrs(spanAttrs, evt);
|
|
2594
2462
|
if (evt.reason) spanAttrs["openclaw.reason"] = normalizeDiagnosticValue(evt.reason, "unknown");
|
|
2595
2463
|
const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
2596
2464
|
const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
|
|
@@ -2702,12 +2570,11 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2702
2570
|
const spanWithDuration = (name, attributes, durationMs, options = {}) => {
|
|
2703
2571
|
const endTimeMs = options.endTimeMs ?? Date.now();
|
|
2704
2572
|
const startTime = typeof options.startTimeMs === "number" ? options.startTimeMs : typeof durationMs === "number" && durationMs >= 0 ? endTimeMs - durationMs : void 0;
|
|
2705
|
-
const parentContext = "parentContext" in options ? options.parentContext ?? void 0 : void 0;
|
|
2706
2573
|
return tracer.startSpan(name, {
|
|
2707
2574
|
attributes: redactOtelAttributes(attributes),
|
|
2708
2575
|
...options.kind !== void 0 ? { kind: options.kind } : {},
|
|
2709
2576
|
...startTime !== void 0 ? { startTime } : {}
|
|
2710
|
-
}, parentContext);
|
|
2577
|
+
}, options.parentContext ?? void 0);
|
|
2711
2578
|
};
|
|
2712
2579
|
const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
|
|
2713
2580
|
const internalOrTrustedTraceContext = (evt, metadata) => metadata.internal ? normalizeTraceContext(evt.trace) : normalizedTrustedTraceContext(evt, metadata);
|
|
@@ -2836,6 +2703,7 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2836
2703
|
if (evt.model) spanAttrs["openclaw.model"] = evt.model;
|
|
2837
2704
|
if (evt.channel) spanAttrs["openclaw.channel"] = evt.channel;
|
|
2838
2705
|
if (evt.trigger) spanAttrs["openclaw.trigger"] = evt.trigger;
|
|
2706
|
+
if (evt.agentId) spanAttrs["openclaw.agent"] = normalizeDiagnosticValue(evt.agentId);
|
|
2839
2707
|
};
|
|
2840
2708
|
const paramsSummaryAttrs = (summary) => {
|
|
2841
2709
|
if (!summary) return {};
|
|
@@ -2892,10 +2760,6 @@ function isOtelSdkDisabled(logger) {
|
|
|
2892
2760
|
logger.warn("diagnostics-otel: invalid OTEL_SDK_DISABLED value; expected true or false, using false");
|
|
2893
2761
|
return false;
|
|
2894
2762
|
}
|
|
2895
|
-
function readNonblankOtelEnv(name) {
|
|
2896
|
-
const value = process.env[name];
|
|
2897
|
-
return value?.trim() ? value : void 0;
|
|
2898
|
-
}
|
|
2899
2763
|
function readPositiveOtelNumber(name, fallback) {
|
|
2900
2764
|
const value = otelCore.getNumberFromEnv(name);
|
|
2901
2765
|
if (value !== void 0 && value <= 0) {
|
|
@@ -2920,7 +2784,7 @@ function resolveResourceDetectors() {
|
|
|
2920
2784
|
});
|
|
2921
2785
|
}
|
|
2922
2786
|
function resolveSignalProtocol(signal, configuredProtocol) {
|
|
2923
|
-
return configuredProtocol ??
|
|
2787
|
+
return configuredProtocol ?? readNonBlankString(process.env[OTEL_SIGNAL_PROTOCOL_ENV[signal]]) ?? readNonBlankString(process.env["OTEL_EXPORTER_OTLP_PROTOCOL"]) ?? OTLP_HTTP_PROTOBUF_PROTOCOL;
|
|
2924
2788
|
}
|
|
2925
2789
|
function createStartupRollbackError(startupError, cleanupError) {
|
|
2926
2790
|
return new AggregateError([startupError, cleanupError], "diagnostics-otel startup failed and rollback cleanup failed", { cause: startupError });
|
|
@@ -3069,10 +2933,13 @@ function createDiagnosticsOtelService() {
|
|
|
3069
2933
|
if (!tracesActive && !metricsActive && !logsActive) return;
|
|
3070
2934
|
const hasOwnedOtlpSignal = ownedOtlpSignals.length > 0;
|
|
3071
2935
|
const sharedEnvEndpoint = hasOwnedOtlpSignal ? process.env[OTEL_EXPORTER_OTLP_ENDPOINT_ENV] : void 0;
|
|
3072
|
-
const endpoint = hasOwnedOtlpSignal ?
|
|
2936
|
+
const endpoint = hasOwnedOtlpSignal ? normalizeOptionalString(otel.endpoint ?? sharedEnvEndpoint) : void 0;
|
|
3073
2937
|
const headers = otel.headers ?? void 0;
|
|
3074
2938
|
const serviceName = otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || "openclaw";
|
|
3075
|
-
const sampleRate =
|
|
2939
|
+
const sampleRate = asFiniteNumberInRange(otel.sampleRate, {
|
|
2940
|
+
min: 0,
|
|
2941
|
+
max: 1
|
|
2942
|
+
});
|
|
3076
2943
|
const contentCapturePolicy = resolveContentCapturePolicy(otel.captureContent);
|
|
3077
2944
|
const resource = resources.resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
|
|
3078
2945
|
const logUrl = logsToOtlp ? resolveSignalOtelUrl({
|
|
@@ -3218,7 +3085,7 @@ function createDiagnosticsOtelService() {
|
|
|
3218
3085
|
recorders,
|
|
3219
3086
|
recordLogRecord,
|
|
3220
3087
|
recordSecurityEvent
|
|
3221
|
-
}), metricsActive ? void 0 : tracesActive ? { exclude: ["gateway.event_loop.sample", "diagnostic.gc"] } : { include: ["log.record", "security.event"] });
|
|
3088
|
+
}), metricsActive ? tracesActive ? void 0 : { exclude: ["diagnostic.phase.completed"] } : tracesActive ? { exclude: ["gateway.event_loop.sample", "diagnostic.gc"] } : { include: ["log.record", "security.event"] });
|
|
3222
3089
|
if (tracesActive) active.unregisterTracePropagationBridge = ctx.internalDiagnostics?.registerTracePropagationBridge?.({
|
|
3223
3090
|
shouldPrepareEvent(event) {
|
|
3224
3091
|
return event.type === "model.call.started" || event.type === "tool.execution.started";
|
package/dist/api.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { emptyPluginConfigSchema } from "openclaw/plugin-sdk/plugin-entry";
|
|
2
|
-
import { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
3
2
|
import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime";
|
|
3
|
+
import { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
4
4
|
export { createChildDiagnosticTraceContext, createDiagnosticTraceContext, emitDiagnosticEvent, emptyPluginConfigSchema, formatDiagnosticTraceparent, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, onDiagnosticEvent, parseDiagnosticTraceparent, redactSensitiveText };
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-ButGOVne.mjs";
|
|
2
2
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
3
3
|
//#region extensions/diagnostics-otel/index.ts
|
|
4
4
|
var diagnostics_otel_default = definePluginEntry({
|
package/dist/runtime-api.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-ButGOVne.mjs";
|
|
2
2
|
export { createDiagnosticsOtelService };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-otel",
|
|
3
|
-
"version": "2026.9.
|
|
3
|
+
"version": "2026.9.7",
|
|
4
4
|
"description": "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -34,10 +34,10 @@
|
|
|
34
34
|
"minHostVersion": ">=2026.4.25"
|
|
35
35
|
},
|
|
36
36
|
"compat": {
|
|
37
|
-
"pluginApi": ">=2026.9.
|
|
37
|
+
"pluginApi": ">=2026.9.7"
|
|
38
38
|
},
|
|
39
39
|
"build": {
|
|
40
|
-
"openclawVersion": "2026.9.
|
|
40
|
+
"openclawVersion": "2026.9.7"
|
|
41
41
|
},
|
|
42
42
|
"release": {
|
|
43
43
|
"publishToClawHub": true,
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"assets/activity.svg"
|
|
56
56
|
],
|
|
57
57
|
"peerDependencies": {
|
|
58
|
-
"openclaw": ">=2026.9.
|
|
58
|
+
"openclaw": ">=2026.9.7"
|
|
59
59
|
},
|
|
60
60
|
"peerDependenciesMeta": {
|
|
61
61
|
"openclaw": {
|