@openclaw/diagnostics-otel 2026.9.8 → 2026.10.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -15,14 +15,15 @@ import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
|
15
15
|
import { readFileSync } from "node:fs";
|
|
16
16
|
import nodePath from "node:path";
|
|
17
17
|
import { isInternalDiagnosticEventMetadata, isValidDiagnosticSpanId, isValidDiagnosticTraceFlags, isValidDiagnosticTraceId, normalizeDiagnosticLane, normalizeDiagnosticValue } from "openclaw/plugin-sdk/diagnostic-runtime";
|
|
18
|
-
import { collectErrorGraphCandidates } from "openclaw/plugin-sdk/error-runtime";
|
|
18
|
+
import { collectErrorGraphCandidates, readErrorName } from "openclaw/plugin-sdk/error-runtime";
|
|
19
19
|
import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime";
|
|
20
20
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
21
21
|
import { BatchLogRecordProcessor, LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
22
22
|
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
|
|
23
23
|
import { B3InjectEncoding, B3Propagator } from "@opentelemetry/propagator-b3";
|
|
24
24
|
import { JaegerPropagator } from "@opentelemetry/propagator-jaeger";
|
|
25
|
-
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES,
|
|
25
|
+
import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_TOOL_DEFINITIONS, GEN_AI_OPERATION_NAME_VALUE_EXECUTE_TOOL, GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT } from "@opentelemetry/semantic-conventions/incubating";
|
|
26
|
+
import { pruneMapToMaxSize } from "openclaw/plugin-sdk/collection-runtime";
|
|
26
27
|
const DROPPED_OTEL_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
|
|
27
28
|
"openclaw.callId",
|
|
28
29
|
"openclaw.call_id",
|
|
@@ -65,8 +66,6 @@ const OTEL_EXPORTER_OTLP_LOGS_PROTOCOL_ENV = "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL";
|
|
|
65
66
|
const OTEL_EXPORTER_OTLP_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CERTIFICATE";
|
|
66
67
|
const OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE";
|
|
67
68
|
const OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV = "OTEL_EXPORTER_OTLP_CLIENT_KEY";
|
|
68
|
-
const OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN";
|
|
69
|
-
const GEN_AI_LATEST_EXPERIMENTAL_OPT_IN = "gen_ai_latest_experimental";
|
|
70
69
|
const GEN_AI_TOKEN_USAGE_BUCKETS = [
|
|
71
70
|
1,
|
|
72
71
|
4,
|
|
@@ -149,44 +148,21 @@ const MAX_RETAINED_TRUSTED_SPAN_CONTEXTS = 1024;
|
|
|
149
148
|
const MAX_OTEL_CONTENT_ATTRIBUTE_CHARS = 131072;
|
|
150
149
|
const MAX_OTEL_ERROR_MESSAGE_CHARS = 4096;
|
|
151
150
|
const PRELOADED_OTEL_SDK_ENV = "OPENCLAW_OTEL_PRELOADED";
|
|
152
|
-
const NO_CONTENT_CAPTURE = {
|
|
153
|
-
inputMessages: false,
|
|
154
|
-
outputMessages: false,
|
|
155
|
-
toolInputs: false,
|
|
156
|
-
toolOutputs: false,
|
|
157
|
-
systemPrompt: false,
|
|
158
|
-
toolDefinitions: false,
|
|
159
|
-
logBodies: false
|
|
160
|
-
};
|
|
161
|
-
function clampOtelLogText(value, maxChars) {
|
|
162
|
-
return value.length > maxChars ? `${truncateUtf16Safe(value, maxChars)}...(truncated)` : value;
|
|
163
|
-
}
|
|
164
151
|
function normalizeOtelLogString(value, maxChars) {
|
|
165
|
-
|
|
152
|
+
const redacted = redactSensitiveText(value);
|
|
153
|
+
return redacted.length > maxChars ? `${truncateUtf16Safe(redacted, maxChars)}...(truncated)` : redacted;
|
|
166
154
|
}
|
|
167
155
|
function normalizeOtelErrorMessage(value) {
|
|
168
156
|
if (!value) return;
|
|
169
157
|
return normalizeOtelLogString(value.trim(), MAX_OTEL_ERROR_MESSAGE_CHARS) || void 0;
|
|
170
158
|
}
|
|
171
|
-
function resolveContentCapturePolicy(value) {
|
|
172
|
-
return value === true ? {
|
|
173
|
-
inputMessages: true,
|
|
174
|
-
outputMessages: true,
|
|
175
|
-
toolInputs: true,
|
|
176
|
-
toolOutputs: true,
|
|
177
|
-
systemPrompt: false,
|
|
178
|
-
toolDefinitions: true,
|
|
179
|
-
logBodies: true
|
|
180
|
-
} : NO_CONTENT_CAPTURE;
|
|
181
|
-
}
|
|
182
159
|
function hasPreloadedOtelSdk() {
|
|
183
160
|
return process.env[PRELOADED_OTEL_SDK_ENV] === "1";
|
|
184
161
|
}
|
|
185
162
|
function normalizeOtelContentValue(value) {
|
|
186
163
|
if (typeof value === "string") return normalizeOtelLogString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
187
164
|
if (Array.isArray(value)) {
|
|
188
|
-
const items =
|
|
189
|
-
for (const item of value.slice(0, 200)) if (typeof item === "string") items.push(item);
|
|
165
|
+
const items = value.slice(0, 200).filter((item) => typeof item === "string");
|
|
190
166
|
if (items.length > 0) return normalizeOtelLogString(items.join("\n"), MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
191
167
|
}
|
|
192
168
|
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
@@ -410,11 +386,6 @@ function errorCategory(err) {
|
|
|
410
386
|
return "unknown";
|
|
411
387
|
}
|
|
412
388
|
}
|
|
413
|
-
function readErrorName(err) {
|
|
414
|
-
if (!err || typeof err !== "object") return;
|
|
415
|
-
const name = err.name;
|
|
416
|
-
return typeof name === "string" && name.trim() ? name : void 0;
|
|
417
|
-
}
|
|
418
389
|
function readErrorCode(err) {
|
|
419
390
|
if (!err || typeof err !== "object") return;
|
|
420
391
|
const code = err.code;
|
|
@@ -432,7 +403,7 @@ function findOtlpExporterError(reason) {
|
|
|
432
403
|
//#endregion
|
|
433
404
|
//#region extensions/diagnostics-otel/src/service-events.ts
|
|
434
405
|
function createDiagnosticsEventHandler(params) {
|
|
435
|
-
const { logger, recorders,
|
|
406
|
+
const { logger, recorders, recordLogEvent } = params;
|
|
436
407
|
return (evt, metadata, privateData) => {
|
|
437
408
|
try {
|
|
438
409
|
switch (evt.type) {
|
|
@@ -450,8 +421,8 @@ function createDiagnosticsEventHandler(params) {
|
|
|
450
421
|
case "message.dispatch.completed": return recorders.recordMessageDispatchCompleted(evt);
|
|
451
422
|
case "message.processed": return recorders.recordMessageProcessed(evt, metadata);
|
|
452
423
|
case "message.delivery.started": return recorders.recordMessageDeliveryStarted(evt);
|
|
453
|
-
case "message.delivery.completed":
|
|
454
|
-
case "message.delivery.error": return recorders.
|
|
424
|
+
case "message.delivery.completed":
|
|
425
|
+
case "message.delivery.error": return recorders.recordMessageDeliveryFinished(evt, metadata);
|
|
455
426
|
case "talk.event": return recorders.recordTalkEvent(evt, metadata);
|
|
456
427
|
case "queue.lane.enqueue": return recorders.recordLaneEnqueue(evt);
|
|
457
428
|
case "queue.lane.dequeue": return recorders.recordLaneDequeue(evt);
|
|
@@ -489,8 +460,8 @@ function createDiagnosticsEventHandler(params) {
|
|
|
489
460
|
case "skill.used": return recorders.recordSkillUsed(evt, metadata);
|
|
490
461
|
case "exec.process.completed": return recorders.recordExecProcessCompleted(evt, metadata);
|
|
491
462
|
case "exec.approval.followup_suppressed": break;
|
|
492
|
-
case "log.record":
|
|
493
|
-
case "security.event": return
|
|
463
|
+
case "log.record":
|
|
464
|
+
case "security.event": return recordLogEvent?.(evt, metadata);
|
|
494
465
|
case "tool.loop": return recorders.recordToolLoop(evt);
|
|
495
466
|
case "diagnostic.memory.sample": return recorders.recordMemoryUsageMetrics(evt);
|
|
496
467
|
case "diagnostic.memory.pressure": return recorders.recordMemoryPressure(evt);
|
|
@@ -707,9 +678,6 @@ function assignOtelEventAttributes(attributes, eventAttributes, keyPrefix, norma
|
|
|
707
678
|
function assignOtelLogEventAttributes(attributes, eventAttributes) {
|
|
708
679
|
assignOtelEventAttributes(attributes, eventAttributes, "openclaw.");
|
|
709
680
|
}
|
|
710
|
-
function assignOtelSecurityEventAttributes(attributes, eventAttributes) {
|
|
711
|
-
assignOtelEventAttributes(attributes, eventAttributes, "openclaw.security.attribute.", normalizeDiagnosticValue);
|
|
712
|
-
}
|
|
713
681
|
function securitySeverityText(severity) {
|
|
714
682
|
switch (severity) {
|
|
715
683
|
case "critical": return "FATAL";
|
|
@@ -721,36 +689,39 @@ function securitySeverityText(severity) {
|
|
|
721
689
|
return severity;
|
|
722
690
|
}
|
|
723
691
|
function assignOtelSecurityAttributes(attributes, evt) {
|
|
692
|
+
const assignOptionalNormalized = (key, value) => {
|
|
693
|
+
if (value) assignOtelLogAttribute(attributes, key, normalizeDiagnosticValue(value));
|
|
694
|
+
};
|
|
724
695
|
assignOtelLogAttribute(attributes, "openclaw.security.event_id", evt.eventId);
|
|
725
696
|
assignOtelLogAttribute(attributes, "openclaw.security.category", evt.category);
|
|
726
697
|
assignOtelLogAttribute(attributes, "openclaw.security.action", normalizeDiagnosticValue(evt.action));
|
|
727
698
|
assignOtelLogAttribute(attributes, "openclaw.security.outcome", evt.outcome);
|
|
728
699
|
assignOtelLogAttribute(attributes, "openclaw.security.severity", evt.severity);
|
|
729
|
-
|
|
700
|
+
assignOptionalNormalized("openclaw.security.reason", evt.reason);
|
|
730
701
|
if (evt.actor) {
|
|
731
702
|
assignOtelLogAttribute(attributes, "openclaw.security.actor.kind", evt.actor.kind);
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
703
|
+
assignOptionalNormalized("openclaw.security.actor.id_hash", evt.actor.idHash);
|
|
704
|
+
assignOptionalNormalized("openclaw.security.actor.device_id_hash", evt.actor.deviceIdHash);
|
|
705
|
+
assignOptionalNormalized("openclaw.security.actor.channel", evt.actor.channel);
|
|
706
|
+
assignOptionalNormalized("openclaw.security.actor.role", evt.actor.role);
|
|
736
707
|
if (evt.actor.scopes?.length) assignOtelLogAttribute(attributes, "openclaw.security.actor.scopes", evt.actor.scopes.map((scope) => normalizeDiagnosticValue(scope)).join(","));
|
|
737
708
|
}
|
|
738
709
|
if (evt.target) {
|
|
739
710
|
assignOtelLogAttribute(attributes, "openclaw.security.target.kind", evt.target.kind);
|
|
740
|
-
|
|
711
|
+
assignOptionalNormalized("openclaw.security.target.id_hash", evt.target.idHash);
|
|
741
712
|
if (evt.target.name) assignOtelLogAttribute(attributes, "openclaw.security.target.name", securityTargetNameAttr(evt.target.name));
|
|
742
|
-
|
|
713
|
+
assignOptionalNormalized("openclaw.security.target.owner", evt.target.owner);
|
|
743
714
|
}
|
|
744
715
|
if (evt.policy) {
|
|
745
|
-
|
|
716
|
+
assignOptionalNormalized("openclaw.security.policy.id", evt.policy.id);
|
|
746
717
|
if (evt.policy.decision) assignOtelLogAttribute(attributes, "openclaw.security.policy.decision", evt.policy.decision);
|
|
747
|
-
|
|
718
|
+
assignOptionalNormalized("openclaw.security.policy.reason", evt.policy.reason);
|
|
748
719
|
}
|
|
749
720
|
if (evt.control) {
|
|
750
|
-
|
|
721
|
+
assignOptionalNormalized("openclaw.security.control.id", evt.control.id);
|
|
751
722
|
if (evt.control.family) assignOtelLogAttribute(attributes, "openclaw.security.control.family", evt.control.family);
|
|
752
723
|
}
|
|
753
|
-
|
|
724
|
+
assignOtelEventAttributes(attributes, evt.attributes, "openclaw.security.attribute.", normalizeDiagnosticValue);
|
|
754
725
|
}
|
|
755
726
|
//#endregion
|
|
756
727
|
//#region extensions/diagnostics-otel/src/service-trace-context.ts
|
|
@@ -772,29 +743,17 @@ function traceFlagsToOtel(traceFlags) {
|
|
|
772
743
|
return (Number.parseInt(traceFlags ?? "00", 16) & TraceFlags.SAMPLED) !== 0 ? TraceFlags.SAMPLED : TraceFlags.NONE;
|
|
773
744
|
}
|
|
774
745
|
function contextForTraceContext(traceContext) {
|
|
775
|
-
|
|
776
|
-
if (!normalized?.spanId) return;
|
|
746
|
+
if (!traceContext?.spanId) return;
|
|
777
747
|
return trace.setSpanContext(context.active(), {
|
|
778
|
-
traceId:
|
|
779
|
-
spanId:
|
|
780
|
-
traceFlags: traceFlagsToOtel(
|
|
748
|
+
traceId: traceContext.traceId,
|
|
749
|
+
spanId: traceContext.spanId,
|
|
750
|
+
traceFlags: traceFlagsToOtel(traceContext.traceFlags),
|
|
781
751
|
isRemote: true
|
|
782
752
|
});
|
|
783
753
|
}
|
|
784
|
-
function contextForTrustedTraceContext(evt, metadata) {
|
|
785
|
-
return metadata.trusted || metadata.trustedTraceContext === true ? contextForTraceContext(evt.trace) : void 0;
|
|
786
|
-
}
|
|
787
754
|
function normalizedTrustedTraceContext(evt, metadata) {
|
|
788
755
|
return metadata.trusted || metadata.trustedTraceContext === true ? normalizeTraceContext(evt.trace) : void 0;
|
|
789
756
|
}
|
|
790
|
-
function addTraceAttributes(attributes, traceContext) {
|
|
791
|
-
const normalized = normalizeTraceContext(traceContext);
|
|
792
|
-
if (!normalized) return;
|
|
793
|
-
attributes["openclaw.traceId"] = normalized.traceId;
|
|
794
|
-
if (normalized.spanId) attributes["openclaw.spanId"] = normalized.spanId;
|
|
795
|
-
if (normalized.parentSpanId) attributes["openclaw.parentSpanId"] = normalized.parentSpanId;
|
|
796
|
-
if (normalized.traceFlags) attributes["openclaw.traceFlags"] = normalized.traceFlags;
|
|
797
|
-
}
|
|
798
757
|
//#endregion
|
|
799
758
|
//#region extensions/diagnostics-otel/src/service-logs.ts
|
|
800
759
|
const LOG_SEVERITY_MAP = {
|
|
@@ -806,202 +765,131 @@ const LOG_SEVERITY_MAP = {
|
|
|
806
765
|
FATAL: 21
|
|
807
766
|
};
|
|
808
767
|
function createDiagnosticsLogExporter(params) {
|
|
809
|
-
const {
|
|
768
|
+
const { captureContent, emitExporterEvent, flushIntervalMs, headers, logger, logsToOtlp, logsToStdout, logHttpAgentOptions, logUrl, resource, serviceName } = params;
|
|
810
769
|
let logProvider = null;
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
770
|
+
if (!logsToOtlp && !logsToStdout) return {
|
|
771
|
+
logProvider,
|
|
772
|
+
recordLogEvent: void 0
|
|
773
|
+
};
|
|
774
|
+
let logRecordExportFailureLastReportedAt = Number.NEGATIVE_INFINITY;
|
|
775
|
+
let otelLogger;
|
|
776
|
+
const activeTransports = [...logsToOtlp ? ["otlp-http-protobuf"] : [], ...logsToStdout ? ["stdout"] : []];
|
|
777
|
+
if (logsToOtlp) {
|
|
778
|
+
const logExporter = observeOtlpExporterHealth(new OTLPLogExporter({
|
|
779
|
+
...logUrl ? { url: logUrl } : {},
|
|
780
|
+
...headers ? { headers } : {},
|
|
781
|
+
...logHttpAgentOptions ? { httpAgentOptions: logHttpAgentOptions } : {}
|
|
782
|
+
}), {
|
|
783
|
+
emitExporterEvent,
|
|
784
|
+
signal: "logs"
|
|
785
|
+
});
|
|
786
|
+
const logProcessor = new BatchLogRecordProcessor({
|
|
787
|
+
exporter: logExporter,
|
|
788
|
+
...typeof flushIntervalMs === "number" ? { scheduledDelayMillis: Math.max(1e3, flushIntervalMs) } : {}
|
|
789
|
+
});
|
|
790
|
+
logProvider = new LoggerProvider({
|
|
791
|
+
resource,
|
|
792
|
+
processors: [logProcessor]
|
|
793
|
+
});
|
|
794
|
+
otelLogger = logProvider.getLogger("openclaw");
|
|
795
|
+
}
|
|
796
|
+
const reportLogExportFailure = (err, label, transport) => {
|
|
797
|
+
emitExporterEvent({
|
|
798
|
+
exporter: "diagnostics-otel",
|
|
799
|
+
signal: "logs",
|
|
800
|
+
transport,
|
|
801
|
+
status: "failure",
|
|
802
|
+
reason: "emit_failed",
|
|
803
|
+
errorCategory: errorCategory(err)
|
|
804
|
+
});
|
|
805
|
+
const now = Date.now();
|
|
806
|
+
if (now - logRecordExportFailureLastReportedAt >= 6e4) {
|
|
807
|
+
logRecordExportFailureLastReportedAt = now;
|
|
808
|
+
logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`);
|
|
835
809
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
status: "failure",
|
|
842
|
-
reason: "emit_failed",
|
|
843
|
-
errorCategory: errorCategory(err)
|
|
844
|
-
});
|
|
845
|
-
const now = Date.now();
|
|
846
|
-
if (now - logRecordExportFailureLastReportedAt >= 6e4) {
|
|
847
|
-
logRecordExportFailureLastReportedAt = now;
|
|
848
|
-
logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`);
|
|
849
|
-
}
|
|
850
|
-
};
|
|
851
|
-
const reportLogExportRecovery = (transport) => {
|
|
852
|
-
emitExporterEvent({
|
|
853
|
-
exporter: "diagnostics-otel",
|
|
854
|
-
signal: "logs",
|
|
855
|
-
transport,
|
|
856
|
-
status: "recovered",
|
|
857
|
-
reason: "emit_failed"
|
|
858
|
-
});
|
|
859
|
-
};
|
|
860
|
-
const reportLogPreparationFailure = (err, label) => {
|
|
861
|
-
for (const transport of activeTransports) reportLogExportFailure(err, label, transport);
|
|
862
|
-
};
|
|
863
|
-
const emitLogRecord = ({ logRecord, traceContext }, label) => {
|
|
864
|
-
if (logsToOtlp) try {
|
|
865
|
-
otelLogger?.emit(logRecord);
|
|
866
|
-
reportLogExportRecovery("otlp-http-protobuf");
|
|
867
|
-
} catch (error) {
|
|
868
|
-
reportLogExportFailure(error, label, "otlp-http-protobuf");
|
|
869
|
-
}
|
|
870
|
-
if (logsToStdout) try {
|
|
871
|
-
writeStdoutDiagnosticLogRecord({
|
|
872
|
-
logRecord,
|
|
873
|
-
serviceName,
|
|
874
|
-
...traceContext ? { traceContext } : {}
|
|
875
|
-
});
|
|
876
|
-
reportLogExportRecovery("stdout");
|
|
877
|
-
} catch (error) {
|
|
878
|
-
reportLogExportFailure(error, label, "stdout");
|
|
879
|
-
}
|
|
880
|
-
};
|
|
881
|
-
const buildDiagnosticLogRecord = (evt, metadata) => {
|
|
882
|
-
const logLevelName = evt.level || "INFO";
|
|
883
|
-
const severityNumber = LOG_SEVERITY_MAP[logLevelName] ?? 9;
|
|
884
|
-
const body = contentCapturePolicy.logBodies ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
810
|
+
};
|
|
811
|
+
const recordLogEvent = (evt, metadata) => {
|
|
812
|
+
if (evt.type === "security.event" && !metadata.trusted) return;
|
|
813
|
+
const label = evt.type === "log.record" ? "log record" : "security event";
|
|
814
|
+
try {
|
|
885
815
|
const attributes = Object.create(null);
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
if (evt.
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
816
|
+
let severityText;
|
|
817
|
+
let body;
|
|
818
|
+
if (evt.type === "log.record") {
|
|
819
|
+
severityText = evt.level || "INFO";
|
|
820
|
+
body = captureContent ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) : "log";
|
|
821
|
+
assignOtelLogAttribute(attributes, "openclaw.log.level", severityText);
|
|
822
|
+
if (evt.loggerName) assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName);
|
|
823
|
+
if (evt.loggerParents?.length) assignOtelLogAttribute(attributes, "openclaw.logger.parents", evt.loggerParents.join("."));
|
|
824
|
+
assignOtelLogEventAttributes(attributes, evt.attributes);
|
|
825
|
+
if (evt.code?.line) assignOtelLogAttribute(attributes, "code.lineno", evt.code.line);
|
|
826
|
+
if (evt.code?.functionName) assignOtelLogAttribute(attributes, "code.function", evt.code.functionName);
|
|
827
|
+
} else {
|
|
828
|
+
severityText = securitySeverityText(evt.severity);
|
|
829
|
+
body = "openclaw.security.event";
|
|
830
|
+
assignOtelSecurityAttributes(attributes, evt);
|
|
831
|
+
}
|
|
892
832
|
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
893
|
-
|
|
833
|
+
if (evt.type === "log.record" && traceContext?.traceFlags) attributes["openclaw.traceFlags"] = traceContext.traceFlags;
|
|
894
834
|
const logRecord = {
|
|
895
835
|
body,
|
|
896
|
-
severityText: logLevelName,
|
|
897
|
-
severityNumber,
|
|
898
|
-
attributes: redactOtelAttributes(attributes),
|
|
899
|
-
timestamp: evt.ts
|
|
900
|
-
};
|
|
901
|
-
const logContext = contextForTrustedTraceContext(evt, metadata);
|
|
902
|
-
if (logContext) logRecord.context = logContext;
|
|
903
|
-
return {
|
|
904
|
-
logRecord,
|
|
905
|
-
...traceContext ? { traceContext } : {}
|
|
906
|
-
};
|
|
907
|
-
};
|
|
908
|
-
const buildSecurityLogRecord = (evt, metadata) => {
|
|
909
|
-
const severityText = securitySeverityText(evt.severity);
|
|
910
|
-
const attributes = Object.create(null);
|
|
911
|
-
assignOtelSecurityAttributes(attributes, evt);
|
|
912
|
-
const traceContext = normalizedTrustedTraceContext(evt, metadata);
|
|
913
|
-
const logRecord = {
|
|
914
|
-
body: "openclaw.security.event",
|
|
915
836
|
severityText,
|
|
916
837
|
severityNumber: LOG_SEVERITY_MAP[severityText] ?? 9,
|
|
917
838
|
attributes: redactOtelAttributes(attributes),
|
|
918
839
|
timestamp: evt.ts
|
|
919
840
|
};
|
|
920
|
-
const logContext =
|
|
841
|
+
const logContext = contextForTraceContext(traceContext);
|
|
921
842
|
if (logContext) logRecord.context = logContext;
|
|
922
|
-
|
|
923
|
-
logRecord
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
const record = buildSecurityLogRecord(evt, metadata);
|
|
939
|
-
emitLogRecord(record, "security event");
|
|
940
|
-
} catch (err) {
|
|
941
|
-
reportLogPreparationFailure(err, "security event");
|
|
843
|
+
for (const transport of activeTransports) try {
|
|
844
|
+
if (transport === "otlp-http-protobuf") otelLogger?.emit(logRecord);
|
|
845
|
+
else writeStdoutDiagnosticLogRecord({
|
|
846
|
+
logRecord,
|
|
847
|
+
serviceName,
|
|
848
|
+
...traceContext ? { traceContext } : {}
|
|
849
|
+
});
|
|
850
|
+
emitExporterEvent({
|
|
851
|
+
exporter: "diagnostics-otel",
|
|
852
|
+
signal: "logs",
|
|
853
|
+
transport,
|
|
854
|
+
status: "recovered",
|
|
855
|
+
reason: "emit_failed"
|
|
856
|
+
});
|
|
857
|
+
} catch (error) {
|
|
858
|
+
reportLogExportFailure(error, label, transport);
|
|
942
859
|
}
|
|
943
|
-
}
|
|
944
|
-
|
|
860
|
+
} catch (err) {
|
|
861
|
+
for (const transport of activeTransports) reportLogExportFailure(err, label, transport);
|
|
862
|
+
}
|
|
863
|
+
};
|
|
945
864
|
return {
|
|
946
865
|
logProvider,
|
|
947
|
-
|
|
948
|
-
recordSecurityEvent
|
|
866
|
+
recordLogEvent
|
|
949
867
|
};
|
|
950
868
|
}
|
|
951
869
|
//#endregion
|
|
952
870
|
//#region extensions/diagnostics-otel/src/service-metrics.ts
|
|
953
871
|
const DEFAULT_METRIC_NAME_PREFIX = "openclaw.";
|
|
954
872
|
function createDiagnosticsMetrics(meter, metricNamePrefix = DEFAULT_METRIC_NAME_PREFIX) {
|
|
955
|
-
const
|
|
956
|
-
|
|
957
|
-
|
|
873
|
+
const createCounter = (name, description, unit = "1") => meter.createCounter(`${metricNamePrefix}${name}`, {
|
|
874
|
+
unit,
|
|
875
|
+
description
|
|
876
|
+
});
|
|
877
|
+
const createHistogram = (name, description, unit = "ms", explicitBucketBoundaries) => meter.createHistogram(`${metricNamePrefix}${name}`, {
|
|
878
|
+
unit,
|
|
879
|
+
description,
|
|
880
|
+
...explicitBucketBoundaries ? { advice: { explicitBucketBoundaries } } : {}
|
|
881
|
+
});
|
|
958
882
|
return {
|
|
959
|
-
gcDurationHistogram: createHistogram("
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
gatewayEventLoopObservedCounter: createCounter("openclaw.gateway.event_loop.observed_ms", {
|
|
970
|
-
unit: "ms",
|
|
971
|
-
description: "Elapsed time covered by completed Gateway event-loop observation windows"
|
|
972
|
-
}),
|
|
973
|
-
gatewayRpcRequestsCounter: createCounter("openclaw.gateway.rpc.requests", {
|
|
974
|
-
unit: "1",
|
|
975
|
-
description: "Authenticated Gateway WebSocket requests received"
|
|
976
|
-
}),
|
|
977
|
-
gatewayRpcOutcomesCounter: createCounter("openclaw.gateway.rpc.outcomes", {
|
|
978
|
-
unit: "1",
|
|
979
|
-
description: "Gateway RPC observations by phase and outcome"
|
|
980
|
-
}),
|
|
981
|
-
gatewayRpcFirstResponseHistogram: createHistogram("openclaw.gateway.rpc.first_response_ms", {
|
|
982
|
-
unit: "ms",
|
|
983
|
-
description: "Elapsed time until the first Gateway RPC response is sent",
|
|
984
|
-
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
985
|
-
}),
|
|
986
|
-
gatewayRpcHandlerHistogram: createHistogram("openclaw.gateway.rpc.handler_ms", {
|
|
987
|
-
unit: "ms",
|
|
988
|
-
description: "Gateway RPC handler duration until return or throw",
|
|
989
|
-
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
990
|
-
}),
|
|
991
|
-
gatewayRpcAdmissionHistogram: createHistogram("openclaw.gateway.rpc.admission_ms", {
|
|
992
|
-
unit: "ms",
|
|
993
|
-
description: "Elapsed time from Gateway RPC receipt until handler invocation",
|
|
994
|
-
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
995
|
-
}),
|
|
996
|
-
gatewayRpcQueueWaitHistogram: createHistogram("openclaw.gateway.rpc.queue_wait_ms", {
|
|
997
|
-
unit: "ms",
|
|
998
|
-
description: "Gateway operator start queue or worker frame queue wait",
|
|
999
|
-
advice: { explicitBucketBoundaries: AGENT_DURATION_MS_BUCKETS }
|
|
1000
|
-
}),
|
|
1001
|
-
tokensCounter: createCounter("openclaw.tokens", {
|
|
1002
|
-
unit: "1",
|
|
1003
|
-
description: "Token usage by type"
|
|
1004
|
-
}),
|
|
883
|
+
gcDurationHistogram: createHistogram("gc.duration_ms", "Elapsed garbage collection duration for the hosting JavaScript isolate", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
884
|
+
gatewayEventLoopDelayMaxHistogram: createHistogram("gateway.event_loop.delay_max_ms", "Maximum event-loop delay per completed Gateway observation window", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
885
|
+
gatewayEventLoopObservedCounter: createCounter("gateway.event_loop.observed_ms", "Elapsed time covered by completed Gateway event-loop observation windows", "ms"),
|
|
886
|
+
gatewayRpcRequestsCounter: createCounter("gateway.rpc.requests", "Authenticated Gateway WebSocket requests received"),
|
|
887
|
+
gatewayRpcOutcomesCounter: createCounter("gateway.rpc.outcomes", "Gateway RPC observations by phase and outcome"),
|
|
888
|
+
gatewayRpcFirstResponseHistogram: createHistogram("gateway.rpc.first_response_ms", "Elapsed time until the first Gateway RPC response is sent", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
889
|
+
gatewayRpcHandlerHistogram: createHistogram("gateway.rpc.handler_ms", "Gateway RPC handler duration until return or throw", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
890
|
+
gatewayRpcAdmissionHistogram: createHistogram("gateway.rpc.admission_ms", "Elapsed time from Gateway RPC receipt until handler invocation", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
891
|
+
gatewayRpcQueueWaitHistogram: createHistogram("gateway.rpc.queue_wait_ms", "Gateway operator start queue or worker frame queue wait", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
892
|
+
tokensCounter: createCounter("tokens", "Token usage by type"),
|
|
1005
893
|
genAiTokenUsageHistogram: meter.createHistogram("gen_ai.client.token.usage", {
|
|
1006
894
|
unit: "{token}",
|
|
1007
895
|
description: "Number of input and output tokens used by GenAI client operations",
|
|
@@ -1012,233 +900,62 @@ function createDiagnosticsMetrics(meter, metricNamePrefix = DEFAULT_METRIC_NAME_
|
|
|
1012
900
|
description: "GenAI client operation duration",
|
|
1013
901
|
advice: { explicitBucketBoundaries: GEN_AI_OPERATION_DURATION_BUCKETS }
|
|
1014
902
|
}),
|
|
1015
|
-
costCounter: createCounter("
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
unit: "ms",
|
|
1072
|
-
description: "Message processing duration"
|
|
1073
|
-
}),
|
|
1074
|
-
messageDeliveryStartedCounter: createCounter("openclaw.message.delivery.started", {
|
|
1075
|
-
unit: "1",
|
|
1076
|
-
description: "Outbound message delivery attempts started"
|
|
1077
|
-
}),
|
|
1078
|
-
messageDeliveryDurationHistogram: createHistogram("openclaw.message.delivery.duration_ms", {
|
|
1079
|
-
unit: "ms",
|
|
1080
|
-
description: "Outbound message delivery duration"
|
|
1081
|
-
}),
|
|
1082
|
-
queueDepthHistogram: createHistogram("openclaw.queue.depth", {
|
|
1083
|
-
unit: "1",
|
|
1084
|
-
description: "Queue depth on enqueue/dequeue"
|
|
1085
|
-
}),
|
|
1086
|
-
queueWaitHistogram: createHistogram("openclaw.queue.wait_ms", {
|
|
1087
|
-
unit: "ms",
|
|
1088
|
-
description: "Queue wait time before execution"
|
|
1089
|
-
}),
|
|
1090
|
-
laneEnqueueCounter: createCounter("openclaw.queue.lane.enqueue", {
|
|
1091
|
-
unit: "1",
|
|
1092
|
-
description: "Command queue lane enqueue events"
|
|
1093
|
-
}),
|
|
1094
|
-
laneDequeueCounter: createCounter("openclaw.queue.lane.dequeue", {
|
|
1095
|
-
unit: "1",
|
|
1096
|
-
description: "Command queue lane dequeue events"
|
|
1097
|
-
}),
|
|
1098
|
-
sessionStateCounter: createCounter("openclaw.session.state", {
|
|
1099
|
-
unit: "1",
|
|
1100
|
-
description: "Session state transitions"
|
|
1101
|
-
}),
|
|
1102
|
-
sessionTurnCreatedCounter: createCounter("openclaw.session.turn.created", {
|
|
1103
|
-
unit: "1",
|
|
1104
|
-
description: "Agent session turns created"
|
|
1105
|
-
}),
|
|
1106
|
-
sessionStuckCounter: createCounter("openclaw.session.stuck", {
|
|
1107
|
-
unit: "1",
|
|
1108
|
-
description: "Sessions stuck in processing"
|
|
1109
|
-
}),
|
|
1110
|
-
sessionStuckAgeHistogram: createHistogram("openclaw.session.stuck_age_ms", {
|
|
1111
|
-
unit: "ms",
|
|
1112
|
-
description: "Age of stuck sessions"
|
|
1113
|
-
}),
|
|
1114
|
-
sessionRecoveryRequestedCounter: createCounter("openclaw.session.recovery.requested", {
|
|
1115
|
-
unit: "1",
|
|
1116
|
-
description: "Session recovery attempts requested"
|
|
1117
|
-
}),
|
|
1118
|
-
sessionRecoveryCompletedCounter: createCounter("openclaw.session.recovery.completed", {
|
|
1119
|
-
unit: "1",
|
|
1120
|
-
description: "Session recovery attempts completed"
|
|
1121
|
-
}),
|
|
1122
|
-
sessionRecoveryAgeHistogram: createHistogram("openclaw.session.recovery.age_ms", {
|
|
1123
|
-
unit: "ms",
|
|
1124
|
-
description: "Age of sessions selected for recovery"
|
|
1125
|
-
}),
|
|
1126
|
-
talkEventCounter: createCounter("openclaw.talk.event", {
|
|
1127
|
-
unit: "1",
|
|
1128
|
-
description: "Talk events emitted by type"
|
|
1129
|
-
}),
|
|
1130
|
-
talkEventDurationHistogram: createHistogram("openclaw.talk.event.duration_ms", {
|
|
1131
|
-
unit: "ms",
|
|
1132
|
-
description: "Talk event duration when reported"
|
|
1133
|
-
}),
|
|
1134
|
-
talkAudioBytesHistogram: createHistogram("openclaw.talk.audio.bytes", {
|
|
1135
|
-
unit: "By",
|
|
1136
|
-
description: "Talk audio frame byte lengths"
|
|
1137
|
-
}),
|
|
1138
|
-
runAttemptCounter: createCounter("openclaw.run.attempt", {
|
|
1139
|
-
unit: "1",
|
|
1140
|
-
description: "Run attempts"
|
|
1141
|
-
}),
|
|
1142
|
-
toolLoopCounter: createCounter("openclaw.tool.loop", {
|
|
1143
|
-
unit: "1",
|
|
1144
|
-
description: "Detected repetitive tool-call loop events"
|
|
1145
|
-
}),
|
|
1146
|
-
skillUsedCounter: createCounter("openclaw.skill.used", {
|
|
1147
|
-
unit: "1",
|
|
1148
|
-
description: "Skills used by agent runs"
|
|
1149
|
-
}),
|
|
1150
|
-
modelCallDurationHistogram: createHistogram("openclaw.model_call.duration_ms", {
|
|
1151
|
-
unit: "ms",
|
|
1152
|
-
description: "Model call duration"
|
|
1153
|
-
}),
|
|
1154
|
-
modelCallRequestBytesHistogram: createHistogram("openclaw.model_call.request_bytes", {
|
|
1155
|
-
unit: "By",
|
|
1156
|
-
description: "UTF-8 byte size of sanitized model request payloads"
|
|
1157
|
-
}),
|
|
1158
|
-
modelCallResponseBytesHistogram: createHistogram("openclaw.model_call.response_bytes", {
|
|
1159
|
-
unit: "By",
|
|
1160
|
-
description: "UTF-8 byte size of bounded streamed model response payloads"
|
|
1161
|
-
}),
|
|
1162
|
-
modelCallTimeToFirstByteHistogram: createHistogram("openclaw.model_call.time_to_first_byte_ms", {
|
|
1163
|
-
unit: "ms",
|
|
1164
|
-
description: "Elapsed time before the first streamed model response event"
|
|
1165
|
-
}),
|
|
1166
|
-
modelFailoverCounter: createCounter("openclaw.model.failover", {
|
|
1167
|
-
unit: "1",
|
|
1168
|
-
description: "Model failovers by source, destination, lane, and reason"
|
|
1169
|
-
}),
|
|
1170
|
-
toolExecutionDurationHistogram: createHistogram("openclaw.tool.execution.duration_ms", {
|
|
1171
|
-
unit: "ms",
|
|
1172
|
-
description: "Tool execution duration"
|
|
1173
|
-
}),
|
|
1174
|
-
toolExecutionBlockedCounter: createCounter("openclaw.tool.execution.blocked", {
|
|
1175
|
-
unit: "1",
|
|
1176
|
-
description: "Tool executions blocked by policy or sandbox diagnostics"
|
|
1177
|
-
}),
|
|
1178
|
-
execProcessDurationHistogram: createHistogram("openclaw.exec.duration_ms", {
|
|
1179
|
-
unit: "ms",
|
|
1180
|
-
description: "Exec process duration"
|
|
1181
|
-
}),
|
|
1182
|
-
memoryRssHistogram: createHistogram("openclaw.memory.rss_bytes", {
|
|
1183
|
-
unit: "By",
|
|
1184
|
-
description: "Resident set size reported by diagnostic memory samples"
|
|
1185
|
-
}),
|
|
1186
|
-
memoryHeapUsedHistogram: createHistogram("openclaw.memory.heap_used_bytes", {
|
|
1187
|
-
unit: "By",
|
|
1188
|
-
description: "Heap used bytes reported by diagnostic memory samples"
|
|
1189
|
-
}),
|
|
1190
|
-
memoryHeapTotalHistogram: createHistogram("openclaw.memory.heap_total_bytes", {
|
|
1191
|
-
unit: "By",
|
|
1192
|
-
description: "Heap total bytes reported by diagnostic memory samples"
|
|
1193
|
-
}),
|
|
1194
|
-
memoryExternalHistogram: createHistogram("openclaw.memory.external_bytes", {
|
|
1195
|
-
unit: "By",
|
|
1196
|
-
description: "External memory bytes reported by diagnostic memory samples"
|
|
1197
|
-
}),
|
|
1198
|
-
memoryArrayBuffersHistogram: createHistogram("openclaw.memory.array_buffers_bytes", {
|
|
1199
|
-
unit: "By",
|
|
1200
|
-
description: "ArrayBuffer bytes reported by diagnostic memory samples"
|
|
1201
|
-
}),
|
|
1202
|
-
memoryPressureCounter: createCounter("openclaw.memory.pressure", {
|
|
1203
|
-
unit: "1",
|
|
1204
|
-
description: "Diagnostic memory pressure events"
|
|
1205
|
-
}),
|
|
1206
|
-
asyncQueueDroppedCounter: createCounter("openclaw.diagnostic.async_queue.dropped", {
|
|
1207
|
-
unit: "1",
|
|
1208
|
-
description: "Async diagnostic queue drops by dropped event class"
|
|
1209
|
-
}),
|
|
1210
|
-
payloadLargeCounter: createCounter("openclaw.payload.large", {
|
|
1211
|
-
unit: "1",
|
|
1212
|
-
description: "Oversized payload diagnostics by surface and action"
|
|
1213
|
-
}),
|
|
1214
|
-
payloadLargeBytesHistogram: createHistogram("openclaw.payload.large_bytes", {
|
|
1215
|
-
unit: "By",
|
|
1216
|
-
description: "Oversized payload byte sizes by surface and action"
|
|
1217
|
-
}),
|
|
1218
|
-
livenessWarningCounter: createCounter("openclaw.liveness.warning", {
|
|
1219
|
-
unit: "1",
|
|
1220
|
-
description: "Diagnostic liveness warning events"
|
|
1221
|
-
}),
|
|
1222
|
-
livenessEventLoopDelayP99Histogram: createHistogram("openclaw.liveness.event_loop_delay_p99_ms", {
|
|
1223
|
-
unit: "ms",
|
|
1224
|
-
description: "P99 event-loop delay reported by diagnostic liveness warnings"
|
|
1225
|
-
}),
|
|
1226
|
-
livenessEventLoopDelayMaxHistogram: createHistogram("openclaw.liveness.event_loop_delay_max_ms", {
|
|
1227
|
-
unit: "ms",
|
|
1228
|
-
description: "Maximum event-loop delay reported by diagnostic liveness warnings"
|
|
1229
|
-
}),
|
|
1230
|
-
livenessEventLoopUtilizationHistogram: createHistogram("openclaw.liveness.event_loop_utilization", {
|
|
1231
|
-
unit: "1",
|
|
1232
|
-
description: "Event-loop utilization reported by diagnostic liveness warnings"
|
|
1233
|
-
}),
|
|
1234
|
-
livenessCpuCoreRatioHistogram: createHistogram("openclaw.liveness.cpu_core_ratio", {
|
|
1235
|
-
unit: "1",
|
|
1236
|
-
description: "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1."
|
|
1237
|
-
}),
|
|
1238
|
-
telemetryExporterCounter: createCounter("openclaw.telemetry.exporter.events", {
|
|
1239
|
-
unit: "1",
|
|
1240
|
-
description: "Diagnostic telemetry exporter lifecycle and failure events"
|
|
1241
|
-
})
|
|
903
|
+
costCounter: createCounter("cost.usd", "Estimated model cost (USD)"),
|
|
904
|
+
durationHistogram: createHistogram("run.duration_ms", "Agent run duration", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
905
|
+
harnessDurationHistogram: createHistogram("harness.duration_ms", "Agent harness lifecycle duration", "ms", AGENT_DURATION_MS_BUCKETS),
|
|
906
|
+
contextHistogram: createHistogram("context.tokens", "Context window size and usage", "1", CONTEXT_TOKENS_BUCKETS),
|
|
907
|
+
webhookReceivedCounter: createCounter("webhook.received", "Webhook requests received"),
|
|
908
|
+
webhookErrorCounter: createCounter("webhook.error", "Webhook processing errors"),
|
|
909
|
+
webhookDurationHistogram: createHistogram("webhook.duration_ms", "Webhook processing duration"),
|
|
910
|
+
messageQueuedCounter: createCounter("message.queued", "Messages queued for processing"),
|
|
911
|
+
messageReceivedCounter: createCounter("message.received", "Inbound messages received"),
|
|
912
|
+
messageDispatchStartedCounter: createCounter("message.dispatch.started", "Inbound message dispatch attempts started"),
|
|
913
|
+
messageDispatchCompletedCounter: createCounter("message.dispatch.completed", "Inbound message dispatch attempts completed"),
|
|
914
|
+
messageDispatchDurationHistogram: createHistogram("message.dispatch.duration_ms", "Inbound message dispatch duration"),
|
|
915
|
+
messageProcessedCounter: createCounter("message.processed", "Messages processed by outcome"),
|
|
916
|
+
messageDurationHistogram: createHistogram("message.duration_ms", "Message processing duration"),
|
|
917
|
+
messageDeliveryStartedCounter: createCounter("message.delivery.started", "Outbound message delivery attempts started"),
|
|
918
|
+
messageDeliveryDurationHistogram: createHistogram("message.delivery.duration_ms", "Outbound message delivery duration"),
|
|
919
|
+
queueDepthHistogram: createHistogram("queue.depth", "Queue depth on enqueue/dequeue", "1"),
|
|
920
|
+
queueWaitHistogram: createHistogram("queue.wait_ms", "Queue wait time before execution"),
|
|
921
|
+
laneEnqueueCounter: createCounter("queue.lane.enqueue", "Command queue lane enqueue events"),
|
|
922
|
+
laneDequeueCounter: createCounter("queue.lane.dequeue", "Command queue lane dequeue events"),
|
|
923
|
+
sessionStateCounter: createCounter("session.state", "Session state transitions"),
|
|
924
|
+
sessionTurnCreatedCounter: createCounter("session.turn.created", "Agent session turns created"),
|
|
925
|
+
sessionStuckCounter: createCounter("session.stuck", "Sessions stuck in processing"),
|
|
926
|
+
sessionStuckAgeHistogram: createHistogram("session.stuck_age_ms", "Age of stuck sessions"),
|
|
927
|
+
sessionRecoveryRequestedCounter: createCounter("session.recovery.requested", "Session recovery attempts requested"),
|
|
928
|
+
sessionRecoveryCompletedCounter: createCounter("session.recovery.completed", "Session recovery attempts completed"),
|
|
929
|
+
sessionRecoveryAgeHistogram: createHistogram("session.recovery.age_ms", "Age of sessions selected for recovery"),
|
|
930
|
+
talkEventCounter: createCounter("talk.event", "Talk events emitted by type"),
|
|
931
|
+
talkEventDurationHistogram: createHistogram("talk.event.duration_ms", "Talk event duration when reported"),
|
|
932
|
+
talkAudioBytesHistogram: createHistogram("talk.audio.bytes", "Talk audio frame byte lengths", "By"),
|
|
933
|
+
runAttemptCounter: createCounter("run.attempt", "Run attempts"),
|
|
934
|
+
toolLoopCounter: createCounter("tool.loop", "Detected repetitive tool-call loop events"),
|
|
935
|
+
skillUsedCounter: createCounter("skill.used", "Skills used by agent runs"),
|
|
936
|
+
modelCallDurationHistogram: createHistogram("model_call.duration_ms", "Model call duration"),
|
|
937
|
+
modelCallRequestBytesHistogram: createHistogram("model_call.request_bytes", "UTF-8 byte size of sanitized model request payloads", "By"),
|
|
938
|
+
modelCallResponseBytesHistogram: createHistogram("model_call.response_bytes", "UTF-8 byte size of bounded streamed model response payloads", "By"),
|
|
939
|
+
modelCallTimeToFirstByteHistogram: createHistogram("model_call.time_to_first_byte_ms", "Elapsed time before the first streamed model response event"),
|
|
940
|
+
modelFailoverCounter: createCounter("model.failover", "Model failovers by source, destination, lane, and reason"),
|
|
941
|
+
toolExecutionDurationHistogram: createHistogram("tool.execution.duration_ms", "Tool execution duration"),
|
|
942
|
+
toolExecutionBlockedCounter: createCounter("tool.execution.blocked", "Tool executions blocked by policy or sandbox diagnostics"),
|
|
943
|
+
execProcessDurationHistogram: createHistogram("exec.duration_ms", "Exec process duration"),
|
|
944
|
+
memoryRssHistogram: createHistogram("memory.rss_bytes", "Resident set size reported by diagnostic memory samples", "By"),
|
|
945
|
+
memoryHeapUsedHistogram: createHistogram("memory.heap_used_bytes", "Heap used bytes reported by diagnostic memory samples", "By"),
|
|
946
|
+
memoryHeapTotalHistogram: createHistogram("memory.heap_total_bytes", "Heap total bytes reported by diagnostic memory samples", "By"),
|
|
947
|
+
memoryExternalHistogram: createHistogram("memory.external_bytes", "External memory bytes reported by diagnostic memory samples", "By"),
|
|
948
|
+
memoryArrayBuffersHistogram: createHistogram("memory.array_buffers_bytes", "ArrayBuffer bytes reported by diagnostic memory samples", "By"),
|
|
949
|
+
memoryPressureCounter: createCounter("memory.pressure", "Diagnostic memory pressure events"),
|
|
950
|
+
asyncQueueDroppedCounter: createCounter("diagnostic.async_queue.dropped", "Async diagnostic queue drops by dropped event class"),
|
|
951
|
+
payloadLargeCounter: createCounter("payload.large", "Oversized payload diagnostics by surface and action"),
|
|
952
|
+
payloadLargeBytesHistogram: createHistogram("payload.large_bytes", "Oversized payload byte sizes by surface and action", "By"),
|
|
953
|
+
livenessWarningCounter: createCounter("liveness.warning", "Diagnostic liveness warning events"),
|
|
954
|
+
livenessEventLoopDelayP99Histogram: createHistogram("liveness.event_loop_delay_p99_ms", "P99 event-loop delay reported by diagnostic liveness warnings"),
|
|
955
|
+
livenessEventLoopDelayMaxHistogram: createHistogram("liveness.event_loop_delay_max_ms", "Maximum event-loop delay reported by diagnostic liveness warnings"),
|
|
956
|
+
livenessEventLoopUtilizationHistogram: createHistogram("liveness.event_loop_utilization", "Event-loop utilization reported by diagnostic liveness warnings", "1"),
|
|
957
|
+
livenessCpuCoreRatioHistogram: createHistogram("liveness.cpu_core_ratio", "Whole-process CPU usage in core equivalents, including worker and native threads; can exceed 1.", "1"),
|
|
958
|
+
telemetryExporterCounter: createCounter("telemetry.exporter.events", "Diagnostic telemetry exporter lifecycle and failure events")
|
|
1242
959
|
};
|
|
1243
960
|
}
|
|
1244
961
|
//#endregion
|
|
@@ -1322,16 +1039,6 @@ function registerOwnedSdkRuntime(warn) {
|
|
|
1322
1039
|
};
|
|
1323
1040
|
}
|
|
1324
1041
|
//#endregion
|
|
1325
|
-
//#region extensions/diagnostics-otel/src/service-recorder-runtime.ts
|
|
1326
|
-
function createDiagnosticsRecorderRuntime(params) {
|
|
1327
|
-
return {
|
|
1328
|
-
...params.metrics,
|
|
1329
|
-
...params.traces,
|
|
1330
|
-
contentCapturePolicy: params.contentCapturePolicy,
|
|
1331
|
-
tracesEnabled: params.tracesEnabled
|
|
1332
|
-
};
|
|
1333
|
-
}
|
|
1334
|
-
//#endregion
|
|
1335
1042
|
//#region extensions/diagnostics-otel/src/service-genai-content.ts
|
|
1336
1043
|
function textPart(content) {
|
|
1337
1044
|
return {
|
|
@@ -1495,32 +1202,6 @@ function assignJsonAttribute(attributes, key, value) {
|
|
|
1495
1202
|
const json = safeJsonString(value, MAX_OTEL_CONTENT_ATTRIBUTE_CHARS);
|
|
1496
1203
|
if (json) attributes[key] = json;
|
|
1497
1204
|
}
|
|
1498
|
-
function assignGenAiModelContentAttributes(attributes, content, policy) {
|
|
1499
|
-
if (policy.systemPrompt && typeof content?.systemPrompt === "string") {
|
|
1500
|
-
const systemInstructions = [textPart(content.systemPrompt)];
|
|
1501
|
-
assignJsonAttribute(attributes, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructions);
|
|
1502
|
-
}
|
|
1503
|
-
if (policy.inputMessages) {
|
|
1504
|
-
const inputMessages = normalizeGenAiMessages(content?.inputMessages, "user");
|
|
1505
|
-
if (inputMessages.length > 0) {
|
|
1506
|
-
assignJsonAttribute(attributes, ATTR_GEN_AI_INPUT_MESSAGES, inputMessages);
|
|
1507
|
-
assignJsonAttribute(attributes, "input.value", inputMessages);
|
|
1508
|
-
attributes["input.mime_type"] = "application/json";
|
|
1509
|
-
}
|
|
1510
|
-
}
|
|
1511
|
-
if (policy.toolDefinitions) {
|
|
1512
|
-
const toolDefinitions = normalizeGenAiToolDefinitions(content?.toolDefinitions);
|
|
1513
|
-
if (toolDefinitions.length > 0) assignJsonAttribute(attributes, ATTR_GEN_AI_TOOL_DEFINITIONS, toolDefinitions);
|
|
1514
|
-
}
|
|
1515
|
-
if (policy.outputMessages) {
|
|
1516
|
-
const outputMessages = normalizeGenAiMessages(content?.outputMessages, "assistant");
|
|
1517
|
-
if (outputMessages.length > 0) {
|
|
1518
|
-
assignJsonAttribute(attributes, ATTR_GEN_AI_OUTPUT_MESSAGES, outputMessages);
|
|
1519
|
-
assignJsonAttribute(attributes, "output.value", outputMessages);
|
|
1520
|
-
attributes["output.mime_type"] = "application/json";
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
}
|
|
1524
1205
|
function assignOtelContentAttribute(attributes, key, value) {
|
|
1525
1206
|
const normalized = normalizeOtelContentValue(value);
|
|
1526
1207
|
if (normalized) attributes[key] = normalized;
|
|
@@ -1530,38 +1211,45 @@ function assignOtelToolIdentityAttributes(attributes, evt) {
|
|
|
1530
1211
|
const toolCallId = evt.toolCallId?.trim();
|
|
1531
1212
|
if (toolCallId) attributes[ATTR_GEN_AI_TOOL_CALL_ID] = toolCallId;
|
|
1532
1213
|
}
|
|
1533
|
-
function assignOtelModelContentAttributes(attributes, content,
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
if (policy.outputMessages) assignOtelContentAttribute(attributes, "openclaw.content.output_messages", redactedContent?.outputMessages);
|
|
1543
|
-
if (policy.systemPrompt) assignOtelContentAttribute(attributes, "openclaw.content.system_prompt", content?.systemPrompt);
|
|
1544
|
-
}
|
|
1545
|
-
function assignOtelToolContentAttributes(attributes, content, policy) {
|
|
1546
|
-
if (policy.toolInputs) {
|
|
1547
|
-
const toolInput = normalizeOtelContentValue(content?.toolInput);
|
|
1548
|
-
if (toolInput) {
|
|
1549
|
-
attributes[ATTR_GEN_AI_TOOL_CALL_ARGUMENTS] = toolInput;
|
|
1550
|
-
attributes["openclaw.content.tool_input"] = toolInput;
|
|
1551
|
-
}
|
|
1214
|
+
function assignOtelModelContentAttributes(attributes, content, captureContent) {
|
|
1215
|
+
if (!captureContent) return;
|
|
1216
|
+
const redactedInput = redactInternalReasoningFromMessages(content?.inputMessages);
|
|
1217
|
+
const redactedOutput = redactInternalReasoningFromMessages(content?.outputMessages);
|
|
1218
|
+
const inputMessages = normalizeGenAiMessages(redactedInput, "user");
|
|
1219
|
+
if (inputMessages.length > 0) {
|
|
1220
|
+
assignJsonAttribute(attributes, ATTR_GEN_AI_INPUT_MESSAGES, inputMessages);
|
|
1221
|
+
assignJsonAttribute(attributes, "input.value", inputMessages);
|
|
1222
|
+
attributes["input.mime_type"] = "application/json";
|
|
1552
1223
|
}
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1224
|
+
const toolDefinitions = normalizeGenAiToolDefinitions(content?.toolDefinitions);
|
|
1225
|
+
if (toolDefinitions.length > 0) assignJsonAttribute(attributes, ATTR_GEN_AI_TOOL_DEFINITIONS, toolDefinitions);
|
|
1226
|
+
const outputMessages = normalizeGenAiMessages(redactedOutput, "assistant");
|
|
1227
|
+
if (outputMessages.length > 0) {
|
|
1228
|
+
assignJsonAttribute(attributes, ATTR_GEN_AI_OUTPUT_MESSAGES, outputMessages);
|
|
1229
|
+
assignJsonAttribute(attributes, "output.value", outputMessages);
|
|
1230
|
+
attributes["output.mime_type"] = "application/json";
|
|
1231
|
+
}
|
|
1232
|
+
assignOtelContentAttribute(attributes, "openclaw.content.input_messages", redactedInput);
|
|
1233
|
+
assignOtelContentAttribute(attributes, "openclaw.content.tool_definitions", content?.toolDefinitions);
|
|
1234
|
+
assignOtelContentAttribute(attributes, "openclaw.content.output_messages", redactedOutput);
|
|
1235
|
+
}
|
|
1236
|
+
function assignOtelToolContentAttributes(attributes, content, captureContent) {
|
|
1237
|
+
if (!captureContent) return;
|
|
1238
|
+
const toolInput = normalizeOtelContentValue(content?.toolInput);
|
|
1239
|
+
if (toolInput) {
|
|
1240
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_ARGUMENTS] = toolInput;
|
|
1241
|
+
attributes["openclaw.content.tool_input"] = toolInput;
|
|
1242
|
+
}
|
|
1243
|
+
const toolOutput = normalizeOtelContentValue(content?.toolOutput);
|
|
1244
|
+
if (toolOutput) {
|
|
1245
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_RESULT] = toolOutput;
|
|
1246
|
+
attributes["openclaw.content.tool_output"] = toolOutput;
|
|
1559
1247
|
}
|
|
1560
1248
|
}
|
|
1561
1249
|
//#endregion
|
|
1562
1250
|
//#region extensions/diagnostics-otel/src/service-recorders-harness.ts
|
|
1563
1251
|
function createHarnessRecorders(runtime) {
|
|
1564
|
-
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled, getTrackedInternalOrTrustedSpan,
|
|
1252
|
+
const { harnessDurationHistogram, modelFailoverCounter, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, trackTrustedSpan, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled, getTrackedInternalOrTrustedSpan, captureContent } = runtime;
|
|
1565
1253
|
const recordAgentCommentary = (evt, metadata, privateData) => {
|
|
1566
1254
|
if (!tracesEnabled || !metadata.trusted) return;
|
|
1567
1255
|
const span = getTrackedInternalOrTrustedSpan(evt, metadata);
|
|
@@ -1572,7 +1260,7 @@ function createHarnessRecorders(runtime) {
|
|
|
1572
1260
|
"openclaw.commentary.text_length": evt.textLength,
|
|
1573
1261
|
"openclaw.commentary.content_truncated": evt.contentTruncated
|
|
1574
1262
|
};
|
|
1575
|
-
assignOtelModelContentAttributes(attrs, privateData.modelContent,
|
|
1263
|
+
assignOtelModelContentAttributes(attrs, privateData.modelContent, captureContent);
|
|
1576
1264
|
span.addEvent("openclaw.agent.commentary", redactOtelAttributes(attrs), evt.sourceTimestampMs);
|
|
1577
1265
|
};
|
|
1578
1266
|
const harnessRunMetricAttrs = (evt) => ({
|
|
@@ -1630,11 +1318,7 @@ function createHarnessRecorders(runtime) {
|
|
|
1630
1318
|
code: SpanStatusCode.ERROR,
|
|
1631
1319
|
message: redactedError ?? errorType
|
|
1632
1320
|
});
|
|
1633
|
-
|
|
1634
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
1635
|
-
return;
|
|
1636
|
-
}
|
|
1637
|
-
span.end(evt.ts);
|
|
1321
|
+
completeTrackedLifecycleSpan(trackedSpan ? trustedTrace : void 0, span, evt.ts);
|
|
1638
1322
|
};
|
|
1639
1323
|
const recordContextAssembled = (evt, metadata) => {
|
|
1640
1324
|
if (!tracesEnabled) return;
|
|
@@ -1690,11 +1374,8 @@ function createHarnessRecorders(runtime) {
|
|
|
1690
1374
|
}
|
|
1691
1375
|
//#endregion
|
|
1692
1376
|
//#region extensions/diagnostics-otel/src/service-genai-attributes.ts
|
|
1693
|
-
function hasOtelSemconvOptIn(value, optIn) {
|
|
1694
|
-
return value?.split(",").map((part) => part.trim()).includes(optIn) ?? false;
|
|
1695
|
-
}
|
|
1696
1377
|
function emitLatestGenAiSemconv() {
|
|
1697
|
-
return
|
|
1378
|
+
return process.env["OTEL_SEMCONV_STABILITY_OPT_IN"]?.split(",").map((part) => part.trim()).includes("gen_ai_latest_experimental") ?? false;
|
|
1698
1379
|
}
|
|
1699
1380
|
function genAiOperationName(api, observationUnit) {
|
|
1700
1381
|
if (observationUnit === "turn") return GEN_AI_OPERATION_NAME_VALUE_INVOKE_AGENT;
|
|
@@ -1786,7 +1467,7 @@ function addUpstreamRequestIdSpanEvent(span, upstreamRequestIdHash) {
|
|
|
1786
1467
|
//#endregion
|
|
1787
1468
|
//#region extensions/diagnostics-otel/src/service-recorders-model.ts
|
|
1788
1469
|
function createModelRecorders(runtime) {
|
|
1789
|
-
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs,
|
|
1470
|
+
const { genAiOperationDurationHistogram, modelCallDurationHistogram, modelCallRequestBytesHistogram, modelCallResponseBytesHistogram, modelCallTimeToFirstByteHistogram, spanWithDuration, activeTrustedParentContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, captureContent, tracesEnabled } = runtime;
|
|
1790
1471
|
const modelCallMetricAttrs = (evt) => ({
|
|
1791
1472
|
"openclaw.provider": evt.provider,
|
|
1792
1473
|
"openclaw.model": evt.model,
|
|
@@ -1853,7 +1534,7 @@ function createModelRecorders(runtime) {
|
|
|
1853
1534
|
assignModelCallSizeTimingAttrs(spanAttrs, evt);
|
|
1854
1535
|
assignModelCallPromptStatsAttrs(spanAttrs, evt);
|
|
1855
1536
|
assignModelCallUsageAttrs(spanAttrs, evt);
|
|
1856
|
-
assignOtelModelContentAttributes(spanAttrs, modelContent,
|
|
1537
|
+
assignOtelModelContentAttributes(spanAttrs, modelContent, captureContent);
|
|
1857
1538
|
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration(modelCallSpanName(evt), spanAttrs, evt.durationMs, {
|
|
1858
1539
|
kind: SpanKind.CLIENT,
|
|
1859
1540
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
@@ -1877,7 +1558,7 @@ function createModelRecorders(runtime) {
|
|
|
1877
1558
|
function createOperationsRecorders(runtime) {
|
|
1878
1559
|
const { durationHistogram, gatewayRpcRequestsCounter, gatewayRpcOutcomesCounter, gatewayRpcFirstResponseHistogram, gatewayRpcHandlerHistogram, gatewayRpcAdmissionHistogram, gatewayRpcQueueWaitHistogram, queueDepthHistogram, queueWaitHistogram, laneEnqueueCounter, laneDequeueCounter, sessionStateCounter, sessionTurnCreatedCounter, sessionStuckCounter, sessionStuckAgeHistogram, sessionRecoveryRequestedCounter, sessionRecoveryCompletedCounter, sessionRecoveryAgeHistogram, talkEventCounter, talkEventDurationHistogram, talkAudioBytesHistogram, runAttemptCounter, toolLoopCounter, memoryRssHistogram, memoryHeapUsedHistogram, memoryHeapTotalHistogram, memoryExternalHistogram, memoryArrayBuffersHistogram, memoryPressureCounter, asyncQueueDroppedCounter, tracer, activeTrustedSpans, spanWithDuration, trustedTraceContext, activeTrustedParentContext, internalOrTrustedExplicitParentContext, setSpanAttrs, completeTrackedLifecycleSpan, addRunAttrs, tracesEnabled } = runtime;
|
|
1879
1560
|
const recordGatewayRpc = (evt, metadata) => {
|
|
1880
|
-
if (!metadata.trusted) return;
|
|
1561
|
+
if (!metadata.trusted || evt.phase === "response" && evt.firstResponse === false) return;
|
|
1881
1562
|
const attrs = { "openclaw.gateway.rpc.method": evt.method };
|
|
1882
1563
|
if (evt.phase === "received") {
|
|
1883
1564
|
gatewayRpcRequestsCounter.add(1, attrs);
|
|
@@ -2078,11 +1759,7 @@ function createOperationsRecorders(runtime) {
|
|
|
2078
1759
|
...message ? { message } : {}
|
|
2079
1760
|
});
|
|
2080
1761
|
}
|
|
2081
|
-
|
|
2082
|
-
completeTrackedLifecycleSpan(trustedTrace, trackedSpan, evt.ts);
|
|
2083
|
-
return;
|
|
2084
|
-
}
|
|
2085
|
-
span.end(evt.ts);
|
|
1762
|
+
completeTrackedLifecycleSpan(trackedSpan ? trustedTrace : void 0, span, evt.ts);
|
|
2086
1763
|
};
|
|
2087
1764
|
return {
|
|
2088
1765
|
recordGatewayRpc,
|
|
@@ -2105,7 +1782,7 @@ function createOperationsRecorders(runtime) {
|
|
|
2105
1782
|
//#endregion
|
|
2106
1783
|
//#region extensions/diagnostics-otel/src/service-recorders-tools.ts
|
|
2107
1784
|
function createToolAndSystemRecorders(runtime) {
|
|
2108
|
-
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, internalOrTrustedExplicitParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs,
|
|
1785
|
+
const { gcDurationHistogram, gatewayEventLoopDelayMaxHistogram, gatewayEventLoopObservedCounter, queueDepthHistogram, skillUsedCounter, toolExecutionDurationHistogram, toolExecutionBlockedCounter, execProcessDurationHistogram, payloadLargeCounter, payloadLargeBytesHistogram, livenessWarningCounter, livenessEventLoopDelayP99Histogram, livenessEventLoopDelayMaxHistogram, livenessEventLoopUtilizationHistogram, livenessCpuCoreRatioHistogram, telemetryExporterCounter, spanWithDuration, activeTrustedParentContext, internalOrTrustedExplicitParentContext, exportedInternalOrTrustedContext, trackTrustedSpan, getTrackedInternalOrTrustedSpan, takeTrackedTrustedSpan, setSpanAttrs, addRunAttrs, paramsSummaryAttrs, captureContent, tracesEnabled } = runtime;
|
|
2109
1786
|
const toolExecutionBaseAttrs = (evt) => ({
|
|
2110
1787
|
"openclaw.toolName": evt.toolName,
|
|
2111
1788
|
"openclaw.tool.source": normalizeDiagnosticValue(evt.toolSource, "core"),
|
|
@@ -2155,7 +1832,7 @@ function createToolAndSystemRecorders(runtime) {
|
|
|
2155
1832
|
addRunAttrs(spanAttrs, evt);
|
|
2156
1833
|
assignOtelToolIdentityAttributes(spanAttrs, evt);
|
|
2157
1834
|
if (evt.type === "tool.execution.error" && evt.errorCode) spanAttrs["openclaw.errorCode"] = normalizeDiagnosticValue(evt.errorCode, "other");
|
|
2158
|
-
assignOtelToolContentAttributes(spanAttrs, toolContent,
|
|
1835
|
+
assignOtelToolContentAttributes(spanAttrs, toolContent, captureContent);
|
|
2159
1836
|
const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.tool.execution", spanAttrs, evt.durationMs, {
|
|
2160
1837
|
parentContext: activeTrustedParentContext(evt, metadata),
|
|
2161
1838
|
endTimeMs: toolTimestampMs(evt)
|
|
@@ -2471,11 +2148,7 @@ function createUsageRecorders(runtime) {
|
|
|
2471
2148
|
message: redactSensitiveText(evt.error)
|
|
2472
2149
|
});
|
|
2473
2150
|
const traceContext = internalOrTrustedTraceContext(evt, metadata);
|
|
2474
|
-
|
|
2475
|
-
completeTrackedLifecycleSpan(traceContext, trackedSpan, evt.ts);
|
|
2476
|
-
return;
|
|
2477
|
-
}
|
|
2478
|
-
span.end(evt.ts);
|
|
2151
|
+
completeTrackedLifecycleSpan(trackedSpan ? traceContext : void 0, span, evt.ts);
|
|
2479
2152
|
};
|
|
2480
2153
|
const messageDeliveryAttrs = (evt) => ({
|
|
2481
2154
|
"openclaw.channel": normalizeDiagnosticValue(evt.channel),
|
|
@@ -2484,34 +2157,22 @@ function createUsageRecorders(runtime) {
|
|
|
2484
2157
|
const recordMessageDeliveryStarted = (evt) => {
|
|
2485
2158
|
messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
|
|
2486
2159
|
};
|
|
2487
|
-
const
|
|
2160
|
+
const recordMessageDeliveryFinished = (evt, metadata) => {
|
|
2488
2161
|
const attrs = {
|
|
2489
2162
|
...messageDeliveryAttrs(evt),
|
|
2490
|
-
"openclaw.outcome": "completed"
|
|
2163
|
+
"openclaw.outcome": evt.type === "message.delivery.error" ? "error" : "completed",
|
|
2164
|
+
...evt.type === "message.delivery.error" ? { "openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other") } : {}
|
|
2491
2165
|
};
|
|
2492
2166
|
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
2493
2167
|
if (!tracesEnabled) return;
|
|
2494
|
-
spanWithDuration("openclaw.message.delivery", {
|
|
2168
|
+
const span = spanWithDuration("openclaw.message.delivery", {
|
|
2495
2169
|
...attrs,
|
|
2496
|
-
"openclaw.delivery.result_count": evt.resultCount
|
|
2170
|
+
...evt.type === "message.delivery.completed" ? { "openclaw.delivery.result_count": evt.resultCount } : {}
|
|
2497
2171
|
}, evt.durationMs, {
|
|
2498
2172
|
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
2499
2173
|
endTimeMs: evt.ts
|
|
2500
|
-
}).end(evt.ts);
|
|
2501
|
-
};
|
|
2502
|
-
const recordMessageDeliveryError = (evt, metadata) => {
|
|
2503
|
-
const attrs = {
|
|
2504
|
-
...messageDeliveryAttrs(evt),
|
|
2505
|
-
"openclaw.outcome": "error",
|
|
2506
|
-
"openclaw.errorCategory": normalizeDiagnosticValue(evt.errorCategory, "other")
|
|
2507
|
-
};
|
|
2508
|
-
messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
|
|
2509
|
-
if (!tracesEnabled) return;
|
|
2510
|
-
const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, {
|
|
2511
|
-
parentContext: activeInternalOrTrustedContext(evt, metadata),
|
|
2512
|
-
endTimeMs: evt.ts
|
|
2513
2174
|
});
|
|
2514
|
-
span.setStatus({
|
|
2175
|
+
if (evt.type === "message.delivery.error") span.setStatus({
|
|
2515
2176
|
code: SpanStatusCode.ERROR,
|
|
2516
2177
|
message: redactSensitiveText(evt.errorCategory)
|
|
2517
2178
|
});
|
|
@@ -2549,8 +2210,7 @@ function createUsageRecorders(runtime) {
|
|
|
2549
2210
|
recordMessageDispatchCompleted,
|
|
2550
2211
|
recordMessageProcessed,
|
|
2551
2212
|
recordMessageDeliveryStarted,
|
|
2552
|
-
|
|
2553
|
-
recordMessageDeliveryError,
|
|
2213
|
+
recordMessageDeliveryFinished,
|
|
2554
2214
|
recordRunStarted
|
|
2555
2215
|
};
|
|
2556
2216
|
}
|
|
@@ -2671,30 +2331,24 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2671
2331
|
spanContext,
|
|
2672
2332
|
...owner ? { owner } : {}
|
|
2673
2333
|
});
|
|
2674
|
-
|
|
2675
|
-
const oldestKey = retainedTrustedSpanContexts.keys().next().value;
|
|
2676
|
-
if (!oldestKey) break;
|
|
2677
|
-
retainedTrustedSpanContexts.delete(oldestKey);
|
|
2678
|
-
}
|
|
2334
|
+
pruneMapToMaxSize(retainedTrustedSpanContexts, MAX_RETAINED_TRUSTED_SPAN_CONTEXTS);
|
|
2679
2335
|
};
|
|
2680
2336
|
const completeTrackedLifecycleSpan = (traceContext, span, endTimeMs) => {
|
|
2681
|
-
|
|
2682
|
-
if (!spanId) {
|
|
2337
|
+
if (!traceContext?.spanId) {
|
|
2683
2338
|
span.end(endTimeMs);
|
|
2684
2339
|
return;
|
|
2685
2340
|
}
|
|
2341
|
+
const spanId = traceContext.spanId;
|
|
2686
2342
|
const spanContext = span.spanContext();
|
|
2687
2343
|
const retainedKeys = [{ spanId }];
|
|
2688
|
-
const retainedAliasKeys = [];
|
|
2689
2344
|
for (const [aliasKey, alias] of activeTrustedSpanAliases) if (alias.span === span) {
|
|
2690
2345
|
retainedKeys.push({
|
|
2691
2346
|
spanId: alias.spanId,
|
|
2692
2347
|
owner: alias.owner
|
|
2693
2348
|
});
|
|
2694
|
-
|
|
2349
|
+
activeTrustedSpanAliases.delete(aliasKey);
|
|
2695
2350
|
}
|
|
2696
2351
|
if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
|
|
2697
|
-
for (const aliasKey of retainedAliasKeys) if (activeTrustedSpanAliases.get(aliasKey)?.span === span) activeTrustedSpanAliases.delete(aliasKey);
|
|
2698
2352
|
span.end(endTimeMs);
|
|
2699
2353
|
for (const retainedKey of retainedKeys) retainTrustedSpanContext(traceContext.traceId, retainedKey.spanId, spanContext, retainedKey.owner);
|
|
2700
2354
|
};
|
|
@@ -2717,11 +2371,9 @@ function createDiagnosticsTraceRuntime(tracer) {
|
|
|
2717
2371
|
activeTrustedSpans,
|
|
2718
2372
|
activeTrustedSpanAliases,
|
|
2719
2373
|
trustedSpanAliasKey,
|
|
2720
|
-
trustedSpanAliasOwner,
|
|
2721
2374
|
spanWithDuration,
|
|
2722
2375
|
trustedTraceContext,
|
|
2723
2376
|
internalOrTrustedTraceContext,
|
|
2724
|
-
internalOrTrustedParentContext,
|
|
2725
2377
|
internalOrTrustedExplicitParentContext,
|
|
2726
2378
|
activeTrustedParentContext,
|
|
2727
2379
|
activeInternalOrTrustedContext,
|
|
@@ -2783,9 +2435,6 @@ function resolveResourceDetectors() {
|
|
|
2783
2435
|
return detector ? [detector] : [];
|
|
2784
2436
|
});
|
|
2785
2437
|
}
|
|
2786
|
-
function resolveSignalProtocol(signal, configuredProtocol) {
|
|
2787
|
-
return configuredProtocol ?? readNonBlankString(process.env[OTEL_SIGNAL_PROTOCOL_ENV[signal]]) ?? readNonBlankString(process.env["OTEL_EXPORTER_OTLP_PROTOCOL"]) ?? OTLP_HTTP_PROTOBUF_PROTOCOL;
|
|
2788
|
-
}
|
|
2789
2438
|
function createStartupRollbackError(startupError, cleanupError) {
|
|
2790
2439
|
return new AggregateError([startupError, cleanupError], "diagnostics-otel startup failed and rollback cleanup failed", { cause: startupError });
|
|
2791
2440
|
}
|
|
@@ -2794,7 +2443,6 @@ function publicExporterEventForHealth(event) {
|
|
|
2794
2443
|
exporter: event.exporter,
|
|
2795
2444
|
signal: event.signal
|
|
2796
2445
|
};
|
|
2797
|
-
if (event.status === "recovered") return;
|
|
2798
2446
|
if (event.status === "started") return {
|
|
2799
2447
|
...base,
|
|
2800
2448
|
status: "started",
|
|
@@ -2812,13 +2460,6 @@ function publicExporterEventForHealth(event) {
|
|
|
2812
2460
|
...event.errorCategory ? { errorCategory: event.errorCategory } : {}
|
|
2813
2461
|
};
|
|
2814
2462
|
}
|
|
2815
|
-
function diagnosticTraceContextFromSpanContext(spanContext) {
|
|
2816
|
-
return {
|
|
2817
|
-
traceId: spanContext.traceId,
|
|
2818
|
-
spanId: spanContext.spanId,
|
|
2819
|
-
traceFlags: spanContext.traceFlags.toString(16).padStart(2, "0")
|
|
2820
|
-
};
|
|
2821
|
-
}
|
|
2822
2463
|
function createDiagnosticsOtelService() {
|
|
2823
2464
|
let state = {};
|
|
2824
2465
|
let preserveExporterRoutesOnNextStop = false;
|
|
@@ -2854,7 +2495,6 @@ function createDiagnosticsOtelService() {
|
|
|
2854
2495
|
const exporterHealthReporter = internalDiagnostics;
|
|
2855
2496
|
const emitPublicExporterEvent = createPublicExporterHealthEventEmitter((event) => {
|
|
2856
2497
|
const publicEvent = publicExporterEventForHealth(event);
|
|
2857
|
-
if (!publicEvent) return;
|
|
2858
2498
|
try {
|
|
2859
2499
|
internalDiagnostics?.emit({
|
|
2860
2500
|
type: "telemetry.exporter",
|
|
@@ -2910,7 +2550,7 @@ function createDiagnosticsOtelService() {
|
|
|
2910
2550
|
];
|
|
2911
2551
|
const supportedOtlpSignals = /* @__PURE__ */ new Set();
|
|
2912
2552
|
for (const signal of ownedOtlpSignals) {
|
|
2913
|
-
const protocol =
|
|
2553
|
+
const protocol = otel.protocol ?? readNonBlankString(process.env[OTEL_SIGNAL_PROTOCOL_ENV[signal]]) ?? readNonBlankString(process.env["OTEL_EXPORTER_OTLP_PROTOCOL"]) ?? OTLP_HTTP_PROTOBUF_PROTOCOL;
|
|
2914
2554
|
if (protocol === OTLP_HTTP_PROTOBUF_PROTOCOL) {
|
|
2915
2555
|
supportedOtlpSignals.add(signal);
|
|
2916
2556
|
continue;
|
|
@@ -2940,7 +2580,7 @@ function createDiagnosticsOtelService() {
|
|
|
2940
2580
|
min: 0,
|
|
2941
2581
|
max: 1
|
|
2942
2582
|
});
|
|
2943
|
-
const
|
|
2583
|
+
const captureContent = otel.captureContent === true;
|
|
2944
2584
|
const resource = resources.resourceFromAttributes({ [ATTR_SERVICE_NAME]: serviceName });
|
|
2945
2585
|
const logUrl = logsToOtlp ? resolveSignalOtelUrl({
|
|
2946
2586
|
signalEndpoint: otel.logsEndpoint,
|
|
@@ -3052,12 +2692,11 @@ function createDiagnosticsOtelService() {
|
|
|
3052
2692
|
active.stopActiveTrustedSpans = diagnosticsTrace.stopActiveTrustedSpans;
|
|
3053
2693
|
const diagnosticMetrics = createDiagnosticsMetrics(meter, otel.metricNamePrefix);
|
|
3054
2694
|
const diagnosticsLogs = createDiagnosticsLogExporter({
|
|
3055
|
-
|
|
2695
|
+
captureContent,
|
|
3056
2696
|
emitExporterEvent,
|
|
3057
2697
|
flushIntervalMs: otel.flushIntervalMs,
|
|
3058
2698
|
headers,
|
|
3059
2699
|
logger: ctx.logger,
|
|
3060
|
-
logsEnabled: logsActive,
|
|
3061
2700
|
logsToOtlp,
|
|
3062
2701
|
logsToStdout,
|
|
3063
2702
|
logHttpAgentOptions,
|
|
@@ -3066,13 +2705,12 @@ function createDiagnosticsOtelService() {
|
|
|
3066
2705
|
serviceName
|
|
3067
2706
|
});
|
|
3068
2707
|
active.logProvider = diagnosticsLogs.logProvider;
|
|
3069
|
-
const
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
traces: diagnosticsTrace,
|
|
2708
|
+
const recorderRuntime = {
|
|
2709
|
+
...diagnosticMetrics,
|
|
2710
|
+
...diagnosticsTrace,
|
|
2711
|
+
captureContent,
|
|
3074
2712
|
tracesEnabled: tracesActive
|
|
3075
|
-
}
|
|
2713
|
+
};
|
|
3076
2714
|
const recorders = {
|
|
3077
2715
|
...createUsageRecorders(recorderRuntime),
|
|
3078
2716
|
...createOperationsRecorders(recorderRuntime),
|
|
@@ -3083,8 +2721,7 @@ function createDiagnosticsOtelService() {
|
|
|
3083
2721
|
active.unsubscribe = subscribe(createDiagnosticsEventHandler({
|
|
3084
2722
|
logger: ctx.logger,
|
|
3085
2723
|
recorders,
|
|
3086
|
-
|
|
3087
|
-
recordSecurityEvent
|
|
2724
|
+
recordLogEvent: diagnosticsLogs.recordLogEvent
|
|
3088
2725
|
}), metricsActive ? tracesActive ? void 0 : { exclude: ["diagnostic.phase.completed"] } : tracesActive ? { exclude: ["gateway.event_loop.sample", "diagnostic.gc"] } : { include: ["log.record", "security.event"] });
|
|
3089
2726
|
if (tracesActive) active.unregisterTracePropagationBridge = ctx.internalDiagnostics?.registerTracePropagationBridge?.({
|
|
3090
2727
|
shouldPrepareEvent(event) {
|
|
@@ -3096,7 +2733,11 @@ function createDiagnosticsOtelService() {
|
|
|
3096
2733
|
},
|
|
3097
2734
|
resolveTraceContext(traceContext) {
|
|
3098
2735
|
const spanContext = diagnosticsTrace.exportedSpanContextForDiagnosticTraceContext(traceContext);
|
|
3099
|
-
return spanContext ?
|
|
2736
|
+
return spanContext ? {
|
|
2737
|
+
traceId: spanContext.traceId,
|
|
2738
|
+
spanId: spanContext.spanId,
|
|
2739
|
+
traceFlags: spanContext.traceFlags.toString(16).padStart(2, "0")
|
|
2740
|
+
} : void 0;
|
|
3100
2741
|
}
|
|
3101
2742
|
}) ?? null;
|
|
3102
2743
|
if (hasOwnedOtlpSignal || sdkPreloaded && !sdkDisabled) active.unregisterUnhandledRejectionHandler = registerUnhandledRejectionHandler((reason) => {
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-C7192_s0.mjs";
|
|
2
2
|
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
3
3
|
//#region extensions/diagnostics-otel/index.ts
|
|
4
4
|
var diagnostics_otel_default = definePluginEntry({
|
package/dist/runtime-api.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-
|
|
1
|
+
import { t as createDiagnosticsOtelService } from "./.setup/runtime-api-C7192_s0.mjs";
|
|
2
2
|
export { createDiagnosticsOtelService };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openclaw/diagnostics-otel",
|
|
3
|
-
"version": "2026.
|
|
3
|
+
"version": "2026.10.1-beta.2",
|
|
4
4
|
"description": "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -34,10 +34,10 @@
|
|
|
34
34
|
"minHostVersion": ">=2026.4.25"
|
|
35
35
|
},
|
|
36
36
|
"compat": {
|
|
37
|
-
"pluginApi": ">=2026.
|
|
37
|
+
"pluginApi": ">=2026.10.1-beta.2"
|
|
38
38
|
},
|
|
39
39
|
"build": {
|
|
40
|
-
"openclawVersion": "2026.
|
|
40
|
+
"openclawVersion": "2026.10.1-beta.2"
|
|
41
41
|
},
|
|
42
42
|
"release": {
|
|
43
43
|
"publishToClawHub": true,
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"assets/activity.svg"
|
|
56
56
|
],
|
|
57
57
|
"peerDependencies": {
|
|
58
|
-
"openclaw": ">=2026.
|
|
58
|
+
"openclaw": ">=2026.10.1-beta.2"
|
|
59
59
|
},
|
|
60
60
|
"peerDependenciesMeta": {
|
|
61
61
|
"openclaw": {
|