@openclaw/diagnostics-otel 2026.6.1-beta.1 → 2026.6.1-beta.3

package/dist/index.js

@@ -11,6 +11,7 @@ import { NodeSDK } from "@opentelemetry/sdk-node";
 import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
 import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_SYSTEM_INSTRUCTIONS, ATTR_GEN_AI_TOOL_DEFINITIONS } from "@opentelemetry/semantic-conventions/incubating";
+import { waitForDiagnosticEventsDrained } from "openclaw/plugin-sdk/diagnostic-runtime";
 import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env";
 //#region extensions/diagnostics-otel/src/service.ts
 const DEFAULT_SERVICE_NAME = "openclaw";
@@ -89,6 +90,8 @@ const GEN_AI_OPERATION_DURATION_BUCKETS = [
 	40.96,
 	81.92
 ];
+const MAX_RETAINED_TRUSTED_SPAN_CONTEXTS = 1024;
+const RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS = 5e3;
 const NO_CONTENT_CAPTURE = {
 	inputMessages: false,
 	outputMessages: false,
@@ -764,12 +767,14 @@ function createDiagnosticsOtelService() {
 			const tracer = trace.getTracer("openclaw");
 			const activeTrustedSpans = /* @__PURE__ */ new Map();
 			const activeTrustedSpanAliases = /* @__PURE__ */ new Map();
-			const pendingTrustedRunFinalizers = /* @__PURE__ */ new Map();
+			const retainedTrustedSpanContexts = /* @__PURE__ */ new Map();
+			const retainedTrustedSpanContextCleanupTimers = /* @__PURE__ */ new Set();
 			stopActiveTrustedSpans = () => {
 				const stopAt = Date.now();
-				for (const handle of pendingTrustedRunFinalizers.values()) clearImmediate(handle);
-				pendingTrustedRunFinalizers.clear();
-				for (const span of new Set([...activeTrustedSpans.values(), ...activeTrustedSpanAliases.values()])) span.end(stopAt);
+				for (const handle of retainedTrustedSpanContextCleanupTimers) clearTimeout(handle);
+				retainedTrustedSpanContextCleanupTimers.clear();
+				retainedTrustedSpanContexts.clear();
+				for (const span of new Set([...activeTrustedSpans.values(), ...Array.from(activeTrustedSpanAliases.values(), (entry) => entry.span)])) span.end(stopAt);
 				activeTrustedSpans.clear();
 				activeTrustedSpanAliases.clear();
 			};
@@ -1072,18 +1077,75 @@ function createDiagnosticsOtelService() {
 				}, parentContext);
 			};
 			const trustedTraceContext = (evt, metadata) => metadata.trusted ? normalizeTraceContext(evt.trace) : void 0;
+			const internalOrTrustedTraceContext = (evt, metadata) => metadata.trusted || metadata.internal ? normalizeTraceContext(evt.trace) : void 0;
+			const trustedSpanAliasOwner = (evt) => {
+				if ("runId" in evt && evt.runId) return {
+					kind: "run",
+					id: evt.runId
+				};
+			};
+			const sameTrustedSpanAliasOwner = (left, right) => Boolean(left && right && left.kind === right.kind && left.id === right.id);
+			const trustedSpanAliasKey = (spanId, owner) => `${spanId}:${owner.kind}:${owner.id}`;
+			const retainedTrustedSpanContextKey = (traceId, spanId, owner) => `${traceId}:${owner ? trustedSpanAliasKey(spanId, owner) : spanId}`;
+			const retainedTrustedSpanContext = (traceContext, spanId, owner) => {
+				if (!traceContext?.traceId || !spanId) return;
+				const retained = (owner ? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId, owner)) : void 0) ?? retainedTrustedSpanContexts.get(retainedTrustedSpanContextKey(traceContext.traceId, spanId));
+				if (retained?.spanContext.traceId !== traceContext.traceId) return;
+				if (retained.owner && !sameTrustedSpanAliasOwner(retained.owner, owner)) return;
+				return retained.spanContext;
+			};
+			const activeTrustedSpanAlias = (spanId, owner) => {
+				if (!owner) return;
+				const alias = activeTrustedSpanAliases.get(trustedSpanAliasKey(spanId, owner));
+				if (!alias || !sameTrustedSpanAliasOwner(alias.owner, owner)) return;
+				return alias.span;
+			};
+			const internalOrTrustedParentContext = (evt, metadata) => {
+				const traceContext = internalOrTrustedTraceContext(evt, metadata);
+				const parentSpanId = traceContext?.parentSpanId ?? traceContext?.spanId;
+				if (!traceContext || !parentSpanId) return;
+				return contextForTraceContext({
+					...traceContext,
+					spanId: parentSpanId
+				});
+			};
+			const internalOrTrustedExplicitParentContext = (evt, metadata) => {
+				const traceContext = internalOrTrustedTraceContext(evt, metadata);
+				if (!traceContext?.parentSpanId) return;
+				return contextForTraceContext({
+					...traceContext,
+					spanId: traceContext.parentSpanId
+				});
+			};
 			const activeTrustedParentContext = (evt, metadata) => {
-				const parentSpanId = trustedTraceContext(evt, metadata)?.parentSpanId;
+				const traceContext = trustedTraceContext(evt, metadata);
+				const parentSpanId = traceContext?.parentSpanId;
 				if (!parentSpanId) return;
-				const activeParentSpan = activeTrustedSpans.get(parentSpanId) ?? activeTrustedSpanAliases.get(parentSpanId);
-				if (!activeParentSpan) return;
-				return trace.setSpanContext(context.active(), activeParentSpan.spanContext());
+				const owner = trustedSpanAliasOwner(evt);
+				const spanContext = (activeTrustedSpans.get(parentSpanId) ?? activeTrustedSpanAlias(parentSpanId, owner))?.spanContext() ?? retainedTrustedSpanContext(traceContext, parentSpanId, owner);
+				if (!spanContext) return;
+				return trace.setSpanContext(context.active(), spanContext);
+			};
+			const activeInternalOrTrustedContext = (evt, metadata) => {
+				const traceContext = internalOrTrustedTraceContext(evt, metadata);
+				if (!traceContext) return;
+				const owner = trustedSpanAliasOwner(evt);
+				const activeSpan = (traceContext.spanId ? activeTrustedSpans.get(traceContext.spanId) ?? activeTrustedSpanAlias(traceContext.spanId, owner) : void 0) ?? (traceContext.parentSpanId ? activeTrustedSpans.get(traceContext.parentSpanId) ?? activeTrustedSpanAlias(traceContext.parentSpanId, owner) : void 0);
+				if (activeSpan) return trace.setSpanContext(context.active(), activeSpan.spanContext());
+				const retainedSpanContext = retainedTrustedSpanContext(traceContext, traceContext.spanId, owner) ?? retainedTrustedSpanContext(traceContext, traceContext.parentSpanId, owner);
+				if (retainedSpanContext) return trace.setSpanContext(context.active(), retainedSpanContext);
+				return internalOrTrustedParentContext(evt, metadata);
 			};
 			const trackTrustedSpan = (evt, metadata, span) => {
 				const spanId = trustedTraceContext(evt, metadata)?.spanId;
 				if (spanId) activeTrustedSpans.set(spanId, span);
 				return span;
 			};
+			const trackInternalOrTrustedSpan = (evt, metadata, span) => {
+				const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
+				if (spanId) activeTrustedSpans.set(spanId, span);
+				return span;
+			};
 			const takeTrackedTrustedSpan = (evt, metadata) => {
 				const spanId = trustedTraceContext(evt, metadata)?.spanId;
 				if (!spanId) return;
@@ -1091,19 +1153,72 @@ function createDiagnosticsOtelService() {
 				if (span) activeTrustedSpans.delete(spanId);
 				return span;
 			};
+			const getTrackedInternalOrTrustedSpan = (evt, metadata) => {
+				const spanId = internalOrTrustedTraceContext(evt, metadata)?.spanId;
+				if (!spanId) return;
+				return activeTrustedSpans.get(spanId);
+			};
 			const setSpanAttrs = (span, attributes) => {
 				span.setAttributes?.(redactOtelAttributes(attributes));
 			};
-			const scheduleTrackedRunSpanFinalize = (spanId, parentSpanId, span, endTimeMs) => {
-				const existingHandle = pendingTrustedRunFinalizers.get(spanId);
-				if (existingHandle) clearImmediate(existingHandle);
-				const handle = setImmediate(() => {
-					pendingTrustedRunFinalizers.delete(spanId);
-					if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
-					if (parentSpanId && activeTrustedSpanAliases.get(parentSpanId) === span) activeTrustedSpanAliases.delete(parentSpanId);
-					span.end(endTimeMs);
+			const retainTrustedSpanContext = (traceId, spanId, spanContext, token, owner) => {
+				retainedTrustedSpanContexts.set(retainedTrustedSpanContextKey(traceId, spanId, owner), {
+					spanContext,
+					token,
+					...owner ? { owner } : {}
 				});
-				pendingTrustedRunFinalizers.set(spanId, handle);
+				while (retainedTrustedSpanContexts.size > MAX_RETAINED_TRUSTED_SPAN_CONTEXTS) {
+					const oldestKey = retainedTrustedSpanContexts.keys().next().value;
+					if (!oldestKey) break;
+					retainedTrustedSpanContexts.delete(oldestKey);
+				}
+			};
+			const scheduleRetainedTrustedSpanContextCleanup = (token) => {
+				let drainHandle;
+				let timeoutHandle;
+				const cleanup = () => {
+					if (drainHandle) {
+						clearTimeout(drainHandle);
+						retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
+						drainHandle = void 0;
+					}
+					if (timeoutHandle) {
+						clearTimeout(timeoutHandle);
+						retainedTrustedSpanContextCleanupTimers.delete(timeoutHandle);
+						timeoutHandle = void 0;
+					}
+					for (const [key, retained] of retainedTrustedSpanContexts) if (retained.token === token) retainedTrustedSpanContexts.delete(key);
+				};
+				drainHandle = setTimeout(() => {
+					if (drainHandle) {
+						retainedTrustedSpanContextCleanupTimers.delete(drainHandle);
+						drainHandle = void 0;
+					}
+					waitForDiagnosticEventsDrained().then(cleanup, cleanup);
+				}, 0);
+				drainHandle.unref?.();
+				retainedTrustedSpanContextCleanupTimers.add(drainHandle);
+				timeoutHandle = setTimeout(cleanup, RETAINED_TRUSTED_SPAN_CONTEXT_TIMEOUT_MS);
+				timeoutHandle.unref?.();
+				retainedTrustedSpanContextCleanupTimers.add(timeoutHandle);
+			};
+			const completeTrackedLifecycleSpan = (spanId, span, endTimeMs) => {
+				const spanContext = span.spanContext();
+				const retainedKeys = [{ spanId }];
+				const retainedAliasKeys = [];
+				for (const [aliasKey, alias] of activeTrustedSpanAliases) if (alias.span === span) {
+					retainedKeys.push({
+						spanId: alias.spanId,
+						owner: alias.owner
+					});
+					retainedAliasKeys.push(aliasKey);
+				}
+				if (activeTrustedSpans.get(spanId) === span) activeTrustedSpans.delete(spanId);
+				for (const aliasKey of retainedAliasKeys) if (activeTrustedSpanAliases.get(aliasKey)?.span === span) activeTrustedSpanAliases.delete(aliasKey);
+				span.end(endTimeMs);
+				const token = Symbol("retainedTrustedSpanContext");
+				for (const retainedKey of retainedKeys) retainTrustedSpanContext(spanContext.traceId, retainedKey.spanId, spanContext, token, retainedKey.owner);
+				scheduleRetainedTrustedSpanContextCleanup(token);
 			};
 			const addRunAttrs = (spanAttrs, evt) => {
 				if (evt.provider) spanAttrs["openclaw.provider"] = evt.provider;
@@ -1246,11 +1361,19 @@ function createDiagnosticsOtelService() {
 					"openclaw.source": lowCardinalityAttr(evt.source)
 				});
 			};
-			const recordMessageDispatchStarted = (evt) => {
-				messageDispatchStartedCounter.add(1, {
+			const recordMessageDispatchStarted = (evt, metadata) => {
+				const attrs = {
 					"openclaw.channel": lowCardinalityAttr(evt.channel),
 					"openclaw.source": lowCardinalityAttr(evt.source)
-				});
+				};
+				messageDispatchStartedCounter.add(1, attrs);
+				if (!tracesEnabled) return;
+				const traceContext = internalOrTrustedTraceContext(evt, metadata);
+				if (!traceContext?.spanId || activeTrustedSpans.has(traceContext.spanId)) return;
+				trackInternalOrTrustedSpan(evt, metadata, spanWithDuration("openclaw.message.processed", attrs, void 0, {
+					parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
+					startTimeMs: evt.ts
+				}));
 			};
 			const recordMessageDispatchCompleted = (evt) => {
 				const attrs = {
@@ -1262,7 +1385,7 @@ function createDiagnosticsOtelService() {
 				messageDispatchCompletedCounter.add(1, attrs);
 				messageDispatchDurationHistogram.record(evt.durationMs, attrs);
 			};
-			const recordMessageProcessed = (evt) => {
+			const recordMessageProcessed = (evt, metadata) => {
 				const attrs = {
 					"openclaw.channel": lowCardinalityAttr(evt.channel),
 					"openclaw.outcome": evt.outcome ?? "unknown"
@@ -1272,12 +1395,22 @@ function createDiagnosticsOtelService() {
 				if (!tracesEnabled) return;
 				const spanAttrs = { ...attrs };
 				if (evt.reason) spanAttrs["openclaw.reason"] = lowCardinalityAttr(evt.reason, "unknown");
-				const span = spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs);
+				const trackedSpan = getTrackedInternalOrTrustedSpan(evt, metadata);
+				const span = trackedSpan ?? spanWithDuration("openclaw.message.processed", spanAttrs, evt.durationMs, {
+					parentContext: internalOrTrustedExplicitParentContext(evt, metadata),
+					endTimeMs: evt.ts
+				});
+				setSpanAttrs(span, spanAttrs);
 				if (evt.outcome === "error" && evt.error) span.setStatus({
 					code: SpanStatusCode.ERROR,
 					message: redactSensitiveText(evt.error)
 				});
-				span.end();
+				const traceContext = internalOrTrustedTraceContext(evt, metadata);
+				if (trackedSpan && traceContext?.spanId) {
+					completeTrackedLifecycleSpan(traceContext.spanId, trackedSpan, evt.ts);
+					return;
+				}
+				span.end(evt.ts);
 			};
 			const messageDeliveryAttrs = (evt) => ({
 				"openclaw.channel": lowCardinalityAttr(evt.channel),
@@ -1286,7 +1419,7 @@ function createDiagnosticsOtelService() {
 			const recordMessageDeliveryStarted = (evt) => {
 				messageDeliveryStartedCounter.add(1, messageDeliveryAttrs(evt));
 			};
-			const recordMessageDeliveryCompleted = (evt) => {
+			const recordMessageDeliveryCompleted = (evt, metadata) => {
 				const attrs = {
 					...messageDeliveryAttrs(evt),
 					"openclaw.outcome": "completed"
@@ -1296,9 +1429,12 @@ function createDiagnosticsOtelService() {
 				spanWithDuration("openclaw.message.delivery", {
 					...attrs,
 					"openclaw.delivery.result_count": evt.resultCount
-				}, evt.durationMs, { endTimeMs: evt.ts }).end(evt.ts);
+				}, evt.durationMs, {
+					parentContext: activeInternalOrTrustedContext(evt, metadata),
+					endTimeMs: evt.ts
+				}).end(evt.ts);
 			};
-			const recordMessageDeliveryError = (evt) => {
+			const recordMessageDeliveryError = (evt, metadata) => {
 				const attrs = {
 					...messageDeliveryAttrs(evt),
 					"openclaw.outcome": "error",
@@ -1306,7 +1442,10 @@ function createDiagnosticsOtelService() {
 				};
 				messageDeliveryDurationHistogram.record(evt.durationMs, attrs);
 				if (!tracesEnabled) return;
-				const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, { endTimeMs: evt.ts });
+				const span = spanWithDuration("openclaw.message.delivery", attrs, evt.durationMs, {
+					parentContext: activeInternalOrTrustedContext(evt, metadata),
+					endTimeMs: evt.ts
+				});
 				span.setStatus({
 					code: SpanStatusCode.ERROR,
 					message: redactSensitiveText(evt.errorCategory)
@@ -1322,7 +1461,17 @@ function createDiagnosticsOtelService() {
 					startTimeMs: evt.ts
 				}));
 				const parentSpanId = trustedTraceContext(evt, metadata)?.parentSpanId;
-				if (parentSpanId && !activeTrustedSpans.has(parentSpanId)) activeTrustedSpanAliases.set(parentSpanId, span);
+				if (parentSpanId && !activeTrustedSpans.has(parentSpanId)) {
+					const owner = {
+						kind: "run",
+						id: evt.runId
+					};
+					activeTrustedSpanAliases.set(trustedSpanAliasKey(parentSpanId, owner), {
+						span,
+						spanId: parentSpanId,
+						owner
+					});
+				}
 			};
 			const recordLaneEnqueue = (evt) => {
 				const attrs = { "openclaw.lane": lowCardinalityQueueLaneAttr(evt.lane) };
@@ -1485,7 +1634,7 @@ function createDiagnosticsOtelService() {
 					...evt.errorCategory ? { message: redactSensitiveText(evt.errorCategory) } : {}
 				});
 				if (trackedSpan && trustedTrace?.spanId) {
-					scheduleTrackedRunSpanFinalize(trustedTrace.spanId, trustedTrace.parentSpanId, trackedSpan, evt.ts);
+					completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
 					return;
 				}
 				span.end(evt.ts);
@@ -1516,7 +1665,9 @@ function createDiagnosticsOtelService() {
 					spanAttrs["openclaw.harness.items.completed"] = evt.itemLifecycle.completedCount;
 					spanAttrs["openclaw.harness.items.active"] = evt.itemLifecycle.activeCount;
 				}
-				const span = takeTrackedTrustedSpan(evt, metadata) ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
+				const trustedTrace = trustedTraceContext(evt, metadata);
+				const trackedSpan = trustedTrace?.spanId ? activeTrustedSpans.get(trustedTrace.spanId) : void 0;
+				const span = trackedSpan ?? spanWithDuration("openclaw.harness.run", spanAttrs, evt.durationMs, {
 					parentContext: activeTrustedParentContext(evt, metadata),
 					endTimeMs: evt.ts
 				});
@@ -1525,6 +1676,10 @@ function createDiagnosticsOtelService() {
 					code: SpanStatusCode.ERROR,
 					message: "error"
 				});
+				if (trackedSpan && trustedTrace?.spanId) {
+					completeTrackedLifecycleSpan(trustedTrace.spanId, trackedSpan, evt.ts);
+					return;
+				}
 				span.end(evt.ts);
 			};
 			const recordHarnessRunError = (evt, metadata) => {
@@ -1901,22 +2056,22 @@ function createDiagnosticsOtelService() {
 							recordMessageReceived(evt);
 							return;
 						case "message.dispatch.started":
-							recordMessageDispatchStarted(evt);
+							recordMessageDispatchStarted(evt, metadata);
 							return;
 						case "message.dispatch.completed":
 							recordMessageDispatchCompleted(evt);
 							return;
 						case "message.processed":
-							recordMessageProcessed(evt);
+							recordMessageProcessed(evt, metadata);
 							return;
 						case "message.delivery.started":
 							recordMessageDeliveryStarted(evt);
 							return;
 						case "message.delivery.completed":
-							recordMessageDeliveryCompleted(evt);
+							recordMessageDeliveryCompleted(evt, metadata);
 							return;
 						case "message.delivery.error":
-							recordMessageDeliveryError(evt);
+							recordMessageDeliveryError(evt, metadata);
 							return;
 						case "talk.event":
 							recordTalkEvent(evt, metadata);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@openclaw/diagnostics-otel",
-  "version": "2026.6.1-beta.1",
+  "version": "2026.6.1-beta.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@openclaw/diagnostics-otel",
-      "version": "2026.6.1-beta.1",
+      "version": "2026.6.1-beta.3",
       "dependencies": {
         "@opentelemetry/api": "1.9.1",
         "@opentelemetry/api-logs": "0.218.0",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openclaw/diagnostics-otel",
-  "version": "2026.6.1-beta.1",
+  "version": "2026.6.1-beta.3",
   "description": "OpenClaw diagnostics OpenTelemetry exporter for metrics and traces.",
   "repository": {
     "type": "git",
@@ -31,10 +31,10 @@
       "minHostVersion": ">=2026.4.25"
     },
     "compat": {
-      "pluginApi": ">=2026.6.1-beta.1"
+      "pluginApi": ">=2026.6.1-beta.3"
     },
     "build": {
-      "openclawVersion": "2026.6.1-beta.1"
+      "openclawVersion": "2026.6.1-beta.3"
     },
     "release": {
       "publishToClawHub": true,
@@ -51,7 +51,7 @@
     "README.md"
   ],
   "peerDependencies": {
-    "openclaw": ">=2026.6.1-beta.1"
+    "openclaw": ">=2026.6.1-beta.3"
   },
   "peerDependenciesMeta": {
     "openclaw": {