@aigentsphere/openclaw-otel-observability 0.2.1

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@aigentsphere/openclaw-otel-observability",
+  "version": "0.2.1",
+  "description": "OpenTelemetry observability plugin for OpenClaw — traces, metrics, and logs for your AI agent using OpenLLMetry",
+  "type": "module",
+  "openclaw": {
+    "extensions": ["./index.ts"]
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/aigentsphere/openclaw-observability-plugin.git"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint src/ index.ts"
+  },
+  "keywords": [
+    "openclaw",
+    "opentelemetry",
+    "observability",
+    "llm",
+    "tracing",
+    "metrics",
+    "dynatrace",
+    "openllmetry"
+  ],
+  "author": "Henrik Rexed",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@traceloop/node-server-sdk": "^0.22.6",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/sdk-node": "^0.203.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@opentelemetry/sdk-metrics": "^2.0.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.203.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.203.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.203.0",
+    "@opentelemetry/resources": "^2.0.1",
+    "@opentelemetry/semantic-conventions": "^1.30.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.0"
+  }
+}

package/src/config.ts

@@ -0,0 +1,72 @@
+/**
+ * Configuration types and defaults for the OTel Observability plugin.
+ */
+
+export interface OtelObservabilityConfig {
+  /** OTLP endpoint URL */
+  endpoint: string;
+  /** OTLP export protocol: 'http' (OTLP/HTTP) or 'grpc' (OTLP/gRPC) */
+  protocol: "http" | "grpc";
+  /** OpenTelemetry service name */
+  serviceName: string;
+  /** Custom headers for OTLP export (e.g., Authorization for Dynatrace) */
+  headers: Record<string, string>;
+  /** Enable trace export */
+  traces: boolean;
+  /** Enable metrics export */
+  metrics: boolean;
+  /** Enable log export */
+  logs: boolean;
+  /** Capture prompt/completion content in spans (disable for privacy) */
+  captureContent: boolean;
+  /** Metrics export interval in milliseconds */
+  metricsIntervalMs: number;
+  /** Additional OTel resource attributes */
+  resourceAttributes: Record<string, string>;
+}
+
+const DEFAULTS: OtelObservabilityConfig = {
+  endpoint: "http://localhost:4318",
+  protocol: "http",
+  serviceName: "openclaw-gateway",
+  headers: {},
+  traces: true,
+  metrics: true,
+  logs: true,
+  captureContent: false,
+  metricsIntervalMs: 30_000,
+  resourceAttributes: {},
+};
+
+export function parseConfig(raw: unknown): OtelObservabilityConfig {
+  const obj =
+    raw && typeof raw === "object" && !Array.isArray(raw)
+      ? (raw as Record<string, unknown>)
+      : {};
+
+  return {
+    endpoint: typeof obj.endpoint === "string" ? obj.endpoint : DEFAULTS.endpoint,
+    protocol: obj.protocol === "grpc" ? "grpc" : DEFAULTS.protocol,
+    serviceName:
+      typeof obj.serviceName === "string" ? obj.serviceName : DEFAULTS.serviceName,
+    headers:
+      obj.headers && typeof obj.headers === "object" && !Array.isArray(obj.headers)
+        ? (obj.headers as Record<string, string>)
+        : DEFAULTS.headers,
+    traces: typeof obj.traces === "boolean" ? obj.traces : DEFAULTS.traces,
+    metrics: typeof obj.metrics === "boolean" ? obj.metrics : DEFAULTS.metrics,
+    logs: typeof obj.logs === "boolean" ? obj.logs : DEFAULTS.logs,
+    captureContent:
+      typeof obj.captureContent === "boolean" ? obj.captureContent : DEFAULTS.captureContent,
+    metricsIntervalMs:
+      typeof obj.metricsIntervalMs === "number" && obj.metricsIntervalMs >= 1000
+        ? obj.metricsIntervalMs
+        : DEFAULTS.metricsIntervalMs,
+    resourceAttributes:
+      obj.resourceAttributes &&
+      typeof obj.resourceAttributes === "object" &&
+      !Array.isArray(obj.resourceAttributes)
+        ? (obj.resourceAttributes as Record<string, string>)
+        : DEFAULTS.resourceAttributes,
+  };
+}

package/src/diagnostics.ts

@@ -0,0 +1,214 @@
+/**
+ * Diagnostic events integration — subscribes to OpenClaw's internal diagnostic
+ * events to get accurate cost/token data, then enriches our connected traces.
+ *
+ * This combines the best of both approaches:
+ * - Our plugin: Connected traces (request → agent turn → tools)
+ * - Official diagnostics: Accurate cost, token counts, context limits
+ */
+
+import type { Span } from "@opentelemetry/api";
+import type { TelemetryRuntime } from "./telemetry.js";
+
+// Import from OpenClaw plugin SDK (loaded lazily)
+let onDiagnosticEvent: ((listener: (evt: any) => void) => () => void) | null = null;
+let sdkLoadAttempted = false;
+
+async function loadSdk(): Promise<void> {
+  if (sdkLoadAttempted) return;
+  sdkLoadAttempted = true;
+  try {
+    // Dynamic import to avoid build issues if SDK not available
+    // @ts-ignore - openclaw/plugin-sdk types not available at build time
+    const sdk = await import("openclaw/plugin-sdk") as any;
+    onDiagnosticEvent = sdk.onDiagnosticEvent;
+  } catch {
+    // SDK not available — will use fallback token extraction
+  }
+}
+
+/** Pending usage data waiting to be attached to spans */
+interface PendingUsageData {
+  costUsd?: number;
+  usage: {
+    input?: number;
+    output?: number;
+    cacheRead?: number;
+    cacheWrite?: number;
+    total?: number;
+  };
+  context?: {
+    limit?: number;
+    used?: number;
+  };
+  durationMs?: number;
+  provider?: string;
+  model?: string;
+}
+
+/** Map of sessionKey → pending usage data from diagnostic events */
+const pendingUsageMap = new Map<string, PendingUsageData>();
+
+/** Map of sessionKey → active agent span (set by hooks.ts) */
+export const activeAgentSpans = new Map<string, Span>();
+
+/**
+ * Register diagnostic event listener to capture model.usage events.
+ * Returns unsubscribe function.
+ */
+export async function registerDiagnosticsListener(
+  telemetry: TelemetryRuntime,
+  logger: any
+): Promise<() => void> {
+  // Load the SDK if not already loaded
+  await loadSdk();
+
+  if (!onDiagnosticEvent) {
+    logger.debug?.("[otel] onDiagnosticEvent not available — using fallback token extraction");
+    return () => {};
+  }
+
+  const { counters, histograms } = telemetry;
+
+  const unsubscribe = onDiagnosticEvent((evt: any) => {
+    if (evt.type !== "model.usage") return;
+
+    const sessionKey = evt.sessionKey || "unknown";
+    const usage = evt.usage || {};
+    const costUsd = evt.costUsd;
+    const model = evt.model || "unknown";
+    const provider = evt.provider || "unknown";
+
+    // Store for later attachment to agent span
+    pendingUsageMap.set(sessionKey, {
+      costUsd,
+      usage,
+      context: evt.context,
+      durationMs: evt.durationMs,
+      provider,
+      model,
+    });
+
+    // Record metrics immediately (don't wait for span)
+    const metricAttrs = {
+      "gen_ai.response.model": model,
+      "openclaw.provider": provider,
+    };
+
+    if (usage.input) {
+      counters.tokensPrompt.add(usage.input, metricAttrs);
+    }
+    if (usage.output) {
+      counters.tokensCompletion.add(usage.output, metricAttrs);
+    }
+    if (usage.cacheRead) {
+      counters.tokensPrompt.add(usage.cacheRead, { ...metricAttrs, "token.type": "cache_read" });
+    }
+    if (usage.cacheWrite) {
+      counters.tokensPrompt.add(usage.cacheWrite, { ...metricAttrs, "token.type": "cache_write" });
+    }
+    if (usage.total) {
+      counters.tokensTotal.add(usage.total, metricAttrs);
+    }
+
+    // Record cost metric
+    if (typeof costUsd === "number" && costUsd > 0) {
+      telemetry.meter.createCounter("openclaw.llm.cost.usd", {
+        description: "Estimated LLM cost in USD",
+        unit: "usd",
+      }).add(costUsd, metricAttrs);
+    }
+
+    // Record LLM duration
+    if (typeof evt.durationMs === "number") {
+      histograms.llmDuration.record(evt.durationMs, metricAttrs);
+    }
+
+    counters.llmRequests.add(1, metricAttrs);
+
+    // If we have an active agent span for this session, enrich it now
+    const agentSpan = activeAgentSpans.get(sessionKey);
+    if (agentSpan) {
+      enrichSpanWithUsage(agentSpan, evt);
+      pendingUsageMap.delete(sessionKey);
+    }
+
+    logger.debug?.(`[otel] model.usage: session=${sessionKey}, model=${model}, cost=$${costUsd?.toFixed(4) || "?"}, tokens=${usage.total || "?"}`);
+  });
+
+  logger.info("[otel] Subscribed to OpenClaw diagnostic events (model.usage, etc.)");
+  return unsubscribe;
+}
+
+/**
+ * Get pending usage data for a session (if any).
+ * Called by agent_end hook to attach data to span.
+ */
+export function getPendingUsage(sessionKey: string): PendingUsageData | undefined {
+  const data = pendingUsageMap.get(sessionKey);
+  if (data) {
+    pendingUsageMap.delete(sessionKey);
+  }
+  return data;
+}
+
+/**
+ * Enrich a span with usage data from diagnostic event.
+ */
+export function enrichSpanWithUsage(span: Span, data: PendingUsageData): void {
+  const usage = data.usage || {};
+
+  // GenAI semantic convention attributes
+  if (usage.input !== undefined) {
+    span.setAttribute("gen_ai.usage.input_tokens", usage.input);
+  }
+  if (usage.output !== undefined) {
+    span.setAttribute("gen_ai.usage.output_tokens", usage.output);
+  }
+  if (usage.total !== undefined) {
+    span.setAttribute("gen_ai.usage.total_tokens", usage.total);
+  }
+  if (usage.cacheRead !== undefined) {
+    span.setAttribute("gen_ai.usage.cache_read_tokens", usage.cacheRead);
+  }
+  if (usage.cacheWrite !== undefined) {
+    span.setAttribute("gen_ai.usage.cache_write_tokens", usage.cacheWrite);
+  }
+
+  // Cost (custom attribute — not in GenAI semconv yet)
+  if (data.costUsd !== undefined) {
+    span.setAttribute("openclaw.llm.cost_usd", data.costUsd);
+  }
+
+  // Context window
+  if (data.context?.limit !== undefined) {
+    span.setAttribute("openclaw.context.limit", data.context.limit);
+  }
+  if (data.context?.used !== undefined) {
+    span.setAttribute("openclaw.context.used", data.context.used);
+  }
+
+  // Provider/model
+  if (data.provider) {
+    span.setAttribute("gen_ai.system", data.provider);
+  }
+  if (data.model) {
+    span.setAttribute("gen_ai.response.model", data.model);
+  }
+}
+
+/**
+ * Check if diagnostic events are available.
+ * Note: Only accurate after registerDiagnosticsListener() has been called.
+ */
+export function hasDiagnosticsSupport(): boolean {
+  return onDiagnosticEvent !== null;
+}
+
+/**
+ * Async check for diagnostics support (loads SDK if needed).
+ */
+export async function checkDiagnosticsSupport(): Promise<boolean> {
+  await loadSdk();
+  return onDiagnosticEvent !== null;
+}