@node-llm/monitor-otel 0.1.0

package/src/NodeLLMSpanProcessor.ts

@@ -0,0 +1,257 @@
+/**
+ * NodeLLM SpanProcessor for OpenTelemetry
+ *
+ * This SpanProcessor intercepts OpenTelemetry spans emitted by the Vercel AI SDK
+ * (and other OTel-instrumented LLM libraries) and converts them to NodeLLM Monitor events.
+ *
+ * Usage:
+ * ```ts
+ * import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+ * import { NodeLLMSpanProcessor } from "@node-llm/monitor/otel";
+ * import { PrismaAdapter } from "@node-llm/monitor/adapters/prisma";
+ *
+ * const store = new PrismaAdapter(prisma);
+ * const provider = new NodeTracerProvider();
+ * provider.addSpanProcessor(new NodeLLMSpanProcessor(store));
+ * provider.register();
+ *
+ * // Now Vercel AI SDK's experimental_telemetry will flow to node-llm-monitor
+ * await generateText({
+ *   model: openai("gpt-4o"),
+ *   prompt: "Hello",
+ *   experimental_telemetry: { isEnabled: true }
+ * });
+ * ```
+ */
+
+import { randomUUID } from "node:crypto";
+import type { MonitoringStore, MonitoringEvent } from "@node-llm/monitor";
+import type {
+  OTelReadableSpan,
+  OTelSpan,
+  OTelContext,
+  OTelSpanProcessor,
+  NodeLLMSpanProcessorOptions
+} from "./types.js";
+import {
+  isAISpan,
+  isTopLevelAISpan,
+  getOperationType,
+  extractAIAttributes,
+  hrTimeToDate,
+  calculateDurationMs,
+  extractProvider,
+  extractModel,
+  mapStatusToEventType,
+  generateRequestId,
+  extractSessionId,
+  parseJsonAttribute
+} from "./utils.js";
+import { estimateCost } from "@node-llm/monitor";
+
+/**
+ * OpenTelemetry SpanProcessor that routes AI spans to NodeLLM Monitor storage.
+ *
+ * This processor:
+ * 1. Filters for AI-related spans (ai.generateText, ai.streamText, etc.)
+ * 2. Extracts LLM-specific attributes (model, tokens, cost, etc.)
+ * 3. Converts to MonitoringEvent format
+ * 4. Saves to the configured MonitoringStore
+ */
+export class NodeLLMSpanProcessor implements OTelSpanProcessor {
+  private readonly store: MonitoringStore;
+  private readonly options: Required<Pick<NodeLLMSpanProcessorOptions, "captureContent">> &
+    NodeLLMSpanProcessorOptions;
+  private pendingWrites: Promise<void>[] = [];
+
+  constructor(store: MonitoringStore, options: NodeLLMSpanProcessorOptions = {}) {
+    this.store = store;
+    this.options = {
+      captureContent: true,
+      ...options
+    };
+  }
+
+  /**
+   * Called when a span starts. We don't need to do anything here
+   * since we process spans on completion.
+   */
+  onStart(_span: OTelSpan, _parentContext: OTelContext): void {
+    // No-op: We process spans when they end
+  }
+
+  /**
+   * Called when a span ends. This is where we extract data and save events.
+   */
+  onEnd(span: OTelReadableSpan): void {
+    // Quick filter: only process AI spans
+    if (!isAISpan(span)) {
+      return;
+    }
+
+    // Apply custom filter if provided
+    if (this.options.filter && !this.options.filter(span)) {
+      return;
+    }
+
+    // Process only top-level spans to avoid duplicates
+    // (doGenerate/doStream contain the detailed info, but top-level has aggregated data)
+    if (!isTopLevelAISpan(span)) {
+      return;
+    }
+
+    // Convert and save asynchronously
+    const writePromise = this.processSpan(span).catch((error) => {
+      if (this.options.onError) {
+        this.options.onError(error as Error, span);
+      } else {
+        console.error("[NodeLLMSpanProcessor] Error processing span:", error);
+      }
+    });
+
+    this.pendingWrites.push(writePromise);
+  }
+
+  /**
+   * Process a single span and save as MonitoringEvent
+   */
+  private async processSpan(span: OTelReadableSpan): Promise<void> {
+    const event = this.spanToEvent(span);
+
+    // Apply custom transform if provided
+    const finalEvent = this.options.transform ? this.options.transform(event, span) : event;
+
+    await this.store.saveEvent(finalEvent);
+  }
+
+  /**
+   * Convert an OTel span to a MonitoringEvent
+   */
+  private spanToEvent(span: OTelReadableSpan): MonitoringEvent {
+    const attrs = extractAIAttributes(span);
+    const operationType = getOperationType(span);
+    const provider = extractProvider(attrs);
+    const model = extractModel(attrs);
+    const duration = calculateDurationMs(span.startTime, span.endTime);
+
+    // Calculate token counts (prefer AI SDK attributes, fallback to GenAI conventions)
+    const promptTokens = attrs.promptTokens ?? attrs.genAiInputTokens ?? 0;
+    const completionTokens = attrs.completionTokens ?? attrs.genAiOutputTokens ?? 0;
+
+    // Estimate cost
+    const cost = estimateCost(provider, model, promptTokens, completionTokens);
+
+    // Extract request ID
+    const requestId = this.options.sessionIdExtractor
+      ? generateRequestId(span)
+      : generateRequestId(span);
+
+    // Extract session ID
+    const sessionId = this.options.sessionIdExtractor
+      ? this.options.sessionIdExtractor(span)
+      : extractSessionId(span);
+
+    // Build payload
+    const payload: Record<string, unknown> = {
+      operationType,
+      functionId: attrs.functionId,
+      finishReason: attrs.responseFinishReason,
+      usage: {
+        promptTokens,
+        completionTokens,
+        totalTokens: promptTokens + completionTokens
+      },
+      // Streaming metrics
+      ...(attrs.msToFirstChunk !== undefined && {
+        msToFirstChunk: attrs.msToFirstChunk
+      }),
+      ...(attrs.msToFinish !== undefined && { msToFinish: attrs.msToFinish }),
+      ...(attrs.avgCompletionTokensPerSecond !== undefined && {
+        avgCompletionTokensPerSecond: attrs.avgCompletionTokensPerSecond
+      }),
+      // Custom metadata
+      ...(attrs.metadata && { metadata: attrs.metadata }),
+      // OpenTelemetry trace context
+      otel: {
+        traceId: span.spanContext().traceId,
+        spanId: span.spanContext().spanId,
+        parentSpanId: span.parentSpanId
+      }
+    };
+
+    // Add content if enabled
+    if (this.options.captureContent) {
+      if (attrs.prompt) {
+        payload.prompt = attrs.prompt;
+      }
+      if (attrs.promptMessages) {
+        payload.messages = parseJsonAttribute(attrs.promptMessages);
+      }
+      if (attrs.responseText) {
+        payload.result = attrs.responseText;
+      }
+      if (attrs.responseObject) {
+        payload.object = parseJsonAttribute(attrs.responseObject);
+      }
+      if (attrs.responseToolCalls) {
+        payload.toolCalls = parseJsonAttribute(attrs.responseToolCalls);
+      }
+    }
+
+    // Tool call specific payload
+    if (operationType === "ai.toolCall") {
+      payload.tool = {
+        name: attrs.toolCallName,
+        id: attrs.toolCallId,
+        ...(this.options.captureContent && {
+          args: parseJsonAttribute(attrs.toolCallArgs),
+          result: parseJsonAttribute(attrs.toolCallResult)
+        })
+      };
+    }
+
+    // Handle errors
+    if (span.status.code === 2) {
+      payload.error = span.status.message || "Unknown error";
+    }
+
+    return {
+      id: randomUUID(),
+      eventType: mapStatusToEventType(span.status.code, operationType),
+      requestId,
+      sessionId,
+      time: hrTimeToDate(span.startTime),
+      duration,
+      cost,
+      payload,
+      createdAt: new Date(),
+      provider,
+      model
+    };
+  }
+
+  /**
+   * Flush all pending writes
+   */
+  async forceFlush(): Promise<void> {
+    await Promise.all(this.pendingWrites);
+    this.pendingWrites = [];
+  }
+
+  /**
+   * Shutdown the processor
+   */
+  async shutdown(): Promise<void> {
+    await this.forceFlush();
+  }
+}
+
+/**
+ * Factory function to create a SpanProcessor with simpler API
+ */
+export function createOTelProcessor(
+  store: MonitoringStore,
+  options?: NodeLLMSpanProcessorOptions
+): NodeLLMSpanProcessor {
+  return new NodeLLMSpanProcessor(store, options);
+}

package/src/index.ts

@@ -0,0 +1 @@
+export * from './NodeLLMSpanProcessor.js';

package/src/types.ts

@@ -0,0 +1,141 @@
+/**
+ * OpenTelemetry Types for NodeLLM Monitor
+ *
+ * These interfaces mirror the OpenTelemetry API to avoid hard dependencies.
+ * Users must provide their own @opentelemetry/api and @opentelemetry/sdk-trace-base.
+ */
+
+import type {
+  Attributes,
+  AttributeValue as OTelAttributeValue,
+  Context,
+  Span,
+  SpanStatus
+} from "@opentelemetry/api";
+import type { ReadableSpan, SpanProcessor } from "@opentelemetry/sdk-trace-base";
+
+/**
+ * Minimal Span interface matching OpenTelemetry ReadableSpan
+ */
+export type OTelReadableSpan = ReadableSpan;
+export type OTelSpan = Span;
+export type OTelContext = Context;
+export type OTelAttributes = Attributes;
+export type AttributeValue = OTelAttributeValue;
+
+export interface SpanEvent {
+  name: string;
+  time: [number, number];
+  attributes?: Attributes;
+}
+
+/**
+ * SpanProcessor interface matching OpenTelemetry SDK
+ */
+export type OTelSpanProcessor = SpanProcessor;
+
+/**
+ * Vercel AI SDK span operation types
+ */
+export type AIOperationType =
+  | "ai.generateText"
+  | "ai.generateText.doGenerate"
+  | "ai.streamText"
+  | "ai.streamText.doStream"
+  | "ai.generateObject"
+  | "ai.generateObject.doGenerate"
+  | "ai.streamObject"
+  | "ai.streamObject.doStream"
+  | "ai.embed"
+  | "ai.embed.doEmbed"
+  | "ai.embedMany"
+  | "ai.embedMany.doEmbed"
+  | "ai.toolCall";
+
+/**
+ * Extracted AI attributes from OTel span
+ */
+export interface AISpanAttributes {
+  // Operation info
+  operationId?: string;
+  functionId?: string;
+
+  // Model info
+  modelId?: string;
+  modelProvider?: string;
+
+  // Request info
+  prompt?: string;
+  promptMessages?: string;
+  promptTools?: string;
+
+  // Response info
+  responseText?: string;
+  responseObject?: string;
+  responseToolCalls?: string;
+  responseFinishReason?: string;
+  responseId?: string;
+  responseModel?: string;
+
+  // Usage
+  promptTokens?: number;
+  completionTokens?: number;
+  totalTokens?: number;
+
+  // Timing (streaming)
+  msToFirstChunk?: number;
+  msToFinish?: number;
+  avgCompletionTokensPerSecond?: number;
+
+  // Tool calls
+  toolCallName?: string;
+  toolCallId?: string;
+  toolCallArgs?: string;
+  toolCallResult?: string;
+
+  // GenAI semantic conventions
+  genAiSystem?: string;
+  genAiRequestModel?: string;
+  genAiResponseModel?: string;
+  genAiInputTokens?: number;
+  genAiOutputTokens?: number;
+
+  // Custom metadata
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Configuration for the NodeLLM SpanProcessor
+ */
+export interface NodeLLMSpanProcessorOptions {
+  /**
+   * Whether to capture prompt/response content.
+   * Default: true (respects Vercel AI SDK's recordInputs/recordOutputs settings)
+   */
+  captureContent?: boolean;
+
+  /**
+   * Filter function to decide which spans to process.
+   * Return true to process, false to skip.
+   * Default: processes all ai.* spans
+   */
+  filter?: (span: OTelReadableSpan) => boolean;
+
+  /**
+   * Custom transform function to modify the monitoring event before saving.
+   */
+  transform?: (
+    event: import("@node-llm/monitor").MonitoringEvent,
+    span: OTelReadableSpan
+  ) => import("@node-llm/monitor").MonitoringEvent;
+
+  /**
+   * Session ID extractor. By default, looks for ai.telemetry.metadata.sessionId
+   */
+  sessionIdExtractor?: (span: OTelReadableSpan) => string | undefined;
+
+  /**
+   * Error handler for storage failures
+   */
+  onError?: (error: Error, span: OTelReadableSpan) => void;
+}

package/src/utils.ts

@@ -0,0 +1,314 @@
+/**
+ * Utility functions for extracting and mapping OpenTelemetry attributes
+ * to NodeLLM Monitor events.
+ */
+
+import type {
+  OTelReadableSpan,
+  AttributeValue,
+  OTelAttributes,
+  AISpanAttributes,
+  AIOperationType
+} from "./types.js";
+
+/**
+ * Known AI operation span names from Vercel AI SDK and standard GenAI conventions
+ */
+const AI_OPERATION_PREFIXES = [
+  "ai.generateText",
+  "ai.streamText",
+  "ai.generateObject",
+  "ai.streamObject",
+  "ai.embed",
+  "ai.embedMany",
+  "ai.toolCall",
+  "gen_ai." // Standard OTel prefix
+];
+
+/**
+ * Check if a span is an AI-related span (Vercel AI SDK or standard GenAI)
+ */
+export function isAISpan(span: OTelReadableSpan): boolean {
+  const hasAIPrefix = AI_OPERATION_PREFIXES.some((prefix) => span.name.startsWith(prefix));
+  const hasAIAttributes =
+    span.attributes &&
+    (span.attributes["ai.operationId"] ||
+      span.attributes["gen_ai.system"] ||
+      span.attributes["gen_ai.request.model"]);
+
+  return !!(hasAIPrefix || hasAIAttributes);
+}
+
+/**
+ * Check if this is a top-level operation span (not a .doGenerate/.doStream child)
+ */
+export function isTopLevelAISpan(span: OTelReadableSpan): boolean {
+  const name = span.name;
+  const operationId = span.attributes["ai.operationId"] as string | undefined;
+
+  // Top-level spans don't have .doGenerate, .doStream, or .doEmbed suffix
+  const isDoSpan =
+    name.includes(".doGenerate") || name.includes(".doStream") || name.includes(".doEmbed");
+
+  // Tool calls are their own top-level spans
+  const isToolCall = operationId === "ai.toolCall" || name === "ai.toolCall";
+
+  return !isDoSpan || isToolCall;
+}
+
+/**
+ * Get the operation type from a span
+ */
+export function getOperationType(span: OTelReadableSpan): AIOperationType | undefined {
+  const operationId = span.attributes["ai.operationId"] as string | undefined;
+  if (operationId) {
+    return operationId as AIOperationType;
+  }
+
+  // Fallback to span name
+  for (const prefix of AI_OPERATION_PREFIXES) {
+    if (span.name.startsWith(prefix)) {
+      return span.name.split(" ")[0] as AIOperationType;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Extract a string attribute safely
+ */
+function getString(attrs: OTelAttributes, key: string): string | undefined {
+  const value = attrs[key];
+  return typeof value === "string" ? value : undefined;
+}
+
+/**
+ * Extract a number attribute safely
+ */
+function getNumber(attrs: OTelAttributes, key: string): number | undefined {
+  const value = attrs[key];
+  return typeof value === "number" ? value : undefined;
+}
+
+/**
+ * Extract all AI-related attributes from a span
+ */
+export function extractAIAttributes(span: OTelReadableSpan): AISpanAttributes {
+  const attrs = span.attributes;
+
+  // Extract custom metadata (ai.telemetry.metadata.*)
+  const metadata: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(attrs)) {
+    if (key.startsWith("ai.telemetry.metadata.")) {
+      const metaKey = key.replace("ai.telemetry.metadata.", "");
+      metadata[metaKey] = value;
+    }
+  }
+
+  return {
+    // Operation info
+    operationId: getString(attrs, "ai.operationId"),
+    functionId: getString(attrs, "ai.telemetry.functionId") || getString(attrs, "resource.name"),
+
+    // Model info
+    modelId: getString(attrs, "ai.model.id"),
+    modelProvider: getString(attrs, "ai.model.provider"),
+
+    // Request info
+    prompt: getString(attrs, "ai.prompt"),
+    promptMessages: getString(attrs, "ai.prompt.messages"),
+    promptTools: getString(attrs, "ai.prompt.tools"),
+
+    // Response info
+    responseText: getString(attrs, "ai.response.text"),
+    responseObject: getString(attrs, "ai.response.object"),
+    responseToolCalls: getString(attrs, "ai.response.toolCalls"),
+    responseFinishReason: getString(attrs, "ai.response.finishReason"),
+    responseId: getString(attrs, "ai.response.id"),
+    responseModel: getString(attrs, "ai.response.model"),
+
+    // Usage - AI SDK attributes
+    promptTokens: getNumber(attrs, "ai.usage.promptTokens"),
+    completionTokens: getNumber(attrs, "ai.usage.completionTokens"),
+
+    // Timing (streaming)
+    msToFirstChunk: getNumber(attrs, "ai.response.msToFirstChunk"),
+    msToFinish: getNumber(attrs, "ai.response.msToFinish"),
+    avgCompletionTokensPerSecond: getNumber(attrs, "ai.response.avgCompletionTokensPerSecond"),
+
+    // Tool calls
+    toolCallName: getString(attrs, "ai.toolCall.name"),
+    toolCallId: getString(attrs, "ai.toolCall.id"),
+    toolCallArgs: getString(attrs, "ai.toolCall.args"),
+    toolCallResult: getString(attrs, "ai.toolCall.result"),
+
+    // GenAI semantic conventions (OpenTelemetry standard)
+    genAiSystem: getString(attrs, "gen_ai.system"),
+    genAiRequestModel: getString(attrs, "gen_ai.request.model"),
+    genAiResponseModel: getString(attrs, "gen_ai.response.model"),
+    genAiInputTokens: getNumber(attrs, "gen_ai.usage.input_tokens"),
+    genAiOutputTokens: getNumber(attrs, "gen_ai.usage.output_tokens"),
+
+    // Custom metadata
+    metadata: Object.keys(metadata).length > 0 ? metadata : undefined
+  };
+}
+
+/**
+ * Convert OTel high-resolution time [seconds, nanoseconds] to Date
+ */
+export function hrTimeToDate(hrTime: [number, number]): Date {
+  const [seconds, nanoseconds] = hrTime;
+  return new Date(seconds * 1000 + nanoseconds / 1_000_000);
+}
+
+/**
+ * Calculate duration in milliseconds from OTel times
+ */
+export function calculateDurationMs(
+  startTime: [number, number],
+  endTime: [number, number]
+): number {
+  const startMs = startTime[0] * 1000 + startTime[1] / 1_000_000;
+  const endMs = endTime[0] * 1000 + endTime[1] / 1_000_000;
+  return Math.round(endMs - startMs);
+}
+
+/**
+ * Extract provider from model ID or attributes
+ * Handles Vercel AI SDK format (e.g., "openai.responses" -> "openai")
+ * Examples:
+ *   - "openai" from ai.model.provider
+ *   - "openai" from "gpt-4o-mini"
+ *   - "anthropic" from "claude-3-5-sonnet"
+ */
+export function extractProvider(attrs: AISpanAttributes): string {
+  // Direct provider attribute
+  if (attrs.modelProvider) {
+    return normalizeProviderName(attrs.modelProvider);
+  }
+
+  // GenAI system
+  if (attrs.genAiSystem) {
+    return normalizeProviderName(attrs.genAiSystem);
+  }
+
+  // Infer from model ID
+  const modelId = attrs.modelId || attrs.genAiRequestModel || "";
+
+  if (modelId.startsWith("gpt-") || modelId.includes("openai")) {
+    return "openai";
+  }
+  if (modelId.startsWith("claude-") || modelId.includes("anthropic")) {
+    return "anthropic";
+  }
+  if (modelId.startsWith("gemini-") || modelId.includes("google")) {
+    return "google";
+  }
+  if (modelId.startsWith("deepseek-")) {
+    return "deepseek";
+  }
+  if (modelId.includes("llama") || modelId.includes("mistral")) {
+    return "meta";
+  }
+
+  return "unknown";
+}
+
+/**
+ * Normalize provider name by stripping operation suffixes
+ * Examples:
+ *   - "openai.responses" -> "openai"
+ *   - "anthropic.messages" -> "anthropic"
+ *   - "google.generativeai" -> "google"
+ *   - "openai" -> "openai" (unchanged)
+ */
+export function normalizeProviderName(provider: string): string {
+  if (provider.includes(".")) {
+    return provider.split(".")[0] || provider;
+  }
+  return provider;
+}
+
+/**
+ * Extract model name from attributes
+ * Handles various formats including:
+ *   - "gpt-4o-mini" (standard)
+ *   - "openai.responses/gpt-4o-mini" (Vercel AI SDK format)
+ */
+export function extractModel(attrs: AISpanAttributes): string {
+  const rawModel =
+    attrs.responseModel ||
+    attrs.modelId ||
+    attrs.genAiResponseModel ||
+    attrs.genAiRequestModel ||
+    "unknown";
+
+  return normalizeModelName(rawModel);
+}
+
+/**
+ * Normalize model name by stripping provider prefixes
+ * Examples:
+ *   - "openai.responses/gpt-4o-mini" -> "gpt-4o-mini"
+ *   - "anthropic.messages/claude-3-5-sonnet" -> "claude-3-5-sonnet"
+ *   - "gpt-4o-mini" -> "gpt-4o-mini" (unchanged)
+ */
+export function normalizeModelName(model: string): string {
+  if (model.includes("/")) {
+    const parts = model.split("/");
+    return parts[parts.length - 1] || model;
+  }
+  return model;
+}
+
+/**
+ * Map span status code to event type
+ */
+export function mapStatusToEventType(statusCode: number, operationType?: AIOperationType): string {
+  // OTel status codes: 0=UNSET, 1=OK, 2=ERROR
+  const isError = statusCode === 2;
+
+  if (operationType?.includes("toolCall")) {
+    return isError ? "tool.error" : "tool.end";
+  }
+
+  return isError ? "request.error" : "request.end";
+}
+
+/**
+ * Generate a deterministic request ID from trace/span IDs
+ */
+export function generateRequestId(span: OTelReadableSpan): string {
+  const ctx = span.spanContext();
+  // Use trace ID + span ID for uniqueness
+  return `otel_${ctx.traceId.slice(0, 8)}_${ctx.spanId.slice(0, 8)}`;
+}
+
+/**
+ * Extract session ID from span attributes
+ */
+export function extractSessionId(span: OTelReadableSpan): string | undefined {
+  const attrs = span.attributes;
+
+  // Check common session ID locations
+  return (
+    getString(attrs, "ai.telemetry.metadata.sessionId") ||
+    getString(attrs, "ai.telemetry.metadata.session_id") ||
+    getString(attrs, "session.id") ||
+    getString(attrs, "sessionId")
+  );
+}
+
+/**
+ * Parse JSON string attributes safely
+ */
+export function parseJsonAttribute<T>(value: string | undefined): T | undefined {
+  if (!value) return undefined;
+  try {
+    return JSON.parse(value) as T;
+  } catch {
+    return undefined;
+  }
+}