@livekit/agents 1.0.21 → 1.0.23

package/src/telemetry/pino_otel_transport.ts

@@ -0,0 +1,265 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Custom Pino OTEL Transport
+ *
+ * Standalone exporter for Pino logs to LiveKit Cloud.
+ * Uses raw HTTP JSON format, bypassing the OTEL SDK.
+ */
+import { SeverityNumber } from '@opentelemetry/api-logs';
+import { AccessToken } from 'livekit-server-sdk';
+
+export interface PinoLogObject {
+  level: number;
+  time: number;
+  msg: string;
+  pid?: number;
+  hostname?: string;
+  [key: string]: unknown;
+}
+
+export interface PinoCloudExporterConfig {
+  cloudHostname: string;
+  roomId: string;
+  jobId: string;
+  loggerName?: string;
+  batchSize?: number;
+  flushIntervalMs?: number;
+}
+
+function mapPinoLevelToSeverity(pinoLevel: number): {
+  severityNumber: SeverityNumber;
+  severityText: string;
+} {
+  if (pinoLevel <= 10) {
+    return { severityNumber: SeverityNumber.TRACE, severityText: 'TRACE' };
+  } else if (pinoLevel <= 20) {
+    return { severityNumber: SeverityNumber.DEBUG, severityText: 'DEBUG' };
+  } else if (pinoLevel <= 30) {
+    return { severityNumber: SeverityNumber.INFO, severityText: 'INFO' };
+  } else if (pinoLevel <= 40) {
+    return { severityNumber: SeverityNumber.WARN, severityText: 'WARN' };
+  } else if (pinoLevel <= 50) {
+    return { severityNumber: SeverityNumber.ERROR, severityText: 'ERROR' };
+  } else {
+    return { severityNumber: SeverityNumber.FATAL, severityText: 'FATAL' };
+  }
+}
+
+const EXCLUDE_FIELDS = new Set(['level', 'time', 'msg', 'pid', 'hostname', 'v']);
+
+function convertValue(value: unknown): unknown {
+  if (value === null || value === undefined) {
+    return { stringValue: '' };
+  }
+  if (typeof value === 'string') {
+    return { stringValue: value };
+  }
+  if (typeof value === 'number') {
+    return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
+  }
+  if (typeof value === 'boolean') {
+    return { boolValue: value };
+  }
+  if (typeof value === 'object') {
+    return { stringValue: JSON.stringify(value) };
+  }
+  return { stringValue: String(value) };
+}
+
+/**
+ * Standalone Pino log exporter for LiveKit Cloud.
+ *
+ * Collects Pino logs, batches them, and sends via raw HTTP JSON.
+ * No OTEL SDK dependency
+ *
+ * @example
+ * ```typescript
+ * const exporter = new PinoCloudExporter({
+ *   cloudHostname: 'cloud.livekit.io',
+ *   roomId: 'RM_xxx',
+ *   jobId: 'AJ_xxx',
+ * });
+ *
+ * // In Pino formatter hook:
+ * exporter.emit(logObj);
+ *
+ * // On session end:
+ * await exporter.flush();
+ * ```
+ */
+export class PinoCloudExporter {
+  private readonly config: PinoCloudExporterConfig;
+  private readonly loggerName: string;
+  private readonly batchSize: number;
+  private readonly flushIntervalMs: number;
+  private jwt: string | null = null;
+  private pendingLogs: any[] = [];
+  private flushTimer: NodeJS.Timeout | null = null;
+
+  constructor(config: PinoCloudExporterConfig) {
+    this.config = config;
+    this.loggerName = config.loggerName || 'livekit.agents';
+    this.batchSize = config.batchSize || 100;
+    this.flushIntervalMs = config.flushIntervalMs || 5000;
+  }
+
+  emit(logObj: PinoLogObject): void {
+    const record = this.convertToOtlpRecord(logObj);
+    this.pendingLogs.push(record);
+
+    if (!this.flushTimer) {
+      this.flushTimer = setTimeout(() => {
+        this.flush().catch(console.error);
+      }, this.flushIntervalMs);
+    }
+
+    if (this.pendingLogs.length >= this.batchSize) {
+      this.flush().catch(console.error);
+    }
+  }
+
+  private convertToOtlpRecord(logObj: PinoLogObject): any {
+    const { severityNumber, severityText } = mapPinoLevelToSeverity(logObj.level);
+
+    const attributes: any[] = [
+      { key: 'room_id', value: { stringValue: this.config.roomId } },
+      { key: 'job_id', value: { stringValue: this.config.jobId } },
+      { key: 'logger.name', value: { stringValue: this.loggerName } },
+    ];
+
+    if (logObj.pid !== undefined) {
+      attributes.push({ key: 'process.pid', value: { intValue: String(logObj.pid) } });
+    }
+    if (logObj.hostname !== undefined) {
+      attributes.push({ key: 'host.name', value: { stringValue: logObj.hostname } });
+    }
+
+    for (const [key, value] of Object.entries(logObj)) {
+      if (!EXCLUDE_FIELDS.has(key)) {
+        attributes.push({ key, value: convertValue(value) });
+      }
+    }
+
+    return {
+      timeUnixNano: String(BigInt(logObj.time) * BigInt(1_000_000)),
+      observedTimeUnixNano: String(BigInt(Date.now()) * BigInt(1_000_000)),
+      severityNumber,
+      severityText,
+      body: { stringValue: logObj.msg || '' },
+      attributes,
+      traceId: '',
+      spanId: '',
+    };
+  }
+
+  async flush(): Promise<void> {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    if (this.pendingLogs.length === 0) {
+      return;
+    }
+
+    const logs = this.pendingLogs;
+    this.pendingLogs = [];
+
+    try {
+      await this.sendLogs(logs);
+    } catch (error) {
+      this.pendingLogs = [...logs, ...this.pendingLogs];
+      console.error('[PinoCloudExporter] Failed to flush logs:', error);
+    }
+  }
+
+  private async sendLogs(logRecords: any[]): Promise<void> {
+    await this.ensureJwt();
+
+    const payload = {
+      resourceLogs: [
+        {
+          resource: {
+            attributes: [
+              { key: 'service.name', value: { stringValue: 'livekit-agents' } },
+              { key: 'room_id', value: { stringValue: this.config.roomId } },
+              { key: 'job_id', value: { stringValue: this.config.jobId } },
+            ],
+          },
+          scopeLogs: [
+            {
+              scope: {
+                name: this.loggerName,
+                attributes: [
+                  { key: 'room_id', value: { stringValue: this.config.roomId } },
+                  { key: 'job_id', value: { stringValue: this.config.jobId } },
+                ],
+              },
+              logRecords,
+            },
+          ],
+        },
+      ],
+    };
+
+    const endpoint = `https://${this.config.cloudHostname}/observability/logs/otlp/v0`;
+
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.jwt}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(`Log export failed: ${response.status} ${response.statusText} - ${text}`);
+    }
+  }
+
+  private async ensureJwt(): Promise<void> {
+    if (this.jwt) return;
+
+    const apiKey = process.env.LIVEKIT_API_KEY;
+    const apiSecret = process.env.LIVEKIT_API_SECRET;
+
+    if (!apiKey || !apiSecret) {
+      throw new Error('LIVEKIT_API_KEY and LIVEKIT_API_SECRET must be set');
+    }
+
+    const token = new AccessToken(apiKey, apiSecret, { ttl: '6h' });
+    token.addObservabilityGrant({ write: true });
+    this.jwt = await token.toJwt();
+  }
+
+  async shutdown(): Promise<void> {
+    await this.flush();
+  }
+}
+
+let globalExporter: PinoCloudExporter | null = null;
+
+export function initPinoCloudExporter(config: PinoCloudExporterConfig): void {
+  globalExporter = new PinoCloudExporter(config);
+}
+
+export function getPinoCloudExporter(): PinoCloudExporter | null {
+  return globalExporter;
+}
+
+export function emitToOtel(logObj: PinoLogObject): void {
+  if (globalExporter) {
+    globalExporter.emit(logObj);
+  }
+}
+
+export async function flushPinoLogs(): Promise<void> {
+  if (globalExporter) {
+    await globalExporter.flush();
+  }
+}

package/src/telemetry/traces.ts

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { MetricsRecordingHeader } from '@livekit/protocol';
 import {
   type Attributes,
   type Context,
@@ -11,13 +12,20 @@ import {
   context as otelContext,
   trace,
 } from '@opentelemetry/api';
-import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { SeverityNumber } from '@opentelemetry/api-logs';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
 import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
 import { Resource } from '@opentelemetry/resources';
 import type { ReadableSpan, SpanProcessor } from '@opentelemetry/sdk-trace-base';
 import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
+import FormData from 'form-data';
 import { AccessToken } from 'livekit-server-sdk';
+import type { ChatContent, ChatItem } from '../llm/index.js';
+import { enableOtelLogging } from '../log.js';
+import type { SessionReport } from '../voice/report.js';
+import { type SimpleLogRecord, SimpleOTLPHttpLogExporter } from './otel_http_exporter.js';
+import { flushPinoLogs, initPinoCloudExporter } from './pino_otel_transport.js';
 
 export interface StartSpanOptions {
   /** Name of the span */
@@ -94,19 +102,15 @@ class DynamicTracer {
     const endOnExit = options.endOnExit === undefined ? true : options.endOnExit; // default true
     const opts: SpanOptions = { attributes: options.attributes };
 
-    return new Promise((resolve, reject) => {
-      this.tracer.startActiveSpan(options.name, opts, ctx, async (span) => {
-        try {
-          const result = await fn(span);
-          resolve(result);
-        } catch (error) {
-          reject(error);
-        } finally {
-          if (endOnExit) {
-            span.end();
-          }
+    // Directly return the tracer's startActiveSpan result - it handles async correctly
+    return await this.tracer.startActiveSpan(options.name, opts, ctx, async (span) => {
+      try {
+        return await fn(span);
+      } finally {
+        if (endOnExit) {
+          span.end();
         }
-      });
+      }
     });
   }
 
@@ -162,10 +166,6 @@ class MetadataSpanProcessor implements SpanProcessor {
   }
 }
 
-// TODO(brian): PR4 - Add MetadataLogProcessor for structured logging
-
-// TODO(brian): PR4 - Add ExtraDetailsProcessor for structured logging
-
 /**
  * Set the tracer provider for the livekit-agents framework.
  * This should be called before agent session start if using custom tracer providers.
@@ -254,13 +254,313 @@ export async function setupCloudTracer(options: {
     });
     tracerProvider.register();
 
-    // Metadata processor is already configured in the constructor above
     setTracerProvider(tracerProvider);
 
-    // TODO(brian): PR4 - Add logger provider setup here for structured logging
-    // Similar to Python's setup: LoggerProvider, OTLPLogExporter, BatchLogRecordProcessor
+    // Initialize standalone Pino cloud exporter (no OTEL SDK dependency)
+    initPinoCloudExporter({
+      cloudHostname,
+      roomId,
+      jobId,
+    });
+
+    enableOtelLogging();
   } catch (error) {
     console.error('Failed to setup cloud tracer:', error);
     throw error;
   }
 }
+
+/**
+ * Flush all pending Pino logs to ensure they are exported.
+ * Call this before session/job ends to ensure all logs are sent.
+ *
+ * @internal
+ */
+export async function flushOtelLogs(): Promise<void> {
+  await flushPinoLogs();
+}
+
+/**
+ * Convert ChatItem to proto-compatible dictionary format.
+ * TODO: Use actual agent_session proto types once @livekit/protocol v1.43.1+ is published
+ */
+function chatItemToProto(item: ChatItem): Record<string, any> {
+  const itemDict: Record<string, any> = {};
+
+  if (item.type === 'message') {
+    const roleMap: Record<string, string> = {
+      developer: 'DEVELOPER',
+      system: 'SYSTEM',
+      user: 'USER',
+      assistant: 'ASSISTANT',
+    };
+
+    const msg: Record<string, any> = {
+      id: item.id,
+      role: roleMap[item.role] || item.role.toUpperCase(),
+      content: item.content.map((c: ChatContent) => ({ text: c })),
+      createdAt: toRFC3339(item.createdAt),
+    };
+
+    if (item.interrupted) {
+      msg.interrupted = item.interrupted;
+    }
+
+    // TODO(brian): Add extra and transcriptConfidence to ChatMessage
+    // if (item.extra && Object.keys(item.extra).length > 0) {
+    //   msg.extra = item.extra;
+    // }
+
+    // if (item.transcriptConfidence !== undefined && item.transcriptConfidence !== null) {
+    //   msg.transcriptConfidence = item.transcriptConfidence;
+    // }
+
+    // TODO(brian): Add metrics to ChatMessage
+    // const metrics = item.metrics || {};
+    // if (Object.keys(metrics).length > 0) {
+    //   msg.metrics = {};
+    //   if (metrics.started_speaking_at) {
+    //     msg.metrics.startedSpeakingAt = toRFC3339(metrics.started_speaking_at);
+    //   }
+    //   if (metrics.stopped_speaking_at) {
+    //     msg.metrics.stoppedSpeakingAt = toRFC3339(metrics.stopped_speaking_at);
+    //   }
+    //   if (metrics.transcription_delay !== undefined) {
+    //     msg.metrics.transcriptionDelay = metrics.transcription_delay;
+    //   }
+    //   if (metrics.end_of_turn_delay !== undefined) {
+    //     msg.metrics.endOfTurnDelay = metrics.end_of_turn_delay;
+    //   }
+    //   if (metrics.on_user_turn_completed_delay !== undefined) {
+    //     msg.metrics.onUserTurnCompletedDelay = metrics.on_user_turn_completed_delay;
+    //   }
+    //   if (metrics.llm_node_ttft !== undefined) {
+    //     msg.metrics.llmNodeTtft = metrics.llm_node_ttft;
+    //   }
+    //   if (metrics.tts_node_ttfb !== undefined) {
+    //     msg.metrics.ttsNodeTtfb = metrics.tts_node_ttfb;
+    //   }
+    //   if (metrics.e2e_latency !== undefined) {
+    //     msg.metrics.e2eLatency = metrics.e2e_latency;
+    //   }
+    // }
+
+    itemDict.message = msg;
+  } else if (item.type === 'function_call') {
+    itemDict.functionCall = {
+      id: item.id,
+      callId: item.callId,
+      arguments: item.args,
+      name: item.name,
+      createdAt: toRFC3339(item.createdAt),
+    };
+  } else if (item.type === 'function_call_output') {
+    itemDict.functionCallOutput = {
+      id: item.id,
+      name: item.name,
+      callId: item.callId,
+      output: item.output,
+      isError: item.isError,
+      createdAt: toRFC3339(item.createdAt),
+    };
+  } else if (item.type === 'agent_handoff') {
+    const handoff: Record<string, any> = {
+      id: item.id,
+      newAgentId: item.newAgentId,
+      createdAt: toRFC3339(item.createdAt),
+    };
+    if (item.oldAgentId !== undefined && item.oldAgentId !== null && item.oldAgentId !== '') {
+      handoff.oldAgentId = item.oldAgentId;
+    }
+    itemDict.agentHandoff = handoff;
+  }
+
+  try {
+    if (item.type === 'function_call' && typeof itemDict.functionCall?.arguments === 'string') {
+      itemDict.functionCall.arguments = JSON.parse(itemDict.functionCall.arguments);
+    } else if (
+      item.type === 'function_call_output' &&
+      typeof itemDict.functionCallOutput?.output === 'string'
+    ) {
+      itemDict.functionCallOutput.output = JSON.parse(itemDict.functionCallOutput.output);
+    }
+  } catch {
+    // ignore parsing errors
+  }
+
+  return itemDict;
+}
+
+/**
+ * Convert timestamp to RFC3339 format matching Python's _to_rfc3339.
+ * Note: TypeScript createdAt is in milliseconds (Date.now()), not seconds like Python.
+ * @internal
+ */
+function toRFC3339(valueMs: number | Date): string {
+  // valueMs is already in milliseconds (from Date.now())
+  const dt = valueMs instanceof Date ? valueMs : new Date(valueMs);
+  // Truncate sub-millisecond precision
+  const truncated = new Date(Math.floor(dt.getTime()));
+  return truncated.toISOString();
+}
+
+/**
+ * Upload session report to LiveKit Cloud observability.
+ * @param options - Configuration with agentName, cloudHostname, and report
+ */
+export async function uploadSessionReport(options: {
+  agentName: string;
+  cloudHostname: string;
+  report: SessionReport;
+}): Promise<void> {
+  const { agentName, cloudHostname, report } = options;
+
+  // Create OTLP HTTP exporter for chat history logs
+  // Uses raw HTTP JSON format which is required by LiveKit Cloud
+  const logExporter = new SimpleOTLPHttpLogExporter({
+    cloudHostname,
+    resourceAttributes: {
+      room_id: report.roomId,
+      job_id: report.jobId,
+    },
+    scopeName: 'chat_history',
+    scopeAttributes: {
+      room_id: report.roomId,
+      job_id: report.jobId,
+      room: report.room,
+    },
+  });
+
+  // Build log records for session report and chat items
+  const logRecords: SimpleLogRecord[] = [];
+
+  const commonAttrs = {
+    room_id: report.roomId,
+    job_id: report.jobId,
+    'logger.name': 'chat_history',
+  };
+
+  logRecords.push({
+    body: 'session report',
+    timestampMs: report.startedAt || report.timestamp || 0,
+    attributes: {
+      ...commonAttrs,
+      'session.options': report.options || {},
+      'session.report_timestamp': report.timestamp,
+      agent_name: agentName,
+    },
+  });
+
+  // Track last timestamp to ensure monotonic ordering when items have identical timestamps
+  // This fixes the issue where function_call and function_call_output with same timestamp
+  // get reordered by the dashboard
+  let lastTimestamp = 0;
+  for (const item of report.chatHistory.items) {
+    // Ensure monotonically increasing timestamps for proper ordering
+    // Add 0.001ms (1 microsecond) offset when timestamps collide
+    let itemTimestamp = item.createdAt;
+    if (itemTimestamp <= lastTimestamp) {
+      itemTimestamp = lastTimestamp + 0.001; // Add 1 microsecond
+    }
+    lastTimestamp = itemTimestamp;
+
+    const itemProto = chatItemToProto(item);
+    let severityNumber = SeverityNumber.UNSPECIFIED;
+    let severityText = 'unspecified';
+
+    if (item.type === 'function_call_output' && item.isError) {
+      severityNumber = SeverityNumber.ERROR;
+      severityText = 'error';
+    }
+
+    logRecords.push({
+      body: 'chat item',
+      timestampMs: itemTimestamp, // Adjusted for monotonic ordering
+      attributes: { 'chat.item': itemProto, ...commonAttrs },
+      severityNumber,
+      severityText,
+    });
+  }
+  await logExporter.export(logRecords);
+
+  const apiKey = process.env.LIVEKIT_API_KEY;
+  const apiSecret = process.env.LIVEKIT_API_SECRET;
+
+  if (!apiKey || !apiSecret) {
+    throw new Error('LIVEKIT_API_KEY and LIVEKIT_API_SECRET must be set for session upload');
+  }
+
+  const token = new AccessToken(apiKey, apiSecret, { ttl: '6h' });
+  token.addObservabilityGrant({ write: true });
+  const jwt = await token.toJwt();
+
+  const formData = new FormData();
+
+  // Add header (protobuf MetricsRecordingHeader)
+  const headerMsg = new MetricsRecordingHeader({
+    roomId: report.roomId,
+    duration: BigInt(0), // TODO: Calculate actual duration from report
+    startTime: {
+      seconds: BigInt(Math.floor(report.timestamp / 1000)),
+      nanos: Math.floor((report.timestamp % 1000) * 1e6),
+    },
+  });
+
+  const headerBytes = Buffer.from(headerMsg.toBinary());
+  formData.append('header', headerBytes, {
+    filename: 'header.binpb',
+    contentType: 'application/protobuf',
+    knownLength: headerBytes.length,
+    header: {
+      'Content-Type': 'application/protobuf',
+      'Content-Length': headerBytes.length.toString(),
+    },
+  });
+
+  // Add chat_history JSON
+  const chatHistoryJson = JSON.stringify(report.chatHistory.toJSON({ excludeTimestamp: false }));
+  const chatHistoryBuffer = Buffer.from(chatHistoryJson, 'utf-8');
+  formData.append('chat_history', chatHistoryBuffer, {
+    filename: 'chat_history.json',
+    contentType: 'application/json',
+    knownLength: chatHistoryBuffer.length,
+    header: {
+      'Content-Type': 'application/json',
+      'Content-Length': chatHistoryBuffer.length.toString(),
+    },
+  });
+
+  // TODO(brian): Add audio recording file when recorder IO is implemented
+
+  // Upload to LiveKit Cloud using form-data's submit method
+  // This properly streams the multipart form with all headers including Content-Length
+  return new Promise<void>((resolve, reject) => {
+    formData.submit(
+      {
+        protocol: 'https:',
+        host: cloudHostname,
+        path: '/observability/recordings/v0',
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${jwt}`,
+        },
+      },
+      (err, res) => {
+        if (err) {
+          reject(new Error(`Failed to upload session report: ${err.message}`));
+          return;
+        }
+
+        if (res.statusCode && res.statusCode >= 400) {
+          reject(
+            new Error(`Failed to upload session report: ${res.statusCode} ${res.statusMessage}`),
+          );
+          return;
+        }
+
+        res.resume(); // Drain the response
+        res.on('end', () => resolve());
+      },
+    );
+  });
+}