@livekit/agents 1.0.21 → 1.0.23

package/src/job.ts

@@ -14,6 +14,8 @@ import { AsyncLocalStorage } from 'node:async_hooks';
 import type { Logger } from 'pino';
 import type { InferenceExecutor } from './ipc/inference_executor.js';
 import { log } from './log.js';
+import { flushOtelLogs, setupCloudTracer, uploadSessionReport } from './telemetry/index.js';
+import { isCloud } from './utils.js';
 import type { AgentSession } from './voice/agent_session.js';
 import { type SessionReport, createSessionReport } from './voice/report.js';
 
@@ -81,8 +83,6 @@ export class FunctionExistsError extends Error {
 }
 
 /** The job and environment context as seen by the agent, accessible by the entrypoint function. */
-// TODO(brian): PR3 - Add @tracer.startActiveSpan('job_entrypoint') wrapper in entrypoint
-// TODO(brian): PR5 - Add uploadSessionReport() call in cleanup/session end
 export class JobContext {
   #proc: JobProcess;
   #info: RunningJobInfo;
@@ -142,6 +142,10 @@ export class JobContext {
     return this.#room;
   }
 
+  get info(): RunningJobInfo {
+    return this.#info;
+  }
+
   /** @returns The agent's participant if connected to the room, otherwise `undefined` */
   get agent(): LocalParticipant | undefined {
     return this.#room.localParticipant;
@@ -247,7 +251,6 @@ export class JobContext {
     }
 
     // TODO(brian): implement and check recorder io
-    // TODO(brian): PR5 - Ensure chat history serialization includes all required fields (use sessionReportToJSON helper)
 
     return createSessionReport({
       jobId: this.job.id,
@@ -257,6 +260,7 @@ export class JobContext {
       events: targetSession._recordedEvents,
       enableUserDataTraining: true,
       chatHistory: targetSession.history.copy(),
+      startedAt: targetSession._startedAt,
     });
   }
 
@@ -270,14 +274,45 @@ export class JobContext {
 
     // TODO(brian): Implement CLI/console
 
-    // TODO(brian): PR5 - Call uploadSessionReport() if report.enableUserDataTraining is true
-    // TODO(brian): PR5 - Upload includes: multipart form with header (protobuf), chat_history (JSON), and audio recording (if available)
+    // Upload session report to LiveKit Cloud if enabled
+    const url = new URL(this.#info.url);
 
-    this.#logger.debug('Session ended, report generated', {
-      jobId: report.jobId,
-      roomId: report.roomId,
-      eventsCount: report.events.length,
-    });
+    if (report.enableRecording && isCloud(url)) {
+      try {
+        await uploadSessionReport({
+          agentName: this.job.agentName,
+          cloudHostname: url.hostname,
+          report,
+        });
+        this.#logger.info(
+          {
+            jobId: report.jobId,
+            roomId: report.roomId,
+          },
+          'Session report uploaded to LiveKit Cloud',
+        );
+      } catch (error) {
+        this.#logger.error({ error }, 'Failed to upload session report');
+      }
+    }
+
+    this.#logger.debug(
+      {
+        jobId: report.jobId,
+        roomId: report.roomId,
+        eventsCount: report.events.length,
+      },
+      'Session ended, report generated',
+    );
+
+    // Explicitly clear the recorded events to avoid leaking memory
+    session._recordedEvents = [];
+
+    try {
+      await flushOtelLogs();
+    } catch (error) {
+      this.#logger.error({ error }, 'Failed to flush OTEL logs');
+    }
   }
 
   /**
@@ -316,6 +351,20 @@ export class JobContext {
 
     this.#participantEntrypoints.push(callback);
   }
+
+  async initRecording() {
+    const url = new URL(this.#info.url);
+    if (!isCloud(url)) {
+      return;
+    }
+
+    this.#logger.debug({ hostname: url.hostname }, 'Configuring session recording (cloud tracer)');
+    await setupCloudTracer({
+      roomId: this.job.room!.sid,
+      jobId: this.job.id,
+      cloudHostname: url.hostname,
+    });
+  }
 }
 
 export class JobProcess {

package/src/llm/llm.ts

@@ -2,10 +2,12 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { Span } from '@opentelemetry/api';
 import { EventEmitter } from 'node:events';
 import { APIConnectionError, APIError } from '../_exceptions.js';
 import { log } from '../log.js';
 import type { LLMMetrics } from '../metrics/base.js';
+import { recordException, traceTypes, tracer } from '../telemetry/index.js';
 import type { APIConnectOptions } from '../types.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
 import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
@@ -104,6 +106,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
   #llm: LLM;
   #chatCtx: ChatContext;
   #toolCtx?: ToolContext;
+  #llmRequestSpan?: Span;
 
   constructor(
     llm: LLM,
@@ -135,12 +138,24 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     startSoon(() => this.mainTask().then(() => this.queue.close()));
   }
 
-  private async mainTask() {
-    // TODO(brian): PR3 - Add span wrapping: tracer.startActiveSpan('llm_request', ..., { endOnExit: false })
+  private _mainTaskImpl = async (span: Span) => {
+    this.#llmRequestSpan = span;
+    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.#llm.model);
+
     for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
       try {
-        // TODO(brian): PR3 - Add span for retry attempts: tracer.startActiveSpan('llm_request_run', ...)
-        return await this.run();
+        return await tracer.startActiveSpan(
+          async (attemptSpan) => {
+            attemptSpan.setAttribute(traceTypes.ATTR_RETRY_COUNT, i);
+            try {
+              return await this.run();
+            } catch (error) {
+              recordException(attemptSpan, toError(error));
+              throw error;
+            }
+          },
+          { name: 'llm_request_run' },
+        );
       } catch (error) {
         if (error instanceof APIError) {
           const retryInterval = this._connOptions._intervalForRetry(i);
@@ -171,7 +186,13 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
         }
       }
     }
-  }
+  };
+
+  private mainTask = async () =>
+    tracer.startActiveSpan(async (span) => this._mainTaskImpl(span), {
+      name: 'llm_request',
+      endOnExit: false,
+    });
 
   private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {
     this.#llm.emit('error', {
@@ -188,6 +209,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
     let ttft: bigint = BigInt(-1);
     let requestId = '';
     let usage: CompletionUsage | undefined;
+    let completionStartTime: string | undefined;
 
     for await (const ev of this.queue) {
       if (this.abortController.signal.aborted) {
@@ -197,6 +219,7 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
       requestId = ev.id;
       if (ttft === BigInt(-1)) {
         ttft = process.hrtime.bigint() - startTime;
+        completionStartTime = new Date().toISOString();
       }
       if (ev.usage) {
         usage = ev.usage;
@@ -225,6 +248,26 @@ export abstract class LLMStream implements AsyncIterableIterator<ChatChunk> {
         return (usage?.completionTokens || 0) / (durationMs / 1000);
       })(),
     };
+
+    if (this.#llmRequestSpan) {
+      this.#llmRequestSpan.setAttribute(traceTypes.ATTR_LLM_METRICS, JSON.stringify(metrics));
+
+      this.#llmRequestSpan.setAttributes({
+        [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.promptTokens,
+        [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.completionTokens,
+      });
+
+      if (completionStartTime) {
+        this.#llmRequestSpan.setAttribute(
+          traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME,
+          completionStartTime,
+        );
+      }
+
+      // End the span now that metrics are collected
+      this.#llmRequestSpan.end();
+    }
+
     this.#llm.emit('metrics_collected', metrics);
   }
 

package/src/log.ts

@@ -1,8 +1,11 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type { Logger } from 'pino';
-import { pino } from 'pino';
+import { Writable } from 'node:stream';
+import type { DestinationStream, Logger } from 'pino';
+import { multistream, pino } from 'pino';
+import { build as pinoPretty } from 'pino-pretty';
+import { type PinoLogObject, emitToOtel } from './telemetry/pino_otel_transport.js';
 
 /** @internal */
 export type LoggerOptions = {
@@ -16,6 +19,9 @@ export let loggerOptions: LoggerOptions;
 /** @internal */
 let logger: Logger | undefined = undefined;
 
+/** @internal */
+let otelEnabled = false;
+
 /** @internal */
 export const log = () => {
   if (!logger) {
@@ -28,19 +34,50 @@ export const log = () => {
 export const initializeLogger = ({ pretty, level }: LoggerOptions) => {
   loggerOptions = { pretty, level };
   logger = pino(
-    pretty
-      ? {
-          transport: {
-            target: 'pino-pretty',
-            options: {
-              colorize: true,
-            },
-          },
-        }
-      : {},
+    { level: level || 'info' },
+    pretty ? pinoPretty({ colorize: true }) : process.stdout,
   );
-  if (level) {
-    logger.level = level;
+};
+
+/**
+ * Custom Pino destination that parses JSON logs and emits to OTEL.
+ * This receives the FULL serialized log including msg, level, time, etc.
+ */
+class OtelDestination extends Writable {
+  _write(chunk: Buffer, _encoding: string, callback: (error?: Error | null) => void): void {
+    try {
+      const line = chunk.toString().trim();
+      if (line) {
+        const logObj = JSON.parse(line) as PinoLogObject;
+        emitToOtel(logObj);
+      }
+    } catch {
+      // Ignore parse errors (e.g., non-JSON lines)
+    }
+    callback();
   }
-  // TODO(brian): PR4 - Add Pino bridge to OTEL LoggingHandler for structured logging integration
+}
+
+/**
+ * Enable OTEL logging by reconfiguring the logger with multistream.
+ * Uses a custom destination that receives full JSON logs (with msg, level, time).
+ *
+ * @internal
+ */
+export const enableOtelLogging = () => {
+  if (otelEnabled || !logger) {
+    console.warn('OTEL logging already enabled or logger not initialized');
+    return;
+  }
+  otelEnabled = true;
+
+  const { pretty, level } = loggerOptions;
+
+  const logLevel = level || 'info';
+  const streams: { stream: DestinationStream; level: string }[] = [
+    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },
+    { stream: new OtelDestination(), level: 'debug' },
+  ];
+
+  logger = pino({ level: logLevel }, multistream(streams));
 };

package/src/stt/stt.ts

@@ -257,7 +257,18 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
 
   protected async monitorMetrics() {
     for await (const event of this.queue) {
-      this.output.put(event);
+      if (!this.output.closed) {
+        try {
+          this.output.put(event);
+        } catch (e) {
+          if (e instanceof Error && e.message.includes('Queue is closed')) {
+            this.logger.warn(
+              { err: e },
+              'Queue closed during transcript processing (expected during disconnect)',
+            );
+          }
+        }
+      }
       if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;
       const metrics: STTMetrics = {
         type: 'stt_metrics',
@@ -270,7 +281,9 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
       };
       this.#stt.emit('metrics_collected', metrics);
     }
-    this.output.close();
+    if (!this.output.closed) {
+      this.output.close();
+    }
   }
 
   protected abstract run(): Promise<void>;
@@ -336,9 +349,9 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
 
   /** Close both the input and output of the STT stream */
   close() {
-    this.input.close();
-    this.queue.close();
-    this.output.close();
+    if (!this.input.closed) this.input.close();
+    if (!this.queue.closed) this.queue.close();
+    if (!this.output.closed) this.output.close();
     this.closed = true;
   }
 

package/src/telemetry/index.ts

@@ -2,9 +2,27 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-// TODO(brian): PR4 - Add logging integration exports
-// TODO(brian): PR5 - Add uploadSessionReport export
-
+export { ExtraDetailsProcessor, MetadataLogProcessor } from './logging.js';
+export {
+  SimpleOTLPHttpLogExporter,
+  type SimpleLogRecord,
+  type SimpleOTLPHttpLogExporterConfig,
+} from './otel_http_exporter.js';
+export {
+  emitToOtel,
+  flushPinoLogs,
+  initPinoCloudExporter,
+  PinoCloudExporter,
+  type PinoCloudExporterConfig,
+  type PinoLogObject,
+} from './pino_otel_transport.js';
 export * as traceTypes from './trace_types.js';
-export { setTracerProvider, setupCloudTracer, tracer, type StartSpanOptions } from './traces.js';
+export {
+  flushOtelLogs,
+  setTracerProvider,
+  setupCloudTracer,
+  tracer,
+  uploadSessionReport,
+  type StartSpanOptions,
+} from './traces.js';
 export { recordException, recordRealtimeMetrics } from './utils.js';

package/src/telemetry/logging.ts

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Attributes } from '@opentelemetry/api';
+import type { LogRecord, LogRecordProcessor } from '@opentelemetry/sdk-logs';
+
+/**
+ * Metadata log processor that injects metadata (room_id, job_id) into all log records.
+ */
+export class MetadataLogProcessor implements LogRecordProcessor {
+  private metadata: Attributes;
+
+  constructor(metadata: Attributes) {
+    this.metadata = metadata;
+  }
+
+  onEmit(logRecord: LogRecord): void {
+    // Add metadata to log record attributes
+    if (logRecord.attributes) {
+      Object.assign(logRecord.attributes, this.metadata);
+    } else {
+      (logRecord as any).attributes = { ...this.metadata };
+    }
+  }
+
+  shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+}
+
+/**
+ * Extra details processor that adds logger name to log records.
+ */
+export class ExtraDetailsProcessor implements LogRecordProcessor {
+  onEmit(logRecord: LogRecord): void {
+    const loggerName = logRecord.instrumentationScope.name;
+    if (logRecord.attributes) {
+      (logRecord.attributes as any)['logger.name'] = loggerName;
+    } else {
+      (logRecord as any).attributes = { 'logger.name': loggerName };
+    }
+  }
+
+  shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+}

package/src/telemetry/otel_http_exporter.ts

@@ -0,0 +1,191 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * OTLP HTTP JSON Log Exporter for LiveKit Cloud
+ *
+ * This module provides a custom OTLP log exporter that uses HTTP with JSON format
+ * instead of the default protobuf format.
+ */
+import { SeverityNumber } from '@opentelemetry/api-logs';
+import { AccessToken } from 'livekit-server-sdk';
+
+export interface SimpleLogRecord {
+  /** Log message body */
+  body: string;
+  /** Timestamp in milliseconds since epoch */
+  timestampMs: number;
+  /** Log attributes */
+  attributes: Record<string, unknown>;
+  /** Severity number (default: UNSPECIFIED) */
+  severityNumber?: SeverityNumber;
+  /** Severity text (default: 'unspecified') */
+  severityText?: string;
+}
+
+export interface SimpleOTLPHttpLogExporterConfig {
+  /** LiveKit Cloud hostname */
+  cloudHostname: string;
+  /** Resource attributes (e.g., room_id, job_id) */
+  resourceAttributes: Record<string, string>;
+  /** Scope name for the logger */
+  scopeName: string;
+  /** Scope attributes */
+  scopeAttributes?: Record<string, string>;
+}
+
+/**
+ * Simple OTLP HTTP Log Exporter for direct log export
+ *
+ * This is a simplified exporter that doesn't require the full SDK infrastructure.
+ * Use this when you need to send logs directly without LoggerProvider.
+ *
+ * @example
+ * ```typescript
+ * const exporter = new SimpleOTLPHttpLogExporter({
+ *   cloudHostname: 'cloud.livekit.io',
+ *   resourceAttributes: { room_id: 'xxx', job_id: 'yyy' },
+ *   scopeName: 'chat_history',
+ * });
+ *
+ * await exporter.export([
+ *   { body: 'Hello', timestampMs: Date.now(), attributes: { test: true } },
+ * ]);
+ * ```
+ */
+export class SimpleOTLPHttpLogExporter {
+  private readonly config: SimpleOTLPHttpLogExporterConfig;
+  private jwt: string | null = null;
+
+  constructor(config: SimpleOTLPHttpLogExporterConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Export simple log records
+   */
+  async export(records: SimpleLogRecord[]): Promise<void> {
+    if (records.length === 0) return;
+
+    await this.ensureJwt();
+
+    const endpoint = `https://${this.config.cloudHostname}/observability/logs/otlp/v0`;
+    const payload = this.buildPayload(records);
+
+    const response = await fetch(endpoint, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.jwt}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `OTLP log export failed: ${response.status} ${response.statusText} - ${text}`,
+      );
+    }
+  }
+
+  private async ensureJwt(): Promise<void> {
+    if (this.jwt) return;
+
+    const apiKey = process.env.LIVEKIT_API_KEY;
+    const apiSecret = process.env.LIVEKIT_API_SECRET;
+
+    if (!apiKey || !apiSecret) {
+      throw new Error('LIVEKIT_API_KEY and LIVEKIT_API_SECRET must be set');
+    }
+
+    const token = new AccessToken(apiKey, apiSecret, { ttl: '6h' });
+    token.addObservabilityGrant({ write: true });
+    this.jwt = await token.toJwt();
+  }
+
+  private buildPayload(records: SimpleLogRecord[]): object {
+    const resourceAttrs = Object.entries(this.config.resourceAttributes).map(([key, value]) => ({
+      key,
+      value: { stringValue: value },
+    }));
+
+    if (!this.config.resourceAttributes['service.name']) {
+      resourceAttrs.push({ key: 'service.name', value: { stringValue: 'livekit-agents' } });
+    }
+
+    const scopeAttrs = this.config.scopeAttributes
+      ? Object.entries(this.config.scopeAttributes).map(([key, value]) => ({
+          key,
+          value: { stringValue: value },
+        }))
+      : [];
+
+    const logRecords = records.map((record) => ({
+      timeUnixNano: String(BigInt(Math.floor(record.timestampMs * 1_000_000))),
+      observedTimeUnixNano: String(BigInt(Date.now()) * BigInt(1_000_000)),
+      severityNumber: record.severityNumber ?? SeverityNumber.UNSPECIFIED,
+      severityText: record.severityText ?? 'unspecified',
+      body: { stringValue: record.body },
+      attributes: this.convertAttributes(record.attributes),
+      traceId: '',
+      spanId: '',
+    }));
+
+    return {
+      resourceLogs: [
+        {
+          resource: { attributes: resourceAttrs },
+          scopeLogs: [
+            {
+              scope: {
+                name: this.config.scopeName,
+                attributes: scopeAttrs,
+              },
+              logRecords,
+            },
+          ],
+        },
+      ],
+    };
+  }
+
+  private convertAttributes(
+    attrs: Record<string, unknown>,
+  ): Array<{ key: string; value: unknown }> {
+    return Object.entries(attrs).map(([key, value]) => ({
+      key,
+      value: this.convertValue(value),
+    }));
+  }
+
+  private convertValue(value: unknown): unknown {
+    if (value === null || value === undefined) {
+      return { stringValue: '' };
+    }
+    if (typeof value === 'string') {
+      return { stringValue: value };
+    }
+    if (typeof value === 'number') {
+      return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
+    }
+    if (typeof value === 'boolean') {
+      return { boolValue: value };
+    }
+    if (Array.isArray(value)) {
+      return { arrayValue: { values: value.map((v) => this.convertValue(v)) } };
+    }
+    if (typeof value === 'object') {
+      return {
+        kvlistValue: {
+          values: Object.entries(value as Record<string, unknown>).map(([k, v]) => ({
+            key: k,
+            value: this.convertValue(v),
+          })),
+        },
+      };
+    }
+    return { stringValue: String(value) };
+  }
+}