@livekit/agents 1.1.0 → 1.2.0

package/src/stream/stream_channel.test.ts

@@ -126,4 +126,41 @@ describe('StreamChannel', () => {
     const nextResult = await reader.read();
     expect(nextResult.done).toBe(true);
   });
+
+  it('should gracefully handle close while read is pending', async () => {
+    const channel = createStreamChannel<string>();
+    const reader = channel.stream().getReader();
+
+    const readPromise = reader.read();
+
+    await channel.close();
+
+    const result = await readPromise;
+    expect(result.done).toBe(true);
+    expect(result.value).toBeUndefined();
+  });
+
+  it('should complete all pending reads when closed', async () => {
+    const channel = createStreamChannel<number>();
+    const reader = channel.stream().getReader();
+
+    const read1 = reader.read();
+    const read2 = reader.read();
+    const read3 = reader.read();
+
+    await channel.write(42);
+    await channel.write(43);
+    await channel.close();
+
+    const result1 = await read1;
+    expect(result1.done).toBe(false);
+    expect(result1.value).toBe(42);
+
+    const result2 = await read2;
+    expect(result2.done).toBe(false);
+    expect(result2.value).toBe(43);
+
+    const result3 = await read3;
+    expect(result3.done).toBe(true);
+  });
 });

package/src/stream/stream_channel.ts

@@ -4,29 +4,69 @@
 import type { ReadableStream } from 'node:stream/web';
 import { IdentityTransform } from './identity_transform.js';
 
-export interface StreamChannel<T> {
+export interface StreamChannel<T, E extends Error = Error> {
   write(chunk: T): Promise<void>;
   close(): Promise<void>;
   stream(): ReadableStream<T>;
+  abort(error: E): Promise<void>;
+  readonly closed: boolean;
+  addStreamInput(stream: ReadableStream<T>): void;
 }
 
-export function createStreamChannel<T>(): StreamChannel<T> {
+export function createStreamChannel<T, E extends Error = Error>(): StreamChannel<T, E> {
   const transform = new IdentityTransform<T>();
   const writer = transform.writable.getWriter();
+  let isClosed = false;
 
   return {
     write: (chunk: T) => writer.write(chunk),
     stream: () => transform.readable,
+    abort: async (error: E) => {
+      if (isClosed) return;
+      isClosed = true;
+      try {
+        await writer.abort(error);
+      } catch (e) {
+        if (e instanceof Error && e.name === 'TypeError') return;
+        throw e;
+      }
+    },
+    addStreamInput: (newInputStream) => {
+      if (isClosed) return;
+      const reader = newInputStream.getReader();
+      (async () => {
+        try {
+          while (!isClosed) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            await writer.write(value);
+          }
+        } catch (err) {
+          if (!isClosed) {
+            isClosed = true;
+            await writer.abort(err as E);
+          }
+        } finally {
+          reader.releaseLock();
+        }
+      })().catch(() => {});
+    },
     close: async () => {
       try {
-        return await writer.close();
+        const result = await writer.close();
+        isClosed = true;
+        return result;
       } catch (e) {
         if (e instanceof Error && e.name === 'TypeError') {
           // Ignore error if the stream is already closed
+          isClosed = true;
           return;
         }
         throw e;
       }
     },
+    get closed() {
+      return isClosed;
+    },
   };
 }

package/src/stt/stream_adapter.ts

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
+import type { APIConnectOptions } from '../types.js';
+import { isStreamClosedError } from '../utils.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -22,14 +24,18 @@ export class StreamAdapter extends STT {
     this.#stt.on('metrics_collected', (metrics) => {
       this.emit('metrics_collected', metrics);
     });
+
+    this.#stt.on('error', (error) => {
+      this.emit('error', error);
+    });
   }
 
-  _recognize(frame: AudioFrame): Promise<SpeechEvent> {
-    return this.#stt.recognize(frame);
+  _recognize(frame: AudioFrame, abortSignal?: AbortSignal): Promise<SpeechEvent> {
+    return this.#stt.recognize(frame, abortSignal);
   }
 
-  stream(): StreamAdapterWrapper {
-    return new StreamAdapterWrapper(this.#stt, this.#vad);
+  stream(options?: { connOptions?: APIConnectOptions }): StreamAdapterWrapper {
+    return new StreamAdapterWrapper(this.#stt, this.#vad, options?.connOptions);
   }
 }
 
@@ -38,13 +44,18 @@ export class StreamAdapterWrapper extends SpeechStream {
   #vadStream: VADStream;
   label: string;
 
-  constructor(stt: STT, vad: VAD) {
-    super(stt);
+  constructor(stt: STT, vad: VAD, connOptions?: APIConnectOptions) {
+    super(stt, undefined, connOptions);
     this.#stt = stt;
     this.#vadStream = vad.stream();
     this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
   }
 
+  close() {
+    super.close();
+    this.#vadStream.close();
+  }
+
   async monitorMetrics() {
     return; // do nothing
   }
@@ -58,7 +69,17 @@ export class StreamAdapterWrapper extends SpeechStream {
           this.#vadStream.pushFrame(input);
         }
       }
-      this.#vadStream.endInput();
+
+      // Guard against calling endInput() on already-closed stream
+      // This happens during handover when close() is called while forwardInput is running
+      try {
+        this.#vadStream.endInput();
+      } catch (e) {
+        if (isStreamClosedError(e)) {
+          return;
+        }
+        throw e;
+      }
     };
 
     const recognize = async () => {
@@ -71,7 +92,7 @@ export class StreamAdapterWrapper extends SpeechStream {
             this.output.put({ type: SpeechEventType.END_OF_SPEECH });
 
             try {
-              const event = await this.#stt.recognize(ev.frames);
+              const event = await this.#stt.recognize(ev.frames, this.abortSignal);
               if (!event.alternatives![0].text) {
                 continue;
               }
@@ -92,6 +113,6 @@ export class StreamAdapterWrapper extends SpeechStream {
       }
     };
 
-    Promise.all([forwardInput(), recognize()]);
+    await Promise.all([forwardInput(), recognize()]);
   }
 }

package/src/stt/stt.ts

@@ -6,13 +6,15 @@ import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import { APIConnectionError, APIError } from '../_exceptions.js';
-import { calculateAudioDuration } from '../audio.js';
+import { calculateAudioDurationSeconds } from '../audio.js';
+import type { LanguageCode } from '../language.js';
 import { log } from '../log.js';
 import type { STTMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
-import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
+import type { TimedString } from '../voice/index.js';
 
 /** Indicates start/middle/end of speech */
 export enum SpeechEventType {
@@ -38,19 +40,37 @@ export enum SpeechEventType {
   END_OF_SPEECH = 3,
   /** Usage event, emitted periodically to indicate usage metrics. */
   RECOGNITION_USAGE = 4,
+  /**
+   * Preflight transcript, emitted before final transcript when STT has high confidence
+   * but hasn't fully committed yet. Includes all pre-committed transcripts including
+   * final transcript from the previous STT run.
+   */
+  PREFLIGHT_TRANSCRIPT = 5,
 }
 
 /** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
-  language: string;
+  /** Language code of the speech. */
+  language: LanguageCode;
+  /** Transcribed text. */
   text: string;
+  /** Start time of the speech segment in seconds. */
   startTime: number;
+  /** End time of the speech segment in seconds. */
   endTime: number;
+  /** Confidence score of the transcription (0-1). */
   confidence: number;
+  /** Word-level timing information. */
+  words?: TimedString[];
 }
 
 export interface RecognitionUsage {
+  /** Duration of the audio that was recognized in seconds. */
   audioDuration: number;
+  /** Input audio tokens (for token-based STT billing). */
+  inputTokens?: number;
+  /** Output text tokens (for token-based STT billing). */
+  outputTokens?: number;
 }
 
 /** SpeechEvent is a packet of speech-to-text data. */
@@ -70,6 +90,13 @@ export interface SpeechEvent {
 export interface STTCapabilities {
   streaming: boolean;
   interimResults: boolean;
+  /**
+   * Whether this STT supports aligned transcripts with word/chunk timestamps.
+   * - 'word': Provider returns word-level timestamps
+   * - 'chunk': Provider returns chunk-level timestamps (e.g., sentence/phrase boundaries)
+   * - false: Provider does not support aligned transcripts
+   */
+  alignedTranscript?: 'word' | 'chunk' | false;
 }
 
 export interface STTError {
@@ -106,29 +133,67 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCal
     return this.#capabilities;
   }
 
+  /**
+   * Get the model name/identifier for this STT instance.
+   *
+   * @returns The model name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their model information.
+   */
+  get model(): string {
+    return 'unknown';
+  }
+
+  /**
+   * Get the provider name for this STT instance.
+   *
+   * @returns The provider name if available, "unknown" otherwise.
+   *
+   * @remarks
+   * Plugins should override this property to provide their provider information.
+   */
+  get provider(): string {
+    return 'unknown';
+  }
+
   /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
-  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {
+  async recognize(frame: AudioBuffer, abortSignal?: AbortSignal): Promise<SpeechEvent> {
     const startTime = process.hrtime.bigint();
-    const event = await this._recognize(frame);
-    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));
+    const event = await this._recognize(frame, abortSignal);
+    const durationMs = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));
     this.emit('metrics_collected', {
       type: 'stt_metrics',
       requestId: event.requestId ?? '',
       timestamp: Date.now(),
-      duration,
+      durationMs,
       label: this.label,
-      audioDuration: calculateAudioDuration(frame),
+      audioDurationMs: Math.round(calculateAudioDurationSeconds(frame) * 1000),
       streamed: false,
+      metadata: {
+        modelProvider: this.provider,
+        modelName: this.model,
+      },
     });
     return event;
   }
-  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;
+
+  protected abstract _recognize(
+    frame: AudioBuffer,
+    abortSignal?: AbortSignal,
+  ): Promise<SpeechEvent>;
 
   /**
    * Returns a {@link SpeechStream} that can be used to push audio frames and receive
    * transcriptions
+   *
+   * @param options - Optional configuration including connection options
    */
-  abstract stream(): SpeechStream;
+  abstract stream(options?: { connOptions?: APIConnectOptions }): SpeechStream;
+
+  async close(): Promise<void> {
+    return;
+  }
 }
 
 /**
@@ -160,6 +225,9 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
   private _connOptions: APIConnectOptions;
+  private _startTimeOffset: number = 0;
+
+  protected abortController = new AbortController();
 
   constructor(
     stt: STT,
@@ -177,7 +245,7 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
     // is run **after** the constructor has finished. Otherwise we get
     // runtime error when trying to access class variables in the
     // `run` method.
-    startSoon(() => this.mainTask().then(() => this.queue.close()));
+    startSoon(() => this.mainTask().finally(() => this.queue.close()));
   }
 
   private async mainTask() {
@@ -186,7 +254,7 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
         return await this.run();
       } catch (error) {
         if (error instanceof APIError) {
-          const retryInterval = this._connOptions._intervalForRetry(i);
+          const retryInterval = intervalForRetry(this._connOptions, i);
 
           if (this._connOptions.maxRetry === 0 || !error.retryable) {
             this.emitError({ error, recoverable: false });
@@ -198,10 +266,11 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
               options: { retryable: false },
             });
           } else {
-            this.emitError({ error, recoverable: true });
+            // Don't emit error event for recoverable errors during retry loop
+            // to avoid ERR_UNHANDLED_ERROR or premature session termination
             this.logger.warn(
-              { tts: this.#stt.label, attempt: i + 1, error },
-              `failed to recognize speech, retrying in ${retryInterval}s`,
+              { stt: this.#stt.label, attempt: i + 1, error },
+              `failed to recognize speech, retrying in ${retryInterval}ms`,
             );
           }
 
@@ -246,24 +315,58 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
 
   protected async monitorMetrics() {
     for await (const event of this.queue) {
-      this.output.put(event);
+      if (!this.output.closed) {
+        try {
+          this.output.put(event);
+        } catch (e) {
+          if (e instanceof Error && e.message.includes('Queue is closed')) {
+            this.logger.warn(
+              { err: e },
+              'Queue closed during transcript processing (expected during disconnect)',
+            );
+          }
+        }
+      }
       if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;
       const metrics: STTMetrics = {
         type: 'stt_metrics',
         timestamp: Date.now(),
         requestId: event.requestId!,
-        duration: 0,
+        durationMs: 0,
         label: this.#stt.label,
-        audioDuration: event.recognitionUsage!.audioDuration,
+        audioDurationMs: Math.round(event.recognitionUsage!.audioDuration * 1000),
+        inputTokens: event.recognitionUsage!.inputTokens ?? 0,
+        outputTokens: event.recognitionUsage!.outputTokens ?? 0,
         streamed: true,
+        metadata: {
+          modelProvider: this.#stt.provider,
+          modelName: this.#stt.model,
+        },
       };
       this.#stt.emit('metrics_collected', metrics);
     }
-    this.output.close();
+    if (!this.output.closed) {
+      this.output.close();
+    }
   }
 
   protected abstract run(): Promise<void>;
 
+  protected get abortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
+  get startTimeOffset(): number {
+    return this._startTimeOffset;
+  }
+
+  set startTimeOffset(value: number) {
+    if (value < 0) {
+      throw new Error('startTimeOffset must be non-negative');
+    }
+    this._startTimeOffset = value;
+  }
+
   updateInputStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }
@@ -287,6 +390,11 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
       }
     }
 
+    if (frame.samplesPerChannel === 0) {
+      this.input.put(frame);
+      return;
+    }
+
     if (this.resampler) {
       const frames = this.resampler.push(frame);
       for (const frame of frames) {
@@ -325,9 +433,10 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
 
   /** Close both the input and output of the STT stream */
   close() {
-    this.input.close();
-    this.queue.close();
-    this.output.close();
+    if (!this.input.closed) this.input.close();
+    if (!this.queue.closed) this.queue.close();
+    if (!this.output.closed) this.output.close();
+    if (!this.abortController.signal.aborted) this.abortController.abort();
     this.closed = true;
   }
 

package/src/telemetry/index.ts

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export { ExtraDetailsProcessor, MetadataLogProcessor } from './logging.js';
+export {
+  SimpleOTLPHttpLogExporter,
+  type SimpleLogRecord,
+  type SimpleOTLPHttpLogExporterConfig,
+} from './otel_http_exporter.js';
+export {
+  emitToOtel,
+  flushPinoLogs,
+  initPinoCloudExporter,
+  PinoCloudExporter,
+  type PinoCloudExporterConfig,
+  type PinoLogObject,
+} from './pino_otel_transport.js';
+export * as traceTypes from './trace_types.js';
+export {
+  flushOtelLogs,
+  setTracerProvider,
+  setupCloudTracer,
+  tracer,
+  uploadSessionReport,
+  type StartSpanOptions,
+} from './traces.js';
+export { recordException, recordRealtimeMetrics } from './utils.js';

package/src/telemetry/logging.ts

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Attributes } from '@opentelemetry/api';
+import type { LogRecord, LogRecordProcessor } from '@opentelemetry/sdk-logs';
+
+/**
+ * Metadata log processor that injects metadata (room_id, job_id) into all log records.
+ */
+export class MetadataLogProcessor implements LogRecordProcessor {
+  private metadata: Attributes;
+
+  constructor(metadata: Attributes) {
+    this.metadata = metadata;
+  }
+
+  onEmit(logRecord: LogRecord): void {
+    // Add metadata to log record attributes
+    if (logRecord.attributes) {
+      Object.assign(logRecord.attributes, this.metadata);
+    } else {
+      (logRecord as any).attributes = { ...this.metadata };
+    }
+  }
+
+  shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+}
+
+/**
+ * Extra details processor that adds logger name to log records.
+ */
+export class ExtraDetailsProcessor implements LogRecordProcessor {
+  onEmit(logRecord: LogRecord): void {
+    const loggerName = logRecord.instrumentationScope.name;
+    if (logRecord.attributes) {
+      (logRecord.attributes as any)['logger.name'] = loggerName;
+    } else {
+      (logRecord as any).attributes = { 'logger.name': loggerName };
+    }
+  }
+
+  shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+}