@livekit/agents 1.0.37 → 1.0.39

package/src/inference/stt.ts

@@ -16,22 +16,30 @@ import {
 } from '../stt/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
+import {
+  type SttServerEvent,
+  type SttTranscriptEvent,
+  sttServerEventSchema,
+} from './api_protos.js';
 import { type AnyString, connectWs, createAccessToken } from './utils.js';
 
 export type DeepgramModels =
-  | 'deepgram'
+  | 'deepgram/flux-general'
   | 'deepgram/nova-3'
-  | 'deepgram/nova-3-general'
   | 'deepgram/nova-3-medical'
-  | 'deepgram/nova-2-conversationalai'
   | 'deepgram/nova-2'
-  | 'deepgram/nova-2-general'
   | 'deepgram/nova-2-medical'
+  | 'deepgram/nova-2-conversationalai'
   | 'deepgram/nova-2-phonecall';
 
-export type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';
+export type CartesiaModels = 'cartesia/ink-whisper';
+
+export type AssemblyaiModels =
+  | 'assemblyai/universal-streaming'
+  | 'assemblyai/universal-streaming-multilingual';
 
-export type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';
+export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime';
 
 export interface CartesiaOptions {
   min_volume?: number; // default: not specified
@@ -71,7 +79,7 @@ export type STTLanguages =
   | 'hi'
   | AnyString;
 
-type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels;
+type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels | ElevenlabsSTTModels;
 
 export type STTModels = _STTModels | 'auto' | AnyString;
 
@@ -122,7 +130,7 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     apiSecret?: string;
     modelOptions?: STTOptions<TModel>;
   }) {
-    super({ streaming: true, interimResults: true });
+    super({ streaming: true, interimResults: true, alignedTranscript: 'word' });
 
     const {
       model,
@@ -271,7 +279,6 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
       let closing = false;
       let finalReceived = false;
 
-      type SttServerEvent = Record<string, any>;
       const eventChannel = createStreamChannel<SttServerEvent>();
 
       const resourceCleanup = () => {
@@ -380,10 +387,19 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
             if (signal.aborted) return;
             if (result.done) return;
 
-            const json = result.value;
-            const type = json.type as string | undefined;
+            // Parse and validate with Zod schema
+            const parseResult = await sttServerEventSchema.safeParseAsync(result.value);
+            if (!parseResult.success) {
+              this.#logger.warn(
+                { error: parseResult.error, rawData: result.value },
+                'Failed to parse STT server event',
+              );
+              continue;
+            }
+
+            const event: SttServerEvent = parseResult.data;
 
-            switch (type) {
+            switch (event.type) {
               case 'session.created':
               case 'session.finalized':
                 break;
@@ -392,21 +408,15 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
                 resourceCleanup();
                 break;
               case 'interim_transcript':
-                this.processTranscript(json, false);
+                this.processTranscript(event, false);
                 break;
               case 'final_transcript':
-                this.processTranscript(json, true);
+                this.processTranscript(event, true);
                 break;
               case 'error':
-                this.#logger.error({ error: json }, 'Received error from LiveKit STT');
+                this.#logger.error({ error: event }, 'Received error from LiveKit STT');
                 resourceCleanup();
-                throw new APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);
-              default:
-                this.#logger.warn(
-                  { message: json },
-                  'Received unexpected message from LiveKit STT',
-                );
-                break;
+                throw new APIError(`LiveKit STT returned error: ${JSON.stringify(event)}`);
             }
           }
         } finally {
@@ -457,13 +467,13 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
     }
   }
 
-  private processTranscript(data: Record<string, any>, isFinal: boolean) {
+  private processTranscript(data: SttTranscriptEvent, isFinal: boolean) {
     // Check if queue is closed to avoid race condition during disconnect
     if (this.queue.closed) return;
 
-    const requestId = data.request_id ?? this.requestId;
-    const text = data.transcript ?? '';
-    const language = data.language ?? this.opts.language ?? 'en';
+    const requestId = data.session_id || this.requestId;
+    const text = data.transcript;
+    const language = data.language || this.opts.language || 'en';
 
     if (!text && !isFinal) return;
 
@@ -476,10 +486,19 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
 
       const speechData: SpeechData = {
         language,
-        startTime: data.start ?? 0,
-        endTime: data.duration ?? 0,
-        confidence: data.confidence ?? 1.0,
+        startTime: this.startTimeOffset + data.start,
+        endTime: this.startTimeOffset + data.start + data.duration,
+        confidence: data.confidence,
         text,
+        words: data.words.map(
+          (word): TimedString => ({
+            text: word.word,
+            startTime: word.start + this.startTimeOffset,
+            endTime: word.end + this.startTimeOffset,
+            startTimeOffset: this.startTimeOffset,
+            confidence: word.confidence,
+          }),
+        ),
       };
 
       if (isFinal) {

package/src/inference/tts.ts

@@ -23,22 +23,27 @@ import {
 import { type AnyString, connectWs, createAccessToken } from './utils.js';
 
 export type CartesiaModels =
-  | 'cartesia'
-  | 'cartesia/sonic'
+  | 'cartesia/sonic-3'
   | 'cartesia/sonic-2'
-  | 'cartesia/sonic-turbo';
+  | 'cartesia/sonic-turbo'
+  | 'cartesia/sonic';
+
+export type DeepgramTTSModels = 'deepgram/aura' | 'deepgram/aura-2';
 
 export type ElevenlabsModels =
-  | 'elevenlabs'
   | 'elevenlabs/eleven_flash_v2'
   | 'elevenlabs/eleven_flash_v2_5'
   | 'elevenlabs/eleven_turbo_v2'
   | 'elevenlabs/eleven_turbo_v2_5'
   | 'elevenlabs/eleven_multilingual_v2';
 
-export type RimeModels = 'rime' | 'rime/mist' | 'rime/mistv2' | 'rime/arcana';
+export type InworldModels =
+  | 'inworld/inworld-tts-1.5-max'
+  | 'inworld/inworld-tts-1.5-mini'
+  | 'inworld/inworld-tts-1-max'
+  | 'inworld/inworld-tts-1';
 
-export type InworldModels = 'inworld' | 'inworld/inworld-tts-1';
+export type RimeModels = 'rime/arcana' | 'rime/mistv2';
 
 export interface CartesiaOptions {
   duration?: number; // max duration of audio in seconds
@@ -50,25 +55,40 @@ export interface ElevenlabsOptions {
   apply_text_normalization?: 'auto' | 'off' | 'on'; // default: "auto"
 }
 
+export interface DeepgramTTSOptions {}
+
 export interface RimeOptions {}
 
 export interface InworldOptions {}
 
-type _TTSModels = CartesiaModels | ElevenlabsModels | RimeModels | InworldModels;
-
-export type TTSModels = CartesiaModels | ElevenlabsModels | RimeModels | InworldModels | AnyString;
+type _TTSModels =
+  | CartesiaModels
+  | DeepgramTTSModels
+  | ElevenlabsModels
+  | RimeModels
+  | InworldModels;
+
+export type TTSModels =
+  | CartesiaModels
+  | DeepgramTTSModels
+  | ElevenlabsModels
+  | RimeModels
+  | InworldModels
+  | AnyString;
 
 export type ModelWithVoice = `${_TTSModels}:${string}` | TTSModels;
 
 export type TTSOptions<TModel extends TTSModels> = TModel extends CartesiaModels
   ? CartesiaOptions
-  : TModel extends ElevenlabsModels
-    ? ElevenlabsOptions
-    : TModel extends RimeOptions
-      ? RimeOptions
-      : TModel extends InworldOptions
-        ? InworldOptions
-        : Record<string, unknown>;
+  : TModel extends DeepgramTTSModels
+    ? DeepgramTTSOptions
+    : TModel extends ElevenlabsModels
+      ? ElevenlabsOptions
+      : TModel extends RimeModels
+        ? RimeOptions
+        : TModel extends InworldModels
+          ? InworldOptions
+          : Record<string, unknown>;
 
 type TTSEncoding = 'pcm_s16le';
 

package/src/stt/stream_adapter.ts

@@ -4,6 +4,7 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import { log } from '../log.js';
 import type { APIConnectOptions } from '../types.js';
+import { isStreamClosedError } from '../utils.js';
 import type { VAD, VADStream } from '../vad.js';
 import { VADEventType } from '../vad.js';
 import type { SpeechEvent } from './stt.js';
@@ -68,7 +69,17 @@ export class StreamAdapterWrapper extends SpeechStream {
           this.#vadStream.pushFrame(input);
         }
       }
-      this.#vadStream.endInput();
+
+      // Guard against calling endInput() on already-closed stream
+      // This happens during handover when close() is called while forwardInput is running
+      try {
+        this.#vadStream.endInput();
+      } catch (e) {
+        if (isStreamClosedError(e)) {
+          return;
+        }
+        throw e;
+      }
     };
 
     const recognize = async () => {

package/src/stt/stt.ts

@@ -13,6 +13,7 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
+import type { TimedString } from '../voice/index.js';
 
 /** Indicates start/middle/end of speech */
 export enum SpeechEventType {
@@ -53,6 +54,7 @@ export interface SpeechData {
   startTime: number;
   endTime: number;
   confidence: number;
+  words?: TimedString[];
 }
 
 export interface RecognitionUsage {
@@ -76,6 +78,13 @@ export interface SpeechEvent {
 export interface STTCapabilities {
   streaming: boolean;
   interimResults: boolean;
+  /**
+   * Whether this STT supports aligned transcripts with word/chunk timestamps.
+   * - 'word': Provider returns word-level timestamps
+   * - 'chunk': Provider returns chunk-level timestamps (e.g., sentence/phrase boundaries)
+   * - false: Provider does not support aligned transcripts
+   */
+  alignedTranscript?: 'word' | 'chunk' | false;
 }
 
 export interface STTError {
@@ -176,6 +185,7 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
   private logger = log();
   private _connOptions: APIConnectOptions;
+  private _startTimeOffset: number = 0;
 
   protected abortController = new AbortController();
 
@@ -300,6 +310,17 @@ export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent>
     return this.abortController.signal;
   }
 
+  get startTimeOffset(): number {
+    return this._startTimeOffset;
+  }
+
+  set startTimeOffset(value: number) {
+    if (value < 0) {
+      throw new Error('startTimeOffset must be non-negative');
+    }
+    this._startTimeOffset = value;
+  }
+
   updateInputStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }

package/src/telemetry/traces.ts

@@ -37,6 +37,8 @@ export interface StartSpanOptions {
   attributes?: Attributes;
   /** Whether to end the span when the function exits (default: true) */
   endOnExit?: boolean;
+  /** Optional start time for the span in milliseconds (Date.now() format) */
+  startTime?: number;
 }
 
 /**
@@ -79,10 +81,12 @@ class DynamicTracer {
    */
   startSpan(options: StartSpanOptions): Span {
     const ctx = options.context || otelContext.active();
+
     const span = this.tracer.startSpan(
       options.name,
       {
         attributes: options.attributes,
+        startTime: options.startTime,
       },
       ctx,
     );
@@ -101,7 +105,7 @@ class DynamicTracer {
   async startActiveSpan<T>(fn: (span: Span) => Promise<T>, options: StartSpanOptions): Promise<T> {
     const ctx = options.context || otelContext.active();
     const endOnExit = options.endOnExit === undefined ? true : options.endOnExit; // default true
-    const opts: SpanOptions = { attributes: options.attributes };
+    const opts: SpanOptions = { attributes: options.attributes, startTime: options.startTime };
 
     // Directly return the tracer's startActiveSpan result - it handles async correctly
     return await this.tracer.startActiveSpan(options.name, opts, ctx, async (span) => {
@@ -125,7 +129,7 @@ class DynamicTracer {
   startActiveSpanSync<T>(fn: (span: Span) => T, options: StartSpanOptions): T {
     const ctx = options.context || otelContext.active();
     const endOnExit = options.endOnExit === undefined ? true : options.endOnExit; // default true
-    const opts: SpanOptions = { attributes: options.attributes };
+    const opts: SpanOptions = { attributes: options.attributes, startTime: options.startTime };
 
     return this.tracer.startActiveSpan(options.name, opts, ctx, (span) => {
       try {

package/src/utils.ts

@@ -125,6 +125,7 @@ export class Future<T = void> {
   #resolvePromise!: (value: T) => void;
   #rejectPromise!: (error: Error) => void;
   #done: boolean = false;
+  #rejected: boolean = false;
 
   constructor() {
     this.#await = new Promise<T>((resolve, reject) => {
@@ -141,6 +142,11 @@ export class Future<T = void> {
     return this.#done;
   }
 
+  /** Whether the future was rejected (cancelled) */
+  get rejected() {
+    return this.#rejected;
+  }
+
   resolve(value: T) {
     this.#done = true;
     this.#resolvePromise(value);
@@ -148,6 +154,7 @@ export class Future<T = void> {
 
   reject(error: Error) {
     this.#done = true;
+    this.#rejected = true;
     this.#rejectPromise(error);
   }
 }
@@ -668,6 +675,20 @@ export class InvalidErrorType extends Error {
   }
 }
 
+/**
+ * Check if an error is a stream closed error that can be safely ignored during cleanup.
+ * This happens during handover/cleanup when close() is called while operations are still running.
+ *
+ * @param error - The error to check.
+ * @returns True if the error is a stream closed error.
+ */
+export function isStreamClosedError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    (error.message === 'Stream is closed' || error.message === 'Input is closed')
+  );
+}
+
 /**
  * In JS an error can be any arbitrary value.
  * This function converts an unknown error to an Error and stores the original value in the error object.

package/src/voice/agent.ts

@@ -271,6 +271,15 @@ export class Agent<UserData = any> {
 
       const connOptions = activity.agentSession.connOptions.sttConnOptions;
       const stream = wrapped_stt.stream({ connOptions });
+
+      // Set startTimeOffset to provide linear timestamps across reconnections
+      const audioInputStartedAt =
+        activity.agentSession._recorderIO?.recordingStartedAt ?? // Use recording start time if available
+        activity.agentSession._startedAt ?? // Fallback to session start time
+        Date.now(); // Fallback to current time
+
+      stream.startTimeOffset = (Date.now() - audioInputStartedAt) / 1000;
+
       stream.updateInputStream(audio);
 
       let cleaned = false;
@@ -316,16 +325,16 @@ export class Agent<UserData = any> {
         );
       }
 
-      // TODO(brian): make parallelToolCalls configurable
       const { toolChoice } = modelSettings;
       const connOptions = activity.agentSession.connOptions.llmConnOptions;
 
+      // parallelToolCalls is not passed here - it will use the value from LLM's modelOptions
+      // This allows users to configure it via: new inference.LLM({ modelOptions: { parallel_tool_calls: false } })
       const stream = activity.llm.chat({
         chatCtx,
         toolCtx,
         toolChoice,
         connOptions,
-        parallelToolCalls: true,
       });
 
       let cleaned = false;