@livekit/agents 1.1.0 → 1.2.0

package/src/inference/stt.ts

@@ -2,10 +2,12 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { type AudioFrame } from '@livekit/rtc-node';
-import { type RawData, WebSocket } from 'ws';
+import type { WebSocket } from 'ws';
 import { APIError, APIStatusError } from '../_exceptions.js';
 import { AudioByteStream } from '../audio.js';
+import { type LanguageCode, areLanguagesEquivalent, normalizeLanguage } from '../language.js';
 import { log } from '../log.js';
+import { createStreamChannel } from '../stream/stream_channel.js';
 import {
   STT as BaseSTT,
   SpeechStream as BaseSpeechStream,
@@ -15,91 +17,178 @@ import {
 } from '../stt/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';
-import { type AnyModels, connectWs, createAccessToken } from './utils.js';
+import { type TimedString, createTimedString } from '../voice/io.js';
+import {
+  type SttServerEvent,
+  type SttTranscriptEvent,
+  sttServerEventSchema,
+} from './api_protos.js';
+import { type AnyString, connectWs, createAccessToken, getDefaultInferenceUrl } from './utils.js';
 
 export type DeepgramModels =
-  | 'deepgram'
+  | 'deepgram/flux-general'
   | 'deepgram/nova-3'
-  | 'deepgram/nova-3-general'
   | 'deepgram/nova-3-medical'
   | 'deepgram/nova-2'
-  | 'deepgram/nova-2-general'
   | 'deepgram/nova-2-medical'
+  | 'deepgram/nova-2-conversationalai'
   | 'deepgram/nova-2-phonecall';
 
-export type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';
+export type CartesiaModels = 'cartesia/ink-whisper';
+
+export type AssemblyaiModels =
+  | 'assemblyai/universal-streaming'
+  | 'assemblyai/universal-streaming-multilingual';
 
-export type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';
+export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime';
 
 export interface CartesiaOptions {
-  min_volume?: number; // default: not specified
-  max_silence_duration_secs?: number; // default: not specified
+  /** Minimum volume threshold. Default: not specified. */
+  min_volume?: number;
+  /** Maximum silence duration in seconds. Default: not specified. */
+  max_silence_duration_secs?: number;
 }
 
 export interface DeepgramOptions {
-  filler_words?: boolean; // default: true
-  interim_results?: boolean; // default: true
-  endpointing?: number; // default: 25 (ms)
-  punctuate?: boolean; // default: false
+  /** Enable filler words. Default: true. */
+  filler_words?: boolean;
+  /** Enable interim results. Default: true. */
+  interim_results?: boolean;
+  /** Endpointing timeout in milliseconds. Default: 25. */
+  endpointing?: number;
+  /** Enable punctuation. Default: false. */
+  punctuate?: boolean;
+  /** Enable smart formatting. */
   smart_format?: boolean;
+  /** Keywords with boost values. */
   keywords?: Array<[string, number]>;
+  /** Key terms for recognition. */
   keyterms?: string[];
+  /** Enable profanity filter. */
   profanity_filter?: boolean;
+  /** Convert spoken numbers to numerals. */
   numerals?: boolean;
+  /** Opt out of model improvement program. */
   mip_opt_out?: boolean;
 }
 
-export interface AssemblyaiOptions {
-  format_turns?: boolean; // default: false
-  end_of_turn_confidence_threshold?: number; // default: 0.01
-  min_end_of_turn_silence_when_confident?: number; // default: 0
-  max_turn_silence?: number; // default: not specified
-  keyterms_prompt?: string[]; // default: not specified
+export interface AssemblyAIOptions {
+  /** Enable turn formatting. Default: false. */
+  format_turns?: boolean;
+  /** End of turn confidence threshold. Default: 0.01. */
+  end_of_turn_confidence_threshold?: number;
+  /** Minimum silence duration in milliseconds when confident about end of turn. Default: 0. */
+  min_end_of_turn_silence_when_confident?: number;
+  /** Maximum turn silence in milliseconds. Default: not specified. */
+  max_turn_silence?: number;
+  /** Key terms prompt for recognition. Default: not specified. */
+  keyterms_prompt?: string[];
 }
 
-export type STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels | AnyModels;
+export type STTLanguages =
+  | 'multi'
+  | 'en'
+  | 'de'
+  | 'es'
+  | 'fr'
+  | 'ja'
+  | 'pt'
+  | 'zh'
+  | 'hi'
+  | AnyString;
+
+type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels | ElevenlabsSTTModels;
+
+export type STTModels = _STTModels | 'auto' | AnyString;
+
+export type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels;
+
 export type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels
   ? DeepgramOptions
   : TModel extends CartesiaModels
     ? CartesiaOptions
     : TModel extends AssemblyaiModels
-      ? AssemblyaiOptions
+      ? AssemblyAIOptions
       : Record<string, unknown>;
 
-export type STTLanguages = 'en' | 'de' | 'es' | 'fr' | 'ja' | 'pt' | 'zh';
+/** A fallback model with optional extra configuration. Extra fields are passed through to the provider. */
+export interface STTFallbackModel {
+  /** Model name (e.g. "deepgram/nova-3", "assemblyai/universal-streaming", "cartesia/ink-whisper"). */
+  model: string;
+  /** Extra configuration for the model. */
+  extraKwargs?: Record<string, unknown>;
+}
+
+export type STTFallbackModelType = STTFallbackModel | string;
+
+/** Parse a model string into [model, language]. Language is undefined if not specified. */
+export function parseSTTModelString(model: string): [string, LanguageCode | undefined] {
+  const idx = model.lastIndexOf(':');
+  if (idx !== -1) {
+    return [model.slice(0, idx), normalizeLanguage(model.slice(idx + 1))];
+  }
+  return [model, undefined];
+}
+
+/** Normalize a single or list of FallbackModelType into STTFallbackModel[]. */
+export function normalizeSTTFallback(
+  fallback: STTFallbackModelType | STTFallbackModelType[],
+): STTFallbackModel[] {
+  const makeFallback = (model: STTFallbackModelType): STTFallbackModel => {
+    if (typeof model === 'string') {
+      const [name] = parseSTTModelString(model);
+      return { model: name };
+    }
+    return model;
+  };
+
+  if (Array.isArray(fallback)) {
+    return fallback.map(makeFallback);
+  }
+  return [makeFallback(fallback)];
+}
+
 export type STTEncoding = 'pcm_s16le';
 
 const DEFAULT_ENCODING: STTEncoding = 'pcm_s16le';
 const DEFAULT_SAMPLE_RATE = 16000;
-const DEFAULT_BASE_URL = 'wss://agent-gateway.livekit.cloud/v1';
 const DEFAULT_CANCEL_TIMEOUT = 5000;
 
 export interface InferenceSTTOptions<TModel extends STTModels> {
-  model: TModel;
-  language?: STTLanguages | string;
+  model?: TModel;
+  language?: LanguageCode;
   encoding: STTEncoding;
   sampleRate: number;
   baseURL: string;
   apiKey: string;
   apiSecret: string;
-  extraKwargs: STTOptions<TModel>;
+  modelOptions: STTOptions<TModel>;
+  fallback?: STTFallbackModel[];
+  connOptions?: APIConnectOptions;
 }
 
+/**
+ * Livekit Cloud Inference STT
+ */
 export class STT<TModel extends STTModels> extends BaseSTT {
   private opts: InferenceSTTOptions<TModel>;
   private streams: Set<SpeechStream<TModel>> = new Set();
 
-  constructor(opts: {
-    model: TModel;
-    language?: STTLanguages | string;
+  #logger = log();
+
+  constructor(opts?: {
+    model?: ModelWithLanguage;
+    language?: STTLanguages;
     baseURL?: string;
     encoding?: STTEncoding;
     sampleRate?: number;
     apiKey?: string;
     apiSecret?: string;
-    extraKwargs?: STTOptions<TModel>;
+    modelOptions?: STTOptions<TModel>;
+    fallback?: STTFallbackModelType | STTFallbackModelType[];
+    connOptions?: APIConnectOptions;
   }) {
-    super({ streaming: true, interimResults: true });
+    super({ streaming: true, interimResults: true, alignedTranscript: 'word' });
 
     const {
       model,
@@ -109,10 +198,12 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       sampleRate = DEFAULT_SAMPLE_RATE,
       apiKey,
       apiSecret,
-      extraKwargs = {} as STTOptions<TModel>,
+      modelOptions = {} as STTOptions<TModel>,
+      fallback,
+      connOptions,
     } = opts || {};
 
-    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkBaseURL = baseURL || getDefaultInferenceUrl();
     const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
     if (!lkApiKey) {
       throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');
@@ -124,15 +215,36 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');
     }
 
+    // Parse language from model string if provided: "provider/model:language"
+    let nextModel = model;
+    let nextLanguage = language;
+    if (typeof nextModel === 'string') {
+      const [parsedModel, parsedLanguage] = parseSTTModelString(nextModel);
+      if (parsedLanguage !== undefined) {
+        if (nextLanguage && !areLanguagesEquivalent(nextLanguage, parsedLanguage)) {
+          this.#logger.warn(
+            '`language` is provided via both argument and model, using the one from the argument',
+            { language: nextLanguage, model: nextModel },
+          );
+        } else {
+          nextLanguage = parsedLanguage as STTLanguages;
+        }
+        nextModel = parsedModel as TModel;
+      }
+    }
+    const normalizedFallback = fallback ? normalizeSTTFallback(fallback) : undefined;
+
     this.opts = {
-      model,
-      language,
+      model: nextModel as TModel,
+      language: nextLanguage ? normalizeLanguage(nextLanguage) : undefined,
       encoding,
       sampleRate,
       baseURL: lkBaseURL,
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
-      extraKwargs,
+      modelOptions,
+      fallback: normalizedFallback,
+      connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS,
     };
   }
 
@@ -140,12 +252,29 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     return 'inference.STT';
   }
 
+  get model(): string {
+    return this.opts.model ?? 'auto';
+  }
+
+  get provider(): string {
+    return 'livekit';
+  }
+
+  static fromModelString(modelString: string): STT<AnyString> {
+    const [model, language] = parseSTTModelString(modelString);
+    return new STT({ model, language });
+  }
+
   protected async _recognize(_: AudioBuffer): Promise<SpeechEvent> {
     throw new Error('LiveKit STT does not support batch recognition, use stream() instead');
   }
 
   updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {
-    this.opts = { ...this.opts, ...opts };
+    this.opts = {
+      ...this.opts,
+      ...opts,
+      language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,
+    };
 
     for (const stream of this.streams) {
       stream.updateOptions(opts);
@@ -156,10 +285,11 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     language?: STTLanguages | string;
     connOptions?: APIConnectOptions;
   }): SpeechStream<TModel> {
-    const { language, connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const { language, connOptions = this.opts.connOptions ?? DEFAULT_API_CONNECT_OPTIONS } =
+      options || {};
     const streamOpts = {
       ...this.opts,
-      language: language ?? this.opts.language,
+      language: language !== undefined ? normalizeLanguage(language) : this.opts.language,
     } as InferenceSTTOptions<TModel>;
 
     const stream = new SpeechStream(this, streamOpts, connOptions);
@@ -167,6 +297,55 @@ export class STT<TModel extends STTModels> extends BaseSTT {
 
     return stream;
   }
+
+  async connectWs(timeout: number): Promise<WebSocket> {
+    const params = {
+      settings: {
+        sample_rate: String(this.opts.sampleRate),
+        encoding: this.opts.encoding,
+        extra: this.opts.modelOptions,
+      },
+    } as Record<string, unknown>;
+
+    if (this.opts.model && this.opts.model !== 'auto') {
+      params.model = this.opts.model;
+    }
+
+    if (this.opts.language) {
+      (params.settings as Record<string, unknown>).language = this.opts.language;
+    }
+
+    if (this.opts.fallback?.length) {
+      params.fallback = {
+        models: this.opts.fallback.map((m) => ({
+          model: m.model,
+          extra: m.extraKwargs ?? {},
+        })),
+      };
+    }
+
+    if (this.opts.connOptions) {
+      params.connection = {
+        timeout: this.opts.connOptions.timeoutMs / 1000,
+        retries: this.opts.connOptions.maxRetry,
+      };
+    }
+
+    let baseURL = this.opts.baseURL;
+    if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {
+      baseURL = baseURL.replace('http', 'ws');
+    }
+
+    const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);
+    const url = `${baseURL}/stt`;
+    const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;
+
+    const socket = await connectWs(url, headers, timeout);
+    const msg = { ...params, type: 'session.create' };
+    socket.send(JSON.stringify(msg));
+
+    return socket;
+  }
 }
 
 export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
@@ -175,6 +354,8 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
   private speaking = false;
   private speechDuration = 0;
   private reconnectEvent = new Event();
+  private stt: STT<TModel>;
+  private connOptions: APIConnectOptions;
 
   #logger = log();
 
@@ -185,6 +366,8 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
   ) {
     super(sttImpl, opts.sampleRate, connOptions);
     this.opts = opts;
+    this.stt = sttImpl;
+    this.connOptions = connOptions;
   }
 
   get label(): string {
@@ -192,223 +375,281 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
   }
 
   updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {
-    this.opts = { ...this.opts, ...opts };
+    this.opts = {
+      ...this.opts,
+      ...opts,
+      language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,
+    };
+    this.reconnectEvent.set();
   }
 
   protected async run(): Promise<void> {
-    let ws: WebSocket | null = null;
-    let closingWs = false;
-
-    this.reconnectEvent.set();
+    while (true) {
+      // Create fresh resources for each connection attempt
+      let ws: WebSocket | null = null;
+      let closing = false;
+      let finalReceived = false;
+
+      const eventChannel = createStreamChannel<SttServerEvent>();
+
+      const resourceCleanup = () => {
+        if (closing) return;
+        closing = true;
+        eventChannel.close();
+        ws?.removeAllListeners();
+        ws?.close();
+      };
+
+      const createWsListener = async (ws: WebSocket, signal: AbortSignal) => {
+        return new Promise<void>((resolve, reject) => {
+          const onAbort = () => {
+            resourceCleanup();
+            reject(new Error('WebSocket connection aborted'));
+          };
 
-    const connect = async () => {
-      const params = {
-        settings: {
-          sample_rate: String(this.opts.sampleRate),
-          encoding: this.opts.encoding,
-          extra: this.opts.extraKwargs,
-        },
-      } as Record<string, unknown>;
-
-      if (this.opts.model) {
-        params.model = this.opts.model;
-      }
+          signal.addEventListener('abort', onAbort, { once: true });
 
-      if (this.opts.language) {
-        (params.settings as Record<string, unknown>).language = this.opts.language;
-      }
+          ws.on('message', (data) => {
+            const json = JSON.parse(data.toString()) as SttServerEvent;
+            eventChannel.write(json);
+          });
 
-      let baseURL = this.opts.baseURL;
-      if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {
-        baseURL = baseURL.replace('http', 'ws');
-      }
+          ws.on('error', (e) => {
+            this.#logger.error({ error: e }, 'WebSocket error');
+            resourceCleanup();
+            reject(e);
+          });
 
-      const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);
-      const url = `${baseURL}/stt`;
-      const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;
+          ws.on('close', (code: number) => {
+            resourceCleanup();
 
-      const socket = await connectWs(url, headers, 10000);
-      const msg = { ...params, type: 'session.create' };
-      socket.send(JSON.stringify(msg));
+            if (!closing) return this.#logger.error('WebSocket closed unexpectedly');
+            if (finalReceived) return resolve();
 
-      return socket;
-    };
+            reject(
+              new APIStatusError({
+                message: 'LiveKit STT connection closed unexpectedly',
+                options: { statusCode: code },
+              }),
+            );
+          });
+        });
+      };
+
+      const send = async (socket: WebSocket, signal: AbortSignal) => {
+        const audioStream = new AudioByteStream(
+          this.opts.sampleRate,
+          1,
+          Math.floor(this.opts.sampleRate / 20), // 50ms
+        );
+
+        // Create abort promise once to avoid memory leak
+        const abortPromise = new Promise<never>((_, reject) => {
+          if (signal.aborted) {
+            return reject(new Error('Send aborted'));
+          }
+          const onAbort = () => reject(new Error('Send aborted'));
+          signal.addEventListener('abort', onAbort, { once: true });
+        });
 
-    const send = async (socket: WebSocket, signal: AbortSignal) => {
-      const audioStream = new AudioByteStream(
-        this.opts.sampleRate,
-        1,
-        Math.floor(this.opts.sampleRate / 20), // 50ms
-      );
+        // Manual iteration to support cancellation
+        const iterator = this.input[Symbol.asyncIterator]();
+        try {
+          while (true) {
+            const result = await Promise.race([iterator.next(), abortPromise]);
 
-      for await (const ev of this.input) {
-        if (signal.aborted) break;
-        let frames: AudioFrame[];
+            if (result.done) break;
+            const ev = result.value;
 
-        if (ev === SpeechStream.FLUSH_SENTINEL) {
-          frames = audioStream.flush();
-        } else {
-          const frame = ev as AudioFrame;
-          frames = audioStream.write(new Int16Array(frame.data).buffer);
-        }
+            let frames: AudioFrame[];
+            if (ev === SpeechStream.FLUSH_SENTINEL) {
+              frames = audioStream.flush();
+            } else {
+              const frame = ev as AudioFrame;
+              frames = audioStream.write(new Int16Array(frame.data).buffer);
+            }
 
-        for (const frame of frames) {
-          this.speechDuration += frame.samplesPerChannel / frame.sampleRate;
-          const base64 = Buffer.from(frame.data.buffer).toString('base64');
-          const msg = { type: 'input_audio', audio: base64 };
-          socket.send(JSON.stringify(msg));
+            for (const frame of frames) {
+              this.speechDuration += frame.samplesPerChannel / frame.sampleRate;
+              const base64 = Buffer.from(frame.data.buffer).toString('base64');
+              const msg = { type: 'input_audio', audio: base64 };
+              socket.send(JSON.stringify(msg));
+            }
+          }
+
+          closing = true;
+          socket.send(JSON.stringify({ type: 'session.finalize' }));
+        } catch (e) {
+          if ((e as Error).message === 'Send aborted') {
+            // Expected abort, don't log
+            return;
+          }
+          throw e;
         }
-      }
+      };
 
-      closingWs = true;
-      socket.send(JSON.stringify({ type: 'session.finalize' }));
-    };
+      const recv = async (signal: AbortSignal) => {
+        const serverEventStream = eventChannel.stream();
+        const reader = serverEventStream.getReader();
 
-    const recv = async (socket: WebSocket, signal: AbortSignal) => {
-      while (!this.closed && !signal.aborted) {
-        const dataPromise = new Promise<string>((resolve, reject) => {
-          const messageHandler = (d: RawData) => {
-            resolve(d.toString());
-            removeListeners();
-          };
-          const errorHandler = (e: Error) => {
-            reject(e);
-            removeListeners();
-          };
-          const closeHandler = (code: number) => {
-            if (closingWs) {
-              resolve('');
-            } else {
-              reject(
-                new APIStatusError({
-                  message: 'LiveKit STT connection closed unexpectedly',
-                  options: { statusCode: code },
-                }),
+        try {
+          while (!this.closed && !signal.aborted) {
+            const result = await reader.read();
+            if (signal.aborted) return;
+            if (result.done) return;
+
+            // Parse and validate with Zod schema
+            const parseResult = await sttServerEventSchema.safeParseAsync(result.value);
+            if (!parseResult.success) {
+              this.#logger.warn(
+                { error: parseResult.error, rawData: result.value },
+                'Failed to parse STT server event',
               );
+              continue;
             }
-            removeListeners();
-          };
-          const removeListeners = () => {
-            socket.removeListener('message', messageHandler);
-            socket.removeListener('error', errorHandler);
-            socket.removeListener('close', closeHandler);
-          };
-          socket.once('message', messageHandler);
-          socket.once('error', errorHandler);
-          socket.once('close', closeHandler);
-        });
 
-        const data = await Promise.race([dataPromise, waitForAbort(signal)]);
-
-        if (!data || signal.aborted) return;
-
-        const json = JSON.parse(data);
-        const type = json.type as string | undefined;
-
-        switch (type) {
-          case 'session.created':
-          case 'session.finalized':
-          case 'session.closed':
-            break;
-          case 'interim_transcript':
-            this.processTranscript(json, false);
-            break;
-          case 'final_transcript':
-            this.processTranscript(json, true);
-            break;
-          case 'error':
-            this.#logger.error('received error from LiveKit STT: %o', json);
-            throw new APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);
-          default:
-            this.#logger.warn('received unexpected message from LiveKit STT: %o', json);
-            break;
+            const event: SttServerEvent = parseResult.data;
+
+            switch (event.type) {
+              case 'session.created':
+              case 'session.finalized':
+                break;
+              case 'session.closed':
+                finalReceived = true;
+                resourceCleanup();
+                break;
+              case 'interim_transcript':
+                this.processTranscript(event, false);
+                break;
+              case 'final_transcript':
+                this.processTranscript(event, true);
+                break;
+              case 'error':
+                this.#logger.error({ error: event }, 'Received error from LiveKit STT');
+                resourceCleanup();
+                throw new APIError(`LiveKit STT returned error: ${JSON.stringify(event)}`);
+            }
+          }
+        } finally {
+          reader.releaseLock();
+          try {
+            await serverEventStream.cancel();
+          } catch (e) {
+            this.#logger.debug('Error cancelling serverEventStream (may already be cancelled):', e);
+          }
         }
-      }
-    };
+      };
 
-    while (true) {
       try {
-        ws = await connect();
+        ws = await this.stt.connectWs(this.connOptions.timeoutMs);
 
-        const sendTask = Task.from(async ({ signal }) => {
-          await send(ws!, signal);
-        });
-
-        const recvTask = Task.from(async ({ signal }) => {
-          await recv(ws!, signal);
-        });
-
-        const tasks = [sendTask, recvTask];
-        const waitReconnectTask = Task.from(async ({ signal }) => {
-          await Promise.race([this.reconnectEvent.wait(), waitForAbort(signal)]);
-        });
+        const controller = this.abortController; // Use base class abortController for proper cancellation
+        const sendTask = Task.from(({ signal }) => send(ws!, signal), controller);
+        const wsListenerTask = Task.from(({ signal }) => createWsListener(ws!, signal), controller);
+        const recvTask = Task.from(({ signal }) => recv(signal), controller);
+        const waitReconnectTask = Task.from(
+          ({ signal }) => Promise.race([this.reconnectEvent.wait(), waitForAbort(signal)]),
+          controller,
+        );
 
         try {
           await Promise.race([
-            Promise.all(tasks.map((task) => task.result)),
+            Promise.all([sendTask.result, wsListenerTask.result, recvTask.result]),
             waitReconnectTask.result,
           ]);
 
+          // If reconnect didn't trigger, tasks finished - exit loop
           if (!waitReconnectTask.done) break;
+
+          // Reconnect triggered - clear event and continue loop
           this.reconnectEvent.clear();
         } finally {
-          await cancelAndWait([sendTask, recvTask, waitReconnectTask], DEFAULT_CANCEL_TIMEOUT);
+          // Cancel all tasks to ensure cleanup
+          await cancelAndWait(
+            [sendTask, wsListenerTask, recvTask, waitReconnectTask],
+            DEFAULT_CANCEL_TIMEOUT,
+          );
+          resourceCleanup();
         }
       } finally {
-        try {
-          if (ws) ws.close();
-        } catch {}
+        // Ensure cleanup even if connectWs throws
+        resourceCleanup();
       }
     }
   }
 
-  private processTranscript(data: Record<string, any>, isFinal: boolean) {
-    const requestId = data.request_id ?? this.requestId;
-    const text = data.transcript ?? '';
-    const language = data.language ?? this.opts.language ?? 'en';
+  private processTranscript(data: SttTranscriptEvent, isFinal: boolean) {
+    // Check if queue is closed to avoid race condition during disconnect
+    if (this.queue.closed) return;
+
+    const requestId = data.session_id || this.requestId;
+    const text = data.transcript;
+    const language = normalizeLanguage(data.language || this.opts.language || 'en');
 
     if (!text && !isFinal) return;
 
-    // We'll have a more accurate way of detecting when speech started when we have VAD
-    if (!this.speaking) {
-      this.speaking = true;
-      this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
-    }
+    try {
+      // We'll have a more accurate way of detecting when speech started when we have VAD
+      if (!this.speaking) {
+        this.speaking = true;
+        this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
+      }
 
-    const speechData: SpeechData = {
-      language,
-      startTime: data.start ?? 0,
-      endTime: data.duration ?? 0,
-      confidence: data.confidence ?? 1.0,
-      text,
-    };
+      const speechData: SpeechData = {
+        language,
+        startTime: this.startTimeOffset + data.start,
+        endTime: this.startTimeOffset + data.start + data.duration,
+        confidence: data.confidence,
+        text,
+        words: data.words.map(
+          (word): TimedString =>
+            createTimedString({
+              text: word.word,
+              startTime: word.start + this.startTimeOffset,
+              endTime: word.end + this.startTimeOffset,
+              startTimeOffset: this.startTimeOffset,
+              confidence: word.confidence,
+            }),
+        ),
+      };
+
+      if (isFinal) {
+        if (this.speechDuration > 0) {
+          this.queue.put({
+            type: SpeechEventType.RECOGNITION_USAGE,
+            requestId,
+            recognitionUsage: { audioDuration: this.speechDuration },
+          });
+          this.speechDuration = 0;
+        }
 
-    if (isFinal) {
-      if (this.speechDuration > 0) {
         this.queue.put({
-          type: SpeechEventType.RECOGNITION_USAGE,
+          type: SpeechEventType.FINAL_TRANSCRIPT,
           requestId,
-          recognitionUsage: { audioDuration: this.speechDuration },
+          alternatives: [speechData],
         });
-        this.speechDuration = 0;
-      }
 
-      this.queue.put({
-        type: SpeechEventType.FINAL_TRANSCRIPT,
-        requestId,
-        alternatives: [speechData],
-      });
-
-      if (this.speaking) {
-        this.speaking = false;
-        this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+        if (this.speaking) {
+          this.speaking = false;
+          this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+        }
+      } else {
+        this.queue.put({
+          type: SpeechEventType.INTERIM_TRANSCRIPT,
+          requestId,
+          alternatives: [speechData],
+        });
+      }
+    } catch (e) {
+      if (e instanceof Error && e.message.includes('Queue is closed')) {
+        // Expected behavior on disconnect, log as warning
+        this.#logger.warn(
+          { err: e },
+          'Queue closed during transcript processing (expected during disconnect)',
+        );
+      } else {
+        this.#logger.error({ err: e }, 'Error putting transcript to queue');
       }
-    } else {
-      this.queue.put({
-        type: SpeechEventType.INTERIM_TRANSCRIPT,
-        requestId,
-        alternatives: [speechData],
-      });
     }
   }
 }