@livekit/agents 1.0.42 → 1.0.44

package/src/inference/stt.test.ts

@@ -0,0 +1,236 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { beforeAll, describe, expect, it } from 'vitest';
+import { initializeLogger } from '../log.js';
+import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
+import { STT, type STTFallbackModel, normalizeSTTFallback, parseSTTModelString } from './stt.js';
+
+beforeAll(() => {
+  initializeLogger({ level: 'silent', pretty: false });
+});
+
+/** Helper to create STT with required credentials. */
+function makeStt(overrides: Record<string, unknown> = {}) {
+  const defaults = {
+    model: 'deepgram' as const,
+    apiKey: 'test-key',
+    apiSecret: 'test-secret',
+    baseURL: 'https://example.livekit.cloud',
+  };
+  return new STT({ ...defaults, ...overrides });
+}
+
+describe('parseSTTModelString', () => {
+  it('simple model without language', () => {
+    const [model, language] = parseSTTModelString('deepgram');
+    expect(model).toBe('deepgram');
+    expect(language).toBeUndefined();
+  });
+
+  it('model with language suffix', () => {
+    const [model, language] = parseSTTModelString('deepgram:en');
+    expect(model).toBe('deepgram');
+    expect(language).toBe('en');
+  });
+
+  it('provider/model format without language', () => {
+    const [model, language] = parseSTTModelString('deepgram/nova-3');
+    expect(model).toBe('deepgram/nova-3');
+    expect(language).toBeUndefined();
+  });
+
+  it('provider/model format with language', () => {
+    const [model, language] = parseSTTModelString('deepgram/nova-3:en');
+    expect(model).toBe('deepgram/nova-3');
+    expect(language).toBe('en');
+  });
+
+  it.each([
+    ['cartesia/ink-whisper:de', 'cartesia/ink-whisper', 'de'],
+    ['assemblyai:es', 'assemblyai', 'es'],
+    ['deepgram/nova-2-medical:ja', 'deepgram/nova-2-medical', 'ja'],
+    ['deepgram/nova-3:multi', 'deepgram/nova-3', 'multi'],
+    ['cartesia:zh', 'cartesia', 'zh'],
+  ])('various providers and languages: %s', (modelStr, expectedModel, expectedLang) => {
+    const [model, language] = parseSTTModelString(modelStr);
+    expect(model).toBe(expectedModel);
+    expect(language).toBe(expectedLang);
+  });
+
+  it('auto model without language', () => {
+    const [model, language] = parseSTTModelString('auto');
+    expect(model).toBe('auto');
+    expect(language).toBeUndefined();
+  });
+
+  it('auto model with language', () => {
+    const [model, language] = parseSTTModelString('auto:pt');
+    expect(model).toBe('auto');
+    expect(language).toBe('pt');
+  });
+});
+
+describe('normalizeSTTFallback', () => {
+  it('single string model', () => {
+    const result = normalizeSTTFallback('deepgram/nova-3');
+    expect(result).toEqual([{ model: 'deepgram/nova-3' }]);
+  });
+
+  it('single FallbackModel dict', () => {
+    const fallback: STTFallbackModel = { model: 'deepgram/nova-3' };
+    const result = normalizeSTTFallback(fallback);
+    expect(result).toEqual([{ model: 'deepgram/nova-3' }]);
+  });
+
+  it('list of string models', () => {
+    const result = normalizeSTTFallback(['deepgram/nova-3', 'cartesia/ink-whisper']);
+    expect(result).toEqual([{ model: 'deepgram/nova-3' }, { model: 'cartesia/ink-whisper' }]);
+  });
+
+  it('list of FallbackModel dicts', () => {
+    const fallbacks: STTFallbackModel[] = [{ model: 'deepgram/nova-3' }, { model: 'assemblyai' }];
+    const result = normalizeSTTFallback(fallbacks);
+    expect(result).toEqual([{ model: 'deepgram/nova-3' }, { model: 'assemblyai' }]);
+  });
+
+  it('mixed list of strings and dicts', () => {
+    const result = normalizeSTTFallback([
+      'deepgram/nova-3',
+      { model: 'cartesia/ink-whisper' } as STTFallbackModel,
+      'assemblyai',
+    ]);
+    expect(result).toEqual([
+      { model: 'deepgram/nova-3' },
+      { model: 'cartesia/ink-whisper' },
+      { model: 'assemblyai' },
+    ]);
+  });
+
+  it('string with language suffix discards language', () => {
+    const result = normalizeSTTFallback('deepgram/nova-3:en');
+    expect(result).toEqual([{ model: 'deepgram/nova-3' }]);
+  });
+
+  it('FallbackModel with extraKwargs is preserved', () => {
+    const fallback: STTFallbackModel = {
+      model: 'deepgram/nova-3',
+      extraKwargs: { keywords: [['livekit', 1.5]], punctuate: true },
+    };
+    const result = normalizeSTTFallback(fallback);
+    expect(result).toEqual([
+      {
+        model: 'deepgram/nova-3',
+        extraKwargs: { keywords: [['livekit', 1.5]], punctuate: true },
+      },
+    ]);
+  });
+
+  it('list with extraKwargs preserved', () => {
+    const result = normalizeSTTFallback([
+      { model: 'deepgram/nova-3', extraKwargs: { punctuate: true } } as STTFallbackModel,
+      'cartesia/ink-whisper',
+      { model: 'assemblyai', extraKwargs: { format_turns: true } } as STTFallbackModel,
+    ]);
+    expect(result).toEqual([
+      { model: 'deepgram/nova-3', extraKwargs: { punctuate: true } },
+      { model: 'cartesia/ink-whisper' },
+      { model: 'assemblyai', extraKwargs: { format_turns: true } },
+    ]);
+  });
+
+  it('empty list returns empty list', () => {
+    const result = normalizeSTTFallback([]);
+    expect(result).toEqual([]);
+  });
+
+  it('multiple colons in model string splits on last', () => {
+    const result = normalizeSTTFallback('some:model:part:fr');
+    expect(result).toEqual([{ model: 'some:model:part' }]);
+  });
+});
+
+describe('STT constructor fallback and connOptions', () => {
+  it('fallback not given defaults to undefined', () => {
+    const stt = makeStt();
+    expect(stt['opts'].fallback).toBeUndefined();
+  });
+
+  it('fallback single string is normalized', () => {
+    const stt = makeStt({ fallback: 'cartesia/ink-whisper' });
+    expect(stt['opts'].fallback).toEqual([{ model: 'cartesia/ink-whisper' }]);
+  });
+
+  it('fallback list of strings is normalized', () => {
+    const stt = makeStt({ fallback: ['deepgram/nova-3', 'assemblyai'] });
+    expect(stt['opts'].fallback).toEqual([{ model: 'deepgram/nova-3' }, { model: 'assemblyai' }]);
+  });
+
+  it('fallback single FallbackModel is normalized to list', () => {
+    const stt = makeStt({ fallback: { model: 'deepgram/nova-3' } });
+    expect(stt['opts'].fallback).toEqual([{ model: 'deepgram/nova-3' }]);
+  });
+
+  it('fallback with extraKwargs is preserved', () => {
+    const stt = makeStt({
+      fallback: {
+        model: 'deepgram/nova-3',
+        extraKwargs: { punctuate: true, keywords: [['livekit', 1.5]] },
+      },
+    });
+    expect(stt['opts'].fallback).toEqual([
+      {
+        model: 'deepgram/nova-3',
+        extraKwargs: { punctuate: true, keywords: [['livekit', 1.5]] },
+      },
+    ]);
+  });
+
+  it('fallback mixed list is normalized', () => {
+    const stt = makeStt({
+      fallback: [
+        'deepgram/nova-3',
+        { model: 'cartesia', extraKwargs: { min_volume: 0.5 } },
+        'assemblyai',
+      ],
+    });
+    expect(stt['opts'].fallback).toEqual([
+      { model: 'deepgram/nova-3' },
+      { model: 'cartesia', extraKwargs: { min_volume: 0.5 } },
+      { model: 'assemblyai' },
+    ]);
+  });
+
+  it('fallback string with language discards language', () => {
+    const stt = makeStt({ fallback: 'deepgram/nova-3:en' });
+    expect(stt['opts'].fallback).toEqual([{ model: 'deepgram/nova-3' }]);
+  });
+
+  it('connOptions not given uses default', () => {
+    const stt = makeStt();
+    expect(stt['opts'].connOptions).toEqual(DEFAULT_API_CONNECT_OPTIONS);
+  });
+
+  it('connOptions custom timeout', () => {
+    const custom: APIConnectOptions = { timeoutMs: 30000, maxRetry: 3, retryIntervalMs: 2000 };
+    const stt = makeStt({ connOptions: custom });
+    expect(stt['opts'].connOptions).toEqual(custom);
+    expect(stt['opts'].connOptions!.timeoutMs).toBe(30000);
+  });
+
+  it('connOptions custom maxRetry', () => {
+    const custom: APIConnectOptions = { timeoutMs: 10000, maxRetry: 5, retryIntervalMs: 2000 };
+    const stt = makeStt({ connOptions: custom });
+    expect(stt['opts'].connOptions).toEqual(custom);
+    expect(stt['opts'].connOptions!.maxRetry).toBe(5);
+  });
+
+  it('connOptions full custom', () => {
+    const custom: APIConnectOptions = { timeoutMs: 60000, maxRetry: 10, retryIntervalMs: 2000 };
+    const stt = makeStt({ connOptions: custom });
+    expect(stt['opts'].connOptions).toEqual(custom);
+    expect(stt['opts'].connOptions!.timeoutMs).toBe(60000);
+    expect(stt['opts'].connOptions!.maxRetry).toBe(10);
+    expect(stt['opts'].connOptions!.retryIntervalMs).toBe(2000);
+  });
+});

package/src/inference/stt.ts

@@ -42,29 +42,46 @@ export type AssemblyaiModels =
 export type ElevenlabsSTTModels = 'elevenlabs/scribe_v2_realtime';
 
 export interface CartesiaOptions {
-  min_volume?: number; // default: not specified
-  max_silence_duration_secs?: number; // default: not specified
+  /** Minimum volume threshold. Default: not specified. */
+  min_volume?: number;
+  /** Maximum silence duration in seconds. Default: not specified. */
+  max_silence_duration_secs?: number;
 }
 
 export interface DeepgramOptions {
-  filler_words?: boolean; // default: true
-  interim_results?: boolean; // default: true
-  endpointing?: number; // default: 25 (ms)
-  punctuate?: boolean; // default: false
+  /** Enable filler words. Default: true. */
+  filler_words?: boolean;
+  /** Enable interim results. Default: true. */
+  interim_results?: boolean;
+  /** Endpointing timeout in milliseconds. Default: 25. */
+  endpointing?: number;
+  /** Enable punctuation. Default: false. */
+  punctuate?: boolean;
+  /** Enable smart formatting. */
   smart_format?: boolean;
+  /** Keywords with boost values. */
   keywords?: Array<[string, number]>;
+  /** Key terms for recognition. */
   keyterms?: string[];
+  /** Enable profanity filter. */
   profanity_filter?: boolean;
+  /** Convert spoken numbers to numerals. */
   numerals?: boolean;
+  /** Opt out of model improvement program. */
   mip_opt_out?: boolean;
 }
 
 export interface AssemblyAIOptions {
-  format_turns?: boolean; // default: false
-  end_of_turn_confidence_threshold?: number; // default: 0.01
-  min_end_of_turn_silence_when_confident?: number; // default: 0
-  max_turn_silence?: number; // default: not specified
-  keyterms_prompt?: string[]; // default: not specified
+  /** Enable turn formatting. Default: false. */
+  format_turns?: boolean;
+  /** End of turn confidence threshold. Default: 0.01. */
+  end_of_turn_confidence_threshold?: number;
+  /** Minimum silence duration in milliseconds when confident about end of turn. Default: 0. */
+  min_end_of_turn_silence_when_confident?: number;
+  /** Maximum turn silence in milliseconds. Default: not specified. */
+  max_turn_silence?: number;
+  /** Key terms prompt for recognition. Default: not specified. */
+  keyterms_prompt?: string[];
 }
 
 export type STTLanguages =
@@ -93,6 +110,43 @@ export type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels
       ? AssemblyAIOptions
       : Record<string, unknown>;
 
+/** A fallback model with optional extra configuration. Extra fields are passed through to the provider. */
+export interface STTFallbackModel {
+  /** Model name (e.g. "deepgram/nova-3", "assemblyai/universal-streaming", "cartesia/ink-whisper"). */
+  model: string;
+  /** Extra configuration for the model. */
+  extraKwargs?: Record<string, unknown>;
+}
+
+export type STTFallbackModelType = STTFallbackModel | string;
+
+/** Parse a model string into [model, language]. Language is undefined if not specified. */
+export function parseSTTModelString(model: string): [string, string | undefined] {
+  const idx = model.lastIndexOf(':');
+  if (idx !== -1) {
+    return [model.slice(0, idx), model.slice(idx + 1)];
+  }
+  return [model, undefined];
+}
+
+/** Normalize a single or list of FallbackModelType into STTFallbackModel[]. */
+export function normalizeSTTFallback(
+  fallback: STTFallbackModelType | STTFallbackModelType[],
+): STTFallbackModel[] {
+  const makeFallback = (model: STTFallbackModelType): STTFallbackModel => {
+    if (typeof model === 'string') {
+      const [name] = parseSTTModelString(model);
+      return { model: name };
+    }
+    return model;
+  };
+
+  if (Array.isArray(fallback)) {
+    return fallback.map(makeFallback);
+  }
+  return [makeFallback(fallback)];
+}
+
 export type STTEncoding = 'pcm_s16le';
 
 const DEFAULT_ENCODING: STTEncoding = 'pcm_s16le';
@@ -109,6 +163,8 @@ export interface InferenceSTTOptions<TModel extends STTModels> {
   apiKey: string;
   apiSecret: string;
   modelOptions: STTOptions<TModel>;
+  fallback?: STTFallbackModel[];
+  connOptions?: APIConnectOptions;
 }
 
 /**
@@ -121,7 +177,7 @@ export class STT<TModel extends STTModels> extends BaseSTT {
   #logger = log();
 
   constructor(opts?: {
-    model?: TModel;
+    model?: ModelWithLanguage;
     language?: STTLanguages;
     baseURL?: string;
     encoding?: STTEncoding;
@@ -129,6 +185,8 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     apiKey?: string;
     apiSecret?: string;
     modelOptions?: STTOptions<TModel>;
+    fallback?: STTFallbackModelType | STTFallbackModelType[];
+    connOptions?: APIConnectOptions;
   }) {
     super({ streaming: true, interimResults: true, alignedTranscript: 'word' });
 
@@ -141,6 +199,8 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       apiKey,
       apiSecret,
       modelOptions = {} as STTOptions<TModel>,
+      fallback,
+      connOptions,
     } = opts || {};
 
     const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
@@ -155,15 +215,37 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');
     }
 
+    // Parse language from model string if provided: "provider/model:language"
+    let nextModel = model;
+    let nextLanguage = language;
+    if (typeof nextModel === 'string') {
+      const idx = nextModel.lastIndexOf(':');
+      if (idx !== -1) {
+        const languageFromModel = nextModel.slice(idx + 1) as STTLanguages;
+        if (nextLanguage && nextLanguage !== languageFromModel) {
+          this.#logger.warn(
+            '`language` is provided via both argument and model, using the one from the argument',
+            { language: nextLanguage, model: nextModel },
+          );
+        } else {
+          nextLanguage = languageFromModel;
+        }
+        nextModel = nextModel.slice(0, idx) as TModel;
+      }
+    }
+    const normalizedFallback = fallback ? normalizeSTTFallback(fallback) : undefined;
+
     this.opts = {
-      model,
-      language,
+      model: nextModel as TModel,
+      language: nextLanguage,
       encoding,
       sampleRate,
       baseURL: lkBaseURL,
       apiKey: lkApiKey,
       apiSecret: lkApiSecret,
       modelOptions,
+      fallback: normalizedFallback,
+      connOptions: connOptions ?? DEFAULT_API_CONNECT_OPTIONS,
     };
   }
 
@@ -172,11 +254,8 @@ export class STT<TModel extends STTModels> extends BaseSTT {
   }
 
   static fromModelString(modelString: string): STT<AnyString> {
-    if (modelString.includes(':')) {
-      const [model, language] = modelString.split(':') as [AnyString, STTLanguages];
-      return new STT({ model, language });
-    }
-    return new STT({ model: modelString });
+    const [model, language] = parseSTTModelString(modelString);
+    return new STT({ model, language });
   }
 
   protected async _recognize(_: AudioBuffer): Promise<SpeechEvent> {
@@ -195,7 +274,8 @@ export class STT<TModel extends STTModels> extends BaseSTT {
     language?: STTLanguages | string;
     connOptions?: APIConnectOptions;
   }): SpeechStream<TModel> {
-    const { language, connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const { language, connOptions = this.opts.connOptions ?? DEFAULT_API_CONNECT_OPTIONS } =
+      options || {};
     const streamOpts = {
       ...this.opts,
       language: language ?? this.opts.language,
@@ -224,6 +304,22 @@ export class STT<TModel extends STTModels> extends BaseSTT {
       (params.settings as Record<string, unknown>).language = this.opts.language;
     }
 
+    if (this.opts.fallback?.length) {
+      params.fallback = {
+        models: this.opts.fallback.map((m) => ({
+          model: m.model,
+          extra: m.extraKwargs ?? {},
+        })),
+      };
+    }
+
+    if (this.opts.connOptions) {
+      params.connection = {
+        timeout: this.opts.connOptions.timeoutMs / 1000,
+        retries: this.opts.connOptions.maxRetry,
+      };
+    }
+
     let baseURL = this.opts.baseURL;
     if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {
       baseURL = baseURL.replace('http', 'ws');