@livekit/agents 1.1.0-dev.0 → 1.2.0

package/src/inference/tts.ts

@@ -6,6 +6,7 @@ import { WebSocket } from 'ws';
 import { APIError, APIStatusError } from '../_exceptions.js';
 import { AudioByteStream } from '../audio.js';
 import { ConnectionPool } from '../connection_pool.js';
+import { type LanguageCode, normalizeLanguage } from '../language.js';
 import { log } from '../log.js';
 import { createStreamChannel } from '../stream/stream_channel.js';
 import { basic as tokenizeBasic } from '../tokenize/index.js';
@@ -62,7 +63,14 @@ export interface DeepgramTTSOptions {}
 
 export interface RimeOptions {}
 
-export interface InworldOptions {}
+export interface InworldOptions {
+  /** Controls how fast the voice speaks. 1.0 is normal speed, 0.5 is half, 1.5 is 1.5x. Default: 1.0. */
+  speaking_rate?: number;
+  /** Controls randomness in the output. Recommended between 0.6 and 1.1. Default: 1.1. */
+  temperature?: number;
+  /** Controls text normalization. "ON" expands numbers, dates, abbreviations. "OFF" reads text as written. Default: "ON". */
+  text_normalization?: 'ON' | 'OFF';
+}
 
 type _TTSModels =
   | CartesiaModels
@@ -142,7 +150,7 @@ const DEFAULT_LANGUAGE = 'en';
 export interface InferenceTTSOptions<TModel extends TTSModels> {
   model?: TModel;
   voice?: string;
-  language?: string;
+  language?: LanguageCode;
   encoding: TTSEncoding;
   sampleRate: number;
   baseURL: string;
@@ -228,7 +236,7 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
     this.opts = {
       model: nextModel,
       voice: nextVoice,
-      language,
+      language: normalizeLanguage(language),
       encoding,
       sampleRate,
       baseURL: lkBaseURL,
@@ -267,9 +275,13 @@ export class TTS<TModel extends TTSModels> extends BaseTTS {
   }
 
   updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {
-    this.opts = { ...this.opts, ...opts };
+    this.opts = {
+      ...this.opts,
+      ...opts,
+      language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,
+    };
     for (const stream of this.streams) {
-      stream.updateOptions(opts);
+      stream.updateOptions(this.opts);
     }
   }
 
@@ -362,7 +374,11 @@ export class SynthesizeStream<TModel extends TTSModels> extends BaseSynthesizeSt
   }
 
   updateOptions(opts: Partial<Pick<InferenceTTSOptions<TModel>, 'model' | 'voice' | 'language'>>) {
-    this.opts = { ...this.opts, ...opts };
+    this.opts = {
+      ...this.opts,
+      ...opts,
+      language: opts.language !== undefined ? normalizeLanguage(opts.language) : this.opts.language,
+    };
   }
 
   protected async run(): Promise<void> {

package/src/ipc/job_proc_lazy_main.ts

@@ -16,11 +16,22 @@ import type { IPCMessage } from './message.js';
 const ORPHANED_TIMEOUT = 15 * 1000;
 
 const safeSend = (msg: IPCMessage): boolean => {
-  if (process.connected && process.send) {
-    process.send(msg);
-    return true;
+  try {
+    if (process.connected && process.send) {
+      process.send(msg);
+      return true;
+    }
+    return false;
+  } catch (error) {
+    // Channel closed is expected during graceful shutdown
+    // Log at debug level to avoid noise in production logs
+    if (error instanceof Error && error.message.includes('Channel closed')) {
+      log().debug({ msgCase: msg.case }, 'IPC channel closed, message not sent');
+    } else {
+      log().error({ error, msgCase: msg.case }, 'IPC send failed unexpectedly');
+    }
+    return false;
   }
-  return false;
 };
 
 type JobTask = {
@@ -109,27 +120,36 @@ const startJob = (
       }
     }, 10000);
 
-    // Run the job function within the AsyncLocalStorage context
-    await runWithJobContextAsync(ctx, async () => {
-      const { tracer, traceTypes } = await import('../telemetry/index.js');
-      return tracer.startActiveSpan(
-        async (span) => {
-          span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);
-          span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);
-          span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');
-          return func(ctx);
-        },
-        { name: 'job_entrypoint' },
-      );
-    }).finally(() => {
-      clearTimeout(unconnectedTimeout);
-    });
-
-    await once(closeEvent, 'close').then((close) => {
-      logger.debug('shutting down');
+    try {
+      // Run the job function within the AsyncLocalStorage context
+      await runWithJobContextAsync(ctx, async () => {
+        const { tracer, traceTypes } = await import('../telemetry/index.js');
+        return tracer.startActiveSpan(
+          async (span) => {
+            span.setAttribute(traceTypes.ATTR_JOB_ID, info.job.id);
+            span.setAttribute(traceTypes.ATTR_AGENT_NAME, info.job.agentName);
+            span.setAttribute(traceTypes.ATTR_ROOM_NAME, info.job.room?.name ?? '');
+            return func(ctx);
+          },
+          { name: 'job_entrypoint' },
+        );
+      }).finally(() => {
+        clearTimeout(unconnectedTimeout);
+      });
+
+      await once(closeEvent, 'close').then((close) => {
+        logger.debug('shutting down');
+        shutdown = true;
+        safeSend({ case: 'exiting', value: { reason: close[1] } });
+      });
+    } catch (error) {
+      logger.error({ error }, 'error in entry function');
       shutdown = true;
-      safeSend({ case: 'exiting', value: { reason: close[1] } });
-    });
+      safeSend({
+        case: 'exiting',
+        value: { reason: error instanceof Error ? error.message : String(error) },
+      });
+    }
 
     // Close the primary agent session if it exists
     if (ctx._primaryAgentSession) {

package/src/job.ts

@@ -276,7 +276,7 @@ export class JobContext {
       jobId: this.job.id,
       roomId: this.job.room?.sid || '',
       room: this.job.room?.name || '',
-      options: targetSession.options,
+      options: targetSession.sessionOptions,
       events: targetSession._recordedEvents,
       enableRecording: targetSession._enableRecording,
       chatHistory: targetSession.history.copy(),

package/src/language.test.ts

@@ -0,0 +1,62 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import {
+  areLanguagesEquivalent,
+  getBaseLanguage,
+  getIsoLanguage,
+  getLanguageRegion,
+  normalizeLanguage,
+  toLanguageName,
+} from './language.js';
+
+describe('normalizeLanguage', () => {
+  it('normalizes language names', () => {
+    expect(normalizeLanguage('english')).toBe('en');
+  });
+
+  it('normalizes iso 639-3 codes', () => {
+    expect(normalizeLanguage('eng')).toBe('en');
+  });
+
+  it('normalizes bcp-47 casing and separators', () => {
+    expect(normalizeLanguage('en_us')).toBe('en-US');
+    expect(normalizeLanguage('zh_hans_cn')).toBe('zh-Hans-CN');
+  });
+
+  it('preserves iso 639-3 in compound tags', () => {
+    expect(normalizeLanguage('cmn_hans_cn')).toBe('cmn-Hans-CN');
+  });
+
+  it('passes unknown codes through in lowercase', () => {
+    expect(normalizeLanguage('MULTI')).toBe('multi');
+  });
+
+  it('preserves empty string sentinel', () => {
+    expect(normalizeLanguage('')).toBe('');
+  });
+});
+
+describe('language helpers', () => {
+  it('extracts base language', () => {
+    expect(getBaseLanguage('cmn-Hans-CN')).toBe('zh');
+  });
+
+  it('builds iso language tag', () => {
+    expect(getIsoLanguage('cmn-Hans-CN')).toBe('zh-CN');
+  });
+
+  it('extracts region', () => {
+    expect(getLanguageRegion('en-US')).toBe('US');
+  });
+
+  it('maps normalized code back to language name', () => {
+    expect(toLanguageName('eng')).toBe('english');
+  });
+
+  it('compares equivalent representations', () => {
+    expect(areLanguagesEquivalent('english', 'en')).toBe(true);
+    expect(areLanguagesEquivalent('en_us', 'en-US')).toBe(true);
+  });
+});

package/src/language.ts

@@ -0,0 +1,380 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export const KNOWN_LANGUAGE_CODES = [
+  'af',
+  'am',
+  'ar',
+  'as',
+  'az',
+  'be',
+  'bg',
+  'bn',
+  'bs',
+  'ca',
+  'cs',
+  'cy',
+  'da',
+  'de',
+  'el',
+  'en',
+  'es',
+  'et',
+  'eu',
+  'fa',
+  'ff',
+  'fi',
+  'fr',
+  'ga',
+  'gl',
+  'gu',
+  'ha',
+  'he',
+  'hi',
+  'hr',
+  'hu',
+  'hy',
+  'id',
+  'ig',
+  'is',
+  'it',
+  'ja',
+  'jv',
+  'ka',
+  'kk',
+  'km',
+  'kn',
+  'ko',
+  'ku',
+  'ky',
+  'lb',
+  'lg',
+  'ln',
+  'lo',
+  'lt',
+  'lv',
+  'mi',
+  'mk',
+  'ml',
+  'mn',
+  'mr',
+  'ms',
+  'mt',
+  'my',
+  'ne',
+  'nl',
+  'no',
+  'ny',
+  'oc',
+  'or',
+  'pa',
+  'pl',
+  'ps',
+  'pt',
+  'ro',
+  'ru',
+  'sd',
+  'sk',
+  'sl',
+  'sn',
+  'so',
+  'sq',
+  'sr',
+  'sv',
+  'sw',
+  'ta',
+  'te',
+  'tg',
+  'th',
+  'tl',
+  'tr',
+  'uk',
+  'ur',
+  'uz',
+  'vi',
+  'wo',
+  'xh',
+  'yo',
+  'zh',
+  'zu',
+] as const;
+
+export type KnownLanguageCode = (typeof KNOWN_LANGUAGE_CODES)[number];
+
+declare const languageCodeBrand: unique symbol;
+
+export type LanguageCode = string & { readonly [languageCodeBrand]: 'LanguageCode' };
+
+export function asLanguageCode(language: string): LanguageCode {
+  return language as LanguageCode;
+}
+
+const ISO_639_3_TO_1: Record<string, string | undefined> = {
+  afr: 'af',
+  amh: 'am',
+  ara: 'ar',
+  hye: 'hy',
+  asm: 'as',
+  ast: undefined,
+  aze: 'az',
+  bel: 'be',
+  ben: 'bn',
+  bos: 'bs',
+  bul: 'bg',
+  mya: 'my',
+  yue: undefined,
+  cat: 'ca',
+  ceb: undefined,
+  cmn: 'zh',
+  nya: 'ny',
+  hrv: 'hr',
+  ces: 'cs',
+  dan: 'da',
+  nld: 'nl',
+  eng: 'en',
+  est: 'et',
+  fil: undefined,
+  fin: 'fi',
+  fra: 'fr',
+  ful: 'ff',
+  glg: 'gl',
+  lug: 'lg',
+  kat: 'ka',
+  deu: 'de',
+  ell: 'el',
+  guj: 'gu',
+  hau: 'ha',
+  heb: 'he',
+  hin: 'hi',
+  hun: 'hu',
+  isl: 'is',
+  ibo: 'ig',
+  ind: 'id',
+  gle: 'ga',
+  ita: 'it',
+  jpn: 'ja',
+  jav: 'jv',
+  kea: undefined,
+  kan: 'kn',
+  kaz: 'kk',
+  khm: 'km',
+  kor: 'ko',
+  kur: 'ku',
+  kir: 'ky',
+  lao: 'lo',
+  lav: 'lv',
+  lin: 'ln',
+  lit: 'lt',
+  luo: undefined,
+  ltz: 'lb',
+  mkd: 'mk',
+  msa: 'ms',
+  mal: 'ml',
+  mlt: 'mt',
+  zho: 'zh',
+  mri: 'mi',
+  mar: 'mr',
+  mon: 'mn',
+  nep: 'ne',
+  nso: undefined,
+  nor: 'no',
+  oci: 'oc',
+  ori: 'or',
+  pus: 'ps',
+  fas: 'fa',
+  pol: 'pl',
+  por: 'pt',
+  pan: 'pa',
+  ron: 'ro',
+  rus: 'ru',
+  srp: 'sr',
+  sna: 'sn',
+  snd: 'sd',
+  slk: 'sk',
+  slv: 'sl',
+  som: 'so',
+  spa: 'es',
+  swa: 'sw',
+  swe: 'sv',
+  tam: 'ta',
+  tgk: 'tg',
+  tel: 'te',
+  tha: 'th',
+  tur: 'tr',
+  ukr: 'uk',
+  umb: undefined,
+  urd: 'ur',
+  uzb: 'uz',
+  vie: 'vi',
+  cym: 'cy',
+  wol: 'wo',
+  xho: 'xh',
+  zul: 'zu',
+};
+
+const LANGUAGE_NAMES_TO_CODE: Record<string, string> = {
+  afrikaans: 'af',
+  albanian: 'sq',
+  amharic: 'am',
+  arabic: 'ar',
+  armenian: 'hy',
+  azerbaijani: 'az',
+  basque: 'eu',
+  belarusian: 'be',
+  bengali: 'bn',
+  bosnian: 'bs',
+  bulgarian: 'bg',
+  burmese: 'my',
+  catalan: 'ca',
+  chinese: 'zh',
+  croatian: 'hr',
+  czech: 'cs',
+  danish: 'da',
+  dutch: 'nl',
+  english: 'en',
+  estonian: 'et',
+  finnish: 'fi',
+  french: 'fr',
+  galician: 'gl',
+  georgian: 'ka',
+  german: 'de',
+  greek: 'el',
+  gujarati: 'gu',
+  hausa: 'ha',
+  hebrew: 'he',
+  hindi: 'hi',
+  hungarian: 'hu',
+  icelandic: 'is',
+  indonesian: 'id',
+  irish: 'ga',
+  italian: 'it',
+  japanese: 'ja',
+  javanese: 'jv',
+  kannada: 'kn',
+  kazakh: 'kk',
+  khmer: 'km',
+  korean: 'ko',
+  kurdish: 'ku',
+  kyrgyz: 'ky',
+  lao: 'lo',
+  latvian: 'lv',
+  lingala: 'ln',
+  lithuanian: 'lt',
+  luxembourgish: 'lb',
+  macedonian: 'mk',
+  malay: 'ms',
+  malayalam: 'ml',
+  maltese: 'mt',
+  maori: 'mi',
+  marathi: 'mr',
+  mongolian: 'mn',
+  nepali: 'ne',
+  norwegian: 'no',
+  occitan: 'oc',
+  oriya: 'or',
+  pashto: 'ps',
+  persian: 'fa',
+  polish: 'pl',
+  portuguese: 'pt',
+  punjabi: 'pa',
+  romanian: 'ro',
+  russian: 'ru',
+  serbian: 'sr',
+  shona: 'sn',
+  sindhi: 'sd',
+  slovak: 'sk',
+  slovene: 'sl',
+  slovenian: 'sl',
+  somali: 'so',
+  spanish: 'es',
+  swahili: 'sw',
+  swedish: 'sv',
+  tagalog: 'tl',
+  tamil: 'ta',
+  tajik: 'tg',
+  telugu: 'te',
+  thai: 'th',
+  turkish: 'tr',
+  ukrainian: 'uk',
+  urdu: 'ur',
+  uzbek: 'uz',
+  vietnamese: 'vi',
+  welsh: 'cy',
+  wolof: 'wo',
+  xhosa: 'xh',
+  yoruba: 'yo',
+  zulu: 'zu',
+};
+
+const CODE_TO_LANGUAGE_NAME: Record<string, string> = Object.fromEntries(
+  Object.entries(LANGUAGE_NAMES_TO_CODE).map(([name, code]) => [code, name]),
+);
+
+CODE_TO_LANGUAGE_NAME.sl = 'slovene';
+
+export function normalizeLanguage(language: string): LanguageCode {
+  const lowered = language.trim().toLowerCase();
+  if (lowered === '') {
+    return asLanguageCode('');
+  }
+
+  if (lowered in LANGUAGE_NAMES_TO_CODE) {
+    return asLanguageCode(LANGUAGE_NAMES_TO_CODE[lowered]!);
+  }
+
+  if (lowered in ISO_639_3_TO_1) {
+    return asLanguageCode(ISO_639_3_TO_1[lowered] ?? lowered);
+  }
+
+  const parts = lowered.replaceAll('_', '-').split('-');
+  if (parts.length >= 2) {
+    const [base, ...rest] = parts;
+    return asLanguageCode(
+      [
+        base,
+        ...rest.map((part) => {
+          if (part.length === 4) {
+            return part.charAt(0).toUpperCase() + part.slice(1);
+          }
+          return part.toUpperCase();
+        }),
+      ].join('-'),
+    );
+  }
+
+  return asLanguageCode(lowered);
+}
+
+export function getBaseLanguage(language: string): string {
+  const normalized = normalizeLanguage(language);
+  const [base = ''] = normalized.split('-');
+  return ISO_639_3_TO_1[base] ?? base;
+}
+
+export function getIsoLanguage(language: string): string {
+  const normalized = normalizeLanguage(language);
+  const region = getLanguageRegion(normalized);
+  const baseLanguage = getBaseLanguage(normalized);
+  return region ? `${baseLanguage}-${region}` : baseLanguage;
+}
+
+export function getLanguageRegion(language: string): string | undefined {
+  const normalized = normalizeLanguage(language);
+  const [, ...parts] = normalized.split('-');
+  return parts.find((part) => part.length === 2);
+}
+
+export function toLanguageName(language: string): string | undefined {
+  return CODE_TO_LANGUAGE_NAME[getBaseLanguage(language)];
+}
+
+export function areLanguagesEquivalent(
+  left: string | null | undefined,
+  right: string | null | undefined,
+): boolean {
+  if (left == null || right == null) {
+    return left === right;
+  }
+  return normalizeLanguage(left) === normalizeLanguage(right);
+}

package/src/llm/index.ts

@@ -65,8 +65,10 @@ export {
   executeToolCall,
   oaiBuildFunctionInfo,
   oaiParams,
+  serializeImage,
   toJsonSchema,
   type OpenAIFunctionParameters,
+  type SerializedImage,
 } from './utils.js';
 
 export {

package/src/stream/deferred_stream.ts

@@ -25,9 +25,13 @@ export function isStreamReaderReleaseError(e: unknown) {
     'Invalid state: The reader is not attached to a stream',
     'Controller is already closed',
     'WritableStream is closed',
+    // bun
+    'Stream reader cancelled via releaseLock()',
+    // deno
+    'The reader was released.',
   ];
 
-  if (e instanceof TypeError) {
+  if (e instanceof TypeError || (e instanceof Error && e.name === 'AbortError')) {
     return allowedMessages.some((message) => e.message.includes(message));
   }
 

package/src/stt/stt.ts

@@ -7,6 +7,7 @@ import { EventEmitter } from 'node:events';
 import type { ReadableStream } from 'node:stream/web';
 import { APIConnectionError, APIError } from '../_exceptions.js';
 import { calculateAudioDurationSeconds } from '../audio.js';
+import type { LanguageCode } from '../language.js';
 import { log } from '../log.js';
 import type { STTMetrics } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
@@ -50,7 +51,7 @@ export enum SpeechEventType {
 /** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
   /** Language code of the speech. */
-  language: string;
+  language: LanguageCode;
   /** Transcribed text. */
   text: string;
   /** Start time of the speech segment in seconds. */

package/src/utils.ts

@@ -846,7 +846,12 @@ export async function waitForParticipant({
     }
   };
 
+  const onDisconnected = () => {
+    fut.reject(new Error('Got disconnected from room while waiting for participant'));
+  };
+
   room.on(RoomEvent.ParticipantConnected, onParticipantConnected);
+  room.on(RoomEvent.Disconnected, onDisconnected);
 
   try {
     for (const p of room.remoteParticipants.values()) {
@@ -859,6 +864,7 @@ export async function waitForParticipant({
     return await fut.await;
   } finally {
     room.off(RoomEvent.ParticipantConnected, onParticipantConnected);
+    room.off(RoomEvent.Disconnected, onDisconnected);
   }
 }
 
@@ -953,3 +959,17 @@ export const isCloud = (url: URL) => {
   const hostname = url.hostname;
   return hostname.endsWith('.livekit.cloud') || hostname.endsWith('.livekit.run');
 };
+
+/**
+ * Whether the agent is running in development mode (launched via `dev` or `connect`).
+ */
+export const isDevMode = (): boolean => {
+  return process.env.LIVEKIT_DEV_MODE === '1';
+};
+
+/**
+ * Whether the agent is hosted on LiveKit Cloud.
+ */
+export const isHosted = (): boolean => {
+  return process.env.LIVEKIT_REMOTE_EOT_URL !== undefined;
+};