@livekit/agents 1.0.5 → 1.1.0

package/dist/inference/stt.js

@@ -0,0 +1,284 @@
+import {} from "@livekit/rtc-node";
+import { WebSocket } from "ws";
+import { APIError, APIStatusError } from "../_exceptions.js";
+import { AudioByteStream } from "../audio.js";
+import { log } from "../log.js";
+import {
+  STT as BaseSTT,
+  SpeechStream as BaseSpeechStream,
+  SpeechEventType
+} from "../stt/index.js";
+import { DEFAULT_API_CONNECT_OPTIONS } from "../types.js";
+import { Event, Task, cancelAndWait, shortuuid, waitForAbort } from "../utils.js";
+import { connectWs, createAccessToken } from "./utils.js";
+const DEFAULT_ENCODING = "pcm_s16le";
+const DEFAULT_SAMPLE_RATE = 16e3;
+const DEFAULT_BASE_URL = "wss://agent-gateway.livekit.cloud/v1";
+const DEFAULT_CANCEL_TIMEOUT = 5e3;
+class STT extends BaseSTT {
+  opts;
+  streams = /* @__PURE__ */ new Set();
+  constructor(opts) {
+    super({ streaming: true, interimResults: true });
+    const {
+      model,
+      language,
+      baseURL,
+      encoding = DEFAULT_ENCODING,
+      sampleRate = DEFAULT_SAMPLE_RATE,
+      apiKey,
+      apiSecret,
+      extraKwargs = {}
+    } = opts || {};
+    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
+    if (!lkApiKey) {
+      throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
+    }
+    const lkApiSecret = apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;
+    if (!lkApiSecret) {
+      throw new Error("apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET");
+    }
+    this.opts = {
+      model,
+      language,
+      encoding,
+      sampleRate,
+      baseURL: lkBaseURL,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      extraKwargs
+    };
+  }
+  get label() {
+    return "inference.STT";
+  }
+  async _recognize(_) {
+    throw new Error("LiveKit STT does not support batch recognition, use stream() instead");
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+    for (const stream of this.streams) {
+      stream.updateOptions(opts);
+    }
+  }
+  stream(options) {
+    const { language, connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const streamOpts = {
+      ...this.opts,
+      language: language ?? this.opts.language
+    };
+    const stream = new SpeechStream(this, streamOpts, connOptions);
+    this.streams.add(stream);
+    return stream;
+  }
+}
+class SpeechStream extends BaseSpeechStream {
+  opts;
+  requestId = shortuuid("stt_request_");
+  speaking = false;
+  speechDuration = 0;
+  reconnectEvent = new Event();
+  #logger = log();
+  constructor(sttImpl, opts, connOptions) {
+    super(sttImpl, opts.sampleRate, connOptions);
+    this.opts = opts;
+  }
+  get label() {
+    return "inference.SpeechStream";
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+  }
+  async run() {
+    let ws = null;
+    let closingWs = false;
+    this.reconnectEvent.set();
+    const connect = async () => {
+      const params = {
+        settings: {
+          sample_rate: String(this.opts.sampleRate),
+          encoding: this.opts.encoding,
+          extra: this.opts.extraKwargs
+        }
+      };
+      if (this.opts.model) {
+        params.model = this.opts.model;
+      }
+      if (this.opts.language) {
+        params.settings.language = this.opts.language;
+      }
+      let baseURL = this.opts.baseURL;
+      if (baseURL.startsWith("http://") || baseURL.startsWith("https://")) {
+        baseURL = baseURL.replace("http", "ws");
+      }
+      const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);
+      const url = `${baseURL}/stt`;
+      const headers = { Authorization: `Bearer ${token}` };
+      const socket = await connectWs(url, headers, 1e4);
+      const msg = { ...params, type: "session.create" };
+      socket.send(JSON.stringify(msg));
+      return socket;
+    };
+    const send = async (socket, signal) => {
+      const audioStream = new AudioByteStream(
+        this.opts.sampleRate,
+        1,
+        Math.floor(this.opts.sampleRate / 20)
+        // 50ms
+      );
+      for await (const ev of this.input) {
+        if (signal.aborted) break;
+        let frames;
+        if (ev === SpeechStream.FLUSH_SENTINEL) {
+          frames = audioStream.flush();
+        } else {
+          const frame = ev;
+          frames = audioStream.write(new Int16Array(frame.data).buffer);
+        }
+        for (const frame of frames) {
+          this.speechDuration += frame.samplesPerChannel / frame.sampleRate;
+          const base64 = Buffer.from(frame.data.buffer).toString("base64");
+          const msg = { type: "input_audio", audio: base64 };
+          socket.send(JSON.stringify(msg));
+        }
+      }
+      closingWs = true;
+      socket.send(JSON.stringify({ type: "session.finalize" }));
+    };
+    const recv = async (socket, signal) => {
+      while (!this.closed && !signal.aborted) {
+        const dataPromise = new Promise((resolve, reject) => {
+          const messageHandler = (d) => {
+            resolve(d.toString());
+            removeListeners();
+          };
+          const errorHandler = (e) => {
+            reject(e);
+            removeListeners();
+          };
+          const closeHandler = (code) => {
+            if (closingWs) {
+              resolve("");
+            } else {
+              reject(
+                new APIStatusError({
+                  message: "LiveKit STT connection closed unexpectedly",
+                  options: { statusCode: code }
+                })
+              );
+            }
+            removeListeners();
+          };
+          const removeListeners = () => {
+            socket.removeListener("message", messageHandler);
+            socket.removeListener("error", errorHandler);
+            socket.removeListener("close", closeHandler);
+          };
+          socket.once("message", messageHandler);
+          socket.once("error", errorHandler);
+          socket.once("close", closeHandler);
+        });
+        const data = await Promise.race([dataPromise, waitForAbort(signal)]);
+        if (!data || signal.aborted) return;
+        const json = JSON.parse(data);
+        const type = json.type;
+        switch (type) {
+          case "session.created":
+          case "session.finalized":
+          case "session.closed":
+            break;
+          case "interim_transcript":
+            this.processTranscript(json, false);
+            break;
+          case "final_transcript":
+            this.processTranscript(json, true);
+            break;
+          case "error":
+            this.#logger.error("received error from LiveKit STT: %o", json);
+            throw new APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);
+          default:
+            this.#logger.warn("received unexpected message from LiveKit STT: %o", json);
+            break;
+        }
+      }
+    };
+    while (true) {
+      try {
+        ws = await connect();
+        const sendTask = Task.from(async ({ signal }) => {
+          await send(ws, signal);
+        });
+        const recvTask = Task.from(async ({ signal }) => {
+          await recv(ws, signal);
+        });
+        const tasks = [sendTask, recvTask];
+        const waitReconnectTask = Task.from(async ({ signal }) => {
+          await Promise.race([this.reconnectEvent.wait(), waitForAbort(signal)]);
+        });
+        try {
+          await Promise.race([
+            Promise.all(tasks.map((task) => task.result)),
+            waitReconnectTask.result
+          ]);
+          if (!waitReconnectTask.done) break;
+          this.reconnectEvent.clear();
+        } finally {
+          await cancelAndWait([sendTask, recvTask, waitReconnectTask], DEFAULT_CANCEL_TIMEOUT);
+        }
+      } finally {
+        try {
+          if (ws) ws.close();
+        } catch {
+        }
+      }
+    }
+  }
+  processTranscript(data, isFinal) {
+    const requestId = data.request_id ?? this.requestId;
+    const text = data.transcript ?? "";
+    const language = data.language ?? this.opts.language ?? "en";
+    if (!text && !isFinal) return;
+    if (!this.speaking) {
+      this.speaking = true;
+      this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
+    }
+    const speechData = {
+      language,
+      startTime: data.start ?? 0,
+      endTime: data.duration ?? 0,
+      confidence: data.confidence ?? 1,
+      text
+    };
+    if (isFinal) {
+      if (this.speechDuration > 0) {
+        this.queue.put({
+          type: SpeechEventType.RECOGNITION_USAGE,
+          requestId,
+          recognitionUsage: { audioDuration: this.speechDuration }
+        });
+        this.speechDuration = 0;
+      }
+      this.queue.put({
+        type: SpeechEventType.FINAL_TRANSCRIPT,
+        requestId,
+        alternatives: [speechData]
+      });
+      if (this.speaking) {
+        this.speaking = false;
+        this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+      }
+    } else {
+      this.queue.put({
+        type: SpeechEventType.INTERIM_TRANSCRIPT,
+        requestId,
+        alternatives: [speechData]
+      });
+    }
+  }
+}
+export {
+  STT,
+  SpeechStream
+};
+//# sourceMappingURL=stt.js.map

package/dist/inference/stt.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame } from '@livekit/rtc-node';\nimport { type RawData, WebSocket } from 'ws';\nimport { APIError, APIStatusError } from '../_exceptions.js';\nimport { AudioByteStream } from '../audio.js';\nimport { log } from '../log.js';\nimport {\n  STT as BaseSTT,\n  SpeechStream as BaseSpeechStream,\n  type SpeechData,\n  type SpeechEvent,\n  SpeechEventType,\n} from '../stt/index.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';\nimport { type AnyModels, connectWs, createAccessToken } from './utils.js';\n\nexport type DeepgramModels =\n  | 'deepgram'\n  | 'deepgram/nova-3'\n  | 'deepgram/nova-3-general'\n  | 'deepgram/nova-3-medical'\n  | 'deepgram/nova-2'\n  | 'deepgram/nova-2-general'\n  | 'deepgram/nova-2-medical'\n  | 'deepgram/nova-2-phonecall';\n\nexport type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';\n\nexport type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';\n\nexport interface CartesiaOptions {\n  min_volume?: number; // default: not specified\n  max_silence_duration_secs?: number; // default: not specified\n}\n\nexport interface DeepgramOptions {\n  filler_words?: boolean; // default: true\n  interim_results?: boolean; // default: true\n  endpointing?: number; // default: 25 (ms)\n  punctuate?: boolean; // default: false\n  smart_format?: boolean;\n  keywords?: Array<[string, number]>;\n  keyterms?: string[];\n  profanity_filter?: boolean;\n  numerals?: boolean;\n  mip_opt_out?: boolean;\n}\n\nexport interface AssemblyaiOptions {\n  format_turns?: boolean; // default: false\n  end_of_turn_confidence_threshold?: number; // default: 0.01\n  min_end_of_turn_silence_when_confident?: number; // default: 0\n  max_turn_silence?: number; // default: not specified\n  keyterms_prompt?: string[]; // default: not specified\n}\n\nexport type STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels | AnyModels;\nexport type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels\n  ? DeepgramOptions\n  : TModel extends CartesiaModels\n    ? CartesiaOptions\n    : TModel extends AssemblyaiModels\n      ? AssemblyaiOptions\n      : Record<string, unknown>;\n\nexport type STTLanguages = 'en' | 'de' | 'es' | 'fr' | 'ja' | 'pt' | 'zh';\nexport type STTEncoding = 'pcm_s16le';\n\nconst DEFAULT_ENCODING: STTEncoding = 'pcm_s16le';\nconst DEFAULT_SAMPLE_RATE = 16000;\nconst DEFAULT_BASE_URL = 'wss://agent-gateway.livekit.cloud/v1';\nconst DEFAULT_CANCEL_TIMEOUT = 5000;\n\nexport interface InferenceSTTOptions<TModel extends STTModels> {\n  model: TModel;\n  language?: STTLanguages | string;\n  encoding: STTEncoding;\n  sampleRate: number;\n  baseURL: string;\n  apiKey: string;\n  apiSecret: string;\n  extraKwargs: STTOptions<TModel>;\n}\n\nexport class STT<TModel extends STTModels> extends BaseSTT {\n  private opts: InferenceSTTOptions<TModel>;\n  private streams: Set<SpeechStream<TModel>> = new Set();\n\n  constructor(opts: {\n    model: TModel;\n    language?: STTLanguages | string;\n    baseURL?: string;\n    encoding?: STTEncoding;\n    sampleRate?: number;\n    apiKey?: string;\n    apiSecret?: string;\n    extraKwargs?: STTOptions<TModel>;\n  }) {\n    super({ streaming: true, interimResults: true });\n\n    const {\n      model,\n      language,\n      baseURL,\n      encoding = DEFAULT_ENCODING,\n      sampleRate = DEFAULT_SAMPLE_RATE,\n      apiKey,\n      apiSecret,\n      extraKwargs = {} as STTOptions<TModel>,\n    } = opts || {};\n\n    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;\n    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n    if (!lkApiKey) {\n      throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n    }\n\n    const lkApiSecret =\n      apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n    if (!lkApiSecret) {\n      throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n    }\n\n    this.opts = {\n      model,\n      language,\n      encoding,\n      sampleRate,\n      baseURL: lkBaseURL,\n      apiKey: lkApiKey,\n      apiSecret: lkApiSecret,\n      extraKwargs,\n    };\n  }\n\n  get label(): string {\n    return 'inference.STT';\n  }\n\n  protected async _recognize(_: AudioBuffer): Promise<SpeechEvent> {\n    throw new Error('LiveKit STT does not support batch recognition, use stream() instead');\n  }\n\n  updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {\n    this.opts = { ...this.opts, ...opts };\n\n    for (const stream of this.streams) {\n      stream.updateOptions(opts);\n    }\n  }\n\n  stream(options?: {\n    language?: STTLanguages | string;\n    connOptions?: APIConnectOptions;\n  }): SpeechStream<TModel> {\n    const { language, connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};\n    const streamOpts = {\n      ...this.opts,\n      language: language ?? this.opts.language,\n    } as InferenceSTTOptions<TModel>;\n\n    const stream = new SpeechStream(this, streamOpts, connOptions);\n    this.streams.add(stream);\n\n    return stream;\n  }\n}\n\nexport class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {\n  private opts: InferenceSTTOptions<TModel>;\n  private requestId = shortuuid('stt_request_');\n  private speaking = false;\n  private speechDuration = 0;\n  private reconnectEvent = new Event();\n\n  #logger = log();\n\n  constructor(\n    sttImpl: STT<TModel>,\n    opts: InferenceSTTOptions<TModel>,\n    connOptions: APIConnectOptions,\n  ) {\n    super(sttImpl, opts.sampleRate, connOptions);\n    this.opts = opts;\n  }\n\n  get label(): string {\n    return 'inference.SpeechStream';\n  }\n\n  updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {\n    this.opts = { ...this.opts, ...opts };\n  }\n\n  protected async run(): Promise<void> {\n    let ws: WebSocket | null = null;\n    let closingWs = false;\n\n    this.reconnectEvent.set();\n\n    const connect = async () => {\n      const params = {\n        settings: {\n          sample_rate: String(this.opts.sampleRate),\n          encoding: this.opts.encoding,\n          extra: this.opts.extraKwargs,\n        },\n      } as Record<string, unknown>;\n\n      if (this.opts.model) {\n        params.model = this.opts.model;\n      }\n\n      if (this.opts.language) {\n        (params.settings as Record<string, unknown>).language = this.opts.language;\n      }\n\n      let baseURL = this.opts.baseURL;\n      if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {\n        baseURL = baseURL.replace('http', 'ws');\n      }\n\n      const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);\n      const url = `${baseURL}/stt`;\n      const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;\n\n      const socket = await connectWs(url, headers, 10000);\n      const msg = { ...params, type: 'session.create' };\n      socket.send(JSON.stringify(msg));\n\n      return socket;\n    };\n\n    const send = async (socket: WebSocket, signal: AbortSignal) => {\n      const audioStream = new AudioByteStream(\n        this.opts.sampleRate,\n        1,\n        Math.floor(this.opts.sampleRate / 20), // 50ms\n      );\n\n      for await (const ev of this.input) {\n        if (signal.aborted) break;\n        let frames: AudioFrame[];\n\n        if (ev === SpeechStream.FLUSH_SENTINEL) {\n          frames = audioStream.flush();\n        } else {\n          const frame = ev as AudioFrame;\n          frames = audioStream.write(new Int16Array(frame.data).buffer);\n        }\n\n        for (const frame of frames) {\n          this.speechDuration += frame.samplesPerChannel / frame.sampleRate;\n          const base64 = Buffer.from(frame.data.buffer).toString('base64');\n          const msg = { type: 'input_audio', audio: base64 };\n          socket.send(JSON.stringify(msg));\n        }\n      }\n\n      closingWs = true;\n      socket.send(JSON.stringify({ type: 'session.finalize' }));\n    };\n\n    const recv = async (socket: WebSocket, signal: AbortSignal) => {\n      while (!this.closed && !signal.aborted) {\n        const dataPromise = new Promise<string>((resolve, reject) => {\n          const messageHandler = (d: RawData) => {\n            resolve(d.toString());\n            removeListeners();\n          };\n          const errorHandler = (e: Error) => {\n            reject(e);\n            removeListeners();\n          };\n          const closeHandler = (code: number) => {\n            if (closingWs) {\n              resolve('');\n            } else {\n              reject(\n                new APIStatusError({\n                  message: 'LiveKit STT connection closed unexpectedly',\n                  options: { statusCode: code },\n                }),\n              );\n            }\n            removeListeners();\n          };\n          const removeListeners = () => {\n            socket.removeListener('message', messageHandler);\n            socket.removeListener('error', errorHandler);\n            socket.removeListener('close', closeHandler);\n          };\n          socket.once('message', messageHandler);\n          socket.once('error', errorHandler);\n          socket.once('close', closeHandler);\n        });\n\n        const data = await Promise.race([dataPromise, waitForAbort(signal)]);\n\n        if (!data || signal.aborted) return;\n\n        const json = JSON.parse(data);\n        const type = json.type as string | undefined;\n\n        switch (type) {\n          case 'session.created':\n          case 'session.finalized':\n          case 'session.closed':\n            break;\n          case 'interim_transcript':\n            this.processTranscript(json, false);\n            break;\n          case 'final_transcript':\n            this.processTranscript(json, true);\n            break;\n          case 'error':\n            this.#logger.error('received error from LiveKit STT: %o', json);\n            throw new APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);\n          default:\n            this.#logger.warn('received unexpected message from LiveKit STT: %o', json);\n            break;\n        }\n      }\n    };\n\n    while (true) {\n      try {\n        ws = await connect();\n\n        const sendTask = Task.from(async ({ signal }) => {\n          await send(ws!, signal);\n        });\n\n        const recvTask = Task.from(async ({ signal }) => {\n          await recv(ws!, signal);\n        });\n\n        const tasks = [sendTask, recvTask];\n        const waitReconnectTask = Task.from(async ({ signal }) => {\n          await Promise.race([this.reconnectEvent.wait(), waitForAbort(signal)]);\n        });\n\n        try {\n          await Promise.race([\n            Promise.all(tasks.map((task) => task.result)),\n            waitReconnectTask.result,\n          ]);\n\n          if (!waitReconnectTask.done) break;\n          this.reconnectEvent.clear();\n        } finally {\n          await cancelAndWait([sendTask, recvTask, waitReconnectTask], DEFAULT_CANCEL_TIMEOUT);\n        }\n      } finally {\n        try {\n          if (ws) ws.close();\n        } catch {}\n      }\n    }\n  }\n\n  private processTranscript(data: Record<string, any>, isFinal: boolean) {\n    const requestId = data.request_id ?? this.requestId;\n    const text = data.transcript ?? '';\n    const language = data.language ?? this.opts.language ?? 'en';\n\n    if (!text && !isFinal) return;\n\n    // We'll have a more accurate way of detecting when speech started when we have VAD\n    if (!this.speaking) {\n      this.speaking = true;\n      this.queue.put({ type: SpeechEventType.START_OF_SPEECH });\n    }\n\n    const speechData: SpeechData = {\n      language,\n      startTime: data.start ?? 0,\n      endTime: data.duration ?? 0,\n      confidence: data.confidence ?? 1.0,\n      text,\n    };\n\n    if (isFinal) {\n      if (this.speechDuration > 0) {\n        this.queue.put({\n          type: SpeechEventType.RECOGNITION_USAGE,\n          requestId,\n          recognitionUsage: { audioDuration: this.speechDuration },\n        });\n        this.speechDuration = 0;\n      }\n\n      this.queue.put({\n        type: SpeechEventType.FINAL_TRANSCRIPT,\n        requestId,\n        alternatives: [speechData],\n      });\n\n      if (this.speaking) {\n        this.speaking = false;\n        this.queue.put({ type: SpeechEventType.END_OF_SPEECH });\n      }\n    } else {\n      this.queue.put({\n        type: SpeechEventType.INTERIM_TRANSCRIPT,\n        requestId,\n        alternatives: [speechData],\n      });\n    }\n  }\n}\n"],"mappings":"AAGA,eAAgC;AAChC,SAAuB,iBAAiB;AACxC,SAAS,UAAU,sBAAsB;AACzC,SAAS,uBAAuB;AAChC,SAAS,WAAW;AACpB;AAAA,EACE,OAAO;AAAA,EACP,gBAAgB;AAAA,EAGhB;AAAA,OACK;AACP,SAAiC,mCAAmC;AACpE,SAA2B,OAAO,MAAM,eAAe,WAAW,oBAAoB;AACtF,SAAyB,WAAW,yBAAyB;AAsD7D,MAAM,mBAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,mBAAmB;AACzB,MAAM,yBAAyB;AAaxB,MAAM,YAAsC,QAAQ;AAAA,EACjD;AAAA,EACA,UAAqC,oBAAI,IAAI;AAAA,EAErD,YAAY,MAST;AACD,UAAM,EAAE,WAAW,MAAM,gBAAgB,KAAK,CAAC;AAE/C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA,cAAc,CAAC;AAAA,IACjB,IAAI,QAAQ,CAAC;AAEb,UAAM,YAAY,WAAW,QAAQ,IAAI,yBAAyB;AAClE,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAEA,SAAK,OAAO;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,WAAW,GAAsC;AAC/D,UAAM,IAAI,MAAM,sEAAsE;AAAA,EACxF;AAAA,EAEA,cAAc,MAA8E;AAC1F,SAAK,OAAO,EAAE,GAAG,KAAK,MAAM,GAAG,KAAK;AAEpC,eAAW,UAAU,KAAK,SAAS;AACjC,aAAO,cAAc,IAAI;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,OAAO,SAGkB;AACvB,UAAM,EAAE,UAAU,cAAc,4BAA4B,IAAI,WAAW,CAAC;AAC5E,UAAM,aAAa;AAAA,MACjB,GAAG,KAAK;AAAA,MACR,UAAU,YAAY,KAAK,KAAK;AAAA,IAClC;AAEA,UAAM,SAAS,IAAI,aAAa,MAAM,YAAY,WAAW;AAC7D,SAAK,QAAQ,IAAI,MAAM;AAEvB,WAAO;AAAA,EACT;AACF;AAEO,MAAM,qBAA+C,iBAAiB;AAAA,EACnE;AAAA,EACA,YAAY,UAAU,cAAc;AAAA,EACpC,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB,IAAI,MAAM;AAAA,EAEnC,UAAU,IAAI;AAAA,EAEd,YACE,SACA,MACA,aACA;AACA,UAAM,SAAS,KAAK,YAAY,WAAW;AAC3C,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,MAA8E;AAC1F,SAAK,OAAO,EAAE,GAAG,KAAK,MAAM,GAAG,KAAK;AAAA,EACtC;AAAA,EAEA,MAAgB,MAAqB;AACnC,QAAI,KAAuB;AAC3B,QAAI,YAAY;AAEhB,SAAK,eAAe,IAAI;AAExB,UAAM,UAAU,YAAY;AAC1B,YAAM,SAAS;AAAA,QACb,UAAU;AAAA,UACR,aAAa,OAAO,KAAK,KAAK,UAAU;AAAA,UACxC,UAAU,KAAK,KAAK;AAAA,UACpB,OAAO,KAAK,KAAK;AAAA,QACnB;AAAA,MACF;AAEA,UAAI,KAAK,KAAK,OAAO;AACnB,eAAO,QAAQ,KAAK,KAAK;AAAA,MAC3B;AAEA,UAAI,KAAK,KAAK,UAAU;AACtB,QAAC,OAAO,SAAqC,WAAW,KAAK,KAAK;AAAA,MACpE;AAEA,UAAI,UAAU,KAAK,KAAK;AACxB,UAAI,QAAQ,WAAW,SAAS,KAAK,QAAQ,WAAW,UAAU,GAAG;AACnE,kBAAU,QAAQ,QAAQ,QAAQ,IAAI;AAAA,MACxC;AAEA,YAAM,QAAQ,MAAM,kBAAkB,KAAK,KAAK,QAAQ,KAAK,KAAK,SAAS;AAC3E,YAAM,MAAM,GAAG,OAAO;AACtB,YAAM,UAAU,EAAE,eAAe,UAAU,KAAK,GAAG;AAEnD,YAAM,SAAS,MAAM,UAAU,KAAK,SAAS,GAAK;AAClD,YAAM,MAAM,EAAE,GAAG,QAAQ,MAAM,iBAAiB;AAChD,aAAO,KAAK,KAAK,UAAU,GAAG,CAAC;AAE/B,aAAO;AAAA,IACT;AAEA,UAAM,OAAO,OAAO,QAAmB,WAAwB;AAC7D,YAAM,cAAc,IAAI;AAAA,QACtB,KAAK,KAAK;AAAA,QACV;AAAA,QACA,KAAK,MAAM,KAAK,KAAK,aAAa,EAAE;AAAA;AAAA,MACtC;AAEA,uBAAiB,MAAM,KAAK,OAAO;AACjC,YAAI,OAAO,QAAS;AACpB,YAAI;AAEJ,YAAI,OAAO,aAAa,gBAAgB;AACtC,mBAAS,YAAY,MAAM;AAAA,QAC7B,OAAO;AACL,gBAAM,QAAQ;AACd,mBAAS,YAAY,MAAM,IAAI,WAAW,MAAM,IAAI,EAAE,MAAM;AAAA,QAC9D;AAEA,mBAAW,SAAS,QAAQ;AAC1B,eAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,gBAAM,SAAS,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAC/D,gBAAM,MAAM,EAAE,MAAM,eAAe,OAAO,OAAO;AACjD,iBAAO,KAAK,KAAK,UAAU,GAAG,CAAC;AAAA,QACjC;AAAA,MACF;AAEA,kBAAY;AACZ,aAAO,KAAK,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC,CAAC;AAAA,IAC1D;AAEA,UAAM,OAAO,OAAO,QAAmB,WAAwB;AAC7D,aAAO,CAAC,KAAK,UAAU,CAAC,OAAO,SAAS;AACtC,cAAM,cAAc,IAAI,QAAgB,CAAC,SAAS,WAAW;AAC3D,gBAAM,iBAAiB,CAAC,MAAe;AACrC,oBAAQ,EAAE,SAAS,CAAC;AACpB,4BAAgB;AAAA,UAClB;AACA,gBAAM,eAAe,CAAC,MAAa;AACjC,mBAAO,CAAC;AACR,4BAAgB;AAAA,UAClB;AACA,gBAAM,eAAe,CAAC,SAAiB;AACrC,gBAAI,WAAW;AACb,sBAAQ,EAAE;AAAA,YACZ,OAAO;AACL;AAAA,gBACE,IAAI,eAAe;AAAA,kBACjB,SAAS;AAAA,kBACT,SAAS,EAAE,YAAY,KAAK;AAAA,gBAC9B,CAAC;AAAA,cACH;AAAA,YACF;AACA,4BAAgB;AAAA,UAClB;AACA,gBAAM,kBAAkB,MAAM;AAC5B,mBAAO,eAAe,WAAW,cAAc;AAC/C,mBAAO,eAAe,SAAS,YAAY;AAC3C,mBAAO,eAAe,SAAS,YAAY;AAAA,UAC7C;AACA,iBAAO,KAAK,WAAW,cAAc;AACrC,iBAAO,KAAK,SAAS,YAAY;AACjC,iBAAO,KAAK,SAAS,YAAY;AAAA,QACnC,CAAC;AAED,cAAM,OAAO,MAAM,QAAQ,KAAK,CAAC,aAAa,aAAa,MAAM,CAAC,CAAC;AAEnE,YAAI,CAAC,QAAQ,OAAO,QAAS;AAE7B,cAAM,OAAO,KAAK,MAAM,IAAI;AAC5B,cAAM,OAAO,KAAK;AAElB,gBAAQ,MAAM;AAAA,UACZ,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AACH;AAAA,UACF,KAAK;AACH,iBAAK,kBAAkB,MAAM,KAAK;AAClC;AAAA,UACF,KAAK;AACH,iBAAK,kBAAkB,MAAM,IAAI;AACjC;AAAA,UACF,KAAK;AACH,iBAAK,QAAQ,MAAM,uCAAuC,IAAI;AAC9D,kBAAM,IAAI,SAAS,+BAA+B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,UAC1E;AACE,iBAAK,QAAQ,KAAK,oDAAoD,IAAI;AAC1E;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,WAAO,MAAM;AACX,UAAI;AACF,aAAK,MAAM,QAAQ;AAEnB,cAAM,WAAW,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC/C,gBAAM,KAAK,IAAK,MAAM;AAAA,QACxB,CAAC;AAED,cAAM,WAAW,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC/C,gBAAM,KAAK,IAAK,MAAM;AAAA,QACxB,CAAC;AAED,cAAM,QAAQ,CAAC,UAAU,QAAQ;AACjC,cAAM,oBAAoB,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AACxD,gBAAM,QAAQ,KAAK,CAAC,KAAK,eAAe,KAAK,GAAG,aAAa,MAAM,CAAC,CAAC;AAAA,QACvE,CAAC;AAED,YAAI;AACF,gBAAM,QAAQ,KAAK;AAAA,YACjB,QAAQ,IAAI,MAAM,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AAAA,YAC5C,kBAAkB;AAAA,UACpB,CAAC;AAED,cAAI,CAAC,kBAAkB,KAAM;AAC7B,eAAK,eAAe,MAAM;AAAA,QAC5B,UAAE;AACA,gBAAM,cAAc,CAAC,UAAU,UAAU,iBAAiB,GAAG,sBAAsB;AAAA,QACrF;AAAA,MACF,UAAE;AACA,YAAI;AACF,cAAI,GAAI,IAAG,MAAM;AAAA,QACnB,QAAQ;AAAA,QAAC;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kBAAkB,MAA2B,SAAkB;AACrE,UAAM,YAAY,KAAK,cAAc,KAAK;AAC1C,UAAM,OAAO,KAAK,cAAc;AAChC,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK,YAAY;AAExD,QAAI,CAAC,QAAQ,CAAC,QAAS;AAGvB,QAAI,CAAC,KAAK,UAAU;AAClB,WAAK,WAAW;AAChB,WAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AAAA,IAC1D;AAEA,UAAM,aAAyB;AAAA,MAC7B;AAAA,MACA,WAAW,KAAK,SAAS;AAAA,MACzB,SAAS,KAAK,YAAY;AAAA,MAC1B,YAAY,KAAK,cAAc;AAAA,MAC/B;AAAA,IACF;AAEA,QAAI,SAAS;AACX,UAAI,KAAK,iBAAiB,GAAG;AAC3B,aAAK,MAAM,IAAI;AAAA,UACb,MAAM,gBAAgB;AAAA,UACtB;AAAA,UACA,kBAAkB,EAAE,eAAe,KAAK,eAAe;AAAA,QACzD,CAAC;AACD,aAAK,iBAAiB;AAAA,MACxB;AAEA,WAAK,MAAM,IAAI;AAAA,QACb,MAAM,gBAAgB;AAAA,QACtB;AAAA,QACA,cAAc,CAAC,UAAU;AAAA,MAC3B,CAAC;AAED,UAAI,KAAK,UAAU;AACjB,aAAK,WAAW;AAChB,aAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAAA,MACxD;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI;AAAA,QACb,MAAM,gBAAgB;AAAA,QACtB;AAAA,QACA,cAAc,CAAC,UAAU;AAAA,MAC3B,CAAC;AAAA,IACH;AAAA,EACF;AACF;","names":[]}

package/dist/inference/tts.cjs

@@ -0,0 +1,317 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tts_exports = {};
+__export(tts_exports, {
+  SynthesizeStream: () => SynthesizeStream,
+  TTS: () => TTS
+});
+module.exports = __toCommonJS(tts_exports);
+var import_ws = require("ws");
+var import_exceptions = require("../_exceptions.cjs");
+var import_audio = require("../audio.cjs");
+var import_log = require("../log.cjs");
+var import_stream_channel = require("../stream/stream_channel.cjs");
+var import_tokenize = require("../tokenize/index.cjs");
+var import_tts = require("../tts/index.cjs");
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_api_protos = require("./api_protos.cjs");
+var import_utils2 = require("./utils.cjs");
+const DEFAULT_ENCODING = "pcm_s16le";
+const DEFAULT_SAMPLE_RATE = 16e3;
+const DEFAULT_BASE_URL = "https://agent-gateway.livekit.cloud/v1";
+const NUM_CHANNELS = 1;
+const DEFAULT_LANGUAGE = "en";
+class TTS extends import_tts.TTS {
+  opts;
+  streams = /* @__PURE__ */ new Set();
+  #logger = (0, import_log.log)();
+  constructor(opts) {
+    const sampleRate = (opts == null ? void 0 : opts.sampleRate) ?? DEFAULT_SAMPLE_RATE;
+    super(sampleRate, 1, { streaming: true });
+    const {
+      model,
+      voice,
+      language = DEFAULT_LANGUAGE,
+      baseURL,
+      encoding = DEFAULT_ENCODING,
+      apiKey,
+      apiSecret,
+      extraKwargs = {}
+    } = opts || {};
+    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
+    if (!lkApiKey) {
+      throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
+    }
+    const lkApiSecret = apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;
+    if (!lkApiSecret) {
+      throw new Error("apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET");
+    }
+    let nextModel = model;
+    let nextVoice = voice;
+    if (typeof nextModel === "string") {
+      const idx = nextModel.lastIndexOf(":");
+      if (idx !== -1) {
+        const voiceFromModel = nextModel.slice(idx + 1);
+        if (nextVoice && nextVoice !== voiceFromModel) {
+          this.#logger.warn(
+            "`voice` is provided via both argument and model, using the one from the argument",
+            { voice: nextVoice, model: nextModel }
+          );
+        } else {
+          nextVoice = voiceFromModel;
+        }
+        nextModel = nextModel.slice(0, idx);
+      }
+    }
+    this.opts = {
+      model: nextModel,
+      voice: nextVoice,
+      language,
+      encoding,
+      sampleRate,
+      baseURL: lkBaseURL,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      extraKwargs
+    };
+  }
+  get label() {
+    return "inference.TTS";
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+    for (const stream of this.streams) {
+      stream.updateOptions(opts);
+    }
+  }
+  synthesize(_) {
+    throw new Error("ChunkedStream is not implemented");
+  }
+  stream(options) {
+    const { connOptions = import_types.DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const stream = new SynthesizeStream(this, { ...this.opts }, connOptions);
+    this.streams.add(stream);
+    return stream;
+  }
+  async connectWs(timeout) {
+    let baseURL = this.opts.baseURL;
+    if (baseURL.startsWith("http://") || baseURL.startsWith("https://")) {
+      baseURL = baseURL.replace("http", "ws");
+    }
+    const token = await (0, import_utils2.createAccessToken)(this.opts.apiKey, this.opts.apiSecret);
+    const url = `${baseURL}/tts`;
+    const headers = { Authorization: `Bearer ${token}` };
+    const params = {
+      type: "session.create",
+      sample_rate: String(this.opts.sampleRate),
+      encoding: this.opts.encoding,
+      extra: this.opts.extraKwargs
+    };
+    if (this.opts.voice) params.voice = this.opts.voice;
+    if (this.opts.model) params.model = this.opts.model;
+    if (this.opts.language) params.language = this.opts.language;
+    const socket = await (0, import_utils2.connectWs)(url, headers, timeout);
+    socket.send(JSON.stringify(params));
+    return socket;
+  }
+  async closeWs(ws) {
+    await ws.close();
+  }
+  async close() {
+    for (const stream of this.streams) {
+      await stream.close();
+    }
+    this.streams.clear();
+  }
+}
+class SynthesizeStream extends import_tts.SynthesizeStream {
+  opts;
+  tts;
+  connOptions;
+  #logger = (0, import_log.log)();
+  constructor(tts, opts, connOptions) {
+    super(tts, connOptions);
+    this.opts = opts;
+    this.tts = tts;
+    this.connOptions = connOptions;
+  }
+  get label() {
+    return "inference.SynthesizeStream";
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+  }
+  async run() {
+    let ws = null;
+    let closing = false;
+    let finalReceived = false;
+    let lastFrame;
+    const sendTokenizerStream = new import_tokenize.basic.SentenceTokenizer().stream();
+    const eventChannel = (0, import_stream_channel.createStreamChannel)();
+    const requestId = (0, import_utils.shortuuid)("tts_request_");
+    const resourceCleanup = () => {
+      if (closing) return;
+      closing = true;
+      sendTokenizerStream.close();
+      eventChannel.close();
+      ws == null ? void 0 : ws.removeAllListeners();
+      ws == null ? void 0 : ws.close();
+    };
+    const sendClientEvent = async (event) => {
+      const validatedEvent = await import_api_protos.ttsClientEventSchema.parseAsync(event);
+      if (!ws || ws.readyState !== import_ws.WebSocket.OPEN) {
+        this.#logger.warn("Trying to send client TTS event to a closed WebSocket");
+        return;
+      }
+      ws.send(JSON.stringify(validatedEvent));
+    };
+    const sendLastFrame = (segmentId, final) => {
+      if (lastFrame) {
+        this.queue.put({ requestId, segmentId, frame: lastFrame, final });
+        lastFrame = void 0;
+      }
+    };
+    const createInputTask = async () => {
+      for await (const data of this.input) {
+        if (this.abortController.signal.aborted) break;
+        if (data === SynthesizeStream.FLUSH_SENTINEL) {
+          sendTokenizerStream.flush();
+          continue;
+        }
+        sendTokenizerStream.pushText(data);
+      }
+      sendTokenizerStream.endInput();
+    };
+    const createSentenceStreamTask = async () => {
+      for await (const ev of sendTokenizerStream) {
+        if (this.abortController.signal.aborted) break;
+        sendClientEvent({
+          type: "input_transcript",
+          transcript: ev.token + " "
+        });
+      }
+      sendClientEvent({ type: "session.flush" });
+    };
+    const createWsListenerTask = async (ws2) => {
+      return new Promise((resolve, reject) => {
+        this.abortController.signal.addEventListener("abort", () => {
+          resourceCleanup();
+          reject(new Error("WebSocket connection aborted"));
+        });
+        ws2.on("message", async (data) => {
+          const eventJson = JSON.parse(data.toString());
+          const validatedEvent = import_api_protos.ttsServerEventSchema.parse(eventJson);
+          eventChannel.write(validatedEvent);
+        });
+        ws2.on("error", (e) => {
+          this.#logger.error({ error: e }, "WebSocket error");
+          resourceCleanup();
+          reject(e);
+        });
+        ws2.on("close", () => {
+          resourceCleanup();
+          if (!closing) return this.#logger.error("WebSocket closed unexpectedly");
+          if (finalReceived) return resolve();
+          reject(
+            new import_exceptions.APIStatusError({
+              message: "Gateway connection closed unexpectedly",
+              options: { requestId }
+            })
+          );
+        });
+      });
+    };
+    const createRecvTask = async () => {
+      let currentSessionId = null;
+      const bstream = new import_audio.AudioByteStream(this.opts.sampleRate, NUM_CHANNELS);
+      const serverEventStream = eventChannel.stream();
+      const reader = serverEventStream.getReader();
+      try {
+        while (!this.closed && !this.abortController.signal.aborted) {
+          const result = await reader.read();
+          if (this.abortController.signal.aborted) return;
+          if (result.done) return;
+          const serverEvent = result.value;
+          switch (serverEvent.type) {
+            case "session.created":
+              currentSessionId = serverEvent.session_id;
+              break;
+            case "output_audio":
+              const base64Data = new Int8Array(Buffer.from(serverEvent.audio, "base64"));
+              for (const frame of bstream.write(base64Data.buffer)) {
+                sendLastFrame(currentSessionId, false);
+                lastFrame = frame;
+              }
+              break;
+            case "done":
+              finalReceived = true;
+              for (const frame of bstream.flush()) {
+                sendLastFrame(currentSessionId, false);
+                lastFrame = frame;
+              }
+              sendLastFrame(currentSessionId, true);
+              this.queue.put(SynthesizeStream.END_OF_STREAM);
+              break;
+            case "session.closed":
+              resourceCleanup();
+              break;
+            case "error":
+              this.#logger.error(
+                { serverEvent },
+                "Received error message from LiveKit TTS WebSocket"
+              );
+              resourceCleanup();
+              throw new import_exceptions.APIError(`LiveKit TTS returned error: ${serverEvent.message}`);
+            default:
+              this.#logger.warn("Unexpected message %s", serverEvent);
+              break;
+          }
+        }
+      } finally {
+        reader.releaseLock();
+        try {
+          await serverEventStream.cancel();
+        } catch (e) {
+          this.#logger.debug("Error cancelling serverEventStream (may already be cancelled):", e);
+        }
+      }
+    };
+    try {
+      ws = await this.tts.connectWs(this.connOptions.timeoutMs);
+      await Promise.all([
+        createInputTask(),
+        createSentenceStreamTask(),
+        createWsListenerTask(ws),
+        createRecvTask()
+      ]);
+    } catch (e) {
+      this.#logger.error("Error in SynthesizeStream", { error: e });
+    } finally {
+      resourceCleanup();
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SynthesizeStream,
+  TTS
+});
+//# sourceMappingURL=tts.cjs.map