@livekit/agents 0.0.0-20260120144724

package/dist/inference/stt.cjs

@@ -0,0 +1,371 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var stt_exports = {};
+__export(stt_exports, {
+  STT: () => STT,
+  SpeechStream: () => SpeechStream
+});
+module.exports = __toCommonJS(stt_exports);
+var import_rtc_node = require("@livekit/rtc-node");
+var import_exceptions = require("../_exceptions.cjs");
+var import_audio = require("../audio.cjs");
+var import_log = require("../log.cjs");
+var import_stream_channel = require("../stream/stream_channel.cjs");
+var import_stt = require("../stt/index.cjs");
+var import_types = require("../types.cjs");
+var import_utils = require("../utils.cjs");
+var import_utils2 = require("./utils.cjs");
+const DEFAULT_ENCODING = "pcm_s16le";
+const DEFAULT_SAMPLE_RATE = 16e3;
+const DEFAULT_BASE_URL = "wss://agent-gateway.livekit.cloud/v1";
+const DEFAULT_CANCEL_TIMEOUT = 5e3;
+class STT extends import_stt.STT {
+  opts;
+  streams = /* @__PURE__ */ new Set();
+  #logger = (0, import_log.log)();
+  constructor(opts) {
+    super({ streaming: true, interimResults: true });
+    const {
+      model,
+      language,
+      baseURL,
+      encoding = DEFAULT_ENCODING,
+      sampleRate = DEFAULT_SAMPLE_RATE,
+      apiKey,
+      apiSecret,
+      modelOptions = {}
+    } = opts || {};
+    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;
+    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;
+    if (!lkApiKey) {
+      throw new Error("apiKey is required: pass apiKey or set LIVEKIT_API_KEY");
+    }
+    const lkApiSecret = apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;
+    if (!lkApiSecret) {
+      throw new Error("apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET");
+    }
+    this.opts = {
+      model,
+      language,
+      encoding,
+      sampleRate,
+      baseURL: lkBaseURL,
+      apiKey: lkApiKey,
+      apiSecret: lkApiSecret,
+      modelOptions
+    };
+  }
+  get label() {
+    return "inference.STT";
+  }
+  static fromModelString(modelString) {
+    if (modelString.includes(":")) {
+      const [model, language] = modelString.split(":");
+      return new STT({ model, language });
+    }
+    return new STT({ model: modelString });
+  }
+  async _recognize(_) {
+    throw new Error("LiveKit STT does not support batch recognition, use stream() instead");
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+    for (const stream of this.streams) {
+      stream.updateOptions(opts);
+    }
+  }
+  stream(options) {
+    const { language, connOptions = import_types.DEFAULT_API_CONNECT_OPTIONS } = options || {};
+    const streamOpts = {
+      ...this.opts,
+      language: language ?? this.opts.language
+    };
+    const stream = new SpeechStream(this, streamOpts, connOptions);
+    this.streams.add(stream);
+    return stream;
+  }
+  async connectWs(timeout) {
+    const params = {
+      settings: {
+        sample_rate: String(this.opts.sampleRate),
+        encoding: this.opts.encoding,
+        extra: this.opts.modelOptions
+      }
+    };
+    if (this.opts.model && this.opts.model !== "auto") {
+      params.model = this.opts.model;
+    }
+    if (this.opts.language) {
+      params.settings.language = this.opts.language;
+    }
+    let baseURL = this.opts.baseURL;
+    if (baseURL.startsWith("http://") || baseURL.startsWith("https://")) {
+      baseURL = baseURL.replace("http", "ws");
+    }
+    const token = await (0, import_utils2.createAccessToken)(this.opts.apiKey, this.opts.apiSecret);
+    const url = `${baseURL}/stt`;
+    const headers = { Authorization: `Bearer ${token}` };
+    const socket = await (0, import_utils2.connectWs)(url, headers, timeout);
+    const msg = { ...params, type: "session.create" };
+    socket.send(JSON.stringify(msg));
+    return socket;
+  }
+}
+class SpeechStream extends import_stt.SpeechStream {
+  opts;
+  requestId = (0, import_utils.shortuuid)("stt_request_");
+  speaking = false;
+  speechDuration = 0;
+  reconnectEvent = new import_utils.Event();
+  stt;
+  connOptions;
+  #logger = (0, import_log.log)();
+  constructor(sttImpl, opts, connOptions) {
+    super(sttImpl, opts.sampleRate, connOptions);
+    this.opts = opts;
+    this.stt = sttImpl;
+    this.connOptions = connOptions;
+  }
+  get label() {
+    return "inference.SpeechStream";
+  }
+  updateOptions(opts) {
+    this.opts = { ...this.opts, ...opts };
+    this.reconnectEvent.set();
+  }
+  async run() {
+    while (true) {
+      let ws = null;
+      let closing = false;
+      let finalReceived = false;
+      const eventChannel = (0, import_stream_channel.createStreamChannel)();
+      const resourceCleanup = () => {
+        if (closing) return;
+        closing = true;
+        eventChannel.close();
+        ws == null ? void 0 : ws.removeAllListeners();
+        ws == null ? void 0 : ws.close();
+      };
+      const createWsListener = async (ws2, signal) => {
+        return new Promise((resolve, reject) => {
+          const onAbort = () => {
+            resourceCleanup();
+            reject(new Error("WebSocket connection aborted"));
+          };
+          signal.addEventListener("abort", onAbort, { once: true });
+          ws2.on("message", (data) => {
+            const json = JSON.parse(data.toString());
+            eventChannel.write(json);
+          });
+          ws2.on("error", (e) => {
+            this.#logger.error({ error: e }, "WebSocket error");
+            resourceCleanup();
+            reject(e);
+          });
+          ws2.on("close", (code) => {
+            resourceCleanup();
+            if (!closing) return this.#logger.error("WebSocket closed unexpectedly");
+            if (finalReceived) return resolve();
+            reject(
+              new import_exceptions.APIStatusError({
+                message: "LiveKit STT connection closed unexpectedly",
+                options: { statusCode: code }
+              })
+            );
+          });
+        });
+      };
+      const send = async (socket, signal) => {
+        const audioStream = new import_audio.AudioByteStream(
+          this.opts.sampleRate,
+          1,
+          Math.floor(this.opts.sampleRate / 20)
+          // 50ms
+        );
+        const abortPromise = new Promise((_, reject) => {
+          if (signal.aborted) {
+            return reject(new Error("Send aborted"));
+          }
+          const onAbort = () => reject(new Error("Send aborted"));
+          signal.addEventListener("abort", onAbort, { once: true });
+        });
+        const iterator = this.input[Symbol.asyncIterator]();
+        try {
+          while (true) {
+            const result = await Promise.race([iterator.next(), abortPromise]);
+            if (result.done) break;
+            const ev = result.value;
+            let frames;
+            if (ev === SpeechStream.FLUSH_SENTINEL) {
+              frames = audioStream.flush();
+            } else {
+              const frame = ev;
+              frames = audioStream.write(new Int16Array(frame.data).buffer);
+            }
+            for (const frame of frames) {
+              this.speechDuration += frame.samplesPerChannel / frame.sampleRate;
+              const base64 = Buffer.from(frame.data.buffer).toString("base64");
+              const msg = { type: "input_audio", audio: base64 };
+              socket.send(JSON.stringify(msg));
+            }
+          }
+          closing = true;
+          socket.send(JSON.stringify({ type: "session.finalize" }));
+        } catch (e) {
+          if (e.message === "Send aborted") {
+            return;
+          }
+          throw e;
+        }
+      };
+      const recv = async (signal) => {
+        const serverEventStream = eventChannel.stream();
+        const reader = serverEventStream.getReader();
+        try {
+          while (!this.closed && !signal.aborted) {
+            const result = await reader.read();
+            if (signal.aborted) return;
+            if (result.done) return;
+            const json = result.value;
+            const type = json.type;
+            switch (type) {
+              case "session.created":
+              case "session.finalized":
+                break;
+              case "session.closed":
+                finalReceived = true;
+                resourceCleanup();
+                break;
+              case "interim_transcript":
+                this.processTranscript(json, false);
+                break;
+              case "final_transcript":
+                this.processTranscript(json, true);
+                break;
+              case "error":
+                this.#logger.error({ error: json }, "Received error from LiveKit STT");
+                resourceCleanup();
+                throw new import_exceptions.APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);
+              default:
+                this.#logger.warn(
+                  { message: json },
+                  "Received unexpected message from LiveKit STT"
+                );
+                break;
+            }
+          }
+        } finally {
+          reader.releaseLock();
+          try {
+            await serverEventStream.cancel();
+          } catch (e) {
+            this.#logger.debug("Error cancelling serverEventStream (may already be cancelled):", e);
+          }
+        }
+      };
+      try {
+        ws = await this.stt.connectWs(this.connOptions.timeoutMs);
+        const controller = this.abortController;
+        const sendTask = import_utils.Task.from(({ signal }) => send(ws, signal), controller);
+        const wsListenerTask = import_utils.Task.from(({ signal }) => createWsListener(ws, signal), controller);
+        const recvTask = import_utils.Task.from(({ signal }) => recv(signal), controller);
+        const waitReconnectTask = import_utils.Task.from(
+          ({ signal }) => Promise.race([this.reconnectEvent.wait(), (0, import_utils.waitForAbort)(signal)]),
+          controller
+        );
+        try {
+          await Promise.race([
+            Promise.all([sendTask.result, wsListenerTask.result, recvTask.result]),
+            waitReconnectTask.result
+          ]);
+          if (!waitReconnectTask.done) break;
+          this.reconnectEvent.clear();
+        } finally {
+          await (0, import_utils.cancelAndWait)(
+            [sendTask, wsListenerTask, recvTask, waitReconnectTask],
+            DEFAULT_CANCEL_TIMEOUT
+          );
+          resourceCleanup();
+        }
+      } finally {
+        resourceCleanup();
+      }
+    }
+  }
+  processTranscript(data, isFinal) {
+    if (this.queue.closed) return;
+    const requestId = data.request_id ?? this.requestId;
+    const text = data.transcript ?? "";
+    const language = data.language ?? this.opts.language ?? "en";
+    if (!text && !isFinal) return;
+    try {
+      if (!this.speaking) {
+        this.speaking = true;
+        this.queue.put({ type: import_stt.SpeechEventType.START_OF_SPEECH });
+      }
+      const speechData = {
+        language,
+        startTime: data.start ?? 0,
+        endTime: data.duration ?? 0,
+        confidence: data.confidence ?? 1,
+        text
+      };
+      if (isFinal) {
+        if (this.speechDuration > 0) {
+          this.queue.put({
+            type: import_stt.SpeechEventType.RECOGNITION_USAGE,
+            requestId,
+            recognitionUsage: { audioDuration: this.speechDuration }
+          });
+          this.speechDuration = 0;
+        }
+        this.queue.put({
+          type: import_stt.SpeechEventType.FINAL_TRANSCRIPT,
+          requestId,
+          alternatives: [speechData]
+        });
+        if (this.speaking) {
+          this.speaking = false;
+          this.queue.put({ type: import_stt.SpeechEventType.END_OF_SPEECH });
+        }
+      } else {
+        this.queue.put({
+          type: import_stt.SpeechEventType.INTERIM_TRANSCRIPT,
+          requestId,
+          alternatives: [speechData]
+        });
+      }
+    } catch (e) {
+      if (e instanceof Error && e.message.includes("Queue is closed")) {
+        this.#logger.warn(
+          { err: e },
+          "Queue closed during transcript processing (expected during disconnect)"
+        );
+      } else {
+        this.#logger.error({ err: e }, "Error putting transcript to queue");
+      }
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  STT,
+  SpeechStream
+});
+//# sourceMappingURL=stt.cjs.map

package/dist/inference/stt.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/inference/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame } from '@livekit/rtc-node';\nimport type { WebSocket } from 'ws';\nimport { APIError, APIStatusError } from '../_exceptions.js';\nimport { AudioByteStream } from '../audio.js';\nimport { log } from '../log.js';\nimport { createStreamChannel } from '../stream/stream_channel.js';\nimport {\n  STT as BaseSTT,\n  SpeechStream as BaseSpeechStream,\n  type SpeechData,\n  type SpeechEvent,\n  SpeechEventType,\n} from '../stt/index.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';\nimport { type AnyString, connectWs, createAccessToken } from './utils.js';\n\nexport type DeepgramModels =\n  | 'deepgram'\n  | 'deepgram/nova-3'\n  | 'deepgram/nova-3-general'\n  | 'deepgram/nova-3-medical'\n  | 'deepgram/nova-2-conversationalai'\n  | 'deepgram/nova-2'\n  | 'deepgram/nova-2-general'\n  | 'deepgram/nova-2-medical'\n  | 'deepgram/nova-2-phonecall';\n\nexport type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';\n\nexport type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';\n\nexport interface CartesiaOptions {\n  min_volume?: number; // default: not specified\n  max_silence_duration_secs?: number; // default: not specified\n}\n\nexport interface DeepgramOptions {\n  filler_words?: boolean; // default: true\n  interim_results?: boolean; // default: true\n  endpointing?: number; // default: 25 (ms)\n  punctuate?: boolean; // default: false\n  smart_format?: boolean;\n  keywords?: Array<[string, number]>;\n  keyterms?: string[];\n  profanity_filter?: boolean;\n  numerals?: boolean;\n  mip_opt_out?: boolean;\n}\n\nexport interface AssemblyAIOptions {\n  format_turns?: boolean; // default: false\n  end_of_turn_confidence_threshold?: number; // default: 0.01\n  min_end_of_turn_silence_when_confident?: number; // default: 0\n  max_turn_silence?: number; // default: not specified\n  keyterms_prompt?: string[]; // default: not specified\n}\n\nexport type STTLanguages =\n  | 'multi'\n  | 'en'\n  | 'de'\n  | 'es'\n  | 'fr'\n  | 'ja'\n  | 'pt'\n  | 'zh'\n  | 'hi'\n  | AnyString;\n\ntype _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels;\n\nexport type STTModels = _STTModels | 'auto' | AnyString;\n\nexport type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels;\n\nexport type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels\n  ? DeepgramOptions\n  : TModel extends CartesiaModels\n    ? CartesiaOptions\n    : TModel extends AssemblyaiModels\n      ? AssemblyAIOptions\n      : Record<string, unknown>;\n\nexport type STTEncoding = 'pcm_s16le';\n\nconst DEFAULT_ENCODING: STTEncoding = 'pcm_s16le';\nconst DEFAULT_SAMPLE_RATE = 16000;\nconst DEFAULT_BASE_URL = 'wss://agent-gateway.livekit.cloud/v1';\nconst DEFAULT_CANCEL_TIMEOUT = 5000;\n\nexport interface InferenceSTTOptions<TModel extends STTModels> {\n  model?: TModel;\n  language?: STTLanguages;\n  encoding: STTEncoding;\n  sampleRate: number;\n  baseURL: string;\n  apiKey: string;\n  apiSecret: string;\n  modelOptions: STTOptions<TModel>;\n}\n\n/**\n * Livekit Cloud Inference STT\n */\nexport class STT<TModel extends STTModels> extends BaseSTT {\n  private opts: InferenceSTTOptions<TModel>;\n  private streams: Set<SpeechStream<TModel>> = new Set();\n\n  #logger = log();\n\n  constructor(opts?: {\n    model?: TModel;\n    language?: STTLanguages;\n    baseURL?: string;\n    encoding?: STTEncoding;\n    sampleRate?: number;\n    apiKey?: string;\n    apiSecret?: string;\n    modelOptions?: STTOptions<TModel>;\n  }) {\n    super({ streaming: true, interimResults: true });\n\n    const {\n      model,\n      language,\n      baseURL,\n      encoding = DEFAULT_ENCODING,\n      sampleRate = DEFAULT_SAMPLE_RATE,\n      apiKey,\n      apiSecret,\n      modelOptions = {} as STTOptions<TModel>,\n    } = opts || {};\n\n    const lkBaseURL = baseURL || process.env.LIVEKIT_INFERENCE_URL || DEFAULT_BASE_URL;\n    const lkApiKey = apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY;\n    if (!lkApiKey) {\n      throw new Error('apiKey is required: pass apiKey or set LIVEKIT_API_KEY');\n    }\n\n    const lkApiSecret =\n      apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET;\n    if (!lkApiSecret) {\n      throw new Error('apiSecret is required: pass apiSecret or set LIVEKIT_API_SECRET');\n    }\n\n    this.opts = {\n      model,\n      language,\n      encoding,\n      sampleRate,\n      baseURL: lkBaseURL,\n      apiKey: lkApiKey,\n      apiSecret: lkApiSecret,\n      modelOptions,\n    };\n  }\n\n  get label(): string {\n    return 'inference.STT';\n  }\n\n  static fromModelString(modelString: string): STT<AnyString> {\n    if (modelString.includes(':')) {\n      const [model, language] = modelString.split(':') as [AnyString, STTLanguages];\n      return new STT({ model, language });\n    }\n    return new STT({ model: modelString });\n  }\n\n  protected async _recognize(_: AudioBuffer): Promise<SpeechEvent> {\n    throw new Error('LiveKit STT does not support batch recognition, use stream() instead');\n  }\n\n  updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {\n    this.opts = { ...this.opts, ...opts };\n\n    for (const stream of this.streams) {\n      stream.updateOptions(opts);\n    }\n  }\n\n  stream(options?: {\n    language?: STTLanguages | string;\n    connOptions?: APIConnectOptions;\n  }): SpeechStream<TModel> {\n    const { language, connOptions = DEFAULT_API_CONNECT_OPTIONS } = options || {};\n    const streamOpts = {\n      ...this.opts,\n      language: language ?? this.opts.language,\n    } as InferenceSTTOptions<TModel>;\n\n    const stream = new SpeechStream(this, streamOpts, connOptions);\n    this.streams.add(stream);\n\n    return stream;\n  }\n\n  async connectWs(timeout: number): Promise<WebSocket> {\n    const params = {\n      settings: {\n        sample_rate: String(this.opts.sampleRate),\n        encoding: this.opts.encoding,\n        extra: this.opts.modelOptions,\n      },\n    } as Record<string, unknown>;\n\n    if (this.opts.model && this.opts.model !== 'auto') {\n      params.model = this.opts.model;\n    }\n\n    if (this.opts.language) {\n      (params.settings as Record<string, unknown>).language = this.opts.language;\n    }\n\n    let baseURL = this.opts.baseURL;\n    if (baseURL.startsWith('http://') || baseURL.startsWith('https://')) {\n      baseURL = baseURL.replace('http', 'ws');\n    }\n\n    const token = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);\n    const url = `${baseURL}/stt`;\n    const headers = { Authorization: `Bearer ${token}` } as Record<string, string>;\n\n    const socket = await connectWs(url, headers, timeout);\n    const msg = { ...params, type: 'session.create' };\n    socket.send(JSON.stringify(msg));\n\n    return socket;\n  }\n}\n\nexport class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {\n  private opts: InferenceSTTOptions<TModel>;\n  private requestId = shortuuid('stt_request_');\n  private speaking = false;\n  private speechDuration = 0;\n  private reconnectEvent = new Event();\n  private stt: STT<TModel>;\n  private connOptions: APIConnectOptions;\n\n  #logger = log();\n\n  constructor(\n    sttImpl: STT<TModel>,\n    opts: InferenceSTTOptions<TModel>,\n    connOptions: APIConnectOptions,\n  ) {\n    super(sttImpl, opts.sampleRate, connOptions);\n    this.opts = opts;\n    this.stt = sttImpl;\n    this.connOptions = connOptions;\n  }\n\n  get label(): string {\n    return 'inference.SpeechStream';\n  }\n\n  updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void {\n    this.opts = { ...this.opts, ...opts };\n    this.reconnectEvent.set();\n  }\n\n  protected async run(): Promise<void> {\n    while (true) {\n      // Create fresh resources for each connection attempt\n      let ws: WebSocket | null = null;\n      let closing = false;\n      let finalReceived = false;\n\n      type SttServerEvent = Record<string, any>;\n      const eventChannel = createStreamChannel<SttServerEvent>();\n\n      const resourceCleanup = () => {\n        if (closing) return;\n        closing = true;\n        eventChannel.close();\n        ws?.removeAllListeners();\n        ws?.close();\n      };\n\n      const createWsListener = async (ws: WebSocket, signal: AbortSignal) => {\n        return new Promise<void>((resolve, reject) => {\n          const onAbort = () => {\n            resourceCleanup();\n            reject(new Error('WebSocket connection aborted'));\n          };\n\n          signal.addEventListener('abort', onAbort, { once: true });\n\n          ws.on('message', (data) => {\n            const json = JSON.parse(data.toString()) as SttServerEvent;\n            eventChannel.write(json);\n          });\n\n          ws.on('error', (e) => {\n            this.#logger.error({ error: e }, 'WebSocket error');\n            resourceCleanup();\n            reject(e);\n          });\n\n          ws.on('close', (code: number) => {\n            resourceCleanup();\n\n            if (!closing) return this.#logger.error('WebSocket closed unexpectedly');\n            if (finalReceived) return resolve();\n\n            reject(\n              new APIStatusError({\n                message: 'LiveKit STT connection closed unexpectedly',\n                options: { statusCode: code },\n              }),\n            );\n          });\n        });\n      };\n\n      const send = async (socket: WebSocket, signal: AbortSignal) => {\n        const audioStream = new AudioByteStream(\n          this.opts.sampleRate,\n          1,\n          Math.floor(this.opts.sampleRate / 20), // 50ms\n        );\n\n        // Create abort promise once to avoid memory leak\n        const abortPromise = new Promise<never>((_, reject) => {\n          if (signal.aborted) {\n            return reject(new Error('Send aborted'));\n          }\n          const onAbort = () => reject(new Error('Send aborted'));\n          signal.addEventListener('abort', onAbort, { once: true });\n        });\n\n        // Manual iteration to support cancellation\n        const iterator = this.input[Symbol.asyncIterator]();\n        try {\n          while (true) {\n            const result = await Promise.race([iterator.next(), abortPromise]);\n\n            if (result.done) break;\n            const ev = result.value;\n\n            let frames: AudioFrame[];\n            if (ev === SpeechStream.FLUSH_SENTINEL) {\n              frames = audioStream.flush();\n            } else {\n              const frame = ev as AudioFrame;\n              frames = audioStream.write(new Int16Array(frame.data).buffer);\n            }\n\n            for (const frame of frames) {\n              this.speechDuration += frame.samplesPerChannel / frame.sampleRate;\n              const base64 = Buffer.from(frame.data.buffer).toString('base64');\n              const msg = { type: 'input_audio', audio: base64 };\n              socket.send(JSON.stringify(msg));\n            }\n          }\n\n          closing = true;\n          socket.send(JSON.stringify({ type: 'session.finalize' }));\n        } catch (e) {\n          if ((e as Error).message === 'Send aborted') {\n            // Expected abort, don't log\n            return;\n          }\n          throw e;\n        }\n      };\n\n      const recv = async (signal: AbortSignal) => {\n        const serverEventStream = eventChannel.stream();\n        const reader = serverEventStream.getReader();\n\n        try {\n          while (!this.closed && !signal.aborted) {\n            const result = await reader.read();\n            if (signal.aborted) return;\n            if (result.done) return;\n\n            const json = result.value;\n            const type = json.type as string | undefined;\n\n            switch (type) {\n              case 'session.created':\n              case 'session.finalized':\n                break;\n              case 'session.closed':\n                finalReceived = true;\n                resourceCleanup();\n                break;\n              case 'interim_transcript':\n                this.processTranscript(json, false);\n                break;\n              case 'final_transcript':\n                this.processTranscript(json, true);\n                break;\n              case 'error':\n                this.#logger.error({ error: json }, 'Received error from LiveKit STT');\n                resourceCleanup();\n                throw new APIError(`LiveKit STT returned error: ${JSON.stringify(json)}`);\n              default:\n                this.#logger.warn(\n                  { message: json },\n                  'Received unexpected message from LiveKit STT',\n                );\n                break;\n            }\n          }\n        } finally {\n          reader.releaseLock();\n          try {\n            await serverEventStream.cancel();\n          } catch (e) {\n            this.#logger.debug('Error cancelling serverEventStream (may already be cancelled):', e);\n          }\n        }\n      };\n\n      try {\n        ws = await this.stt.connectWs(this.connOptions.timeoutMs);\n\n        const controller = this.abortController; // Use base class abortController for proper cancellation\n        const sendTask = Task.from(({ signal }) => send(ws!, signal), controller);\n        const wsListenerTask = Task.from(({ signal }) => createWsListener(ws!, signal), controller);\n        const recvTask = Task.from(({ signal }) => recv(signal), controller);\n        const waitReconnectTask = Task.from(\n          ({ signal }) => Promise.race([this.reconnectEvent.wait(), waitForAbort(signal)]),\n          controller,\n        );\n\n        try {\n          await Promise.race([\n            Promise.all([sendTask.result, wsListenerTask.result, recvTask.result]),\n            waitReconnectTask.result,\n          ]);\n\n          // If reconnect didn't trigger, tasks finished - exit loop\n          if (!waitReconnectTask.done) break;\n\n          // Reconnect triggered - clear event and continue loop\n          this.reconnectEvent.clear();\n        } finally {\n          // Cancel all tasks to ensure cleanup\n          await cancelAndWait(\n            [sendTask, wsListenerTask, recvTask, waitReconnectTask],\n            DEFAULT_CANCEL_TIMEOUT,\n          );\n          resourceCleanup();\n        }\n      } finally {\n        // Ensure cleanup even if connectWs throws\n        resourceCleanup();\n      }\n    }\n  }\n\n  private processTranscript(data: Record<string, any>, isFinal: boolean) {\n    // Check if queue is closed to avoid race condition during disconnect\n    if (this.queue.closed) return;\n\n    const requestId = data.request_id ?? this.requestId;\n    const text = data.transcript ?? '';\n    const language = data.language ?? this.opts.language ?? 'en';\n\n    if (!text && !isFinal) return;\n\n    try {\n      // We'll have a more accurate way of detecting when speech started when we have VAD\n      if (!this.speaking) {\n        this.speaking = true;\n        this.queue.put({ type: SpeechEventType.START_OF_SPEECH });\n      }\n\n      const speechData: SpeechData = {\n        language,\n        startTime: data.start ?? 0,\n        endTime: data.duration ?? 0,\n        confidence: data.confidence ?? 1.0,\n        text,\n      };\n\n      if (isFinal) {\n        if (this.speechDuration > 0) {\n          this.queue.put({\n            type: SpeechEventType.RECOGNITION_USAGE,\n            requestId,\n            recognitionUsage: { audioDuration: this.speechDuration },\n          });\n          this.speechDuration = 0;\n        }\n\n        this.queue.put({\n          type: SpeechEventType.FINAL_TRANSCRIPT,\n          requestId,\n          alternatives: [speechData],\n        });\n\n        if (this.speaking) {\n          this.speaking = false;\n          this.queue.put({ type: SpeechEventType.END_OF_SPEECH });\n        }\n      } else {\n        this.queue.put({\n          type: SpeechEventType.INTERIM_TRANSCRIPT,\n          requestId,\n          alternatives: [speechData],\n        });\n      }\n    } catch (e) {\n      if (e instanceof Error && e.message.includes('Queue is closed')) {\n        // Expected behavior on disconnect, log as warning\n        this.#logger.warn(\n          { err: e },\n          'Queue closed during transcript processing (expected during disconnect)',\n        );\n      } else {\n        this.#logger.error({ err: e }, 'Error putting transcript to queue');\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAAgC;AAEhC,wBAAyC;AACzC,mBAAgC;AAChC,iBAAoB;AACpB,4BAAoC;AACpC,iBAMO;AACP,mBAAoE;AACpE,mBAAsF;AACtF,IAAAA,gBAA6D;AAuE7D,MAAM,mBAAgC;AACtC,MAAM,sBAAsB;AAC5B,MAAM,mBAAmB;AACzB,MAAM,yBAAyB;AAgBxB,MAAM,YAAsC,WAAAC,IAAQ;AAAA,EACjD;AAAA,EACA,UAAqC,oBAAI,IAAI;AAAA,EAErD,cAAU,gBAAI;AAAA,EAEd,YAAY,MAST;AACD,UAAM,EAAE,WAAW,MAAM,gBAAgB,KAAK,CAAC;AAE/C,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA,eAAe,CAAC;AAAA,IAClB,IAAI,QAAQ,CAAC;AAEb,UAAM,YAAY,WAAW,QAAQ,IAAI,yBAAyB;AAClE,UAAM,WAAW,UAAU,QAAQ,IAAI,6BAA6B,QAAQ,IAAI;AAChF,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAEA,UAAM,cACJ,aAAa,QAAQ,IAAI,gCAAgC,QAAQ,IAAI;AACvE,QAAI,CAAC,aAAa;AAChB,YAAM,IAAI,MAAM,iEAAiE;AAAA,IACnF;AAEA,SAAK,OAAO;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,gBAAgB,aAAqC;AAC1D,QAAI,YAAY,SAAS,GAAG,GAAG;AAC7B,YAAM,CAAC,OAAO,QAAQ,IAAI,YAAY,MAAM,GAAG;AAC/C,aAAO,IAAI,IAAI,EAAE,OAAO,SAAS,CAAC;AAAA,IACpC;AACA,WAAO,IAAI,IAAI,EAAE,OAAO,YAAY,CAAC;AAAA,EACvC;AAAA,EAEA,MAAgB,WAAW,GAAsC;AAC/D,UAAM,IAAI,MAAM,sEAAsE;AAAA,EACxF;AAAA,EAEA,cAAc,MAA8E;AAC1F,SAAK,OAAO,EAAE,GAAG,KAAK,MAAM,GAAG,KAAK;AAEpC,eAAW,UAAU,KAAK,SAAS;AACjC,aAAO,cAAc,IAAI;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,OAAO,SAGkB;AACvB,UAAM,EAAE,UAAU,cAAc,yCAA4B,IAAI,WAAW,CAAC;AAC5E,UAAM,aAAa;AAAA,MACjB,GAAG,KAAK;AAAA,MACR,UAAU,YAAY,KAAK,KAAK;AAAA,IAClC;AAEA,UAAM,SAAS,IAAI,aAAa,MAAM,YAAY,WAAW;AAC7D,SAAK,QAAQ,IAAI,MAAM;AAEvB,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,SAAqC;AACnD,UAAM,SAAS;AAAA,MACb,UAAU;AAAA,QACR,aAAa,OAAO,KAAK,KAAK,UAAU;AAAA,QACxC,UAAU,KAAK,KAAK;AAAA,QACpB,OAAO,KAAK,KAAK;AAAA,MACnB;AAAA,IACF;AAEA,QAAI,KAAK,KAAK,SAAS,KAAK,KAAK,UAAU,QAAQ;AACjD,aAAO,QAAQ,KAAK,KAAK;AAAA,IAC3B;AAEA,QAAI,KAAK,KAAK,UAAU;AACtB,MAAC,OAAO,SAAqC,WAAW,KAAK,KAAK;AAAA,IACpE;AAEA,QAAI,UAAU,KAAK,KAAK;AACxB,QAAI,QAAQ,WAAW,SAAS,KAAK,QAAQ,WAAW,UAAU,GAAG;AACnE,gBAAU,QAAQ,QAAQ,QAAQ,IAAI;AAAA,IACxC;AAEA,UAAM,QAAQ,UAAM,iCAAkB,KAAK,KAAK,QAAQ,KAAK,KAAK,SAAS;AAC3E,UAAM,MAAM,GAAG,OAAO;AACtB,UAAM,UAAU,EAAE,eAAe,UAAU,KAAK,GAAG;AAEnD,UAAM,SAAS,UAAM,yBAAU,KAAK,SAAS,OAAO;AACpD,UAAM,MAAM,EAAE,GAAG,QAAQ,MAAM,iBAAiB;AAChD,WAAO,KAAK,KAAK,UAAU,GAAG,CAAC;AAE/B,WAAO;AAAA,EACT;AACF;AAEO,MAAM,qBAA+C,WAAAC,aAAiB;AAAA,EACnE;AAAA,EACA,gBAAY,wBAAU,cAAc;AAAA,EACpC,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB,IAAI,mBAAM;AAAA,EAC3B;AAAA,EACA;AAAA,EAER,cAAU,gBAAI;AAAA,EAEd,YACE,SACA,MACA,aACA;AACA,UAAM,SAAS,KAAK,YAAY,WAAW;AAC3C,SAAK,OAAO;AACZ,SAAK,MAAM;AACX,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO;AAAA,EACT;AAAA,EAEA,cAAc,MAA8E;AAC1F,SAAK,OAAO,EAAE,GAAG,KAAK,MAAM,GAAG,KAAK;AACpC,SAAK,eAAe,IAAI;AAAA,EAC1B;AAAA,EAEA,MAAgB,MAAqB;AACnC,WAAO,MAAM;AAEX,UAAI,KAAuB;AAC3B,UAAI,UAAU;AACd,UAAI,gBAAgB;AAGpB,YAAM,mBAAe,2CAAoC;AAEzD,YAAM,kBAAkB,MAAM;AAC5B,YAAI,QAAS;AACb,kBAAU;AACV,qBAAa,MAAM;AACnB,iCAAI;AACJ,iCAAI;AAAA,MACN;AAEA,YAAM,mBAAmB,OAAOC,KAAe,WAAwB;AACrE,eAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,gBAAM,UAAU,MAAM;AACpB,4BAAgB;AAChB,mBAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,UAClD;AAEA,iBAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAExD,UAAAA,IAAG,GAAG,WAAW,CAAC,SAAS;AACzB,kBAAM,OAAO,KAAK,MAAM,KAAK,SAAS,CAAC;AACvC,yBAAa,MAAM,IAAI;AAAA,UACzB,CAAC;AAED,UAAAA,IAAG,GAAG,SAAS,CAAC,MAAM;AACpB,iBAAK,QAAQ,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB;AAClD,4BAAgB;AAChB,mBAAO,CAAC;AAAA,UACV,CAAC;AAED,UAAAA,IAAG,GAAG,SAAS,CAAC,SAAiB;AAC/B,4BAAgB;AAEhB,gBAAI,CAAC,QAAS,QAAO,KAAK,QAAQ,MAAM,+BAA+B;AACvE,gBAAI,cAAe,QAAO,QAAQ;AAElC;AAAA,cACE,IAAI,iCAAe;AAAA,gBACjB,SAAS;AAAA,gBACT,SAAS,EAAE,YAAY,KAAK;AAAA,cAC9B,CAAC;AAAA,YACH;AAAA,UACF,CAAC;AAAA,QACH,CAAC;AAAA,MACH;AAEA,YAAM,OAAO,OAAO,QAAmB,WAAwB;AAC7D,cAAM,cAAc,IAAI;AAAA,UACtB,KAAK,KAAK;AAAA,UACV;AAAA,UACA,KAAK,MAAM,KAAK,KAAK,aAAa,EAAE;AAAA;AAAA,QACtC;AAGA,cAAM,eAAe,IAAI,QAAe,CAAC,GAAG,WAAW;AACrD,cAAI,OAAO,SAAS;AAClB,mBAAO,OAAO,IAAI,MAAM,cAAc,CAAC;AAAA,UACzC;AACA,gBAAM,UAAU,MAAM,OAAO,IAAI,MAAM,cAAc,CAAC;AACtD,iBAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,QAC1D,CAAC;AAGD,cAAM,WAAW,KAAK,MAAM,OAAO,aAAa,EAAE;AAClD,YAAI;AACF,iBAAO,MAAM;AACX,kBAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,SAAS,KAAK,GAAG,YAAY,CAAC;AAEjE,gBAAI,OAAO,KAAM;AACjB,kBAAM,KAAK,OAAO;AAElB,gBAAI;AACJ,gBAAI,OAAO,aAAa,gBAAgB;AACtC,uBAAS,YAAY,MAAM;AAAA,YAC7B,OAAO;AACL,oBAAM,QAAQ;AACd,uBAAS,YAAY,MAAM,IAAI,WAAW,MAAM,IAAI,EAAE,MAAM;AAAA,YAC9D;AAEA,uBAAW,SAAS,QAAQ;AAC1B,mBAAK,kBAAkB,MAAM,oBAAoB,MAAM;AACvD,oBAAM,SAAS,OAAO,KAAK,MAAM,KAAK,MAAM,EAAE,SAAS,QAAQ;AAC/D,oBAAM,MAAM,EAAE,MAAM,eAAe,OAAO,OAAO;AACjD,qBAAO,KAAK,KAAK,UAAU,GAAG,CAAC;AAAA,YACjC;AAAA,UACF;AAEA,oBAAU;AACV,iBAAO,KAAK,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC,CAAC;AAAA,QAC1D,SAAS,GAAG;AACV,cAAK,EAAY,YAAY,gBAAgB;AAE3C;AAAA,UACF;AACA,gBAAM;AAAA,QACR;AAAA,MACF;AAEA,YAAM,OAAO,OAAO,WAAwB;AAC1C,cAAM,oBAAoB,aAAa,OAAO;AAC9C,cAAM,SAAS,kBAAkB,UAAU;AAE3C,YAAI;AACF,iBAAO,CAAC,KAAK,UAAU,CAAC,OAAO,SAAS;AACtC,kBAAM,SAAS,MAAM,OAAO,KAAK;AACjC,gBAAI,OAAO,QAAS;AACpB,gBAAI,OAAO,KAAM;AAEjB,kBAAM,OAAO,OAAO;AACpB,kBAAM,OAAO,KAAK;AAElB,oBAAQ,MAAM;AAAA,cACZ,KAAK;AAAA,cACL,KAAK;AACH;AAAA,cACF,KAAK;AACH,gCAAgB;AAChB,gCAAgB;AAChB;AAAA,cACF,KAAK;AACH,qBAAK,kBAAkB,MAAM,KAAK;AAClC;AAAA,cACF,KAAK;AACH,qBAAK,kBAAkB,MAAM,IAAI;AACjC;AAAA,cACF,KAAK;AACH,qBAAK,QAAQ,MAAM,EAAE,OAAO,KAAK,GAAG,iCAAiC;AACrE,gCAAgB;AAChB,sBAAM,IAAI,2BAAS,+BAA+B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,cAC1E;AACE,qBAAK,QAAQ;AAAA,kBACX,EAAE,SAAS,KAAK;AAAA,kBAChB;AAAA,gBACF;AACA;AAAA,YACJ;AAAA,UACF;AAAA,QACF,UAAE;AACA,iBAAO,YAAY;AACnB,cAAI;AACF,kBAAM,kBAAkB,OAAO;AAAA,UACjC,SAAS,GAAG;AACV,iBAAK,QAAQ,MAAM,kEAAkE,CAAC;AAAA,UACxF;AAAA,QACF;AAAA,MACF;AAEA,UAAI;AACF,aAAK,MAAM,KAAK,IAAI,UAAU,KAAK,YAAY,SAAS;AAExD,cAAM,aAAa,KAAK;AACxB,cAAM,WAAW,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,IAAK,MAAM,GAAG,UAAU;AACxE,cAAM,iBAAiB,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,iBAAiB,IAAK,MAAM,GAAG,UAAU;AAC1F,cAAM,WAAW,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,MAAM,GAAG,UAAU;AACnE,cAAM,oBAAoB,kBAAK;AAAA,UAC7B,CAAC,EAAE,OAAO,MAAM,QAAQ,KAAK,CAAC,KAAK,eAAe,KAAK,OAAG,2BAAa,MAAM,CAAC,CAAC;AAAA,UAC/E;AAAA,QACF;AAEA,YAAI;AACF,gBAAM,QAAQ,KAAK;AAAA,YACjB,QAAQ,IAAI,CAAC,SAAS,QAAQ,eAAe,QAAQ,SAAS,MAAM,CAAC;AAAA,YACrE,kBAAkB;AAAA,UACpB,CAAC;AAGD,cAAI,CAAC,kBAAkB,KAAM;AAG7B,eAAK,eAAe,MAAM;AAAA,QAC5B,UAAE;AAEA,oBAAM;AAAA,YACJ,CAAC,UAAU,gBAAgB,UAAU,iBAAiB;AAAA,YACtD;AAAA,UACF;AACA,0BAAgB;AAAA,QAClB;AAAA,MACF,UAAE;AAEA,wBAAgB;AAAA,MAClB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kBAAkB,MAA2B,SAAkB;AAErE,QAAI,KAAK,MAAM,OAAQ;AAEvB,UAAM,YAAY,KAAK,cAAc,KAAK;AAC1C,UAAM,OAAO,KAAK,cAAc;AAChC,UAAM,WAAW,KAAK,YAAY,KAAK,KAAK,YAAY;AAExD,QAAI,CAAC,QAAQ,CAAC,QAAS;AAEvB,QAAI;AAEF,UAAI,CAAC,KAAK,UAAU;AAClB,aAAK,WAAW;AAChB,aAAK,MAAM,IAAI,EAAE,MAAM,2BAAgB,gBAAgB,CAAC;AAAA,MAC1D;AAEA,YAAM,aAAyB;AAAA,QAC7B;AAAA,QACA,WAAW,KAAK,SAAS;AAAA,QACzB,SAAS,KAAK,YAAY;AAAA,QAC1B,YAAY,KAAK,cAAc;AAAA,QAC/B;AAAA,MACF;AAEA,UAAI,SAAS;AACX,YAAI,KAAK,iBAAiB,GAAG;AAC3B,eAAK,MAAM,IAAI;AAAA,YACb,MAAM,2BAAgB;AAAA,YACtB;AAAA,YACA,kBAAkB,EAAE,eAAe,KAAK,eAAe;AAAA,UACzD,CAAC;AACD,eAAK,iBAAiB;AAAA,QACxB;AAEA,aAAK,MAAM,IAAI;AAAA,UACb,MAAM,2BAAgB;AAAA,UACtB;AAAA,UACA,cAAc,CAAC,UAAU;AAAA,QAC3B,CAAC;AAED,YAAI,KAAK,UAAU;AACjB,eAAK,WAAW;AAChB,eAAK,MAAM,IAAI,EAAE,MAAM,2BAAgB,cAAc,CAAC;AAAA,QACxD;AAAA,MACF,OAAO;AACL,aAAK,MAAM,IAAI;AAAA,UACb,MAAM,2BAAgB;AAAA,UACtB;AAAA,UACA,cAAc,CAAC,UAAU;AAAA,QAC3B,CAAC;AAAA,MACH;AAAA,IACF,SAAS,GAAG;AACV,UAAI,aAAa,SAAS,EAAE,QAAQ,SAAS,iBAAiB,GAAG;AAE/D,aAAK,QAAQ;AAAA,UACX,EAAE,KAAK,EAAE;AAAA,UACT;AAAA,QACF;AAAA,MACF,OAAO;AACL,aAAK,QAAQ,MAAM,EAAE,KAAK,EAAE,GAAG,mCAAmC;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AACF;","names":["import_utils","BaseSTT","BaseSpeechStream","ws"]}

package/dist/inference/stt.d.cts

@@ -0,0 +1,91 @@
+import type { WebSocket } from 'ws';
+import { STT as BaseSTT, SpeechStream as BaseSpeechStream, type SpeechEvent } from '../stt/index.js';
+import { type APIConnectOptions } from '../types.js';
+import { type AudioBuffer } from '../utils.js';
+import { type AnyString } from './utils.js';
+export type DeepgramModels = 'deepgram' | 'deepgram/nova-3' | 'deepgram/nova-3-general' | 'deepgram/nova-3-medical' | 'deepgram/nova-2-conversationalai' | 'deepgram/nova-2' | 'deepgram/nova-2-general' | 'deepgram/nova-2-medical' | 'deepgram/nova-2-phonecall';
+export type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';
+export type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';
+export interface CartesiaOptions {
+    min_volume?: number;
+    max_silence_duration_secs?: number;
+}
+export interface DeepgramOptions {
+    filler_words?: boolean;
+    interim_results?: boolean;
+    endpointing?: number;
+    punctuate?: boolean;
+    smart_format?: boolean;
+    keywords?: Array<[string, number]>;
+    keyterms?: string[];
+    profanity_filter?: boolean;
+    numerals?: boolean;
+    mip_opt_out?: boolean;
+}
+export interface AssemblyAIOptions {
+    format_turns?: boolean;
+    end_of_turn_confidence_threshold?: number;
+    min_end_of_turn_silence_when_confident?: number;
+    max_turn_silence?: number;
+    keyterms_prompt?: string[];
+}
+export type STTLanguages = 'multi' | 'en' | 'de' | 'es' | 'fr' | 'ja' | 'pt' | 'zh' | 'hi' | AnyString;
+type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels;
+export type STTModels = _STTModels | 'auto' | AnyString;
+export type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels;
+export type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels ? DeepgramOptions : TModel extends CartesiaModels ? CartesiaOptions : TModel extends AssemblyaiModels ? AssemblyAIOptions : Record<string, unknown>;
+export type STTEncoding = 'pcm_s16le';
+export interface InferenceSTTOptions<TModel extends STTModels> {
+    model?: TModel;
+    language?: STTLanguages;
+    encoding: STTEncoding;
+    sampleRate: number;
+    baseURL: string;
+    apiKey: string;
+    apiSecret: string;
+    modelOptions: STTOptions<TModel>;
+}
+/**
+ * Livekit Cloud Inference STT
+ */
+export declare class STT<TModel extends STTModels> extends BaseSTT {
+    #private;
+    private opts;
+    private streams;
+    constructor(opts?: {
+        model?: TModel;
+        language?: STTLanguages;
+        baseURL?: string;
+        encoding?: STTEncoding;
+        sampleRate?: number;
+        apiKey?: string;
+        apiSecret?: string;
+        modelOptions?: STTOptions<TModel>;
+    });
+    get label(): string;
+    static fromModelString(modelString: string): STT<AnyString>;
+    protected _recognize(_: AudioBuffer): Promise<SpeechEvent>;
+    updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void;
+    stream(options?: {
+        language?: STTLanguages | string;
+        connOptions?: APIConnectOptions;
+    }): SpeechStream<TModel>;
+    connectWs(timeout: number): Promise<WebSocket>;
+}
+export declare class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
+    #private;
+    private opts;
+    private requestId;
+    private speaking;
+    private speechDuration;
+    private reconnectEvent;
+    private stt;
+    private connOptions;
+    constructor(sttImpl: STT<TModel>, opts: InferenceSTTOptions<TModel>, connOptions: APIConnectOptions);
+    get label(): string;
+    updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void;
+    protected run(): Promise<void>;
+    private processTranscript;
+}
+export {};
+//# sourceMappingURL=stt.d.ts.map

package/dist/inference/stt.d.ts

@@ -0,0 +1,91 @@
+import type { WebSocket } from 'ws';
+import { STT as BaseSTT, SpeechStream as BaseSpeechStream, type SpeechEvent } from '../stt/index.js';
+import { type APIConnectOptions } from '../types.js';
+import { type AudioBuffer } from '../utils.js';
+import { type AnyString } from './utils.js';
+export type DeepgramModels = 'deepgram' | 'deepgram/nova-3' | 'deepgram/nova-3-general' | 'deepgram/nova-3-medical' | 'deepgram/nova-2-conversationalai' | 'deepgram/nova-2' | 'deepgram/nova-2-general' | 'deepgram/nova-2-medical' | 'deepgram/nova-2-phonecall';
+export type CartesiaModels = 'cartesia' | 'cartesia/ink-whisper';
+export type AssemblyaiModels = 'assemblyai' | 'assemblyai/universal-streaming';
+export interface CartesiaOptions {
+    min_volume?: number;
+    max_silence_duration_secs?: number;
+}
+export interface DeepgramOptions {
+    filler_words?: boolean;
+    interim_results?: boolean;
+    endpointing?: number;
+    punctuate?: boolean;
+    smart_format?: boolean;
+    keywords?: Array<[string, number]>;
+    keyterms?: string[];
+    profanity_filter?: boolean;
+    numerals?: boolean;
+    mip_opt_out?: boolean;
+}
+export interface AssemblyAIOptions {
+    format_turns?: boolean;
+    end_of_turn_confidence_threshold?: number;
+    min_end_of_turn_silence_when_confident?: number;
+    max_turn_silence?: number;
+    keyterms_prompt?: string[];
+}
+export type STTLanguages = 'multi' | 'en' | 'de' | 'es' | 'fr' | 'ja' | 'pt' | 'zh' | 'hi' | AnyString;
+type _STTModels = DeepgramModels | CartesiaModels | AssemblyaiModels;
+export type STTModels = _STTModels | 'auto' | AnyString;
+export type ModelWithLanguage = `${_STTModels}:${STTLanguages}` | STTModels;
+export type STTOptions<TModel extends STTModels> = TModel extends DeepgramModels ? DeepgramOptions : TModel extends CartesiaModels ? CartesiaOptions : TModel extends AssemblyaiModels ? AssemblyAIOptions : Record<string, unknown>;
+export type STTEncoding = 'pcm_s16le';
+export interface InferenceSTTOptions<TModel extends STTModels> {
+    model?: TModel;
+    language?: STTLanguages;
+    encoding: STTEncoding;
+    sampleRate: number;
+    baseURL: string;
+    apiKey: string;
+    apiSecret: string;
+    modelOptions: STTOptions<TModel>;
+}
+/**
+ * Livekit Cloud Inference STT
+ */
+export declare class STT<TModel extends STTModels> extends BaseSTT {
+    #private;
+    private opts;
+    private streams;
+    constructor(opts?: {
+        model?: TModel;
+        language?: STTLanguages;
+        baseURL?: string;
+        encoding?: STTEncoding;
+        sampleRate?: number;
+        apiKey?: string;
+        apiSecret?: string;
+        modelOptions?: STTOptions<TModel>;
+    });
+    get label(): string;
+    static fromModelString(modelString: string): STT<AnyString>;
+    protected _recognize(_: AudioBuffer): Promise<SpeechEvent>;
+    updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void;
+    stream(options?: {
+        language?: STTLanguages | string;
+        connOptions?: APIConnectOptions;
+    }): SpeechStream<TModel>;
+    connectWs(timeout: number): Promise<WebSocket>;
+}
+export declare class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
+    #private;
+    private opts;
+    private requestId;
+    private speaking;
+    private speechDuration;
+    private reconnectEvent;
+    private stt;
+    private connOptions;
+    constructor(sttImpl: STT<TModel>, opts: InferenceSTTOptions<TModel>, connOptions: APIConnectOptions);
+    get label(): string;
+    updateOptions(opts: Partial<Pick<InferenceSTTOptions<TModel>, 'model' | 'language'>>): void;
+    protected run(): Promise<void>;
+    private processTranscript;
+}
+export {};
+//# sourceMappingURL=stt.d.ts.map

package/dist/inference/stt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/inference/stt.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAKpC,OAAO,EACL,GAAG,IAAI,OAAO,EACd,YAAY,IAAI,gBAAgB,EAEhC,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,KAAK,WAAW,EAAuD,MAAM,aAAa,CAAC;AACpG,OAAO,EAAE,KAAK,SAAS,EAAgC,MAAM,YAAY,CAAC;AAE1E,MAAM,MAAM,cAAc,GACtB,UAAU,GACV,iBAAiB,GACjB,yBAAyB,GACzB,yBAAyB,GACzB,kCAAkC,GAClC,iBAAiB,GACjB,yBAAyB,GACzB,yBAAyB,GACzB,2BAA2B,CAAC;AAEhC,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,sBAAsB,CAAC;AAEjE,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,gCAAgC,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gCAAgC,CAAC,EAAE,MAAM,CAAC;IAC1C,sCAAsC,CAAC,EAAE,MAAM,CAAC;IAChD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,MAAM,YAAY,GACpB,OAAO,GACP,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,SAAS,CAAC;AAEd,KAAK,UAAU,GAAG,cAAc,GAAG,cAAc,GAAG,gBAAgB,CAAC;AAErE,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,CAAC;AAExD,MAAM,MAAM,iBAAiB,GAAG,GAAG,UAAU,IAAI,YAAY,EAAE,GAAG,SAAS,CAAC;AAE5E,MAAM,MAAM,UAAU,CAAC,MAAM,SAAS,SAAS,IAAI,MAAM,SAAS,cAAc,GAC5E,eAAe,GACf,MAAM,SAAS,cAAc,GAC3B,eAAe,GACf,MAAM,SAAS,gBAAgB,GAC7B,iBAAiB,GACjB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAEhC,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC;AAOtC,MAAM,WAAW,mBAAmB,CAAC,MAAM,SAAS,SAAS;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,QAAQ,EAAE,WAAW,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,qBAAa,GAAG,CAAC,MAAM,SAAS,SAAS,CAAE,SAAQ,OAAO;;IACxD,OAAO,CAAC,IAAI,CAA8B;IAC1C,OAAO,CAAC,OAAO,CAAwC;gBAI3C,IAAI,CAAC,EAAE;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,WAAW,CAAC;QACvB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;KACnC;IAsCD,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,MAAM,CAAC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,GAAG,CAAC,SAAS,CAAC;cAQ3C,UAAU,CAAC,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAIhE,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI;IAQ3F,MAAM,CAAC,OAAO,CAAC,EAAE;QACf,QAAQ,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC;QACjC,WAAW,CAAC,EAAE,iBAAiB,CAAC;KACjC,GAAG,YAAY,CAAC,MAAM,CAAC;IAalB,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;CAgCrD;AAED,qBAAa,YAAY,CAAC,MAAM,SAAS,SAAS,CAAE,SAAQ,gBAAgB;;IAC1E,OAAO,CAAC,IAAI,CAA8B;IAC1C,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,cAAc,CAAe;IACrC,OAAO,CAAC,GAAG,CAAc;IACzB,OAAO,CAAC,WAAW,CAAoB;gBAKrC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,EACjC,WAAW,EAAE,iBAAiB;IAQhC,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,CAAC,GAAG,IAAI;cAK3E,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAiMpC,OAAO,CAAC,iBAAiB;CAgE1B"}