@livekit/agents-plugin-assemblyai 0.0.0 → 1.2.8

package/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 -->
@@ -9,8 +9,10 @@ The Agents Framework is designed for building realtime, programmable
 participants that run on servers. Use it to create conversational, multi-modal
 voice agents that can see, hear, and understand.
 
-This package contains the AssemblyAI plugin, which allows for speech recognition.
-Refer to the [documentation](https://docs.livekit.io/agents/overview/) for
-information on how to use it.
+This package contains the AssemblyAI plugin, which allows for speech recognition
+via AssemblyAI's Universal-Streaming (v3) API. Refer to the
+[documentation](https://docs.livekit.io/agents/overview/) for information on how
+to use it, or browse the [API
+reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_assemblyai.html).
 See the [repository](https://github.com/livekit/agents-js) for more information
 about the framework as a whole.

package/dist/index.cjs

@@ -13,11 +13,24 @@ var __copyProps = (to, from, except, desc) => {
 };
 var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var src_exports = {};
-module.exports = __toCommonJS(src_exports);
-__reExport(src_exports, require("./stt.cjs"), module.exports);
+var index_exports = {};
+module.exports = __toCommonJS(index_exports);
+var import_agents = require("@livekit/agents");
+__reExport(index_exports, require("./models.cjs"), module.exports);
+__reExport(index_exports, require("./stt.cjs"), module.exports);
+class AssemblyAIPlugin extends import_agents.Plugin {
+  constructor() {
+    super({
+      title: "assemblyai",
+      version: "1.2.8",
+      package: "@livekit/agents-plugin-assemblyai"
+    });
+  }
+}
+import_agents.Plugin.registerPlugin(new AssemblyAIPlugin());
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  ...require("./models.cjs"),
   ...require("./stt.cjs")
 });
 //# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport * from './stt.js';\n"],"mappings":";;;;;;;;;;;;;;;AAAA;AAAA;AAIA,wBAAc,qBAJd;","names":[]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * from './models.js';\nexport * from './stt.js';\n\nclass AssemblyAIPlugin extends Plugin {\n  constructor() {\n    super({\n      title: 'assemblyai',\n      version: __PACKAGE_VERSION__,\n      package: __PACKAGE_NAME__,\n    });\n  }\n}\n\nPlugin.registerPlugin(new AssemblyAIPlugin());\n"],"mappings":";;;;;;;;;;;;;;;AAAA;AAAA;AAGA,oBAAuB;AAEvB,0BAAc,wBALd;AAMA,0BAAc,qBANd;AAQA,MAAM,yBAAyB,qBAAO;AAAA,EACpC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,qBAAO,eAAe,IAAI,iBAAiB,CAAC;","names":[]}

package/dist/index.d.cts

@@ -1,2 +1,3 @@
+export * from './models.js';
 export * from './stt.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts

@@ -1,2 +1,3 @@
+export * from './models.js';
 export * from './stt.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,cAAc,aAAa,CAAC;AAC5B,cAAc,UAAU,CAAC"}

package/dist/index.js

@@ -1,2 +1,14 @@
+import { Plugin } from "@livekit/agents";
+export * from "./models.js";
 export * from "./stt.js";
+class AssemblyAIPlugin extends Plugin {
+  constructor() {
+    super({
+      title: "assemblyai",
+      version: "1.2.8",
+      package: "@livekit/agents-plugin-assemblyai"
+    });
+  }
+}
+Plugin.registerPlugin(new AssemblyAIPlugin());
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport * from './stt.js';\n"],"mappings":"AAIA,cAAc;","names":[]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Plugin } from '@livekit/agents';\n\nexport * from './models.js';\nexport * from './stt.js';\n\nclass AssemblyAIPlugin extends Plugin {\n  constructor() {\n    super({\n      title: 'assemblyai',\n      version: __PACKAGE_VERSION__,\n      package: __PACKAGE_NAME__,\n    });\n  }\n}\n\nPlugin.registerPlugin(new AssemblyAIPlugin());\n"],"mappings":"AAGA,SAAS,cAAc;AAEvB,cAAc;AACd,cAAc;AAEd,MAAM,yBAAyB,OAAO;AAAA,EACpC,cAAc;AACZ,UAAM;AAAA,MACJ,OAAO;AAAA,MACP,SAAS;AAAA,MACT,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;AAEA,OAAO,eAAe,IAAI,iBAAiB,CAAC;","names":[]}

package/dist/models.cjs

@@ -0,0 +1,17 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var models_exports = {};
+module.exports = __toCommonJS(models_exports);
+//# sourceMappingURL=models.cjs.map

package/dist/models.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/models.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport type STTModels =\n  | 'universal-streaming-english'\n  | 'universal-streaming-multilingual'\n  | 'u3-rt-pro'\n  // Deprecated alias — AssemblyAI maps this to `u3-rt-pro` server-side, but the\n  // Python plugin emits a warning and rewrites it. Kept here so TS users don't\n  // break if they already pass it.\n  | 'u3-pro';\n\nexport type STTEncoding = 'pcm_s16le' | 'pcm_mulaw';\n"],"mappings":";;;;;;;;;;;;;;AAAA;AAAA;","names":[]}

package/dist/models.d.cts

@@ -0,0 +1,3 @@
+export type STTModels = 'universal-streaming-english' | 'universal-streaming-multilingual' | 'u3-rt-pro' | 'u3-pro';
+export type STTEncoding = 'pcm_s16le' | 'pcm_mulaw';
+//# sourceMappingURL=models.d.ts.map

package/dist/models.d.ts

@@ -0,0 +1,3 @@
+export type STTModels = 'universal-streaming-english' | 'universal-streaming-multilingual' | 'u3-rt-pro' | 'u3-pro';
+export type STTEncoding = 'pcm_s16le' | 'pcm_mulaw';
+//# sourceMappingURL=models.d.ts.map

package/dist/models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,SAAS,GACjB,6BAA6B,GAC7B,kCAAkC,GAClC,WAAW,GAIX,QAAQ,CAAC;AAEb,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC"}

package/dist/models.js

@@ -0,0 +1 @@
+//# sourceMappingURL=models.js.map

package/dist/models.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/stt.cjs

@@ -23,144 +23,423 @@ __export(stt_exports, {
 });
 module.exports = __toCommonJS(stt_exports);
 var import_agents = require("@livekit/agents");
-var import_assemblyai = require("assemblyai");
+var import_ws = require("ws");
 const defaultSTTOptions = {
-  apiKey: process.env.ASSEMBLY_AI_KEY,
-  interimResults: true,
+  apiKey: process.env.ASSEMBLYAI_API_KEY,
   sampleRate: 16e3,
-  keywords: [],
-  // NOTE:
-  // The default is 700ms from AssemblyAI.
-  // We use a low default of 300ms here because we also use
-  // the new end-of-utterance model from LiveKit to handle
-  // turn detection in my agent. Which means that even though
-  // this will quickly return a final transcript EVEN THOUGH
-  // USER IS NOT DONE SPEAKING, the EOU model from LiveKit
-  // DOES properly differentiate and doesn't interrupt (magically!)
-  // Ref: https://blog.livekit.io/using-a-transformer-to-improve-end-of-turn-detection/
-  endUtteranceSilenceThreshold: 200
+  bufferSizeMs: 50,
+  encoding: "pcm_s16le",
+  speechModel: "universal-streaming-english",
+  baseUrl: "wss://streaming.assemblyai.com"
 };
 class STT extends import_agents.stt.STT {
   #opts;
-  #logger = (0, import_agents.log)();
+  #streams = /* @__PURE__ */ new Set();
   label = "assemblyai.STT";
-  constructor(opts = defaultSTTOptions) {
+  get model() {
+    return this.#opts.speechModel;
+  }
+  get provider() {
+    return "AssemblyAI";
+  }
+  constructor(opts = {}) {
     super({
       streaming: true,
-      interimResults: opts.interimResults ?? defaultSTTOptions.interimResults
+      interimResults: true,
+      alignedTranscript: "word"
     });
-    if (opts.apiKey === void 0 && defaultSTTOptions.apiKey === void 0) {
+    if (opts.speechModel === "u3-pro") {
+      (0, import_agents.log)().warn("'u3-pro' is deprecated, use 'u3-rt-pro' instead.");
+      opts.speechModel = "u3-rt-pro";
+    }
+    if (opts.prompt !== void 0 && opts.speechModel !== "u3-rt-pro") {
+      throw new Error("The 'prompt' parameter is only supported with the 'u3-rt-pro' model.");
+    }
+    const apiKey = opts.apiKey ?? defaultSTTOptions.apiKey;
+    if (!apiKey) {
       throw new Error(
-        "AssemblyAI API key is required, whether as an argument or as $ASSEMBLY_AI_KEY"
+        "AssemblyAI API key is required. Pass one in via the `apiKey` parameter, or set it as the `ASSEMBLYAI_API_KEY` environment variable"
       );
     }
-    this.#opts = { ...defaultSTTOptions, ...opts };
+    const minTurnSilence = opts.minTurnSilence ?? 100;
+    this.#opts = {
+      ...defaultSTTOptions,
+      ...opts,
+      apiKey,
+      minTurnSilence
+    };
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async _recognize(_) {
-    throw new Error("Recognize is not supported on AssemblyAI STT");
+    throw new Error("Non-streaming recognize is not supported on AssemblyAI STT");
   }
-  stream() {
-    return new SpeechStream(this, this.#opts);
+  updateOptions(opts) {
+    this.#opts = { ...this.#opts, ...opts };
+    for (const ref of this.#streams) {
+      const stream = ref.deref();
+      if (stream) {
+        stream.updateOptions(opts);
+      } else {
+        this.#streams.delete(ref);
+      }
+    }
+  }
+  stream(options) {
+    const stream = new SpeechStream(this, this.#opts, options == null ? void 0 : options.connOptions);
+    this.#streams.add(new WeakRef(stream));
+    return stream;
   }
 }
 class SpeechStream extends import_agents.stt.SpeechStream {
+  static CLOSE_MSG = JSON.stringify({ type: "Terminate" });
   #opts;
   #logger = (0, import_agents.log)();
-  #speaking = false;
-  #client;
-  #transcriber;
+  #speechDurationInS = 0;
+  #lastPreflightStartTime = 0;
+  #pendingConfigMessages = [];
+  #configMessagePending = new import_agents.Future();
+  #sessionId = null;
+  #expiresAt = null;
   label = "assemblyai.SpeechStream";
-  constructor(stt2, opts) {
-    super(stt2);
+  constructor(stt2, opts, connOptions) {
+    super(stt2, opts.sampleRate, connOptions);
     this.#opts = opts;
     this.closed = false;
-    this.#client = new import_assemblyai.AssemblyAI({
-      apiKey: this.#opts.apiKey || ""
-    });
-    this.#run();
   }
-  async #run() {
-    try {
-      this.#transcriber = this.#client.realtime.transcriber({
-        sampleRate: this.#opts.sampleRate,
-        wordBoost: this.#opts.keywords.map((k) => k[0]),
-        endUtteranceSilenceThreshold: this.#opts.endUtteranceSilenceThreshold
-      });
-      this.#transcriber.on("open", (data) => {
-        this.#logger.child({ sessionId: data.sessionId, expiresAt: data.expiresAt }).debug(`AssemblyAI session opened`);
-      });
-      this.#transcriber.on("close", (code, reason) => {
-        this.#logger.child({ code, reason }).debug(`AssemblyAI session closed`);
-        if (!this.closed) {
-          this.#run();
-        }
-      });
-      this.#transcriber.on("error", (error) => {
-        this.#logger.child({ error: error.message }).error(`AssemblyAI error`);
-      });
-      this.#transcriber.on("transcript", (transcript) => {
-        if (this.closed) return;
-        if (!transcript.text || transcript.text.trim() === "") {
-          return;
-        }
-        if (!this.#speaking) {
-          this.#speaking = true;
-          this.queue.put({ type: import_agents.stt.SpeechEventType.START_OF_SPEECH });
-        }
-        if (transcript.message_type === "PartialTranscript") {
-          this.queue.put({
-            type: import_agents.stt.SpeechEventType.INTERIM_TRANSCRIPT,
-            alternatives: [assemblyTranscriptToSpeechData(transcript)]
-          });
-        } else if (transcript.message_type === "FinalTranscript") {
-          this.queue.put({
-            type: import_agents.stt.SpeechEventType.FINAL_TRANSCRIPT,
-            alternatives: [assemblyTranscriptToSpeechData(transcript)]
-          });
+  /**
+   * The AssemblyAI session ID. Set when the WebSocket connection is established
+   * (before any speech events). Null until the connection completes.
+   * Share this with the AssemblyAI team when reporting issues.
+   */
+  get sessionId() {
+    return this.#sessionId;
+  }
+  /**
+   * Unix timestamp when the AssemblyAI session expires. Set alongside
+   * {@link sessionId} when the WebSocket connection is established.
+   */
+  get expiresAt() {
+    return this.#expiresAt;
+  }
+  updateOptions(opts) {
+    this.#opts = { ...this.#opts, ...opts };
+    const configMsg = { type: "UpdateConfiguration" };
+    if (opts.prompt !== void 0) configMsg.prompt = opts.prompt;
+    if (opts.keytermsPrompt !== void 0) configMsg.keyterms_prompt = opts.keytermsPrompt;
+    if (opts.maxTurnSilence !== void 0) configMsg.max_turn_silence = opts.maxTurnSilence;
+    if (opts.minTurnSilence !== void 0) configMsg.min_turn_silence = opts.minTurnSilence;
+    if (opts.endOfTurnConfidenceThreshold !== void 0) {
+      configMsg.end_of_turn_confidence_threshold = opts.endOfTurnConfidenceThreshold;
+    }
+    if (opts.vadThreshold !== void 0) configMsg.vad_threshold = opts.vadThreshold;
+    if (Object.keys(configMsg).length > 1) {
+      this.#pendingConfigMessages.push(configMsg);
+      if (!this.#configMessagePending.done) this.#configMessagePending.resolve();
+    }
+  }
+  /**
+   * Force-finalize the current turn immediately.
+   */
+  forceEndpoint() {
+    this.#pendingConfigMessages.push({ type: "ForceEndpoint" });
+    if (!this.#configMessagePending.done) this.#configMessagePending.resolve();
+  }
+  // Deepgram-style reconnect loop around a single websocket lifetime.
+  async run() {
+    const maxRetry = 32;
+    let retries = 0;
+    while (!this.input.closed && !this.closed) {
+      try {
+        const ws = await this.#connectWS();
+        await this.#runWS(ws);
+        retries = 0;
+      } catch (e) {
+        if (!this.closed && !this.input.closed) {
+          if (retries >= maxRetry) {
+            throw new Error(`failed to connect to AssemblyAI after ${retries} attempts: ${e}`);
+          }
+          const retryDelaySeconds = Math.min(retries * 5, 10);
+          retries++;
+          this.#logger.warn(
+            `failed to connect to AssemblyAI, retrying in ${retryDelaySeconds} seconds: ${e} (${retries}/${maxRetry})`
+          );
+          await (0, import_agents.delay)(retryDelaySeconds * 1e3);
+        } else {
+          this.#logger.warn(
+            `AssemblyAI disconnected, connection is closed: ${e} (inputClosed: ${this.input.closed}, isClosed: ${this.closed})`
+          );
         }
+      }
+    }
+    this.closed = true;
+  }
+  async #connectWS() {
+    let minSilence = this.#opts.minTurnSilence;
+    let maxSilence = this.#opts.maxTurnSilence;
+    if (this.#opts.speechModel === "u3-rt-pro") {
+      if (minSilence === void 0) minSilence = 100;
+      if (maxSilence === void 0) maxSilence = minSilence;
+    }
+    const defaultLanguageDetection = this.#opts.speechModel.includes("multilingual") || this.#opts.speechModel === "u3-rt-pro";
+    const languageDetection = this.#opts.languageDetection ?? defaultLanguageDetection;
+    const liveConfig = {
+      sample_rate: this.#opts.sampleRate,
+      encoding: this.#opts.encoding,
+      speech_model: this.#opts.speechModel,
+      format_turns: this.#opts.formatTurns,
+      end_of_turn_confidence_threshold: this.#opts.endOfTurnConfidenceThreshold,
+      min_turn_silence: minSilence,
+      max_turn_silence: maxSilence,
+      keyterms_prompt: this.#opts.keytermsPrompt !== void 0 ? JSON.stringify(this.#opts.keytermsPrompt) : void 0,
+      language_detection: languageDetection,
+      prompt: this.#opts.prompt,
+      vad_threshold: this.#opts.vadThreshold,
+      speaker_labels: this.#opts.speakerLabels,
+      max_speakers: this.#opts.maxSpeakers,
+      domain: this.#opts.domain
+    };
+    const url = new URL(`${this.#opts.baseUrl}/v3/ws`);
+    for (const [key, value] of Object.entries(liveConfig)) {
+      if (value === void 0 || value === null) continue;
+      if (typeof value === "boolean") {
+        url.searchParams.append(key, value ? "true" : "false");
+      } else {
+        url.searchParams.append(key, String(value));
+      }
+    }
+    const ws = new import_ws.WebSocket(url, {
+      headers: {
+        Authorization: this.#opts.apiKey,
+        "Content-Type": "application/json",
+        "User-Agent": "AssemblyAI/1.0 (integration=Livekit)"
+      }
+    });
+    await new Promise((resolve, reject) => {
+      ws.on("open", () => resolve());
+      ws.on("error", (error) => reject(error));
+      ws.on("close", (code) => reject(new Error(`WebSocket returned ${code}`)));
+    });
+    return ws;
+  }
+  async #runWS(ws) {
+    let closing = false;
+    const sessionController = new AbortController();
+    const wsMonitor = import_agents.Task.from(async (controller) => {
+      const closed = new Promise((_, reject) => {
+        ws.once("close", (code, reason) => {
+          if (!closing) {
+            this.#logger.error(`WebSocket closed with code ${code}: ${reason}`);
+            reject(new Error("WebSocket closed"));
+          }
+        });
       });
-      await this.#transcriber.connect();
-      const sendTask = async () => {
-        var _a;
-        const samples100Ms = Math.floor(this.#opts.sampleRate / 10);
-        const stream = new import_agents.AudioByteStream(this.#opts.sampleRate, 1, samples100Ms);
-        for await (const data of this.input) {
-          if (this.closed) break;
+      await Promise.race([closed, (0, import_agents.waitForAbort)(controller.signal)]);
+    });
+    const sendTask = async () => {
+      const samplesPerBuffer = Math.floor(this.#opts.sampleRate * this.#opts.bufferSizeMs / 1e3);
+      const audioStream = new import_agents.AudioByteStream(this.#opts.sampleRate, 1, samplesPerBuffer);
+      const abortPromise = (0, import_agents.waitForAbort)(this.abortSignal);
+      const sessionAbort = (0, import_agents.waitForAbort)(sessionController.signal);
+      try {
+        while (!this.closed) {
+          const result = await Promise.race([this.input.next(), abortPromise, sessionAbort]);
+          if (result === void 0) return;
+          if (result.done) break;
+          const data = result.value;
           let frames;
           if (data === SpeechStream.FLUSH_SENTINEL) {
-            frames = stream.flush();
-          } else if (data.sampleRate === this.#opts.sampleRate) {
-            frames = stream.write(data.data.buffer);
+            frames = audioStream.flush();
+          } else if (data.sampleRate === this.#opts.sampleRate && data.channels === 1) {
+            frames = audioStream.write(data.data.buffer);
           } else {
-            throw new Error(`Sample rate or channel count of frame does not match`);
+            throw new Error("sample rate or channel count of frame does not match");
           }
-          for await (const frame of frames) {
-            (_a = this.#transcriber) == null ? void 0 : _a.sendAudio(new Uint8Array(frame.data.buffer));
+          for (const frame of frames) {
+            this.#speechDurationInS += frame.samplesPerChannel / frame.sampleRate;
+            ws.send(frame.data.buffer);
           }
         }
-        if (this.#transcriber) {
-          await this.#transcriber.close();
+      } finally {
+        closing = true;
+        try {
+          ws.send(SpeechStream.CLOSE_MSG);
+        } catch {
+        }
+        wsMonitor.cancel();
+      }
+    };
+    let messageHandler = null;
+    const listenTask = import_agents.Task.from(async (controller) => {
+      const listenMessage = new Promise((resolve, reject) => {
+        messageHandler = (msg, isBinary) => {
+          if (isBinary) {
+            this.#logger.error("unexpected binary message from AssemblyAI");
+            return;
+          }
+          try {
+            const json = JSON.parse(msg.toString());
+            this.#processStreamEvent(json);
+            if (this.closed || closing) {
+              resolve();
+            }
+          } catch (err) {
+            this.#logger.error(`AssemblyAI: error processing message: ${msg}`);
+            reject(err);
+          }
+        };
+        ws.on("message", messageHandler);
+      });
+      await Promise.race([listenMessage, (0, import_agents.waitForAbort)(controller.signal)]);
+    });
+    const configTask = import_agents.Task.from(async (controller) => {
+      while (this.#pendingConfigMessages.length > 0) {
+        const msg = this.#pendingConfigMessages.shift();
+        ws.send(JSON.stringify(msg));
+      }
+      while (!controller.signal.aborted) {
+        await Promise.race([this.#configMessagePending.await, (0, import_agents.waitForAbort)(controller.signal)]);
+        if (controller.signal.aborted) return;
+        this.#configMessagePending = new import_agents.Future();
+        while (this.#pendingConfigMessages.length > 0) {
+          const msg = this.#pendingConfigMessages.shift();
+          ws.send(JSON.stringify(msg));
         }
-      };
-      await sendTask();
-    } catch (error) {
-      this.#logger.child({ error }).error(`Error in AssemblyAI STT`);
-      if (!this.closed) {
-        setTimeout(() => this.#run(), 5e3);
+      }
+    });
+    try {
+      await Promise.all([sendTask(), listenTask.result, wsMonitor.result]);
+    } finally {
+      closing = true;
+      sessionController.abort();
+      listenTask.cancel();
+      configTask.cancel();
+      if (messageHandler) ws.off("message", messageHandler);
+      try {
+        ws.close();
+      } catch {
+      }
+    }
+  }
+  #averageConfidence(words) {
+    if (words.length === 0) return 0;
+    return words.reduce((sum, w) => sum + (w.confidence ?? 0), 0) / words.length;
+  }
+  #processStreamEvent(data) {
+    const messageType = data.type;
+    if (messageType === "Begin") {
+      this.#sessionId = data.id ?? null;
+      this.#expiresAt = data.expires_at ?? null;
+      this.#logger.info(
+        `AssemblyAI session started id=${this.#sessionId} expires_at=${this.#expiresAt}`
+      );
+      return;
+    }
+    if (messageType === "SpeechStarted") {
+      this.queue.put({ type: import_agents.stt.SpeechEventType.START_OF_SPEECH });
+      return;
+    }
+    if (messageType === "Termination") {
+      this.#logger.debug(
+        `AssemblyAI session terminated audio_duration=${data.audio_duration_seconds}s session_duration=${data.session_duration_seconds}s`
+      );
+      return;
+    }
+    if (messageType !== "Turn") {
+      return;
+    }
+    const words = data.words ?? [];
+    const endOfTurn = Boolean(data.end_of_turn);
+    const turnIsFormatted = Boolean(data.turn_is_formatted);
+    const utterance = data.utterance ?? "";
+    const transcript = data.transcript ?? "";
+    const language = (0, import_agents.normalizeLanguage)(data.language_code ?? "en");
+    const timedWords = words.map(
+      (word) => (0, import_agents.createTimedString)({
+        text: word.text ?? "",
+        startTime: (word.start ?? 0) / 1e3 + this.startTimeOffset,
+        endTime: (word.end ?? 0) / 1e3 + this.startTimeOffset,
+        confidence: word.confidence ?? 0,
+        startTimeOffset: this.startTimeOffset
+      })
+    );
+    let startTime = 0;
+    let endTime = 0;
+    let confidence = 0;
+    if (timedWords.length > 0) {
+      const interimText = timedWords.map((w) => w.text).join(" ");
+      startTime = timedWords[0].startTime ?? 0;
+      endTime = timedWords[timedWords.length - 1].endTime ?? 0;
+      confidence = this.#averageConfidence(timedWords);
+      this.queue.put({
+        type: import_agents.stt.SpeechEventType.INTERIM_TRANSCRIPT,
+        alternatives: [
+          {
+            language,
+            text: interimText,
+            startTime,
+            endTime,
+            confidence,
+            words: timedWords
+          }
+        ]
+      });
+    }
+    if (utterance) {
+      if (this.#lastPreflightStartTime === 0) {
+        this.#lastPreflightStartTime = startTime;
+      }
+      const utteranceWords = timedWords.filter(
+        (w) => w.startTime !== void 0 && w.startTime >= this.#lastPreflightStartTime
+      );
+      const utteranceConfidence = this.#averageConfidence(utteranceWords);
+      this.queue.put({
+        type: import_agents.stt.SpeechEventType.PREFLIGHT_TRANSCRIPT,
+        alternatives: [
+          {
+            language,
+            text: utterance,
+            startTime: this.#lastPreflightStartTime,
+            endTime,
+            confidence: utteranceConfidence,
+            words: utteranceWords
+          }
+        ]
+      });
+      this.#lastPreflightStartTime = endTime;
+    }
+    const waitingForFormatted = this.#opts.formatTurns === true && !turnIsFormatted;
+    if (endOfTurn && !waitingForFormatted) {
+      this.queue.put({
+        type: import_agents.stt.SpeechEventType.FINAL_TRANSCRIPT,
+        alternatives: [
+          {
+            language,
+            text: transcript,
+            startTime,
+            endTime,
+            confidence,
+            words: timedWords
+          }
+        ]
+      });
+      this.queue.put({ type: import_agents.stt.SpeechEventType.END_OF_SPEECH });
+      if (this.#speechDurationInS > 0) {
+        this.queue.put({
+          type: import_agents.stt.SpeechEventType.RECOGNITION_USAGE,
+          // Propagate the AssemblyAI session id as the request id so metrics
+          // can be correlated back to a specific connection, mirroring how
+          // Deepgram surfaces its `request_id`.
+          requestId: this.#sessionId ?? void 0,
+          recognitionUsage: {
+            audioDuration: this.#speechDurationInS
+          }
+        });
+        this.#speechDurationInS = 0;
+        this.#lastPreflightStartTime = 0;
       }
     }
   }
 }
-const assemblyTranscriptToSpeechData = (transcript) => {
-  return {
-    language: "en-US",
-    startTime: transcript.audio_start || 0,
-    endTime: transcript.audio_end || 0,
-    confidence: transcript.confidence || 1,
-    text: transcript.text || ""
-  };
-};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   STT,