@livekit/agents-plugin-assemblyai 1.4.11 → 1.5.1

package/src/stt-metadata.test.ts

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { stt as sttLib } from '@livekit/agents';
+import { AudioFrame } from '@livekit/rtc-node';
+import { once } from 'node:events';
+import type { AddressInfo } from 'node:net';
+import { describe, expect, it } from 'vitest';
+import { WebSocketServer } from 'ws';
+import { STT } from './stt.js';
+
+function makeFrame(samplesPerChannel = 800, sampleRate = 16000): AudioFrame {
+  const data = new Int16Array(samplesPerChannel);
+  data.fill(1);
+  return new AudioFrame(data, sampleRate, 1, samplesPerChannel);
+}
+
+async function startWebSocketServer() {
+  const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 });
+  await once(wss, 'listening');
+  const address = wss.address() as AddressInfo;
+  return { wss, baseUrl: `ws://127.0.0.1:${address.port}` };
+}
+
+async function closeWebSocketServer(wss: WebSocketServer): Promise<void> {
+  await new Promise<void>((resolve) => wss.close(() => resolve()));
+}
+
+async function waitUntil(predicate: () => boolean, timeoutMs = 1000): Promise<void> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error('timed out waiting for condition');
+}
+
+async function collectUntilEnd(stream: sttLib.SpeechStream): Promise<sttLib.SpeechEvent[]> {
+  const events: sttLib.SpeechEvent[] = [];
+  for await (const event of stream) {
+    events.push(event);
+    if (event.type === sttLib.SpeechEventType.END_OF_SPEECH) break;
+  }
+  return events;
+}
+
+describe('AssemblyAI STT metadata', () => {
+  it('maps turn confidence fields onto speech data metadata', async () => {
+    const { wss, baseUrl } = await startWebSocketServer();
+    let requestUrl = '';
+
+    wss.on('connection', (ws, req) => {
+      requestUrl = req.url ?? '';
+      ws.on('message', () => {
+        ws.send(
+          JSON.stringify({
+            type: 'Turn',
+            transcript: 'hola mundo',
+            utterance: 'hola mundo',
+            end_of_turn: true,
+            language_code: 'es',
+            language_confidence: 0.94,
+            words: [
+              { text: 'hola', start: 0, end: 200, confidence: 0.96 },
+              { text: 'mundo', start: 200, end: 500, confidence: 0.98 },
+            ],
+          }),
+        );
+      });
+    });
+
+    try {
+      const assemblyai = new STT({
+        apiKey: 'test-key',
+        baseUrl,
+        speechModel: 'u3-rt-pro',
+      });
+      const stream = assemblyai.stream({
+        connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 1000 },
+      });
+
+      await waitUntil(() => requestUrl !== '');
+
+      stream.pushFrame(makeFrame());
+      stream.endInput();
+
+      const events = await collectUntilEnd(stream);
+      stream.close();
+
+      expect(new URL(`ws://127.0.0.1${requestUrl}`).pathname).toBe('/v3/ws');
+      expect(
+        events
+          .filter((event) => event.alternatives?.[0])
+          .map((event) => ({
+            type: event.type,
+            language: event.alternatives?.[0]?.language,
+            metadata: event.alternatives?.[0]?.metadata,
+          })),
+      ).toEqual([
+        {
+          type: sttLib.SpeechEventType.INTERIM_TRANSCRIPT,
+          language: 'es',
+          metadata: {
+            assemblyai: {
+              languageConfidence: 0.94,
+            },
+          },
+        },
+        {
+          type: sttLib.SpeechEventType.PREFLIGHT_TRANSCRIPT,
+          language: 'es',
+          metadata: {
+            assemblyai: {
+              languageConfidence: 0.94,
+            },
+          },
+        },
+        {
+          type: sttLib.SpeechEventType.FINAL_TRANSCRIPT,
+          language: 'es',
+          metadata: {
+            assemblyai: {
+              languageConfidence: 0.94,
+            },
+          },
+        },
+      ]);
+    } finally {
+      await closeWebSocketServer(wss);
+    }
+  });
+});

package/src/stt.test.ts

@@ -3,9 +3,33 @@
 // SPDX-License-Identifier: Apache-2.0
 import { VAD } from '@livekit/agents-plugin-silero';
 import { stt } from '@livekit/agents-plugins-test';
+import { once } from 'node:events';
+import type { AddressInfo } from 'node:net';
 import { describe, expect, it } from 'vitest';
+import { WebSocketServer } from 'ws';
 import { STT } from './stt.js';
 
+async function startWebSocketServer() {
+  const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 });
+  await once(wss, 'listening');
+  const address = wss.address() as AddressInfo;
+  return { wss, baseUrl: `ws://127.0.0.1:${address.port}` };
+}
+
+async function closeWebSocketServer(wss: WebSocketServer): Promise<void> {
+  for (const client of wss.clients) client.close();
+  await new Promise<void>((resolve) => wss.close(() => resolve()));
+}
+
+async function waitUntil(predicate: () => boolean, timeoutMs = 1000): Promise<void> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error('timed out waiting for condition');
+}
+
 describe('AssemblyAI options', () => {
   it('accepts u3-rt-pro-beta-1', () => {
     const stt = new STT({ apiKey: 'test-key', speechModel: 'u3-rt-pro-beta-1' });
@@ -47,6 +71,32 @@ describe('AssemblyAI options', () => {
         }),
     ).toThrow(/previousContextNTurns/);
   });
+
+  it('forwards inactivity timeout to the streaming query', async () => {
+    const { wss, baseUrl } = await startWebSocketServer();
+    let requestUrl = '';
+
+    wss.on('connection', (_ws, req) => {
+      requestUrl = req.url ?? '';
+    });
+
+    try {
+      const stream = new STT({
+        apiKey: 'test-key',
+        baseUrl,
+        inactivityTimeout: 45,
+      }).stream();
+
+      await waitUntil(() => requestUrl !== '');
+      stream.close();
+
+      const url = new URL(`ws://127.0.0.1${requestUrl}`);
+      expect(url.pathname).toBe('/v3/ws');
+      expect(url.searchParams.get('inactivity_timeout')).toBe('45');
+    } finally {
+      await closeWebSocketServer(wss);
+    }
+  });
 });
 
 const hasAssemblyAIApiKey = Boolean(process.env.ASSEMBLYAI_API_KEY);

package/src/stt.ts

@@ -6,6 +6,8 @@ import {
   type APIConnectOptions,
   type AudioBuffer,
   AudioByteStream,
+  ChatMessage,
+  type ConversationItemAddedEvent,
   Future,
   Task,
   createTimedString,
@@ -41,6 +43,7 @@ interface StreamEventMessage {
   end_of_turn_confidence?: number;
   turn_is_formatted?: boolean;
   language_code?: string;
+  language_confidence?: number;
   speaker_label?: string;
   words?: Array<{
     text?: string;
@@ -54,6 +57,18 @@ interface StreamEventMessage {
   session_duration_seconds?: number;
 }
 
+function speechDataMetadata(data: StreamEventMessage): stt.SpeechData['metadata'] | undefined {
+  const assemblyai: Record<string, number> = {};
+
+  if (typeof data.language_confidence === 'number') {
+    assemblyai.languageConfidence = data.language_confidence;
+  }
+
+  if (Object.keys(assemblyai).length === 0) return undefined;
+
+  return { assemblyai };
+}
+
 export interface STTOptions {
   apiKey?: string;
   sampleRate: number;
@@ -66,6 +81,11 @@ export interface STTOptions {
   encoding: STTEncoding;
   speechModel: STTModels;
   languageDetection?: boolean;
+  /**
+   * Session inactivity timeout in seconds. AssemblyAI accepts integer values
+   * from 5 to 3600; when unset, no inactivity timeout is applied.
+   */
+  inactivityTimeout?: number;
   endOfTurnConfidenceThreshold?: number;
   /** Minimum silence (ms) before a confident end-of-turn is finalized. */
   minTurnSilence?: number;
@@ -100,6 +120,13 @@ export interface STTOptions {
    * or `max_accuracy`. Explicit turn-silence values still take precedence over mode defaults.
    */
   mode?: 'min_latency' | 'balanced' | 'max_accuracy';
+  /**
+   * When the model supports it, let an `AgentSession` push each assistant reply into
+   * `agentContext` so it is carried into the model's conversation context. Defaults to false;
+   * set true to enable. Prior user turns are carried automatically by the model regardless of
+   * this flag. Ignored on models without context support.
+   */
+  agentContextCarryover?: boolean;
   baseUrl: string;
 }
 
@@ -115,6 +142,9 @@ const defaultSTTOptions: STTOptions = {
 export class STT extends stt.STT {
   #opts: STTOptions;
   #streams = new Set<WeakRef<SpeechStream>>();
+  // set (user + session))
+  #userKeyterms: string[];
+  #sessionKeyterms: string[] = [];
   label = 'assemblyai.STT';
 
   get model(): string {
@@ -126,10 +156,20 @@ export class STT extends stt.STT {
   }
 
   constructor(opts: Partial<STTOptions> = {}) {
+    // u3-rt-pro family — "u3-pro" is normalized below — and is opt-in via the user)
+    const rawModel = opts.speechModel ?? defaultSTTOptions.speechModel;
+    const supportsCarryover = isU3ProModel(rawModel) || rawModel === 'u3-pro';
+    if (opts.agentContextCarryover && !supportsCarryover) {
+      log().warn(
+        `agentContextCarryover is enabled but model '${rawModel}' does not support it; ignoring`,
+      );
+    }
     super({
       streaming: true,
       interimResults: true,
       alignedTranscript: 'word',
+      keyterms: true,
+      chatContext: (opts.agentContextCarryover ?? false) && supportsCarryover,
     });
 
     if (opts.speechModel === 'u3-pro') {
@@ -171,6 +211,7 @@ export class STT extends stt.STT {
       apiKey,
       minTurnSilence,
     };
+    this.#userKeyterms = [...(this.#opts.keytermsPrompt ?? [])];
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -179,17 +220,51 @@ export class STT extends stt.STT {
   }
 
   updateOptions(opts: Partial<STTOptions>) {
-    this.#opts = { ...this.#opts, ...opts };
+    // session keyterms so a user update doesn't drop them)
+    const nextOpts = { ...opts };
+    if (nextOpts.keytermsPrompt !== undefined) {
+      this.#userKeyterms = [...nextOpts.keytermsPrompt];
+      nextOpts.keytermsPrompt = [...new Set([...this.#userKeyterms, ...this.#sessionKeyterms])];
+    }
+    this.#opts = { ...this.#opts, ...nextOpts };
     for (const ref of this.#streams) {
       const stream = ref.deref();
       if (stream) {
-        stream.updateOptions(opts);
+        stream.updateOptions(nextOpts);
       } else {
         this.#streams.delete(ref);
       }
     }
   }
 
+  override _updateSessionKeyterms(keyterms: string[]): void {
+    if (
+      keyterms.length === this.#sessionKeyterms.length &&
+      keyterms.every((t, i) => t === this.#sessionKeyterms[i])
+    ) {
+      return;
+    }
+    this.#sessionKeyterms = [...keyterms];
+    const merged = [...new Set([...this.#userKeyterms, ...keyterms])];
+    this.#opts.keytermsPrompt = merged;
+    // applied live via the stream's UpdateConfiguration (no reconnect)
+    for (const ref of this.#streams) {
+      const stream = ref.deref();
+      if (stream) {
+        stream.updateOptions({ keytermsPrompt: merged });
+      } else {
+        this.#streams.delete(ref);
+      }
+    }
+  }
+
+  override _pushConversationItem(ev: ConversationItemAddedEvent): void {
+    const chatItem = ev.item;
+    if (chatItem instanceof ChatMessage && chatItem.role === 'assistant' && chatItem.textContent) {
+      this.updateOptions({ agentContext: chatItem.textContent });
+    }
+  }
+
   stream(options?: { connOptions?: APIConnectOptions }): SpeechStream {
     const stream = new SpeechStream(this, this.#opts, options?.connOptions);
     this.#streams.add(new WeakRef(stream));
@@ -321,10 +396,11 @@ export class SpeechStream extends stt.SpeechStream {
       min_turn_silence: minSilence,
       max_turn_silence: maxSilence,
       keyterms_prompt:
-        this.#opts.keytermsPrompt !== undefined
+        this.#opts.keytermsPrompt !== undefined && this.#opts.keytermsPrompt.length > 0
           ? JSON.stringify(this.#opts.keytermsPrompt)
           : undefined,
       language_detection: languageDetection,
+      inactivity_timeout: this.#opts.inactivityTimeout,
       prompt: this.#opts.prompt,
       agent_context: this.#opts.agentContext,
       previous_context_n_turns: this.#opts.previousContextNTurns,
@@ -525,6 +601,7 @@ export class SpeechStream extends stt.SpeechStream {
     const utterance = data.utterance ?? '';
     const transcript = data.transcript ?? '';
     const language = normalizeLanguage(data.language_code ?? 'en');
+    const metadata = speechDataMetadata(data);
 
     // Word timestamps are in milliseconds:
     // https://www.assemblyai.com/docs/api-reference/streaming-api/streaming-api#receive.receiveTurn.words
@@ -559,6 +636,7 @@ export class SpeechStream extends stt.SpeechStream {
             endTime,
             confidence,
             words: timedWords,
+            ...(metadata ? { metadata } : {}),
           },
         ],
       });
@@ -586,6 +664,7 @@ export class SpeechStream extends stt.SpeechStream {
             endTime,
             confidence: utteranceConfidence,
             words: utteranceWords,
+            ...(metadata ? { metadata } : {}),
           },
         ],
       });
@@ -606,6 +685,7 @@ export class SpeechStream extends stt.SpeechStream {
             endTime,
             confidence,
             words: timedWords,
+            ...(metadata ? { metadata } : {}),
           },
         ],
       });