@livekit/agents-plugin-openai 0.4.3 → 0.5.0

package/dist/tts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.js","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AACrC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAW9B,MAAM,iBAAiB,GAAe;IACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;IAClC,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,OAAO,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,KAAK,CAAa;IAClB,OAAO,CAAS;IAEhB;;;;;;OAMG;IACH,YAAY,OAA4B,iBAAiB;QACvD,KAAK,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAEzE,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;QAED,IAAI,CAAC,OAAO;YACV,IAAI,CAAC,KAAK,CAAC,MAAM;gBACjB,IAAI,MAAM,CAAC;oBACT,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;IACP,CAAC;IAED,aAAa,CAAC,IAAuE;QACnF,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,aAAa,CACtB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,eAAe,EAAE,KAAK;YACtB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;SACxB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,MAAM,OAAO,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,YAAY,MAAyB;QACnC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAyB;QAClC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,sBAAsB,EAAE,mBAAmB,CAAC,CAAC;QACzF,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gBACb,KAAK;gBACL,SAAS;gBACT,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -16,7 +16,7 @@
     "openai": "^4.70.2",
     "sharp": "^0.33.5",
     "ws": "^8.16.0",
-    "@livekit/agents": "0.4.3"
+    "@livekit/agents": "0.4.4"
   },
   "scripts": {
     "build": "tsc -b tsconfig.json",

package/src/index.ts

@@ -4,3 +4,5 @@
 export * as realtime from './realtime/index.js';
 export * from './models.js';
 export { type LLMOptions, LLM, LLMStream } from './llm.js';
+export { type STTOptions, STT } from './stt.js';
+export { type TTSOptions, TTS, ChunkedStream } from './tts.js';

package/src/models.ts

@@ -27,7 +27,13 @@ export type ChatModels =
   | 'gpt-3.5-turbo-1106'
   | 'gpt-3.5-turbo-16k-0613';
 
-// adapters for OpenAI-compatible LLMs
+export type WhisperModels = 'whisper-1';
+
+export type TTSModels = 'tts-1' | 'tts-1-hd';
+
+export type TTSVoices = 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+
+// adapters for OpenAI-compatible LLMs, TTSs, STTs
 
 export type TelnyxChatModels =
   | 'meta-llama/Meta-Llama-3.1-8B-Instruct'
@@ -56,6 +62,11 @@ export type GroqChatModels =
   | 'gemma-7b-it'
   | 'gemma2-9b-it';
 
+export type GroqAudioModels =
+  | 'whisper-large-v3'
+  | 'distil-whisper-large-v3-en'
+  | 'whisper-large-v3-turbo';
+
 export type DeepSeekChatModels = 'deepseek-coder' | 'deepseek-chat';
 
 export type TogetherChatModels =

package/src/realtime/api_proto.ts

@@ -208,17 +208,32 @@ export type ResponseStatusDetails =
       reason: 'turn_detected' | 'client_cancelled' | string;
     };
 
+export interface ModelUsage {
+  total_tokens: number;
+  input_tokens: number;
+  output_tokens: number;
+  input_token_details: {
+    text_tokens: number;
+    audio_tokens: number;
+    cached_tokens: number;
+    cached_tokens_details: {
+      text_tokens: number;
+      audio_tokens: number;
+    };
+  };
+  output_token_details: {
+    text_tokens: number;
+    audio_tokens: number;
+  };
+}
+
 export interface ResponseResource {
   id: string;
   object: 'realtime.response';
   status: ResponseStatus;
   status_details: ResponseStatusDetails;
   output: ItemResource[];
-  usage?: {
-    total_tokens: number;
-    input_tokens: number;
-    output_tokens: number;
-  };
+  usage?: ModelUsage;
 }
 
 // Client Events

package/src/realtime/realtime_model.ts

@@ -37,7 +37,7 @@ export interface RealtimeResponse {
   id: string;
   status: api_proto.ResponseStatus;
   statusDetails: api_proto.ResponseStatusDetails | null;
-  usage: api_proto.ResponseResource['usage'] | null;
+  usage: api_proto.ModelUsage | null;
   output: RealtimeOutput[];
   doneFut: Future;
 }
@@ -939,7 +939,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     const response = this.#pendingResponses[responseId];
     response.status = responseData.status;
     response.statusDetails = responseData.status_details;
-    response.usage = responseData.usage;
+    response.usage = responseData.usage ?? null;
     this.#pendingResponses[responseId] = response;
     response.doneFut.resolve();
     this.emit('response_done', response);

package/src/stt.ts

@@ -0,0 +1,140 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type AudioBuffer, mergeFrames, stt } from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
+import { OpenAI } from 'openai';
+import type { GroqAudioModels, WhisperModels } from './models.js';
+
+export interface STTOptions {
+  apiKey?: string;
+  language: string;
+  detectLanguage: boolean;
+  model: WhisperModels | string;
+  baseURL?: string;
+  client?: OpenAI;
+}
+
+const defaultSTTOptions: STTOptions = {
+  apiKey: process.env.OPENAI_API_KEY,
+  language: 'en',
+  detectLanguage: false,
+  model: 'whisper-1',
+};
+
+export class STT extends stt.STT {
+  #opts: STTOptions;
+  #client: OpenAI;
+
+  /**
+   * Create a new instance of OpenAI STT.
+   *
+   * @remarks
+   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+   * `OPENAI_API_KEY` environmental variable.
+   */
+  constructor(opts: Partial<STTOptions> = defaultSTTOptions) {
+    super({ streaming: false, interimResults: false });
+
+    this.#opts = { ...defaultSTTOptions, ...opts };
+    if (this.#opts.apiKey === undefined) {
+      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
+    }
+
+    this.#client =
+      this.#opts.client ||
+      new OpenAI({
+        baseURL: opts.baseURL,
+        apiKey: opts.apiKey,
+      });
+  }
+
+  /**
+   * Create a new instance of Groq STT.
+   *
+   * @remarks
+   * `apiKey` must be set to your Groq API key, either using the argument or by setting the
+   * `GROQ_API_KEY` environmental variable.
+   */
+  static withGroq(
+    opts: Partial<{
+      model: string | GroqAudioModels;
+      apiKey?: string;
+      baseURL?: string;
+      client: OpenAI;
+      language: string;
+      detectLanguage: boolean;
+    }> = {},
+  ): STT {
+    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;
+    if (opts.apiKey === undefined) {
+      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');
+    }
+
+    return new STT({
+      model: 'whisper-large-v3-turbo',
+      baseURL: 'https://api.groq.com/openai/v1',
+      ...opts,
+    });
+  }
+
+  #sanitizeOptions(language?: string): STTOptions {
+    if (language) {
+      return { ...this.#opts, language };
+    } else {
+      return this.#opts;
+    }
+  }
+
+  #createWav(frame: AudioFrame): Buffer {
+    const bitsPerSample = 16;
+    const byteRate = (frame.sampleRate * frame.channels * bitsPerSample) / 8;
+    const blockAlign = (frame.channels * bitsPerSample) / 8;
+
+    const header = Buffer.alloc(44);
+    header.write('RIFF', 0);
+    header.writeUInt32LE(36 + frame.data.byteLength, 4);
+    header.write('WAVE', 8);
+    header.write('fmt ', 12);
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(frame.channels, 22);
+    header.writeUInt32LE(frame.sampleRate, 24);
+    header.writeUInt32LE(byteRate, 28);
+    header.writeUInt16LE(blockAlign, 32);
+    header.writeUInt16LE(16, 34);
+    header.write('data', 36);
+    header.writeUInt32LE(frame.data.byteLength, 40);
+    return Buffer.concat([header, Buffer.from(frame.data.buffer)]);
+  }
+
+  async recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
+    const config = this.#sanitizeOptions(language);
+    buffer = mergeFrames(buffer);
+    const file = new File([this.#createWav(buffer)], 'audio.wav', { type: 'audio/wav' });
+    const resp = await this.#client.audio.transcriptions.create({
+      file,
+      model: this.#opts.model,
+      language: config.language,
+      response_format: 'json',
+    });
+
+    return {
+      type: stt.SpeechEventType.FINAL_TRANSCRIPT,
+      alternatives: [
+        {
+          text: resp.text || '',
+          language: language || '',
+          startTime: 0,
+          endTime: 0,
+          confidence: 0,
+        },
+      ],
+    };
+  }
+
+  /** This method throws an error; streaming is unsupported on OpenAI STT. */
+  stream(): stt.SpeechStream {
+    throw new Error('Streaming is not supported on OpenAI STT');
+  }
+}

package/src/tts.ts

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AudioByteStream, tts } from '@livekit/agents';
+import { OpenAI } from 'openai';
+import type { TTSModels, TTSVoices } from './models.js';
+
+const OPENAI_TTS_SAMPLE_RATE = 24000;
+const OPENAI_TTS_CHANNELS = 1;
+
+export interface TTSOptions {
+  model: TTSModels | string;
+  voice: TTSVoices;
+  speed: number;
+  baseURL?: string;
+  client?: OpenAI;
+  apiKey?: string;
+}
+
+const defaultTTSOptions: TTSOptions = {
+  apiKey: process.env.OPENAI_API_KEY,
+  model: 'tts-1',
+  voice: 'alloy',
+  speed: 1,
+};
+
+export class TTS extends tts.TTS {
+  #opts: TTSOptions;
+  #client: OpenAI;
+
+  /**
+   * Create a new instance of OpenAI TTS.
+   *
+   * @remarks
+   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+   * `OPENAI_API_KEY` environmental variable.
+   */
+  constructor(opts: Partial<TTSOptions> = defaultTTSOptions) {
+    super(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS, { streaming: false });
+
+    this.#opts = { ...defaultTTSOptions, ...opts };
+    if (this.#opts.apiKey === undefined) {
+      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
+    }
+
+    this.#client =
+      this.#opts.client ||
+      new OpenAI({
+        baseURL: opts.baseURL,
+        apiKey: opts.apiKey,
+      });
+  }
+
+  updateOptions(opts: { model?: TTSModels | string; voice?: TTSVoices; speed?: number }) {
+    this.#opts = { ...this.#opts, ...opts };
+  }
+
+  synthesize(text: string): ChunkedStream {
+    return new ChunkedStream(
+      this.#client.audio.speech.create({
+        input: text,
+        model: this.#opts.model,
+        voice: this.#opts.voice,
+        response_format: 'pcm',
+        speed: this.#opts.speed,
+      }),
+    );
+  }
+
+  stream(): tts.SynthesizeStream {
+    throw new Error('Streaming is not supported on OpenAI TTS');
+  }
+}
+
+export class ChunkedStream extends tts.ChunkedStream {
+  constructor(stream: Promise<Response>) {
+    super();
+    this.#run(stream);
+  }
+
+  async #run(stream: Promise<Response>) {
+    const buffer = await stream.then((r) => r.arrayBuffer());
+    const requestId = crypto.randomUUID();
+    const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);
+    const frames = audioByteStream.write(buffer);
+
+    for (const frame of frames) {
+      this.queue.put({
+        frame,
+        requestId,
+        segmentId: requestId,
+      });
+    }
+    this.queue.close();
+  }
+}