@livekit/agents-plugin-openai 1.0.51 → 1.1.0

package/dist/tts.d.ts

@@ -14,6 +14,8 @@ export declare class TTS extends tts.TTS {
     #private;
     label: string;
     private abortController;
+    get model(): string;
+    get provider(): string;
     /**
      * Create a new instance of OpenAI TTS.
      *

package/dist/tts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,iBAAiB,EAA8B,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE1F,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,SAAS,GAAG,MAAM,CAAC;IAC1B,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AASD,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAG9B,KAAK,SAAgB;IACrB,OAAO,CAAC,eAAe,CAAyB;IAEhD;;;;;;OAMG;gBACS,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAgBzD,aAAa,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,SAAS,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAIrF,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW,GACxB,aAAa;IAoBhB,MAAM,IAAI,GAAG,CAAC,gBAAgB;IAIxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,KAAK,SAA0B;IAC/B,OAAO,CAAC,MAAM,CAAe;gBAI3B,GAAG,EAAE,GAAG,EACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,EACpB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW;cAMX,GAAG;CA+BpB"}
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,iBAAiB,EAA8B,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE1F,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,SAAS,GAAG,MAAM,CAAC;IAC1B,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AASD,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAG9B,KAAK,SAAgB;IACrB,OAAO,CAAC,eAAe,CAAyB;IAEhD,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,QAAQ,IAAI,MAAM,CAOrB;IAED;;;;;;OAMG;gBACS,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAgBzD,aAAa,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,SAAS,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAIrF,UAAU,CACR,IAAI,EAAE,MAAM,EACZ,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW,GACxB,aAAa;IAoBhB,MAAM,IAAI,GAAG,CAAC,gBAAgB;IAIxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,KAAK,SAA0B;IAC/B,OAAO,CAAC,MAAM,CAAe;gBAI3B,GAAG,EAAE,GAAG,EACR,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,EACpB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,WAAW,CAAC,EAAE,WAAW;cAMX,GAAG;CA+BpB"}

package/dist/tts.js

@@ -13,6 +13,17 @@ class TTS extends tts.TTS {
   #client;
   label = "openai.TTS";
   abortController = new AbortController();
+  get model() {
+    return this.#opts.model;
+  }
+  get provider() {
+    try {
+      const url = new URL(this.#client.baseURL);
+      return url.host;
+    } catch {
+      return "api.openai.com";
+    }
+  }
   /**
    * Create a new instance of OpenAI TTS.
    *

package/dist/tts.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type APIConnectOptions, AudioByteStream, shortuuid, tts } from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { OpenAI } from 'openai';\nimport type { TTSModels, TTSVoices } from './models.js';\n\nconst OPENAI_TTS_SAMPLE_RATE = 24000;\nconst OPENAI_TTS_CHANNELS = 1;\n\nexport interface TTSOptions {\n  model: TTSModels | string;\n  voice: TTSVoices;\n  speed: number;\n  instructions?: string;\n  baseURL?: string;\n  client?: OpenAI;\n  apiKey?: string;\n}\n\nconst defaultTTSOptions: TTSOptions = {\n  apiKey: process.env.OPENAI_API_KEY,\n  model: 'tts-1',\n  voice: 'alloy',\n  speed: 1,\n};\n\nexport class TTS extends tts.TTS {\n  #opts: TTSOptions;\n  #client: OpenAI;\n  label = 'openai.TTS';\n  private abortController = new AbortController();\n\n  /**\n   * Create a new instance of OpenAI TTS.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environment variable.\n   */\n  constructor(opts: Partial<TTSOptions> = defaultTTSOptions) {\n    super(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS, { streaming: false });\n\n    this.#opts = { ...defaultTTSOptions, ...opts };\n    if (this.#opts.apiKey === undefined && !this.#opts.client) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: this.#opts.baseURL,\n        apiKey: this.#opts.apiKey,\n      });\n  }\n\n  updateOptions(opts: { model?: TTSModels | string; voice?: TTSVoices; speed?: number }) {\n    this.#opts = { ...this.#opts, ...opts };\n  }\n\n  synthesize(\n    text: string,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ): ChunkedStream {\n    return new ChunkedStream(\n      this,\n      text,\n      this.#client.audio.speech.create(\n        {\n          input: text,\n          model: this.#opts.model,\n          voice: this.#opts.voice,\n          instructions: this.#opts.instructions,\n          response_format: 'pcm',\n          speed: this.#opts.speed,\n        },\n        { signal: abortSignal },\n      ),\n      connOptions,\n      abortSignal,\n    );\n  }\n\n  stream(): tts.SynthesizeStream {\n    throw new Error('Streaming is not supported on OpenAI TTS');\n  }\n\n  async close(): Promise<void> {\n    this.abortController.abort();\n  }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n  label = 'openai.ChunkedStream';\n  private stream: Promise<any>;\n\n  // set Promise<T> to any because OpenAI returns an annoying Response type\n  constructor(\n    tts: TTS,\n    text: string,\n    stream: Promise<any>,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ) {\n    super(text, tts, connOptions, abortSignal);\n    this.stream = stream;\n  }\n\n  protected async run() {\n    try {\n      const buffer = await this.stream.then((r) => r.arrayBuffer());\n      const requestId = shortuuid();\n      const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);\n      const frames = audioByteStream.write(buffer);\n\n      let lastFrame: AudioFrame | undefined;\n      const sendLastFrame = (segmentId: string, final: boolean) => {\n        if (lastFrame) {\n          this.queue.put({ requestId, segmentId, frame: lastFrame, final });\n          lastFrame = undefined;\n        }\n      };\n\n      for (const frame of frames) {\n        sendLastFrame(requestId, false);\n        lastFrame = frame;\n      }\n      sendLastFrame(requestId, true);\n\n      this.queue.close();\n    } catch (error) {\n      if (error instanceof Error && error.name === 'AbortError') {\n        return;\n      }\n      throw error;\n    } finally {\n      this.queue.close();\n    }\n  }\n}\n"],"mappings":"AAGA,SAAiC,iBAAiB,WAAW,WAAW;AAExE,SAAS,cAAc;AAGvB,MAAM,yBAAyB;AAC/B,MAAM,sBAAsB;AAY5B,MAAM,oBAAgC;AAAA,EACpC,QAAQ,QAAQ,IAAI;AAAA,EACpB,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AACT;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA,EACA,kBAAkB,IAAI,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAS9C,YAAY,OAA4B,mBAAmB;AACzD,UAAM,wBAAwB,qBAAqB,EAAE,WAAW,MAAM,CAAC;AAEvE,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,UAAa,CAAC,KAAK,MAAM,QAAQ;AACzD,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK,MAAM;AAAA,MACpB,QAAQ,KAAK,MAAM;AAAA,IACrB,CAAC;AAAA,EACL;AAAA,EAEA,cAAc,MAAyE;AACrF,SAAK,QAAQ,EAAE,GAAG,KAAK,OAAO,GAAG,KAAK;AAAA,EACxC;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA,KAAK,QAAQ,MAAM,OAAO;AAAA,QACxB;AAAA,UACE,OAAO;AAAA,UACP,OAAO,KAAK,MAAM;AAAA,UAClB,OAAO,KAAK,MAAM;AAAA,UAClB,cAAc,KAAK,MAAM;AAAA,UACzB,iBAAiB;AAAA,UACjB,OAAO,KAAK,MAAM;AAAA,QACpB;AAAA,QACA,EAAE,QAAQ,YAAY;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,MAAM,QAAuB;AAC3B,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AACF;AAEO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,QAAQ;AAAA,EACA;AAAA;AAAA,EAGR,YACEA,MACA,MACA,QACA,aACA,aACA;AACA,UAAM,MAAMA,MAAK,aAAa,WAAW;AACzC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAgB,MAAM;AACpB,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC;AAC5D,YAAM,YAAY,UAAU;AAC5B,YAAM,kBAAkB,IAAI,gBAAgB,wBAAwB,mBAAmB;AACvF,YAAM,SAAS,gBAAgB,MAAM,MAAM;AAE3C,UAAI;AACJ,YAAM,gBAAgB,CAAC,WAAmB,UAAmB;AAC3D,YAAI,WAAW;AACb,eAAK,MAAM,IAAI,EAAE,WAAW,WAAW,OAAO,WAAW,MAAM,CAAC;AAChE,sBAAY;AAAA,QACd;AAAA,MACF;AAEA,iBAAW,SAAS,QAAQ;AAC1B,sBAAc,WAAW,KAAK;AAC9B,oBAAY;AAAA,MACd;AACA,oBAAc,WAAW,IAAI;AAE7B,WAAK,MAAM,MAAM;AAAA,IACnB,SAAS,OAAO;AACd,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AACF;","names":["tts"]}
+{"version":3,"sources":["../src/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type APIConnectOptions, AudioByteStream, shortuuid, tts } from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { OpenAI } from 'openai';\nimport type { TTSModels, TTSVoices } from './models.js';\n\nconst OPENAI_TTS_SAMPLE_RATE = 24000;\nconst OPENAI_TTS_CHANNELS = 1;\n\nexport interface TTSOptions {\n  model: TTSModels | string;\n  voice: TTSVoices;\n  speed: number;\n  instructions?: string;\n  baseURL?: string;\n  client?: OpenAI;\n  apiKey?: string;\n}\n\nconst defaultTTSOptions: TTSOptions = {\n  apiKey: process.env.OPENAI_API_KEY,\n  model: 'tts-1',\n  voice: 'alloy',\n  speed: 1,\n};\n\nexport class TTS extends tts.TTS {\n  #opts: TTSOptions;\n  #client: OpenAI;\n  label = 'openai.TTS';\n  private abortController = new AbortController();\n\n  get model(): string {\n    return this.#opts.model;\n  }\n\n  get provider(): string {\n    try {\n      const url = new URL(this.#client.baseURL);\n      return url.host;\n    } catch {\n      return 'api.openai.com';\n    }\n  }\n\n  /**\n   * Create a new instance of OpenAI TTS.\n   *\n   * @remarks\n   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the\n   * `OPENAI_API_KEY` environment variable.\n   */\n  constructor(opts: Partial<TTSOptions> = defaultTTSOptions) {\n    super(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS, { streaming: false });\n\n    this.#opts = { ...defaultTTSOptions, ...opts };\n    if (this.#opts.apiKey === undefined && !this.#opts.client) {\n      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');\n    }\n\n    this.#client =\n      this.#opts.client ||\n      new OpenAI({\n        baseURL: this.#opts.baseURL,\n        apiKey: this.#opts.apiKey,\n      });\n  }\n\n  updateOptions(opts: { model?: TTSModels | string; voice?: TTSVoices; speed?: number }) {\n    this.#opts = { ...this.#opts, ...opts };\n  }\n\n  synthesize(\n    text: string,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ): ChunkedStream {\n    return new ChunkedStream(\n      this,\n      text,\n      this.#client.audio.speech.create(\n        {\n          input: text,\n          model: this.#opts.model,\n          voice: this.#opts.voice,\n          instructions: this.#opts.instructions,\n          response_format: 'pcm',\n          speed: this.#opts.speed,\n        },\n        { signal: abortSignal },\n      ),\n      connOptions,\n      abortSignal,\n    );\n  }\n\n  stream(): tts.SynthesizeStream {\n    throw new Error('Streaming is not supported on OpenAI TTS');\n  }\n\n  async close(): Promise<void> {\n    this.abortController.abort();\n  }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n  label = 'openai.ChunkedStream';\n  private stream: Promise<any>;\n\n  // set Promise<T> to any because OpenAI returns an annoying Response type\n  constructor(\n    tts: TTS,\n    text: string,\n    stream: Promise<any>,\n    connOptions?: APIConnectOptions,\n    abortSignal?: AbortSignal,\n  ) {\n    super(text, tts, connOptions, abortSignal);\n    this.stream = stream;\n  }\n\n  protected async run() {\n    try {\n      const buffer = await this.stream.then((r) => r.arrayBuffer());\n      const requestId = shortuuid();\n      const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);\n      const frames = audioByteStream.write(buffer);\n\n      let lastFrame: AudioFrame | undefined;\n      const sendLastFrame = (segmentId: string, final: boolean) => {\n        if (lastFrame) {\n          this.queue.put({ requestId, segmentId, frame: lastFrame, final });\n          lastFrame = undefined;\n        }\n      };\n\n      for (const frame of frames) {\n        sendLastFrame(requestId, false);\n        lastFrame = frame;\n      }\n      sendLastFrame(requestId, true);\n\n      this.queue.close();\n    } catch (error) {\n      if (error instanceof Error && error.name === 'AbortError') {\n        return;\n      }\n      throw error;\n    } finally {\n      this.queue.close();\n    }\n  }\n}\n"],"mappings":"AAGA,SAAiC,iBAAiB,WAAW,WAAW;AAExE,SAAS,cAAc;AAGvB,MAAM,yBAAyB;AAC/B,MAAM,sBAAsB;AAY5B,MAAM,oBAAgC;AAAA,EACpC,QAAQ,QAAQ,IAAI;AAAA,EACpB,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AACT;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA;AAAA,EACA,QAAQ;AAAA,EACA,kBAAkB,IAAI,gBAAgB;AAAA,EAE9C,IAAI,QAAgB;AAClB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,IAAI,WAAmB;AACrB,QAAI;AACF,YAAM,MAAM,IAAI,IAAI,KAAK,QAAQ,OAAO;AACxC,aAAO,IAAI;AAAA,IACb,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,YAAY,OAA4B,mBAAmB;AACzD,UAAM,wBAAwB,qBAAqB,EAAE,WAAW,MAAM,CAAC;AAEvE,SAAK,QAAQ,EAAE,GAAG,mBAAmB,GAAG,KAAK;AAC7C,QAAI,KAAK,MAAM,WAAW,UAAa,CAAC,KAAK,MAAM,QAAQ;AACzD,YAAM,IAAI,MAAM,0EAA0E;AAAA,IAC5F;AAEA,SAAK,UACH,KAAK,MAAM,UACX,IAAI,OAAO;AAAA,MACT,SAAS,KAAK,MAAM;AAAA,MACpB,QAAQ,KAAK,MAAM;AAAA,IACrB,CAAC;AAAA,EACL;AAAA,EAEA,cAAc,MAAyE;AACrF,SAAK,QAAQ,EAAE,GAAG,KAAK,OAAO,GAAG,KAAK;AAAA,EACxC;AAAA,EAEA,WACE,MACA,aACA,aACe;AACf,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA,KAAK,QAAQ,MAAM,OAAO;AAAA,QACxB;AAAA,UACE,OAAO;AAAA,UACP,OAAO,KAAK,MAAM;AAAA,UAClB,OAAO,KAAK,MAAM;AAAA,UAClB,cAAc,KAAK,MAAM;AAAA,UACzB,iBAAiB;AAAA,UACjB,OAAO,KAAK,MAAM;AAAA,QACpB;AAAA,QACA,EAAE,QAAQ,YAAY;AAAA,MACxB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,SAA+B;AAC7B,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AAAA,EAEA,MAAM,QAAuB;AAC3B,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AACF;AAEO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,QAAQ;AAAA,EACA;AAAA;AAAA,EAGR,YACEA,MACA,MACA,QACA,aACA,aACA;AACA,UAAM,MAAMA,MAAK,aAAa,WAAW;AACzC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAgB,MAAM;AACpB,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,OAAO,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC;AAC5D,YAAM,YAAY,UAAU;AAC5B,YAAM,kBAAkB,IAAI,gBAAgB,wBAAwB,mBAAmB;AACvF,YAAM,SAAS,gBAAgB,MAAM,MAAM;AAE3C,UAAI;AACJ,YAAM,gBAAgB,CAAC,WAAmB,UAAmB;AAC3D,YAAI,WAAW;AACb,eAAK,MAAM,IAAI,EAAE,WAAW,WAAW,OAAO,WAAW,MAAM,CAAC;AAChE,sBAAY;AAAA,QACd;AAAA,MACF;AAEA,iBAAW,SAAS,QAAQ;AAC1B,sBAAc,WAAW,KAAK;AAC9B,oBAAY;AAAA,MACd;AACA,oBAAc,WAAW,IAAI;AAE7B,WAAK,MAAM,MAAM;AAAA,IACnB,SAAS,OAAO;AACd,UAAI,iBAAiB,SAAS,MAAM,SAAS,cAAc;AACzD;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,WAAK,MAAM,MAAM;AAAA,IACnB;AAAA,EACF;AACF;","names":["tts"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "1.0.51",
+  "version": "1.1.0",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -30,9 +30,9 @@
     "@types/ws": "^8.5.10",
     "tsup": "^8.3.5",
     "typescript": "^5.0.0",
-    "@livekit/agents": "1.0.51",
-    "@livekit/agents-plugin-silero": "1.0.51",
-    "@livekit/agents-plugins-test": "1.0.51"
+    "@livekit/agents": "1.1.0",
+    "@livekit/agents-plugin-silero": "1.1.0",
+    "@livekit/agents-plugins-test": "1.1.0"
   },
   "dependencies": {
     "@livekit/mutex": "^1.1.1",
@@ -42,7 +42,7 @@
   "peerDependencies": {
     "@livekit/rtc-node": "^0.13.24",
     "zod": "^3.25.76 || ^4.1.8",
-    "@livekit/agents": "1.0.51"
+    "@livekit/agents": "1.1.0"
   },
   "scripts": {
     "build": "tsup --onSuccess \"pnpm build:types\"",

package/src/llm.ts

@@ -86,6 +86,15 @@ export class LLM extends llm.LLM {
     return this.#opts.model;
   }
 
+  get provider(): string {
+    try {
+      const url = new URL(this.#client.baseURL);
+      return url.host;
+    } catch {
+      return 'api.openai.com';
+    }
+  }
+
   /**
    * Create a new instance of OpenAI LLM with Azure.
    *

package/src/realtime/api_proto.ts

@@ -162,6 +162,11 @@ export interface InputAudioContent {
   audio: AudioBase64Bytes;
 }
 
+export interface InputImageContent {
+  type: 'input_image';
+  image_url: string;
+}
+
 export interface TextContent {
   type: 'text';
   text: string;
@@ -181,6 +186,7 @@ export interface AudioContent {
 export type Content =
   | InputTextContent
   | InputAudioContent
+  | InputImageContent
   | TextContent
   | OutputTextContent
   | AudioContent;
@@ -206,7 +212,7 @@ export interface SystemItem extends BaseItem {
 export interface UserItem extends BaseItem {
   type: 'message';
   role: 'user';
-  content: (InputTextContent | InputAudioContent)[];
+  content: (InputTextContent | InputAudioContent | InputImageContent)[];
 }
 
 export interface AssistantItem extends BaseItem {
@@ -361,7 +367,7 @@ export interface UserItemCreate {
   id: string;
   type: 'message';
   role: 'user';
-  content: (InputTextContent | InputAudioContent)[];
+  content: (InputTextContent | InputAudioContent | InputImageContent)[];
 }
 
 export interface AssistantItemCreate {

package/src/realtime/realtime_model.test.ts

@@ -8,14 +8,14 @@ import { livekitItemToOpenAIItem } from './realtime_model.js';
 
 describe('livekitItemToOpenAIItem', () => {
   describe('message items', () => {
-    it('should use output_text type for assistant messages', () => {
+    it('should use output_text type for assistant messages', async () => {
       const assistantMessage = new llm.ChatMessage({
         role: 'assistant',
         content: 'Hello, how can I help you?',
         id: 'test-assistant-msg',
       });
 
-      const result = livekitItemToOpenAIItem(assistantMessage) as api_proto.AssistantItem;
+      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;
 
       expect(result.type).toBe('message');
       expect(result.role).toBe('assistant');
@@ -25,14 +25,14 @@ describe('livekitItemToOpenAIItem', () => {
       expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');
     });
 
-    it('should use input_text type for user messages', () => {
+    it('should use input_text type for user messages', async () => {
       const userMessage = new llm.ChatMessage({
         role: 'user',
         content: 'What is the weather like?',
         id: 'test-user-msg',
       });
 
-      const result = livekitItemToOpenAIItem(userMessage) as api_proto.UserItem;
+      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
 
       expect(result.type).toBe('message');
       expect(result.role).toBe('user');
@@ -42,14 +42,14 @@ describe('livekitItemToOpenAIItem', () => {
       expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');
     });
 
-    it('should use input_text type for system messages', () => {
+    it('should use input_text type for system messages', async () => {
       const systemMessage = new llm.ChatMessage({
         role: 'system',
         content: 'You are a helpful assistant.',
         id: 'test-system-msg',
       });
 
-      const result = livekitItemToOpenAIItem(systemMessage) as api_proto.UserItem;
+      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.UserItem;
 
       expect(result.type).toBe('message');
       expect(result.role).toBe('system');
@@ -58,14 +58,14 @@ describe('livekitItemToOpenAIItem', () => {
       expect(content.type).toBe('input_text');
     });
 
-    it('should convert developer role to system role', () => {
+    it('should convert developer role to system role', async () => {
       const developerMessage = new llm.ChatMessage({
         role: 'developer',
         content: 'System instructions.',
         id: 'test-developer-msg',
       });
 
-      const result = livekitItemToOpenAIItem(developerMessage) as api_proto.UserItem;
+      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.UserItem;
 
       expect(result.type).toBe('message');
       expect(result.role).toBe('system');
@@ -73,14 +73,16 @@ describe('livekitItemToOpenAIItem', () => {
       expect(content.type).toBe('input_text');
     });
 
-    it('should handle multiple content items for assistant', () => {
+    it('should handle multiple content items for assistant', async () => {
       const multiContentMessage = new llm.ChatMessage({
         role: 'assistant',
         content: ['First part.', 'Second part.'],
         id: 'test-multi-msg',
       });
 
-      const result = livekitItemToOpenAIItem(multiContentMessage) as api_proto.AssistantItem;
+      const result = (await livekitItemToOpenAIItem(
+        multiContentMessage,
+      )) as api_proto.AssistantItem;
 
       expect(result.content).toHaveLength(2);
       const content0 = result.content[0]!;
@@ -88,10 +90,121 @@ describe('livekitItemToOpenAIItem', () => {
       expect(content0.type).toBe('output_text');
       expect(content1.type).toBe('output_text');
     });
+
+    it('should convert image content to input_image for user messages', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: 'image/png',
+      });
+
+      const userMessage = new llm.ChatMessage({
+        role: 'user',
+        content: [imageContent],
+        id: 'test-image-msg',
+      });
+
+      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
+
+      expect(result.type).toBe('message');
+      expect(result.role).toBe('user');
+      expect(result.content).toHaveLength(1);
+      const content = result.content[0]!;
+      expect(content.type).toBe('input_image');
+      expect((content as api_proto.InputImageContent).image_url).toBe(
+        `data:image/png;base64,${base64Data}`,
+      );
+    });
+
+    it('should ignore image content for assistant messages', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: 'image/png',
+      });
+
+      const assistantMessage = new llm.ChatMessage({
+        role: 'assistant',
+        content: [imageContent],
+        id: 'test-assistant-image-msg',
+      });
+
+      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;
+
+      expect(result.type).toBe('message');
+      expect(result.role).toBe('assistant');
+      expect(result.content).toHaveLength(0);
+    });
+
+    it('should ignore image content for system messages', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: 'image/png',
+      });
+
+      const systemMessage = new llm.ChatMessage({
+        role: 'system',
+        content: [imageContent],
+        id: 'test-system-image-msg',
+      });
+
+      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.SystemItem;
+
+      expect(result.type).toBe('message');
+      expect(result.role).toBe('system');
+      expect(result.content).toHaveLength(0);
+    });
+
+    it('should ignore image content for developer messages mapped to system', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: 'image/png',
+      });
+
+      const developerMessage = new llm.ChatMessage({
+        role: 'developer',
+        content: [imageContent],
+        id: 'test-developer-image-msg',
+      });
+
+      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.SystemItem;
+
+      expect(result.type).toBe('message');
+      expect(result.role).toBe('system');
+      expect(result.content).toHaveLength(0);
+    });
+
+    it('should handle mixed text and image content', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: 'image/png',
+      });
+
+      const userMessage = new llm.ChatMessage({
+        role: 'user',
+        content: ['Describe this image:', imageContent],
+        id: 'test-mixed-msg',
+      });
+
+      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
+
+      expect(result.type).toBe('message');
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]!.type).toBe('input_text');
+      expect(result.content[1]!.type).toBe('input_image');
+    });
   });
 
   describe('function_call items', () => {
-    it('should convert function call items correctly', () => {
+    it('should convert function call items correctly', async () => {
       const functionCall = new llm.FunctionCall({
         callId: 'call-123',
         name: 'get_weather',
@@ -99,7 +212,7 @@ describe('livekitItemToOpenAIItem', () => {
         id: 'test-func-call',
       });
 
-      const result = livekitItemToOpenAIItem(functionCall) as api_proto.FunctionCallItem;
+      const result = (await livekitItemToOpenAIItem(functionCall)) as api_proto.FunctionCallItem;
 
       expect(result.type).toBe('function_call');
       expect(result.id).toBe('test-func-call');
@@ -110,7 +223,7 @@ describe('livekitItemToOpenAIItem', () => {
   });
 
   describe('function_call_output items', () => {
-    it('should convert function call output items correctly', () => {
+    it('should convert function call output items correctly', async () => {
       const functionOutput = new llm.FunctionCallOutput({
         callId: 'call-123',
         output: 'The weather in San Francisco is sunny.',
@@ -118,7 +231,9 @@ describe('livekitItemToOpenAIItem', () => {
         id: 'test-func-output',
       });
 
-      const result = livekitItemToOpenAIItem(functionOutput) as api_proto.FunctionCallOutputItem;
+      const result = (await livekitItemToOpenAIItem(
+        functionOutput,
+      )) as api_proto.FunctionCallOutputItem;
 
       expect(result.type).toBe('function_call_output');
       expect(result.id).toBe('test-func-output');

package/src/realtime/realtime_model.ts

@@ -144,6 +144,15 @@ export class RealtimeModel extends llm.RealtimeModel {
     return this._options.model;
   }
 
+  get provider(): string {
+    try {
+      const url = new URL(this._options.baseURL);
+      return url.host;
+    } catch {
+      return 'api.openai.com';
+    }
+  }
+
   constructor(
     options: {
       model?: string;
@@ -461,28 +470,27 @@ export class RealtimeSession extends llm.RealtimeSession {
 
   async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {
     const unlock = await this.updateChatCtxLock.lock();
-    const events = this.createChatCtxUpdateEvents(_chatCtx);
-    const futures: Future<void>[] = [];
+    try {
+      const events = await this.createChatCtxUpdateEvents(_chatCtx);
+      const futures: Future<void>[] = [];
 
-    for (const event of events) {
-      const future = new Future<void>();
-      futures.push(future);
+      for (const event of events) {
+        const future = new Future<void>();
+        futures.push(future);
 
-      if (event.type === 'conversation.item.create') {
-        this.itemCreateFutures[event.item.id] = future;
-      } else if (event.type == 'conversation.item.delete') {
-        this.itemDeleteFutures[event.item_id] = future;
-      }
+        if (event.type === 'conversation.item.create') {
+          this.itemCreateFutures[event.item.id] = future;
+        } else if (event.type == 'conversation.item.delete') {
+          this.itemDeleteFutures[event.item_id] = future;
+        }
 
-      this.sendEvent(event);
-    }
+        this.sendEvent(event);
+      }
 
-    if (futures.length === 0) {
-      unlock();
-      return;
-    }
+      if (futures.length === 0) {
+        return;
+      }
 
-    try {
       // wait for futures to resolve or timeout
       await Promise.race([
         Promise.all(futures),
@@ -498,10 +506,10 @@ export class RealtimeSession extends llm.RealtimeSession {
     }
   }
 
-  private createChatCtxUpdateEvents(
+  private async createChatCtxUpdateEvents(
     chatCtx: llm.ChatContext,
     addMockAudio: boolean = false,
-  ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {
+  ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> {
     const newChatCtx = chatCtx.copy();
     if (addMockAudio) {
       newChatCtx.items.push(createMockAudioItem());
@@ -533,7 +541,7 @@ export class RealtimeSession extends llm.RealtimeSession {
       }
       events.push({
         type: 'conversation.item.create',
-        item: livekitItemToOpenAIItem(chatItem),
+        item: await livekitItemToOpenAIItem(chatItem),
         previous_item_id: previousId ?? undefined,
         event_id: shortuuid('chat_ctx_create_'),
       } as api_proto.ConversationItemCreateEvent);
@@ -704,7 +712,7 @@ export class RealtimeSession extends llm.RealtimeSession {
             content: [_options.audioTranscript],
           });
           chatCtx.items[idx] = newItem;
-          const events = this.createChatCtxUpdateEvents(chatCtx);
+          const events = await this.createChatCtxUpdateEvents(chatCtx);
           for (const ev of events) {
             this.sendEvent(ev);
           }
@@ -845,7 +853,7 @@ export class RealtimeSession extends llm.RealtimeSession {
 
       const oldChatCtx = this.remoteChatCtx;
       this.remoteChatCtx = new llm.RemoteChatContext();
-      events.push(...this.createChatCtxUpdateEvents(chatCtx));
+      events.push(...(await this.createChatCtxUpdateEvents(chatCtx)));
 
       try {
         for (const ev of events) {
@@ -1560,7 +1568,6 @@ export class RealtimeSession extends llm.RealtimeSession {
     if (event.error.message.startsWith('Cancellation failed')) {
       return;
     }
-
     this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');
     this.emitError({
       error: new APIError(event.error.message, {
@@ -1622,7 +1629,7 @@ export class RealtimeSession extends llm.RealtimeSession {
 }
 
 /** @internal Exported for testing purposes */
-export function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {
+export async function livekitItemToOpenAIItem(item: llm.ChatItem): Promise<api_proto.ItemResource> {
   switch (item.type) {
     case 'function_call':
       return {
@@ -1649,8 +1656,22 @@ export function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResou
             text: c,
           } as api_proto.InputTextContent | api_proto.OutputTextContent);
         } else if (c.type === 'image_content') {
-          // not supported for now
-          continue;
+          // only user can send image
+          if (role !== 'user') continue;
+
+          const serialized = await llm.serializeImage(c);
+          if (serialized.externalUrl) {
+            log().warn('External URL is not supported for input_image in realtime API');
+            continue;
+          }
+          if (!serialized.base64Data) {
+            log().warn('Serialized image has no data bytes');
+            continue;
+          }
+          contentList.push({
+            type: 'input_image',
+            image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
+          } as api_proto.InputImageContent);
         } else if (c.type === 'audio_content') {
           if (role === 'user') {
             const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');
@@ -1699,6 +1720,10 @@ function openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {
       for (const c of contents) {
         if (c.type === 'text' || c.type === 'input_text') {
           content.push(c.text);
+        } else if (c.type === 'input_image' && (c as api_proto.InputImageContent).image_url) {
+          content.push(
+            llm.createImageContent({ image: (c as api_proto.InputImageContent).image_url }),
+          );
         }
       }
       return llm.ChatMessage.create({

package/src/realtime/realtime_model_beta.ts

@@ -448,28 +448,27 @@ export class RealtimeSession extends llm.RealtimeSession {
 
   async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {
     const unlock = await this.updateChatCtxLock.lock();
-    const events = this.createChatCtxUpdateEvents(_chatCtx);
-    const futures: Future<void>[] = [];
+    try {
+      const events = await this.createChatCtxUpdateEvents(_chatCtx);
+      const futures: Future<void>[] = [];
 
-    for (const event of events) {
-      const future = new Future<void>();
-      futures.push(future);
+      for (const event of events) {
+        const future = new Future<void>();
+        futures.push(future);
 
-      if (event.type === 'conversation.item.create') {
-        this.itemCreateFutures[event.item.id] = future;
-      } else if (event.type == 'conversation.item.delete') {
-        this.itemDeleteFutures[event.item_id] = future;
-      }
+        if (event.type === 'conversation.item.create') {
+          this.itemCreateFutures[event.item.id] = future;
+        } else if (event.type == 'conversation.item.delete') {
+          this.itemDeleteFutures[event.item_id] = future;
+        }
 
-      this.sendEvent(event);
-    }
+        this.sendEvent(event);
+      }
 
-    if (futures.length === 0) {
-      unlock();
-      return;
-    }
+      if (futures.length === 0) {
+        return;
+      }
 
-    try {
       // wait for futures to resolve or timeout
       await Promise.race([
         Promise.all(futures),
@@ -485,10 +484,10 @@ export class RealtimeSession extends llm.RealtimeSession {
     }
   }
 
-  private createChatCtxUpdateEvents(
+  private async createChatCtxUpdateEvents(
     chatCtx: llm.ChatContext,
     addMockAudio: boolean = false,
-  ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {
+  ): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> {
     const newChatCtx = chatCtx.copy();
     if (addMockAudio) {
       newChatCtx.items.push(createMockAudioItem());
@@ -520,7 +519,7 @@ export class RealtimeSession extends llm.RealtimeSession {
       }
       events.push({
         type: 'conversation.item.create',
-        item: livekitItemToOpenAIItem(chatItem),
+        item: await livekitItemToOpenAIItem(chatItem),
         previous_item_id: previousId ?? undefined,
         event_id: shortuuid('chat_ctx_create_'),
       } as api_proto.ConversationItemCreateEvent);
@@ -682,7 +681,7 @@ export class RealtimeSession extends llm.RealtimeSession {
             content: [_options.audioTranscript],
           });
           chatCtx.items[idx] = newItem;
-          const events = this.createChatCtxUpdateEvents(chatCtx);
+          const events = await this.createChatCtxUpdateEvents(chatCtx);
           for (const ev of events) {
             this.sendEvent(ev);
           }
@@ -805,7 +804,7 @@ export class RealtimeSession extends llm.RealtimeSession {
 
       const oldChatCtx = this.remoteChatCtx;
       this.remoteChatCtx = new llm.RemoteChatContext();
-      events.push(...this.createChatCtxUpdateEvents(chatCtx));
+      events.push(...(await this.createChatCtxUpdateEvents(chatCtx)));
 
       try {
         for (const ev of events) {
@@ -1521,7 +1520,7 @@ export class RealtimeSession extends llm.RealtimeSession {
   }
 }
 
-function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {
+async function livekitItemToOpenAIItem(item: llm.ChatItem): Promise<api_proto.ItemResource> {
   switch (item.type) {
     case 'function_call':
       return {
@@ -1548,8 +1547,22 @@ function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {
             text: c,
           } as api_proto.InputTextContent);
         } else if (c.type === 'image_content') {
-          // not supported for now
-          continue;
+          if (role !== 'user') {
+            continue;
+          }
+          const serialized = await llm.serializeImage(c);
+          if (serialized.externalUrl) {
+            log().warn('External URL is not supported for input_image in realtime API');
+            continue;
+          }
+          if (!serialized.base64Data) {
+            log().warn('Serialized image has no data bytes');
+            continue;
+          }
+          contentList.push({
+            type: 'input_image',
+            image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
+          } as api_proto.InputImageContent);
         } else if (c.type === 'audio_content') {
           if (role === 'user') {
             const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');
@@ -1598,6 +1611,10 @@ function openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {
       for (const c of contents) {
         if (c.type === 'text' || c.type === 'input_text') {
           content.push(c.text);
+        } else if (c.type === 'input_image' && (c as api_proto.InputImageContent).image_url) {
+          content.push(
+            llm.createImageContent({ image: (c as api_proto.InputImageContent).image_url }),
+          );
         }
       }
       return llm.ChatMessage.create({