@livekit/agents-plugin-openai 0.4.3 → 0.6.0

package/dist/stt.js

@@ -0,0 +1,109 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { mergeFrames, stt } from '@livekit/agents';
+import { OpenAI } from 'openai';
+const defaultSTTOptions = {
+    apiKey: process.env.OPENAI_API_KEY,
+    language: 'en',
+    detectLanguage: false,
+    model: 'whisper-1',
+};
+export class STT extends stt.STT {
+    #opts;
+    #client;
+    /**
+     * Create a new instance of OpenAI STT.
+     *
+     * @remarks
+     * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+     * `OPENAI_API_KEY` environmental variable.
+     */
+    constructor(opts = defaultSTTOptions) {
+        super({ streaming: false, interimResults: false });
+        this.#opts = { ...defaultSTTOptions, ...opts };
+        if (this.#opts.apiKey === undefined) {
+            throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
+        }
+        this.#client =
+            this.#opts.client ||
+                new OpenAI({
+                    baseURL: opts.baseURL,
+                    apiKey: opts.apiKey,
+                });
+    }
+    /**
+     * Create a new instance of Groq STT.
+     *
+     * @remarks
+     * `apiKey` must be set to your Groq API key, either using the argument or by setting the
+     * `GROQ_API_KEY` environmental variable.
+     */
+    static withGroq(opts = {}) {
+        opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;
+        if (opts.apiKey === undefined) {
+            throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');
+        }
+        return new STT({
+            model: 'whisper-large-v3-turbo',
+            baseURL: 'https://api.groq.com/openai/v1',
+            ...opts,
+        });
+    }
+    #sanitizeOptions(language) {
+        if (language) {
+            return { ...this.#opts, language };
+        }
+        else {
+            return this.#opts;
+        }
+    }
+    #createWav(frame) {
+        const bitsPerSample = 16;
+        const byteRate = (frame.sampleRate * frame.channels * bitsPerSample) / 8;
+        const blockAlign = (frame.channels * bitsPerSample) / 8;
+        const header = Buffer.alloc(44);
+        header.write('RIFF', 0);
+        header.writeUInt32LE(36 + frame.data.byteLength, 4);
+        header.write('WAVE', 8);
+        header.write('fmt ', 12);
+        header.writeUInt32LE(16, 16);
+        header.writeUInt16LE(1, 20);
+        header.writeUInt16LE(frame.channels, 22);
+        header.writeUInt32LE(frame.sampleRate, 24);
+        header.writeUInt32LE(byteRate, 28);
+        header.writeUInt16LE(blockAlign, 32);
+        header.writeUInt16LE(16, 34);
+        header.write('data', 36);
+        header.writeUInt32LE(frame.data.byteLength, 40);
+        return Buffer.concat([header, Buffer.from(frame.data.buffer)]);
+    }
+    async recognize(buffer, language) {
+        const config = this.#sanitizeOptions(language);
+        buffer = mergeFrames(buffer);
+        const file = new File([this.#createWav(buffer)], 'audio.wav', { type: 'audio/wav' });
+        const resp = await this.#client.audio.transcriptions.create({
+            file,
+            model: this.#opts.model,
+            language: config.language,
+            response_format: 'json',
+        });
+        return {
+            type: stt.SpeechEventType.FINAL_TRANSCRIPT,
+            alternatives: [
+                {
+                    text: resp.text || '',
+                    language: language || '',
+                    startTime: 0,
+                    endTime: 0,
+                    confidence: 0,
+                },
+            ],
+        };
+    }
+    /** This method throws an error; streaming is unsupported on OpenAI STT. */
+    stream() {
+        throw new Error('Streaming is not supported on OpenAI STT');
+    }
+}
+//# sourceMappingURL=stt.js.map

package/dist/stt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stt.js","sourceRoot":"","sources":["../src/stt.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAoB,WAAW,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAErE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAYhC,MAAM,iBAAiB,GAAe;IACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;IAClC,QAAQ,EAAE,IAAI;IACd,cAAc,EAAE,KAAK;IACrB,KAAK,EAAE,WAAW;CACnB,CAAC;AAEF,MAAM,OAAO,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,KAAK,CAAa;IAClB,OAAO,CAAS;IAEhB;;;;;;OAMG;IACH,YAAY,OAA4B,iBAAiB;QACvD,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QAEnD,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;QAED,IAAI,CAAC,OAAO;YACV,IAAI,CAAC,KAAK,CAAC,MAAM;gBACjB,IAAI,MAAM,CAAC;oBACT,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CACb,OAOK,EAAE;QAEP,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QACtD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,IAAI,GAAG,CAAC;YACb,KAAK,EAAE,wBAAwB;YAC/B,OAAO,EAAE,gCAAgC;YACzC,GAAG,IAAI;SACR,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CAAC,QAAiB;QAChC,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,UAAU,CAAC,KAAiB;QAC1B,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAExD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxB,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAmB,EAAE,QAAiB;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACrF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;YAC1D,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,eAAe,EAAE,MAAM;SACxB,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;YAC1C,YAAY,EAAE;gBACZ;oBACE,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;oBACrB,QAAQ,EAAE,QAAQ,IAAI,EAAE;oBACxB,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;oBACV,UAAU,EAAE,CAAC;iBACd;aACF;SACF,CAAC;IACJ,CAAC;IAED,2EAA2E;IAC3E,MAAM;QACJ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/dist/tts.d.ts

@@ -0,0 +1,34 @@
+import { tts } from '@livekit/agents';
+import { OpenAI } from 'openai';
+import type { TTSModels, TTSVoices } from './models.js';
+export interface TTSOptions {
+    model: TTSModels | string;
+    voice: TTSVoices;
+    speed: number;
+    baseURL?: string;
+    client?: OpenAI;
+    apiKey?: string;
+}
+export declare class TTS extends tts.TTS {
+    #private;
+    /**
+     * Create a new instance of OpenAI TTS.
+     *
+     * @remarks
+     * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+     * `OPENAI_API_KEY` environmental variable.
+     */
+    constructor(opts?: Partial<TTSOptions>);
+    updateOptions(opts: {
+        model?: TTSModels | string;
+        voice?: TTSVoices;
+        speed?: number;
+    }): void;
+    synthesize(text: string): ChunkedStream;
+    stream(): tts.SynthesizeStream;
+}
+export declare class ChunkedStream extends tts.ChunkedStream {
+    #private;
+    constructor(stream: Promise<Response>);
+}
+//# sourceMappingURL=tts.d.ts.map

package/dist/tts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAGA,OAAO,EAAmB,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAKxD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,SAAS,GAAG,MAAM,CAAC;IAC1B,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AASD,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;;IAI9B;;;;;;OAMG;gBACS,IAAI,GAAE,OAAO,CAAC,UAAU,CAAqB;IAgBzD,aAAa,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,SAAS,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAIrF,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAYvC,MAAM,IAAI,GAAG,CAAC,gBAAgB;CAG/B;AAED,qBAAa,aAAc,SAAQ,GAAG,CAAC,aAAa;;gBACtC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC;CAoBtC"}

package/dist/tts.js

@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AudioByteStream, tts } from '@livekit/agents';
+import { OpenAI } from 'openai';
+const OPENAI_TTS_SAMPLE_RATE = 24000;
+const OPENAI_TTS_CHANNELS = 1;
+const defaultTTSOptions = {
+    apiKey: process.env.OPENAI_API_KEY,
+    model: 'tts-1',
+    voice: 'alloy',
+    speed: 1,
+};
+export class TTS extends tts.TTS {
+    #opts;
+    #client;
+    /**
+     * Create a new instance of OpenAI TTS.
+     *
+     * @remarks
+     * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+     * `OPENAI_API_KEY` environmental variable.
+     */
+    constructor(opts = defaultTTSOptions) {
+        super(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS, { streaming: false });
+        this.#opts = { ...defaultTTSOptions, ...opts };
+        if (this.#opts.apiKey === undefined) {
+            throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
+        }
+        this.#client =
+            this.#opts.client ||
+                new OpenAI({
+                    baseURL: opts.baseURL,
+                    apiKey: opts.apiKey,
+                });
+    }
+    updateOptions(opts) {
+        this.#opts = { ...this.#opts, ...opts };
+    }
+    synthesize(text) {
+        return new ChunkedStream(this.#client.audio.speech.create({
+            input: text,
+            model: this.#opts.model,
+            voice: this.#opts.voice,
+            response_format: 'pcm',
+            speed: this.#opts.speed,
+        }));
+    }
+    stream() {
+        throw new Error('Streaming is not supported on OpenAI TTS');
+    }
+}
+export class ChunkedStream extends tts.ChunkedStream {
+    constructor(stream) {
+        super();
+        this.#run(stream);
+    }
+    async #run(stream) {
+        const buffer = await stream.then((r) => r.arrayBuffer());
+        const requestId = crypto.randomUUID();
+        const audioByteStream = new AudioByteStream(OPENAI_TTS_SAMPLE_RATE, OPENAI_TTS_CHANNELS);
+        const frames = audioByteStream.write(buffer);
+        for (const frame of frames) {
+            this.queue.put({
+                frame,
+                requestId,
+                segmentId: requestId,
+            });
+        }
+        this.queue.close();
+    }
+}
+//# sourceMappingURL=tts.js.map

package/dist/tts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.js","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AACrC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAW9B,MAAM,iBAAiB,GAAe;IACpC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;IAClC,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,OAAO,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,KAAK,CAAa;IAClB,OAAO,CAAS;IAEhB;;;;;;OAMG;IACH,YAAY,OAA4B,iBAAiB;QACvD,KAAK,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAEzE,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAC9F,CAAC;QAED,IAAI,CAAC,OAAO;YACV,IAAI,CAAC,KAAK,CAAC,MAAM;gBACjB,IAAI,MAAM,CAAC;oBACT,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;IACP,CAAC;IAED,aAAa,CAAC,IAAuE;QACnF,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,aAAa,CACtB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,eAAe,EAAE,KAAK;YACtB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;SACxB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,MAAM;QACJ,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;CACF;AAED,MAAM,OAAO,aAAc,SAAQ,GAAG,CAAC,aAAa;IAClD,YAAY,MAAyB;QACnC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAyB;QAClC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC,sBAAsB,EAAE,mBAAmB,CAAC,CAAC;QACzF,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;gBACb,KAAK;gBACL,SAAS;gBACT,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "0.4.3",
+  "version": "0.6.0",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,15 +8,19 @@
   "type": "module",
   "devDependencies": {
     "@microsoft/api-extractor": "^7.35.0",
+    "@livekit/rtc-node": "^0.11.1",
     "@types/ws": "^8.5.10",
-    "typescript": "^5.0.0"
+    "typescript": "^5.0.0",
+    "@livekit/agents": "^0.4.5"
   },
   "dependencies": {
-    "@livekit/rtc-node": "^0.11.1",
     "openai": "^4.70.2",
     "sharp": "^0.33.5",
-    "ws": "^8.16.0",
-    "@livekit/agents": "0.4.3"
+    "ws": "^8.16.0"
+  },
+  "peerDependencies": {
+    "@livekit/rtc-node": "^0.11.1",
+    "@livekit/agents": "^0.4.5"
   },
   "scripts": {
     "build": "tsc -b tsconfig.json",

package/src/index.ts

@@ -4,3 +4,5 @@
 export * as realtime from './realtime/index.js';
 export * from './models.js';
 export { type LLMOptions, LLM, LLMStream } from './llm.js';
+export { type STTOptions, STT } from './stt.js';
+export { type TTSOptions, TTS, ChunkedStream } from './tts.js';

package/src/llm.ts

@@ -438,7 +438,11 @@ export class LLMStream extends llm.LLMStream {
           function: {
             name,
             description: func.description,
-            parameters: llm.oaiParams(func.parameters),
+            // don't format parameters if they are raw openai params
+            parameters:
+              func.parameters.type == ('object' as const)
+                ? func.parameters
+                : llm.oaiParams(func.parameters),
           },
         }))
       : undefined;

package/src/models.ts

@@ -27,7 +27,13 @@ export type ChatModels =
   | 'gpt-3.5-turbo-1106'
   | 'gpt-3.5-turbo-16k-0613';
 
-// adapters for OpenAI-compatible LLMs
+export type WhisperModels = 'whisper-1';
+
+export type TTSModels = 'tts-1' | 'tts-1-hd';
+
+export type TTSVoices = 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
+
+// adapters for OpenAI-compatible LLMs, TTSs, STTs
 
 export type TelnyxChatModels =
   | 'meta-llama/Meta-Llama-3.1-8B-Instruct'
@@ -56,6 +62,11 @@ export type GroqChatModels =
   | 'gemma-7b-it'
   | 'gemma2-9b-it';
 
+export type GroqAudioModels =
+  | 'whisper-large-v3'
+  | 'distil-whisper-large-v3-en'
+  | 'whisper-large-v3-turbo';
+
 export type DeepSeekChatModels = 'deepseek-coder' | 'deepseek-chat';
 
 export type TogetherChatModels =

package/src/realtime/api_proto.ts

@@ -208,17 +208,32 @@ export type ResponseStatusDetails =
       reason: 'turn_detected' | 'client_cancelled' | string;
     };
 
+export interface ModelUsage {
+  total_tokens: number;
+  input_tokens: number;
+  output_tokens: number;
+  input_token_details: {
+    text_tokens: number;
+    audio_tokens: number;
+    cached_tokens: number;
+    cached_tokens_details: {
+      text_tokens: number;
+      audio_tokens: number;
+    };
+  };
+  output_token_details: {
+    text_tokens: number;
+    audio_tokens: number;
+  };
+}
+
 export interface ResponseResource {
   id: string;
   object: 'realtime.response';
   status: ResponseStatus;
   status_details: ResponseStatusDetails;
   output: ItemResource[];
-  usage?: {
-    total_tokens: number;
-    input_tokens: number;
-    output_tokens: number;
-  };
+  usage?: ModelUsage;
 }
 
 // Client Events

package/src/realtime/realtime_model.ts

@@ -37,7 +37,7 @@ export interface RealtimeResponse {
   id: string;
   status: api_proto.ResponseStatus;
   statusDetails: api_proto.ResponseStatusDetails | null;
-  usage: api_proto.ResponseResource['usage'] | null;
+  usage: api_proto.ModelUsage | null;
   output: RealtimeOutput[];
   doneFut: Future;
 }
@@ -630,7 +630,11 @@ export class RealtimeSession extends multimodal.RealtimeSession {
           type: 'function' as const,
           name,
           description: func.description,
-          parameters: llm.oaiParams(func.parameters),
+          parameters:
+            // don't format parameters if they are raw openai params
+            func.parameters.type == ('object' as const)
+              ? func.parameters
+              : llm.oaiParams(func.parameters),
         }))
       : [];
 
@@ -842,8 +846,8 @@ export class RealtimeSession extends multimodal.RealtimeSession {
 
   #getContent(ptr: ContentPtr): RealtimeContent {
     const response = this.#pendingResponses[ptr.response_id];
-    const output = response.output[ptr.output_index];
-    const content = output.content[ptr.content_index];
+    const output = response!.output[ptr.output_index];
+    const content = output!.content[ptr.content_index]!;
     return content;
   }
 
@@ -936,10 +940,10 @@ export class RealtimeSession extends multimodal.RealtimeSession {
   #handleResponseDone(event: api_proto.ResponseDoneEvent): void {
     const responseData = event.response;
     const responseId = responseData.id;
-    const response = this.#pendingResponses[responseId];
+    const response = this.#pendingResponses[responseId]!;
     response.status = responseData.status;
     response.statusDetails = responseData.status_details;
-    response.usage = responseData.usage;
+    response.usage = responseData.usage ?? null;
     this.#pendingResponses[responseId] = response;
     response.doneFut.resolve();
     this.emit('response_done', response);
@@ -970,7 +974,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       content: [],
       doneFut: new Future(),
     };
-    response.output.push(newOutput);
+    response?.output.push(newOutput);
     this.emit('response_output_added', newOutput);
   }
 
@@ -978,9 +982,9 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     const responseId = event.response_id;
     const response = this.#pendingResponses[responseId];
     const outputIndex = event.output_index;
-    const output = response.output[outputIndex];
+    const output = response!.output[outputIndex];
 
-    if (output.type === 'function_call') {
+    if (output?.type === 'function_call') {
       if (!this.#fncCtx) {
         this.#logger.error('function call received but no fncCtx is available');
         return;
@@ -991,6 +995,11 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       if (item.type !== 'function_call') {
         throw new Error('Expected function_call item');
       }
+      const func = this.#fncCtx[item.name];
+      if (!func) {
+        this.#logger.error(`no function with name ${item.name} in fncCtx`);
+        return;
+      }
 
       this.emit('function_call_started', {
         callId: item.call_id,
@@ -1002,7 +1011,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
         `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`,
       );
 
-      this.#fncCtx[item.name].execute(parsedArgs).then(
+      func.execute(parsedArgs).then(
         (content) => {
           this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);
           this.emit('function_call_completed', {
@@ -1028,7 +1037,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       );
     }
 
-    output.doneFut.resolve();
+    output?.doneFut.resolve();
     this.emit('response_output_done', output);
   }
 
@@ -1036,7 +1045,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     const responseId = event.response_id;
     const response = this.#pendingResponses[responseId];
     const outputIndex = event.output_index;
-    const output = response.output[outputIndex];
+    const output = response!.output[outputIndex];
 
     const textStream = new AsyncIterableQueue<string>();
     const audioStream = new AsyncIterableQueue<AudioFrame>();
@@ -1052,7 +1061,7 @@ export class RealtimeSession extends multimodal.RealtimeSession {
       audioStream: audioStream,
       toolCalls: [],
     };
-    output.content.push(newContent);
+    output?.content.push(newContent);
     this.emit('response_content_added', newContent);
   }
 
@@ -1061,11 +1070,13 @@ export class RealtimeSession extends multimodal.RealtimeSession {
     this.emit('response_content_done', content);
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {}
+  #handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {
+    this.emit('response_text_delta', event);
+  }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {}
+  #handleResponseTextDone(event: api_proto.ResponseTextDoneEvent): void {
+    this.emit('response_text_done', event);
+  }
 
   #handleResponseAudioTranscriptDelta(event: api_proto.ResponseAudioTranscriptDeltaEvent): void {
     const content = this.#getContent(event);

package/src/stt.ts

@@ -0,0 +1,140 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type AudioBuffer, mergeFrames, stt } from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
+import { OpenAI } from 'openai';
+import type { GroqAudioModels, WhisperModels } from './models.js';
+
+export interface STTOptions {
+  apiKey?: string;
+  language: string;
+  detectLanguage: boolean;
+  model: WhisperModels | string;
+  baseURL?: string;
+  client?: OpenAI;
+}
+
+const defaultSTTOptions: STTOptions = {
+  apiKey: process.env.OPENAI_API_KEY,
+  language: 'en',
+  detectLanguage: false,
+  model: 'whisper-1',
+};
+
+export class STT extends stt.STT {
+  #opts: STTOptions;
+  #client: OpenAI;
+
+  /**
+   * Create a new instance of OpenAI STT.
+   *
+   * @remarks
+   * `apiKey` must be set to your OpenAI API key, either using the argument or by setting the
+   * `OPENAI_API_KEY` environmental variable.
+   */
+  constructor(opts: Partial<STTOptions> = defaultSTTOptions) {
+    super({ streaming: false, interimResults: false });
+
+    this.#opts = { ...defaultSTTOptions, ...opts };
+    if (this.#opts.apiKey === undefined) {
+      throw new Error('OpenAI API key is required, whether as an argument or as $OPENAI_API_KEY');
+    }
+
+    this.#client =
+      this.#opts.client ||
+      new OpenAI({
+        baseURL: opts.baseURL,
+        apiKey: opts.apiKey,
+      });
+  }
+
+  /**
+   * Create a new instance of Groq STT.
+   *
+   * @remarks
+   * `apiKey` must be set to your Groq API key, either using the argument or by setting the
+   * `GROQ_API_KEY` environmental variable.
+   */
+  static withGroq(
+    opts: Partial<{
+      model: string | GroqAudioModels;
+      apiKey?: string;
+      baseURL?: string;
+      client: OpenAI;
+      language: string;
+      detectLanguage: boolean;
+    }> = {},
+  ): STT {
+    opts.apiKey = opts.apiKey || process.env.GROQ_API_KEY;
+    if (opts.apiKey === undefined) {
+      throw new Error('Groq API key is required, whether as an argument or as $GROQ_API_KEY');
+    }
+
+    return new STT({
+      model: 'whisper-large-v3-turbo',
+      baseURL: 'https://api.groq.com/openai/v1',
+      ...opts,
+    });
+  }
+
+  #sanitizeOptions(language?: string): STTOptions {
+    if (language) {
+      return { ...this.#opts, language };
+    } else {
+      return this.#opts;
+    }
+  }
+
+  #createWav(frame: AudioFrame): Buffer {
+    const bitsPerSample = 16;
+    const byteRate = (frame.sampleRate * frame.channels * bitsPerSample) / 8;
+    const blockAlign = (frame.channels * bitsPerSample) / 8;
+
+    const header = Buffer.alloc(44);
+    header.write('RIFF', 0);
+    header.writeUInt32LE(36 + frame.data.byteLength, 4);
+    header.write('WAVE', 8);
+    header.write('fmt ', 12);
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(frame.channels, 22);
+    header.writeUInt32LE(frame.sampleRate, 24);
+    header.writeUInt32LE(byteRate, 28);
+    header.writeUInt16LE(blockAlign, 32);
+    header.writeUInt16LE(16, 34);
+    header.write('data', 36);
+    header.writeUInt32LE(frame.data.byteLength, 40);
+    return Buffer.concat([header, Buffer.from(frame.data.buffer)]);
+  }
+
+  async recognize(buffer: AudioBuffer, language?: string): Promise<stt.SpeechEvent> {
+    const config = this.#sanitizeOptions(language);
+    buffer = mergeFrames(buffer);
+    const file = new File([this.#createWav(buffer)], 'audio.wav', { type: 'audio/wav' });
+    const resp = await this.#client.audio.transcriptions.create({
+      file,
+      model: this.#opts.model,
+      language: config.language,
+      response_format: 'json',
+    });
+
+    return {
+      type: stt.SpeechEventType.FINAL_TRANSCRIPT,
+      alternatives: [
+        {
+          text: resp.text || '',
+          language: language || '',
+          startTime: 0,
+          endTime: 0,
+          confidence: 0,
+        },
+      ],
+    };
+  }
+
+  /** This method throws an error; streaming is unsupported on OpenAI STT. */
+  stream(): stt.SpeechStream {
+    throw new Error('Streaming is not supported on OpenAI STT');
+  }
+}