@livekit/agents-plugin-elevenlabs 0.1.0

package/.turbo/turbo-build.log

@@ -0,0 +1,4 @@
+
+> @livekit/agents-plugin-elevenlabs@0.1.0 build /home/runner/work/agents-js/agents-js/plugins/elevenlabs
+> tsc
+

package/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from './tts.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './tts.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AAEtC,cAAc,UAAU,CAAC"}

package/dist/models.d.ts

@@ -0,0 +1,2 @@
+export type TTSModels = 'eleven_monolingual_v1' | 'eleven_multilingual_v1' | 'eleven_multilingual_v2' | 'eleven_turbo_v2';
+//# sourceMappingURL=models.d.ts.map

package/dist/models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,SAAS,GACjB,uBAAuB,GACvB,wBAAwB,GACxB,wBAAwB,GACxB,iBAAiB,CAAC"}

package/dist/models.js

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export {};
+//# sourceMappingURL=models.js.map

package/dist/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC"}

package/dist/tts.d.ts

@@ -0,0 +1,51 @@
+import { tts } from '@livekit/agents';
+import { WebSocket } from 'ws';
+import type { TTSModels } from './models.js';
+type Voice = {
+    id: string;
+    name: string;
+    category: string;
+    settings?: VoiceSettings;
+};
+type VoiceSettings = {
+    stability: number;
+    similarity_boost: number;
+    style?: number;
+    use_speaker_boost: boolean;
+};
+type TTSOptions = {
+    apiKey: string;
+    voice: Voice;
+    modelID: TTSModels;
+    baseURL: string;
+    sampleRate: number;
+    latency: number;
+};
+export declare class TTS extends tts.TTS {
+    config: TTSOptions;
+    constructor(voice?: Voice, modelID?: TTSModels, apiKey?: string, baseURL?: string, sampleRate?: number, latency?: number);
+    listVoices(): Promise<Voice[]>;
+    synthesize(text: string): Promise<tts.ChunkedStream>;
+    stream(): tts.SynthesizeStream;
+}
+export declare class SynthesizeStream extends tts.SynthesizeStream {
+    closed: boolean;
+    config: TTSOptions;
+    text: string;
+    task: {
+        run: Promise<void>;
+        cancel: () => void;
+    };
+    queue: string[];
+    eventQueue: (tts.SynthesisEvent | undefined)[];
+    constructor(config: TTSOptions);
+    get streamURL(): string;
+    pushText(token?: string | undefined): void;
+    run(maxRetry: number): Promise<void>;
+    listenTask(ws: WebSocket): Promise<void>;
+    flush(): void;
+    next(): IteratorResult<tts.SynthesisEvent>;
+    close(wait: boolean): Promise<void>;
+}
+export {};
+//# sourceMappingURL=tts.d.ts.map

package/dist/tts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.d.ts","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAGA,OAAO,EAAO,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,EAAgB,SAAS,EAAE,MAAM,IAAI,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,KAAK,KAAK,GAAG;IACX,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAkBF,KAAK,UAAU,GAAG;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;IACb,OAAO,EAAE,SAAS,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,qBAAa,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,MAAM,EAAE,UAAU,CAAC;gBAGjB,KAAK,QAAgB,EACrB,OAAO,GAAE,SAAoC,EAC7C,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,UAAU,SAAQ,EAClB,OAAO,SAAI;IAoBP,UAAU,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;IAuB9B,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;IAI1D,MAAM,IAAI,GAAG,CAAC,gBAAgB;CAG/B;AAED,qBAAa,gBAAiB,SAAQ,GAAG,CAAC,gBAAgB;IACxD,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE;QACJ,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACnB,MAAM,EAAE,MAAM,IAAI,CAAC;KACpB,CAAC;IACF,KAAK,EAAE,MAAM,EAAE,CAAM;IACrB,UAAU,EAAE,CAAC,GAAG,CAAC,cAAc,GAAG,SAAS,CAAC,EAAE,CAAM;gBAExC,MAAM,EAAE,UAAU;IAa9B,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAYpC,GAAG,CAAC,QAAQ,EAAE,MAAM;IAgEpB,UAAU,CAAC,EAAE,EAAE,SAAS;IA2B9B,KAAK;IAML,IAAI,IAAI,cAAc,CAAC,GAAG,CAAC,cAAc,CAAC;IASpC,KAAK,CAAC,IAAI,EAAE,OAAO;CAW1B"}

package/dist/tts.js

@@ -0,0 +1,251 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log, tts } from '@livekit/agents';
+import { AudioFrame } from '@livekit/rtc-node';
+import { URL } from 'url';
+import { WebSocket } from 'ws';
+const DEFAULT_VOICE = {
+    id: 'EXAVITQu4vr4xnSDxMaL',
+    name: 'Bella',
+    category: 'premade',
+    settings: {
+        stability: 0.71,
+        similarity_boost: 0.5,
+        style: 0.0,
+        use_speaker_boost: true,
+    },
+};
+const API_BASE_URL_V1 = 'https://api.elevenlabs.io/v1';
+const AUTHORIZATION_HEADER = 'xi-api-key';
+const STREAM_EOS = '';
+export class TTS extends tts.TTS {
+    config;
+    constructor(voice = DEFAULT_VOICE, modelID = 'eleven_multilingual_v2', apiKey, baseURL, sampleRate = 24000, latency = 2) {
+        super(true);
+        apiKey = apiKey || process.env.ELEVEN_API_KEY;
+        if (apiKey === undefined) {
+            throw new Error('ElevenLabs API key is required, whether as an argument or as $ELEVEN_API_KEY');
+        }
+        this.config = {
+            voice,
+            modelID,
+            apiKey,
+            baseURL: baseURL || API_BASE_URL_V1,
+            sampleRate,
+            latency,
+        };
+    }
+    async listVoices() {
+        return fetch(this.config.baseURL + '/voices', {
+            headers: {
+                [AUTHORIZATION_HEADER]: this.config.apiKey,
+            },
+        })
+            .then((data) => data.json())
+            .then((data) => {
+            const voices = [];
+            for (const voice of data.voices) {
+                voices.push({
+                    id: voice.voice_id,
+                    name: voice.name,
+                    category: voice.category,
+                    settings: undefined,
+                });
+            }
+            return voices;
+        });
+    }
+    async synthesize(text) {
+        return new ChunkedStream(text, this.config);
+    }
+    stream() {
+        return new SynthesizeStream(this.config);
+    }
+}
+export class SynthesizeStream extends tts.SynthesizeStream {
+    closed;
+    config;
+    text;
+    task;
+    queue = [];
+    eventQueue = [];
+    constructor(config) {
+        super();
+        this.config = config;
+        this.closed = false;
+        this.text = '';
+        this.task = {
+            run: new Promise(() => {
+                this.run(32);
+            }),
+            cancel: () => { },
+        };
+    }
+    get streamURL() {
+        return `${this.config.baseURL}/text-to-speech/${this.config.voice.id}/stream-input?model_id=${this.config.modelID}&optimize_streaming_latency=${this.config.latency}`;
+    }
+    pushText(token) {
+        if (this.closed)
+            throw new Error('cannot push to a closed stream');
+        if (!token || token.length === 0)
+            return;
+        const splitters = '.,?!;:—-()[]} ';
+        this.text += token;
+        if (splitters.includes(token[token.length - 1])) {
+            this.queue.push(this.text);
+            this.text = '';
+        }
+    }
+    async run(maxRetry) {
+        let retries = 0;
+        while (!this.closed) {
+            const url = new URL(this.streamURL);
+            url.protocol = url.protocol.replace('http', 'ws');
+            const ws = new WebSocket(url, {
+                headers: { [AUTHORIZATION_HEADER]: this.config.apiKey },
+            });
+            try {
+                await new Promise((resolve, reject) => {
+                    ws.on('open', resolve);
+                    ws.on('error', (error) => reject(error));
+                    ws.on('close', (code) => reject(`WebSocket returned ${code}`));
+                });
+                ws.send(JSON.stringify({ text: ' ', voice_settings: this.config.voice }));
+                let started = false;
+                const retryQueue = [];
+                const task = this.listenTask(ws);
+                while (ws.readyState !== ws.CLOSED) {
+                    let text = undefined;
+                    if (retryQueue.length === 0) {
+                        text = this.queue.shift();
+                    }
+                    else {
+                        text = retryQueue.shift();
+                    }
+                    if (!started) {
+                        this.eventQueue.push(new tts.SynthesisEvent(tts.SynthesisEventType.STARTED));
+                        started = true;
+                    }
+                    try {
+                        ws.send(JSON.stringify({ text, try_trigger_generation: true }));
+                    }
+                    catch (e) {
+                        // XI closes idle connections after a while.
+                        retryQueue.push(text);
+                        break;
+                    }
+                    if (text == STREAM_EOS) {
+                        await task;
+                        this.eventQueue.push(new tts.SynthesisEvent(tts.SynthesisEventType.FINISHED));
+                        break;
+                    }
+                }
+            }
+            catch (e) {
+                if (retries >= maxRetry) {
+                    throw new Error(`failed to connect to ElevenLabs after ${retries} attempts: ${e}`);
+                }
+                const delay = Math.min(retries * 5, 5);
+                retries++;
+                log.warn(`failed to connect to ElevenLabs, retrying in ${delay} seconds: ${e} (${retries}/${maxRetry})`);
+                await new Promise((resolve) => setTimeout(resolve, delay * 1000));
+            }
+        }
+        this.closed = true;
+    }
+    async listenTask(ws) {
+        while (!this.closed) {
+            try {
+                await new Promise((resolve, reject) => {
+                    ws.on('message', (data) => resolve(data));
+                    ws.on('close', (code, reason) => reject(`WebSocket closed with code ${code}: ${reason}`));
+                }).then((msg) => {
+                    const json = JSON.parse(msg.toString());
+                    if ('audio' in json) {
+                        const data = new Uint16Array(Buffer.from(json.audio, 'base64'));
+                        const audioFrame = new AudioFrame(data, this.config.sampleRate, 1, Math.trunc(data.length / 2));
+                        this.eventQueue.push(new tts.SynthesisEvent(tts.SynthesisEventType.AUDIO, { text: '', data: audioFrame }));
+                    }
+                });
+            }
+            catch {
+                break;
+            }
+        }
+    }
+    flush() {
+        this.queue.push(this.text + ' ');
+        this.text = '';
+        this.queue.push('');
+    }
+    next() {
+        const event = this.eventQueue.shift();
+        if (event) {
+            return { done: false, value: event };
+        }
+        else {
+            return { done: true, value: undefined };
+        }
+    }
+    async close(wait) {
+        if (wait) {
+            log.warn('wait is not yet supported for ElevenLabs TTS');
+        }
+        try {
+            await this.task.run;
+        }
+        finally {
+            this.eventQueue.push(undefined);
+        }
+    }
+}
+class ChunkedStream extends tts.ChunkedStream {
+    config;
+    text;
+    queue = [];
+    constructor(text, config) {
+        super();
+        this.config = config;
+        this.text = text;
+    }
+    async next() {
+        await this.run();
+        const audio = this.queue.shift();
+        if (audio) {
+            return { done: false, value: audio };
+        }
+        else {
+            return { done: true, value: undefined };
+        }
+    }
+    async close() {
+        this.queue.push(undefined);
+    }
+    async run() {
+        const voice = this.config.voice;
+        const url = new URL(`${this.config.baseURL}/text-to-speech/${voice.id}/stream`);
+        url.searchParams.append('output_format', 'pcm_' + this.config.sampleRate);
+        url.searchParams.append('optimize_streaming_latency', this.config.latency.toString());
+        await fetch(url.toString(), {
+            method: 'POST',
+            headers: {
+                [AUTHORIZATION_HEADER]: this.config.apiKey,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                text: this.text,
+                model_id: this.config.modelID,
+                voice_settings: this.config.voice.settings || undefined,
+            }),
+        })
+            .then((data) => data.arrayBuffer())
+            .then((data) => new DataView(data, 0, data.byteLength))
+            .then((data) => this.queue.push({
+            text: this.text,
+            data: new AudioFrame(new Uint16Array(data.buffer), this.config.sampleRate, 1, data.byteLength / 2),
+        }, undefined))
+            .catch(() => this.queue.push(undefined));
+    }
+}
+//# sourceMappingURL=tts.js.map

package/dist/tts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.js","sourceRoot":"","sources":["../src/tts.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAgB,SAAS,EAAE,MAAM,IAAI,CAAC;AAiB7C,MAAM,aAAa,GAAU;IAC3B,EAAE,EAAE,sBAAsB;IAC1B,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,SAAS;IACnB,QAAQ,EAAE;QACR,SAAS,EAAE,IAAI;QACf,gBAAgB,EAAE,GAAG;QACrB,KAAK,EAAE,GAAG;QACV,iBAAiB,EAAE,IAAI;KACxB;CACF,CAAC;AAEF,MAAM,eAAe,GAAG,8BAA8B,CAAC;AACvD,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAC1C,MAAM,UAAU,GAAG,EAAE,CAAC;AAWtB,MAAM,OAAO,GAAI,SAAQ,GAAG,CAAC,GAAG;IAC9B,MAAM,CAAa;IAEnB,YACE,KAAK,GAAG,aAAa,EACrB,UAAqB,wBAAwB,EAC7C,MAAe,EACf,OAAgB,EAChB,UAAU,GAAG,KAAK,EAClB,OAAO,GAAG,CAAC;QAEX,KAAK,CAAC,IAAI,CAAC,CAAC;QACZ,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,8EAA8E,CAC/E,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,GAAG;YACZ,KAAK;YACL,OAAO;YACP,MAAM;YACN,OAAO,EAAE,OAAO,IAAI,eAAe;YACnC,UAAU;YACV,OAAO;SACR,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU;QACd,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,SAAS,EAAE;YAC5C,OAAO,EAAE;gBACP,CAAC,oBAAoB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;aAC3C;SACF,CAAC;aACC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC3B,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IACd,IACD,CAAC,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,KAAK,CAAC,QAAQ;oBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,SAAS;iBACpB,CAAC,CAAC;YACL,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,OAAO,IAAI,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,OAAO,gBAAiB,SAAQ,GAAG,CAAC,gBAAgB;IACxD,MAAM,CAAU;IAChB,MAAM,CAAa;IACnB,IAAI,CAAS;IACb,IAAI,CAGF;IACF,KAAK,GAAa,EAAE,CAAC;IACrB,UAAU,GAAuC,EAAE,CAAC;IAEpD,YAAY,MAAkB;QAC5B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,IAAI,GAAG;YACV,GAAG,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE;gBACpB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACf,CAAC,CAAC;YACF,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC;SACjB,CAAC;IACJ,CAAC;IAED,IAAI,SAAS;QACX,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,mBAAmB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,0BAA0B,IAAI,CAAC,MAAM,CAAC,OAAO,+BAA+B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACxK,CAAC;IAED,QAAQ,CAAC,KAA0B;QACjC,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEzC,MAAM,SAAS,GAAG,gBAAgB,CAAC;QACnC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC;QACnB,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB;QACxB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,EAAE,CAAC,oBAAoB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;aACxD,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACpC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACvB,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC,CAAC;gBACjE,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC1E,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,MAAM,UAAU,GAAa,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBACjC,OAAO,EAAE,CAAC,UAAU,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC;oBACnC,IAAI,IAAI,GAAG,SAAS,CAAC;oBACrB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC5B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC5B,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC;oBAC5B,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;wBAC7E,OAAO,GAAG,IAAI,CAAC;oBACjB,CAAC;oBAED,IAAI,CAAC;wBACH,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,4CAA4C;wBAC5C,UAAU,CAAC,IAAI,CAAC,IAAK,CAAC,CAAC;wBACvB,MAAM;oBACR,CAAC;oBAED,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;wBACvB,MAAM,IAAI,CAAC;wBACX,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAC9E,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;oBACxB,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC;gBACrF,CAAC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,OAAO,EAAE,CAAC;gBAEV,GAAG,CAAC,IAAI,CACN,gDAAgD,KAAK,aAAa,CAAC,KAAK,OAAO,IAAI,QAAQ,GAAG,CAC/F,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,EAAa;QAC5B,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC7C,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC1C,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,8BAA8B,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC5F,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;oBACd,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;oBACxC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;wBACpB,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAChE,MAAM,UAAU,GAAG,IAAI,UAAU,CAC/B,IAAI,EACJ,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAC5B,CAAC;wBACF,IAAI,CAAC,UAAU,CAAC,IAAI,CAClB,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CACrF,CAAC;oBACJ,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,CAAC;IAED,IAAI;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAa;QACvB,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;CACF;AAED,MAAM,aAAc,SAAQ,GAAG,CAAC,aAAa;IAC3C,MAAM,CAAa;IACnB,IAAI,CAAS;IACb,KAAK,GAAyC,EAAE,CAAC;IAEjD,YAAY,IAAY,EAAE,MAAkB;QAC1C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACjC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAEhC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,mBAAmB,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QAChF,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1E,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,4BAA4B,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEtF,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC1B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,CAAC,oBAAoB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1C,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC7B,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS;aACxD,CAAC;SACH,CAAC;aACC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;aAClC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;aACtD,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACb,IAAI,CAAC,KAAK,CAAC,IAAI,CACb;YACE,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,UAAU,CAClB,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAC5B,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,CAAC,EACD,IAAI,CAAC,UAAU,GAAG,CAAC,CACpB;SACF,EACD,SAAS,CACV,CACF;aACA,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF"}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@livekit/agents-plugin-elevenlabs",
+  "version": "0.1.0",
+  "description": "ElevenLabs plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "aoife cassidy <aoife@cassidy.sh>",
+  "type": "module",
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/rtc-node": "^0.1.0",
+    "ws": "^8.16.0",
+    "@livekit/agents": "0.1.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './tts.js';

package/src/models.ts

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type TTSModels =
+  | 'eleven_monolingual_v1'
+  | 'eleven_multilingual_v1'
+  | 'eleven_multilingual_v2'
+  | 'eleven_turbo_v2';

package/src/tts.ts

@@ -0,0 +1,331 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log, tts } from '@livekit/agents';
+import { AudioFrame } from '@livekit/rtc-node';
+import { URL } from 'url';
+import { type RawData, WebSocket } from 'ws';
+import type { TTSModels } from './models.js';
+
+type Voice = {
+  id: string;
+  name: string;
+  category: string;
+  settings?: VoiceSettings;
+};
+
+type VoiceSettings = {
+  stability: number; // 0..1
+  similarity_boost: number; // 0..1
+  style?: number; // 0..1
+  use_speaker_boost: boolean;
+};
+
+const DEFAULT_VOICE: Voice = {
+  id: 'EXAVITQu4vr4xnSDxMaL',
+  name: 'Bella',
+  category: 'premade',
+  settings: {
+    stability: 0.71,
+    similarity_boost: 0.5,
+    style: 0.0,
+    use_speaker_boost: true,
+  },
+};
+
+const API_BASE_URL_V1 = 'https://api.elevenlabs.io/v1';
+const AUTHORIZATION_HEADER = 'xi-api-key';
+const STREAM_EOS = '';
+
+type TTSOptions = {
+  apiKey: string;
+  voice: Voice;
+  modelID: TTSModels;
+  baseURL: string;
+  sampleRate: number;
+  latency: number;
+};
+
+export class TTS extends tts.TTS {
+  config: TTSOptions;
+
+  constructor(
+    voice = DEFAULT_VOICE,
+    modelID: TTSModels = 'eleven_multilingual_v2',
+    apiKey?: string,
+    baseURL?: string,
+    sampleRate = 24000,
+    latency = 2,
+  ) {
+    super(true);
+    apiKey = apiKey || process.env.ELEVEN_API_KEY;
+    if (apiKey === undefined) {
+      throw new Error(
+        'ElevenLabs API key is required, whether as an argument or as $ELEVEN_API_KEY',
+      );
+    }
+
+    this.config = {
+      voice,
+      modelID,
+      apiKey,
+      baseURL: baseURL || API_BASE_URL_V1,
+      sampleRate,
+      latency,
+    };
+  }
+
+  async listVoices(): Promise<Voice[]> {
+    return fetch(this.config.baseURL + '/voices', {
+      headers: {
+        [AUTHORIZATION_HEADER]: this.config.apiKey,
+      },
+    })
+      .then((data) => data.json())
+      .then((data) => {
+        const voices: Voice[] = [];
+        for (const voice of (
+          data as { voices: { voice_id: string; name: string; category: string }[] }
+        ).voices) {
+          voices.push({
+            id: voice.voice_id,
+            name: voice.name,
+            category: voice.category,
+            settings: undefined,
+          });
+        }
+        return voices;
+      });
+  }
+
+  async synthesize(text: string): Promise<tts.ChunkedStream> {
+    return new ChunkedStream(text, this.config);
+  }
+
+  stream(): tts.SynthesizeStream {
+    return new SynthesizeStream(this.config);
+  }
+}
+
+export class SynthesizeStream extends tts.SynthesizeStream {
+  closed: boolean;
+  config: TTSOptions;
+  text: string;
+  task: {
+    run: Promise<void>;
+    cancel: () => void;
+  };
+  queue: string[] = [];
+  eventQueue: (tts.SynthesisEvent | undefined)[] = [];
+
+  constructor(config: TTSOptions) {
+    super();
+    this.config = config;
+    this.closed = false;
+    this.text = '';
+    this.task = {
+      run: new Promise(() => {
+        this.run(32);
+      }),
+      cancel: () => {},
+    };
+  }
+
+  get streamURL(): string {
+    return `${this.config.baseURL}/text-to-speech/${this.config.voice.id}/stream-input?model_id=${this.config.modelID}&optimize_streaming_latency=${this.config.latency}`;
+  }
+
+  pushText(token?: string | undefined): void {
+    if (this.closed) throw new Error('cannot push to a closed stream');
+    if (!token || token.length === 0) return;
+
+    const splitters = '.,?!;:—-()[]} ';
+    this.text += token;
+    if (splitters.includes(token[token.length - 1])) {
+      this.queue.push(this.text);
+      this.text = '';
+    }
+  }
+
+  async run(maxRetry: number) {
+    let retries = 0;
+    while (!this.closed) {
+      const url = new URL(this.streamURL);
+      url.protocol = url.protocol.replace('http', 'ws');
+      const ws = new WebSocket(url, {
+        headers: { [AUTHORIZATION_HEADER]: this.config.apiKey },
+      });
+
+      try {
+        await new Promise((resolve, reject) => {
+          ws.on('open', resolve);
+          ws.on('error', (error) => reject(error));
+          ws.on('close', (code) => reject(`WebSocket returned ${code}`));
+        });
+
+        ws.send(JSON.stringify({ text: ' ', voice_settings: this.config.voice }));
+        let started = false;
+        const retryQueue: string[] = [];
+        const task = this.listenTask(ws);
+        while (ws.readyState !== ws.CLOSED) {
+          let text = undefined;
+          if (retryQueue.length === 0) {
+            text = this.queue.shift();
+          } else {
+            text = retryQueue.shift();
+          }
+
+          if (!started) {
+            this.eventQueue.push(new tts.SynthesisEvent(tts.SynthesisEventType.STARTED));
+            started = true;
+          }
+
+          try {
+            ws.send(JSON.stringify({ text, try_trigger_generation: true }));
+          } catch (e) {
+            // XI closes idle connections after a while.
+            retryQueue.push(text!);
+            break;
+          }
+
+          if (text == STREAM_EOS) {
+            await task;
+            this.eventQueue.push(new tts.SynthesisEvent(tts.SynthesisEventType.FINISHED));
+            break;
+          }
+        }
+      } catch (e) {
+        if (retries >= maxRetry) {
+          throw new Error(`failed to connect to ElevenLabs after ${retries} attempts: ${e}`);
+        }
+
+        const delay = Math.min(retries * 5, 5);
+        retries++;
+
+        log.warn(
+          `failed to connect to ElevenLabs, retrying in ${delay} seconds: ${e} (${retries}/${maxRetry})`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay * 1000));
+      }
+    }
+    this.closed = true;
+  }
+
+  async listenTask(ws: WebSocket) {
+    while (!this.closed) {
+      try {
+        await new Promise<RawData>((resolve, reject) => {
+          ws.on('message', (data) => resolve(data));
+          ws.on('close', (code, reason) => reject(`WebSocket closed with code ${code}: ${reason}`));
+        }).then((msg) => {
+          const json = JSON.parse(msg.toString());
+          if ('audio' in json) {
+            const data = new Uint16Array(Buffer.from(json.audio, 'base64'));
+            const audioFrame = new AudioFrame(
+              data,
+              this.config.sampleRate,
+              1,
+              Math.trunc(data.length / 2),
+            );
+            this.eventQueue.push(
+              new tts.SynthesisEvent(tts.SynthesisEventType.AUDIO, { text: '', data: audioFrame }),
+            );
+          }
+        });
+      } catch {
+        break;
+      }
+    }
+  }
+
+  flush() {
+    this.queue.push(this.text + ' ');
+    this.text = '';
+    this.queue.push('');
+  }
+
+  next(): IteratorResult<tts.SynthesisEvent> {
+    const event = this.eventQueue.shift();
+    if (event) {
+      return { done: false, value: event };
+    } else {
+      return { done: true, value: undefined };
+    }
+  }
+
+  async close(wait: boolean) {
+    if (wait) {
+      log.warn('wait is not yet supported for ElevenLabs TTS');
+    }
+
+    try {
+      await this.task.run;
+    } finally {
+      this.eventQueue.push(undefined);
+    }
+  }
+}
+
+class ChunkedStream extends tts.ChunkedStream {
+  config: TTSOptions;
+  text: string;
+  queue: (tts.SynthesizedAudio | undefined)[] = [];
+
+  constructor(text: string, config: TTSOptions) {
+    super();
+    this.config = config;
+    this.text = text;
+  }
+
+  async next(): Promise<IteratorResult<tts.SynthesizedAudio>> {
+    await this.run();
+    const audio = this.queue.shift();
+    if (audio) {
+      return { done: false, value: audio };
+    } else {
+      return { done: true, value: undefined };
+    }
+  }
+
+  async close() {
+    this.queue.push(undefined);
+  }
+
+  async run() {
+    const voice = this.config.voice;
+
+    const url = new URL(`${this.config.baseURL}/text-to-speech/${voice.id}/stream`);
+    url.searchParams.append('output_format', 'pcm_' + this.config.sampleRate);
+    url.searchParams.append('optimize_streaming_latency', this.config.latency.toString());
+
+    await fetch(url.toString(), {
+      method: 'POST',
+      headers: {
+        [AUTHORIZATION_HEADER]: this.config.apiKey,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        text: this.text,
+        model_id: this.config.modelID,
+        voice_settings: this.config.voice.settings || undefined,
+      }),
+    })
+      .then((data) => data.arrayBuffer())
+      .then((data) => new DataView(data, 0, data.byteLength))
+      .then((data) =>
+        this.queue.push(
+          {
+            text: this.text,
+            data: new AudioFrame(
+              new Uint16Array(data.buffer),
+              this.config.sampleRate,
+              1,
+              data.byteLength / 2,
+            ),
+          },
+          undefined,
+        ),
+      )
+      .catch(() => this.queue.push(undefined));
+  }
+}

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["./src"],
+  "compilerOptions": {
+    // match output dir to input dir. e.g. dist/index instead of dist/src/index
+    "rootDir": "./src",
+    "declarationDir": "./dist",
+    "outDir": "./dist"
+  }
+}