@livekit/agents-plugin-cartesia 0.1.0

package/dist/tts.js

@@ -0,0 +1,234 @@
+import { AudioByteStream, log, tokenize, tts } from "@livekit/agents";
+import { randomUUID } from "node:crypto";
+import { request } from "node:https";
+import { WebSocket } from "ws";
+import {
+  TTSDefaultVoiceId
+} from "./models.js";
+const AUTHORIZATION_HEADER = "X-API-Key";
+const VERSION_HEADER = "Cartesia-Version";
+const VERSION = "2024-06-10";
+const NUM_CHANNELS = 1;
+const BUFFERED_WORDS_COUNT = 8;
+const defaultTTSOptions = {
+  model: "sonic-english",
+  encoding: "pcm_s16le",
+  sampleRate: 24e3,
+  voice: TTSDefaultVoiceId,
+  apiKey: process.env.CARTESIA_API_KEY,
+  language: "en"
+};
+class TTS extends tts.TTS {
+  #opts;
+  label = "cartesia.TTS";
+  constructor(opts = {}) {
+    super(opts.sampleRate || defaultTTSOptions.sampleRate, NUM_CHANNELS, {
+      streaming: true
+    });
+    this.#opts = {
+      ...defaultTTSOptions,
+      ...opts
+    };
+    if (this.#opts.apiKey === void 0) {
+      throw new Error(
+        "Cartesia API key is required, whether as an argument or as $CARTESIA_API_KEY"
+      );
+    }
+  }
+  // TODO(nbsp): updateOptions
+  synthesize(text) {
+    return new ChunkedStream(this, text, this.#opts);
+  }
+  stream() {
+    return new SynthesizeStream(this, this.#opts);
+  }
+}
+class ChunkedStream extends tts.ChunkedStream {
+  label = "cartesia.ChunkedStream";
+  #opts;
+  #text;
+  // set Promise<T> to any because OpenAI returns an annoying Response type
+  constructor(tts2, text, opts) {
+    super(text, tts2);
+    this.#text = text;
+    this.#opts = opts;
+    this.#run();
+  }
+  async #run() {
+    const requestId = randomUUID();
+    const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);
+    const json = toCartesiaOptions(this.#opts);
+    json.transcript = this.#text;
+    const req = request(
+      {
+        hostname: "api.cartesia.ai",
+        port: 443,
+        path: "/tts/bytes",
+        method: "POST",
+        headers: {
+          [AUTHORIZATION_HEADER]: this.#opts.apiKey,
+          [VERSION_HEADER]: VERSION
+        }
+      },
+      (res) => {
+        res.on("data", (chunk) => {
+          for (const frame of bstream.write(chunk)) {
+            this.queue.put({
+              requestId,
+              frame,
+              final: false,
+              segmentId: requestId
+            });
+          }
+        });
+        res.on("close", () => {
+          for (const frame of bstream.flush()) {
+            this.queue.put({
+              requestId,
+              frame,
+              final: false,
+              segmentId: requestId
+            });
+          }
+          this.queue.close();
+        });
+      }
+    );
+    req.write(JSON.stringify(json));
+    req.end();
+  }
+}
+class SynthesizeStream extends tts.SynthesizeStream {
+  #opts;
+  #logger = log();
+  #tokenizer = new tokenize.basic.SentenceTokenizer(void 0, BUFFERED_WORDS_COUNT).stream();
+  label = "cartesia.SynthesizeStream";
+  constructor(tts2, opts) {
+    super(tts2);
+    this.#opts = opts;
+    this.#run();
+  }
+  async #run() {
+    const requestId = randomUUID();
+    let closing = false;
+    const sentenceStreamTask = async (ws2) => {
+      const packet = toCartesiaOptions(this.#opts);
+      for await (const event of this.#tokenizer) {
+        ws2.send(
+          JSON.stringify({
+            ...packet,
+            context_id: requestId,
+            transcript: event.token + " ",
+            continue: true
+          })
+        );
+      }
+      ws2.send(
+        JSON.stringify({
+          ...packet,
+          context_id: requestId,
+          transcript: " ",
+          continue: false
+        })
+      );
+    };
+    const inputTask = async () => {
+      for await (const data of this.input) {
+        if (data === SynthesizeStream.FLUSH_SENTINEL) {
+          this.#tokenizer.flush();
+          continue;
+        }
+        this.#tokenizer.pushText(data);
+      }
+      this.#tokenizer.endInput();
+      this.#tokenizer.close();
+    };
+    const recvTask = async (ws2) => {
+      const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);
+      let lastFrame;
+      const sendLastFrame = (segmentId, final) => {
+        if (lastFrame) {
+          this.queue.put({ requestId, segmentId, frame: lastFrame, final });
+          lastFrame = void 0;
+        }
+      };
+      ws2.on("message", (data) => {
+        const json = JSON.parse(data.toString());
+        const segmentId = json.context_id;
+        if ("data" in json) {
+          const data2 = new Int8Array(Buffer.from(json.data, "base64"));
+          for (const frame of bstream.write(data2)) {
+            sendLastFrame(segmentId, false);
+            lastFrame = frame;
+          }
+        } else if ("done" in json) {
+          for (const frame of bstream.flush()) {
+            sendLastFrame(segmentId, false);
+            lastFrame = frame;
+          }
+          sendLastFrame(segmentId, true);
+          this.queue.put(SynthesizeStream.END_OF_STREAM);
+          if (segmentId === requestId) {
+            closing = true;
+            ws2.close();
+            return;
+          }
+        }
+      });
+      ws2.on("close", (code, reason) => {
+        if (!closing) {
+          this.#logger.error(`WebSocket closed with code ${code}: ${reason}`);
+        }
+        ws2.removeAllListeners();
+      });
+    };
+    const url = `wss://api.cartesia.ai/tts/websocket?api_key=${this.#opts.apiKey}&cartesia_version=${VERSION}`;
+    const ws = new WebSocket(url);
+    try {
+      await new Promise((resolve, reject) => {
+        ws.on("open", resolve);
+        ws.on("error", (error) => reject(error));
+        ws.on("close", (code) => reject(`WebSocket returned ${code}`));
+      });
+      await Promise.all([inputTask(), sentenceStreamTask(ws), recvTask(ws)]);
+    } catch (e) {
+      throw new Error(`failed to connect to Cartesia: ${e}`);
+    }
+  }
+}
+const toCartesiaOptions = (opts) => {
+  const voice = {};
+  if (typeof opts.voice === "string") {
+    voice.mode = "id";
+    voice.id = opts.voice;
+  } else {
+    voice.mode = "embedding";
+    voice.embedding = opts.voice;
+  }
+  const voiceControls = {};
+  if (opts.speed) {
+    voiceControls.speed = opts.speed;
+  }
+  if (opts.emotion) {
+    voiceControls.emotion = opts.emotion;
+  }
+  if (Object.keys({}).length) {
+    voice.__experimental_controls = voiceControls;
+  }
+  return {
+    model_id: opts.model,
+    voice,
+    output_format: {
+      container: "raw",
+      encoding: opts.encoding,
+      sample_rate: opts.sampleRate
+    },
+    language: opts.language
+  };
+};
+export {
+  ChunkedStream,
+  SynthesizeStream,
+  TTS
+};
+//# sourceMappingURL=tts.js.map

package/dist/tts.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/tts.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioByteStream, log, tokenize, tts } from '@livekit/agents';\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { randomUUID } from 'node:crypto';\nimport { request } from 'node:https';\nimport { WebSocket } from 'ws';\nimport {\n  TTSDefaultVoiceId,\n  type TTSEncoding,\n  type TTSModels,\n  type TTSVoiceEmotion,\n  type TTSVoiceSpeed,\n} from './models.js';\n\nconst AUTHORIZATION_HEADER = 'X-API-Key';\nconst VERSION_HEADER = 'Cartesia-Version';\nconst VERSION = '2024-06-10';\nconst NUM_CHANNELS = 1;\nconst BUFFERED_WORDS_COUNT = 8;\n\nexport interface TTSOptions {\n  model: TTSModels | string;\n  encoding: TTSEncoding;\n  sampleRate: number;\n  voice: string | number[];\n  speed?: TTSVoiceSpeed | number;\n  emotion?: (TTSVoiceEmotion | string)[];\n  apiKey?: string;\n  language: string;\n}\n\nconst defaultTTSOptions: TTSOptions = {\n  model: 'sonic-english',\n  encoding: 'pcm_s16le',\n  sampleRate: 24000,\n  voice: TTSDefaultVoiceId,\n  apiKey: process.env.CARTESIA_API_KEY,\n  language: 'en',\n};\n\nexport class TTS extends tts.TTS {\n  #opts: TTSOptions;\n  label = 'cartesia.TTS';\n\n  constructor(opts: Partial<TTSOptions> = {}) {\n    super(opts.sampleRate || defaultTTSOptions.sampleRate, NUM_CHANNELS, {\n      streaming: true,\n    });\n\n    this.#opts = {\n      ...defaultTTSOptions,\n      ...opts,\n    };\n\n    if (this.#opts.apiKey === undefined) {\n      throw new Error(\n        'Cartesia API key is required, whether as an argument or as $CARTESIA_API_KEY',\n      );\n    }\n  }\n\n  // TODO(nbsp): updateOptions\n\n  synthesize(text: string): tts.ChunkedStream {\n    return new ChunkedStream(this, text, this.#opts);\n  }\n\n  stream(): tts.SynthesizeStream {\n    return new SynthesizeStream(this, this.#opts);\n  }\n}\n\nexport class ChunkedStream extends tts.ChunkedStream {\n  label = 'cartesia.ChunkedStream';\n  #opts: TTSOptions;\n  #text: string;\n\n  // set Promise<T> to any because OpenAI returns an annoying Response type\n  constructor(tts: TTS, text: string, opts: TTSOptions) {\n    super(text, tts);\n    this.#text = text;\n    this.#opts = opts;\n    this.#run();\n  }\n\n  async #run() {\n    const requestId = randomUUID();\n    const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);\n    const json = toCartesiaOptions(this.#opts);\n    json.transcript = this.#text;\n\n    const req = request(\n      {\n        hostname: 'api.cartesia.ai',\n        port: 443,\n        path: '/tts/bytes',\n        method: 'POST',\n        headers: {\n          [AUTHORIZATION_HEADER]: this.#opts.apiKey!,\n          [VERSION_HEADER]: VERSION,\n        },\n      },\n      (res) => {\n        res.on('data', (chunk) => {\n          for (const frame of bstream.write(chunk)) {\n            this.queue.put({\n              requestId,\n              frame,\n              final: false,\n              segmentId: requestId,\n            });\n          }\n        });\n        res.on('close', () => {\n          for (const frame of bstream.flush()) {\n            this.queue.put({\n              requestId,\n              frame,\n              final: false,\n              segmentId: requestId,\n            });\n          }\n          this.queue.close();\n        });\n      },\n    );\n\n    req.write(JSON.stringify(json));\n    req.end();\n  }\n}\n\nexport class SynthesizeStream extends tts.SynthesizeStream {\n  #opts: TTSOptions;\n  #logger = log();\n  #tokenizer = new tokenize.basic.SentenceTokenizer(undefined, BUFFERED_WORDS_COUNT).stream();\n  label = 'cartesia.SynthesizeStream';\n\n  constructor(tts: TTS, opts: TTSOptions) {\n    super(tts);\n    this.#opts = opts;\n    this.#run();\n  }\n\n  async #run() {\n    const requestId = randomUUID();\n    let closing = false;\n\n    const sentenceStreamTask = async (ws: WebSocket) => {\n      const packet = toCartesiaOptions(this.#opts);\n      for await (const event of this.#tokenizer) {\n        ws.send(\n          JSON.stringify({\n            ...packet,\n            context_id: requestId,\n            transcript: event.token + ' ',\n            continue: true,\n          }),\n        );\n      }\n\n      ws.send(\n        JSON.stringify({\n          ...packet,\n          context_id: requestId,\n          transcript: ' ',\n          continue: false,\n        }),\n      );\n    };\n\n    const inputTask = async () => {\n      for await (const data of this.input) {\n        if (data === SynthesizeStream.FLUSH_SENTINEL) {\n          this.#tokenizer.flush();\n          continue;\n        }\n        this.#tokenizer.pushText(data);\n      }\n      this.#tokenizer.endInput();\n      this.#tokenizer.close();\n    };\n\n    const recvTask = async (ws: WebSocket) => {\n      const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);\n\n      let lastFrame: AudioFrame | undefined;\n      const sendLastFrame = (segmentId: string, final: boolean) => {\n        if (lastFrame) {\n          this.queue.put({ requestId, segmentId, frame: lastFrame, final });\n          lastFrame = undefined;\n        }\n      };\n\n      ws.on('message', (data) => {\n        const json = JSON.parse(data.toString());\n        const segmentId = json.context_id;\n        if ('data' in json) {\n          const data = new Int8Array(Buffer.from(json.data, 'base64'));\n          for (const frame of bstream.write(data)) {\n            sendLastFrame(segmentId, false);\n            lastFrame = frame;\n          }\n        } else if ('done' in json) {\n          for (const frame of bstream.flush()) {\n            sendLastFrame(segmentId, false);\n            lastFrame = frame;\n          }\n          sendLastFrame(segmentId, true);\n          this.queue.put(SynthesizeStream.END_OF_STREAM);\n\n          if (segmentId === requestId) {\n            closing = true;\n            ws.close();\n            return;\n          }\n        }\n      });\n      ws.on('close', (code, reason) => {\n        if (!closing) {\n          this.#logger.error(`WebSocket closed with code ${code}: ${reason}`);\n        }\n        ws.removeAllListeners();\n      });\n    };\n\n    const url = `wss://api.cartesia.ai/tts/websocket?api_key=${this.#opts.apiKey}&cartesia_version=${VERSION}`;\n    const ws = new WebSocket(url);\n\n    try {\n      await new Promise((resolve, reject) => {\n        ws.on('open', resolve);\n        ws.on('error', (error) => reject(error));\n        ws.on('close', (code) => reject(`WebSocket returned ${code}`));\n      });\n\n      await Promise.all([inputTask(), sentenceStreamTask(ws), recvTask(ws)]);\n    } catch (e) {\n      throw new Error(`failed to connect to Cartesia: ${e}`);\n    }\n  }\n}\n\nconst toCartesiaOptions = (opts: TTSOptions): { [id: string]: unknown } => {\n  const voice: { [id: string]: unknown } = {};\n  if (typeof opts.voice === 'string') {\n    voice.mode = 'id';\n    voice.id = opts.voice;\n  } else {\n    voice.mode = 'embedding';\n    voice.embedding = opts.voice;\n  }\n\n  const voiceControls: { [id: string]: unknown } = {};\n  if (opts.speed) {\n    voiceControls.speed = opts.speed;\n  }\n  if (opts.emotion) {\n    voiceControls.emotion = opts.emotion;\n  }\n\n  if (Object.keys({}).length) {\n    voice.__experimental_controls = voiceControls;\n  }\n\n  return {\n    model_id: opts.model,\n    voice,\n    output_format: {\n      container: 'raw',\n      encoding: opts.encoding,\n      sample_rate: opts.sampleRate,\n    },\n    language: opts.language,\n  };\n};\n"],"mappings":"AAGA,SAAS,iBAAiB,KAAK,UAAU,WAAW;AAEpD,SAAS,kBAAkB;AAC3B,SAAS,eAAe;AACxB,SAAS,iBAAiB;AAC1B;AAAA,EACE;AAAA,OAKK;AAEP,MAAM,uBAAuB;AAC7B,MAAM,iBAAiB;AACvB,MAAM,UAAU;AAChB,MAAM,eAAe;AACrB,MAAM,uBAAuB;AAa7B,MAAM,oBAAgC;AAAA,EACpC,OAAO;AAAA,EACP,UAAU;AAAA,EACV,YAAY;AAAA,EACZ,OAAO;AAAA,EACP,QAAQ,QAAQ,IAAI;AAAA,EACpB,UAAU;AACZ;AAEO,MAAM,YAAY,IAAI,IAAI;AAAA,EAC/B;AAAA,EACA,QAAQ;AAAA,EAER,YAAY,OAA4B,CAAC,GAAG;AAC1C,UAAM,KAAK,cAAc,kBAAkB,YAAY,cAAc;AAAA,MACnE,WAAW;AAAA,IACb,CAAC;AAED,SAAK,QAAQ;AAAA,MACX,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAEA,QAAI,KAAK,MAAM,WAAW,QAAW;AACnC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAIA,WAAW,MAAiC;AAC1C,WAAO,IAAI,cAAc,MAAM,MAAM,KAAK,KAAK;AAAA,EACjD;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,iBAAiB,MAAM,KAAK,KAAK;AAAA,EAC9C;AACF;AAEO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,QAAQ;AAAA,EACR;AAAA,EACA;AAAA;AAAA,EAGA,YAAYA,MAAU,MAAc,MAAkB;AACpD,UAAM,MAAMA,IAAG;AACf,SAAK,QAAQ;AACb,SAAK,QAAQ;AACb,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,YAAY,WAAW;AAC7B,UAAM,UAAU,IAAI,gBAAgB,KAAK,MAAM,YAAY,YAAY;AACvE,UAAM,OAAO,kBAAkB,KAAK,KAAK;AACzC,SAAK,aAAa,KAAK;AAEvB,UAAM,MAAM;AAAA,MACV;AAAA,QACE,UAAU;AAAA,QACV,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,CAAC,oBAAoB,GAAG,KAAK,MAAM;AAAA,UACnC,CAAC,cAAc,GAAG;AAAA,QACpB;AAAA,MACF;AAAA,MACA,CAAC,QAAQ;AACP,YAAI,GAAG,QAAQ,CAAC,UAAU;AACxB,qBAAW,SAAS,QAAQ,MAAM,KAAK,GAAG;AACxC,iBAAK,MAAM,IAAI;AAAA,cACb;AAAA,cACA;AAAA,cACA,OAAO;AAAA,cACP,WAAW;AAAA,YACb,CAAC;AAAA,UACH;AAAA,QACF,CAAC;AACD,YAAI,GAAG,SAAS,MAAM;AACpB,qBAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,iBAAK,MAAM,IAAI;AAAA,cACb;AAAA,cACA;AAAA,cACA,OAAO;AAAA,cACP,WAAW;AAAA,YACb,CAAC;AAAA,UACH;AACA,eAAK,MAAM,MAAM;AAAA,QACnB,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,MAAM,KAAK,UAAU,IAAI,CAAC;AAC9B,QAAI,IAAI;AAAA,EACV;AACF;AAEO,MAAM,yBAAyB,IAAI,iBAAiB;AAAA,EACzD;AAAA,EACA,UAAU,IAAI;AAAA,EACd,aAAa,IAAI,SAAS,MAAM,kBAAkB,QAAW,oBAAoB,EAAE,OAAO;AAAA,EAC1F,QAAQ;AAAA,EAER,YAAYA,MAAU,MAAkB;AACtC,UAAMA,IAAG;AACT,SAAK,QAAQ;AACb,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,YAAY,WAAW;AAC7B,QAAI,UAAU;AAEd,UAAM,qBAAqB,OAAOC,QAAkB;AAClD,YAAM,SAAS,kBAAkB,KAAK,KAAK;AAC3C,uBAAiB,SAAS,KAAK,YAAY;AACzC,QAAAA,IAAG;AAAA,UACD,KAAK,UAAU;AAAA,YACb,GAAG;AAAA,YACH,YAAY;AAAA,YACZ,YAAY,MAAM,QAAQ;AAAA,YAC1B,UAAU;AAAA,UACZ,CAAC;AAAA,QACH;AAAA,MACF;AAEA,MAAAA,IAAG;AAAA,QACD,KAAK,UAAU;AAAA,UACb,GAAG;AAAA,UACH,YAAY;AAAA,UACZ,YAAY;AAAA,UACZ,UAAU;AAAA,QACZ,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,QAAQ,KAAK,OAAO;AACnC,YAAI,SAAS,iBAAiB,gBAAgB;AAC5C,eAAK,WAAW,MAAM;AACtB;AAAA,QACF;AACA,aAAK,WAAW,SAAS,IAAI;AAAA,MAC/B;AACA,WAAK,WAAW,SAAS;AACzB,WAAK,WAAW,MAAM;AAAA,IACxB;AAEA,UAAM,WAAW,OAAOA,QAAkB;AACxC,YAAM,UAAU,IAAI,gBAAgB,KAAK,MAAM,YAAY,YAAY;AAEvE,UAAI;AACJ,YAAM,gBAAgB,CAAC,WAAmB,UAAmB;AAC3D,YAAI,WAAW;AACb,eAAK,MAAM,IAAI,EAAE,WAAW,WAAW,OAAO,WAAW,MAAM,CAAC;AAChE,sBAAY;AAAA,QACd;AAAA,MACF;AAEA,MAAAA,IAAG,GAAG,WAAW,CAAC,SAAS;AACzB,cAAM,OAAO,KAAK,MAAM,KAAK,SAAS,CAAC;AACvC,cAAM,YAAY,KAAK;AACvB,YAAI,UAAU,MAAM;AAClB,gBAAMC,QAAO,IAAI,UAAU,OAAO,KAAK,KAAK,MAAM,QAAQ,CAAC;AAC3D,qBAAW,SAAS,QAAQ,MAAMA,KAAI,GAAG;AACvC,0BAAc,WAAW,KAAK;AAC9B,wBAAY;AAAA,UACd;AAAA,QACF,WAAW,UAAU,MAAM;AACzB,qBAAW,SAAS,QAAQ,MAAM,GAAG;AACnC,0BAAc,WAAW,KAAK;AAC9B,wBAAY;AAAA,UACd;AACA,wBAAc,WAAW,IAAI;AAC7B,eAAK,MAAM,IAAI,iBAAiB,aAAa;AAE7C,cAAI,cAAc,WAAW;AAC3B,sBAAU;AACV,YAAAD,IAAG,MAAM;AACT;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AACD,MAAAA,IAAG,GAAG,SAAS,CAAC,MAAM,WAAW;AAC/B,YAAI,CAAC,SAAS;AACZ,eAAK,QAAQ,MAAM,8BAA8B,IAAI,KAAK,MAAM,EAAE;AAAA,QACpE;AACA,QAAAA,IAAG,mBAAmB;AAAA,MACxB,CAAC;AAAA,IACH;AAEA,UAAM,MAAM,+CAA+C,KAAK,MAAM,MAAM,qBAAqB,OAAO;AACxG,UAAM,KAAK,IAAI,UAAU,GAAG;AAE5B,QAAI;AACF,YAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,WAAG,GAAG,QAAQ,OAAO;AACrB,WAAG,GAAG,SAAS,CAAC,UAAU,OAAO,KAAK,CAAC;AACvC,WAAG,GAAG,SAAS,CAAC,SAAS,OAAO,sBAAsB,IAAI,EAAE,CAAC;AAAA,MAC/D,CAAC;AAED,YAAM,QAAQ,IAAI,CAAC,UAAU,GAAG,mBAAmB,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;AAAA,IACvE,SAAS,GAAG;AACV,YAAM,IAAI,MAAM,kCAAkC,CAAC,EAAE;AAAA,IACvD;AAAA,EACF;AACF;AAEA,MAAM,oBAAoB,CAAC,SAAgD;AACzE,QAAM,QAAmC,CAAC;AAC1C,MAAI,OAAO,KAAK,UAAU,UAAU;AAClC,UAAM,OAAO;AACb,UAAM,KAAK,KAAK;AAAA,EAClB,OAAO;AACL,UAAM,OAAO;AACb,UAAM,YAAY,KAAK;AAAA,EACzB;AAEA,QAAM,gBAA2C,CAAC;AAClD,MAAI,KAAK,OAAO;AACd,kBAAc,QAAQ,KAAK;AAAA,EAC7B;AACA,MAAI,KAAK,SAAS;AAChB,kBAAc,UAAU,KAAK;AAAA,EAC/B;AAEA,MAAI,OAAO,KAAK,CAAC,CAAC,EAAE,QAAQ;AAC1B,UAAM,0BAA0B;AAAA,EAClC;AAEA,SAAO;AAAA,IACL,UAAU,KAAK;AAAA,IACf;AAAA,IACA,eAAe;AAAA,MACb,WAAW;AAAA,MACX,UAAU,KAAK;AAAA,MACf,aAAa,KAAK;AAAA,IACpB;AAAA,IACA,UAAU,KAAK;AAAA,EACjB;AACF;","names":["tts","ws","data"]}

package/dist/tts.test.cjs

@@ -0,0 +1,9 @@
+"use strict";
+var import_agents_plugin_openai = require("@livekit/agents-plugin-openai");
+var import_agents_plugins_test = require("@livekit/agents-plugins-test");
+var import_vitest = require("vitest");
+var import_tts = require("./tts.cjs");
+(0, import_vitest.describe)("Cartesia", async () => {
+  await (0, import_agents_plugins_test.tts)(new import_tts.TTS(), new import_agents_plugin_openai.STT());
+});
+//# sourceMappingURL=tts.test.cjs.map

package/dist/tts.test.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './tts.js';\n\ndescribe('Cartesia', async () => {\n  await tts(new TTS(), new STT());\n});\n"],"mappings":";AAGA,kCAAoB;AACpB,iCAAoB;AACpB,oBAAyB;AACzB,iBAAoB;AAAA,IAEpB,wBAAS,YAAY,YAAY;AAC/B,YAAM,gCAAI,IAAI,eAAI,GAAG,IAAI,gCAAI,CAAC;AAChC,CAAC;","names":[]}

package/dist/tts.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tts.test.d.ts.map

package/dist/tts.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tts.test.d.ts","sourceRoot":"","sources":["../src/tts.test.ts"],"names":[],"mappings":""}

package/dist/tts.test.js

@@ -0,0 +1,8 @@
+import { STT } from "@livekit/agents-plugin-openai";
+import { tts } from "@livekit/agents-plugins-test";
+import { describe } from "vitest";
+import { TTS } from "./tts.js";
+describe("Cartesia", async () => {
+  await tts(new TTS(), new STT());
+});
+//# sourceMappingURL=tts.test.js.map

package/dist/tts.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/tts.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { STT } from '@livekit/agents-plugin-openai';\nimport { tts } from '@livekit/agents-plugins-test';\nimport { describe } from 'vitest';\nimport { TTS } from './tts.js';\n\ndescribe('Cartesia', async () => {\n  await tts(new TTS(), new STT());\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,WAAW;AACpB,SAAS,gBAAgB;AACzB,SAAS,WAAW;AAEpB,SAAS,YAAY,YAAY;AAC/B,QAAM,IAAI,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC;AAChC,CAAC;","names":[]}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@livekit/agents-plugin-cartesia",
+  "version": "0.1.0",
+  "description": "Cartesia plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "devDependencies": {
+    "@livekit/agents": "^x",
+    "@livekit/agents-plugin-openai": "^x",
+    "@livekit/agents-plugins-test": "^x",
+    "@livekit/rtc-node": "^0.13.1",
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "ws": "^8.16.0"
+  },
+  "peerDependencies": {
+    "@livekit/rtc-node": "^0.13.1",
+    "@livekit/agents": "^0.6.1x"
+  },
+  "scripts": {
+    "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './tts.js';

package/src/models.ts

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type TTSModels = 'sonic-english' | 'sonic-multilingual';
+
+export type TTSLanguages = 'en' | 'es' | 'fr' | 'de' | 'pt' | 'zh' | 'ja';
+
+export const TTSDefaultVoiceId = 'c2ac25f9-ecc4-4f56-9095-651354df60c0';
+
+export type TTSVoiceSpeed = 'fastest' | 'fast' | 'normal' | 'slow' | 'slowest';
+
+export type TTSVoiceEmotion =
+  | 'anger:lowest'
+  | 'anger:low'
+  | 'anger'
+  | 'anger:high'
+  | 'anger:highest'
+  | 'positivity:lowest'
+  | 'positivity:low'
+  | 'positivity'
+  | 'positivity:high'
+  | 'positivity:highest'
+  | 'surprise:lowest'
+  | 'surprise:low'
+  | 'surprise'
+  | 'surprise:high'
+  | 'surprise:highest'
+  | 'sadness:lowest'
+  | 'sadness:low'
+  | 'sadness'
+  | 'sadness:high'
+  | 'sadness:highest'
+  | 'curiosity:lowest'
+  | 'curiosity:low'
+  | 'curiosity'
+  | 'curiosity:high'
+  | 'curiosity:highest';
+
+export type TTSEncoding =
+  // XXX(nbsp): not yet supported
+  // | 'pcm_f32le'
+  // | 'pcm_mulaw'
+  // | 'pcm_alaw'
+  'pcm_s16le';

package/src/tts.test.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { STT } from '@livekit/agents-plugin-openai';
+import { tts } from '@livekit/agents-plugins-test';
+import { describe } from 'vitest';
+import { TTS } from './tts.js';
+
+describe('Cartesia', async () => {
+  await tts(new TTS(), new STT());
+});

package/src/tts.ts

@@ -0,0 +1,278 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AudioByteStream, log, tokenize, tts } from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
+import { randomUUID } from 'node:crypto';
+import { request } from 'node:https';
+import { WebSocket } from 'ws';
+import {
+  TTSDefaultVoiceId,
+  type TTSEncoding,
+  type TTSModels,
+  type TTSVoiceEmotion,
+  type TTSVoiceSpeed,
+} from './models.js';
+
+const AUTHORIZATION_HEADER = 'X-API-Key';
+const VERSION_HEADER = 'Cartesia-Version';
+const VERSION = '2024-06-10';
+const NUM_CHANNELS = 1;
+const BUFFERED_WORDS_COUNT = 8;
+
+export interface TTSOptions {
+  model: TTSModels | string;
+  encoding: TTSEncoding;
+  sampleRate: number;
+  voice: string | number[];
+  speed?: TTSVoiceSpeed | number;
+  emotion?: (TTSVoiceEmotion | string)[];
+  apiKey?: string;
+  language: string;
+}
+
+const defaultTTSOptions: TTSOptions = {
+  model: 'sonic-english',
+  encoding: 'pcm_s16le',
+  sampleRate: 24000,
+  voice: TTSDefaultVoiceId,
+  apiKey: process.env.CARTESIA_API_KEY,
+  language: 'en',
+};
+
+export class TTS extends tts.TTS {
+  #opts: TTSOptions;
+  label = 'cartesia.TTS';
+
+  constructor(opts: Partial<TTSOptions> = {}) {
+    super(opts.sampleRate || defaultTTSOptions.sampleRate, NUM_CHANNELS, {
+      streaming: true,
+    });
+
+    this.#opts = {
+      ...defaultTTSOptions,
+      ...opts,
+    };
+
+    if (this.#opts.apiKey === undefined) {
+      throw new Error(
+        'Cartesia API key is required, whether as an argument or as $CARTESIA_API_KEY',
+      );
+    }
+  }
+
+  // TODO(nbsp): updateOptions
+
+  synthesize(text: string): tts.ChunkedStream {
+    return new ChunkedStream(this, text, this.#opts);
+  }
+
+  stream(): tts.SynthesizeStream {
+    return new SynthesizeStream(this, this.#opts);
+  }
+}
+
+export class ChunkedStream extends tts.ChunkedStream {
+  label = 'cartesia.ChunkedStream';
+  #opts: TTSOptions;
+  #text: string;
+
+  // set Promise<T> to any because OpenAI returns an annoying Response type
+  constructor(tts: TTS, text: string, opts: TTSOptions) {
+    super(text, tts);
+    this.#text = text;
+    this.#opts = opts;
+    this.#run();
+  }
+
+  async #run() {
+    const requestId = randomUUID();
+    const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);
+    const json = toCartesiaOptions(this.#opts);
+    json.transcript = this.#text;
+
+    const req = request(
+      {
+        hostname: 'api.cartesia.ai',
+        port: 443,
+        path: '/tts/bytes',
+        method: 'POST',
+        headers: {
+          [AUTHORIZATION_HEADER]: this.#opts.apiKey!,
+          [VERSION_HEADER]: VERSION,
+        },
+      },
+      (res) => {
+        res.on('data', (chunk) => {
+          for (const frame of bstream.write(chunk)) {
+            this.queue.put({
+              requestId,
+              frame,
+              final: false,
+              segmentId: requestId,
+            });
+          }
+        });
+        res.on('close', () => {
+          for (const frame of bstream.flush()) {
+            this.queue.put({
+              requestId,
+              frame,
+              final: false,
+              segmentId: requestId,
+            });
+          }
+          this.queue.close();
+        });
+      },
+    );
+
+    req.write(JSON.stringify(json));
+    req.end();
+  }
+}
+
+export class SynthesizeStream extends tts.SynthesizeStream {
+  #opts: TTSOptions;
+  #logger = log();
+  #tokenizer = new tokenize.basic.SentenceTokenizer(undefined, BUFFERED_WORDS_COUNT).stream();
+  label = 'cartesia.SynthesizeStream';
+
+  constructor(tts: TTS, opts: TTSOptions) {
+    super(tts);
+    this.#opts = opts;
+    this.#run();
+  }
+
+  async #run() {
+    const requestId = randomUUID();
+    let closing = false;
+
+    const sentenceStreamTask = async (ws: WebSocket) => {
+      const packet = toCartesiaOptions(this.#opts);
+      for await (const event of this.#tokenizer) {
+        ws.send(
+          JSON.stringify({
+            ...packet,
+            context_id: requestId,
+            transcript: event.token + ' ',
+            continue: true,
+          }),
+        );
+      }
+
+      ws.send(
+        JSON.stringify({
+          ...packet,
+          context_id: requestId,
+          transcript: ' ',
+          continue: false,
+        }),
+      );
+    };
+
+    const inputTask = async () => {
+      for await (const data of this.input) {
+        if (data === SynthesizeStream.FLUSH_SENTINEL) {
+          this.#tokenizer.flush();
+          continue;
+        }
+        this.#tokenizer.pushText(data);
+      }
+      this.#tokenizer.endInput();
+      this.#tokenizer.close();
+    };
+
+    const recvTask = async (ws: WebSocket) => {
+      const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS);
+
+      let lastFrame: AudioFrame | undefined;
+      const sendLastFrame = (segmentId: string, final: boolean) => {
+        if (lastFrame) {
+          this.queue.put({ requestId, segmentId, frame: lastFrame, final });
+          lastFrame = undefined;
+        }
+      };
+
+      ws.on('message', (data) => {
+        const json = JSON.parse(data.toString());
+        const segmentId = json.context_id;
+        if ('data' in json) {
+          const data = new Int8Array(Buffer.from(json.data, 'base64'));
+          for (const frame of bstream.write(data)) {
+            sendLastFrame(segmentId, false);
+            lastFrame = frame;
+          }
+        } else if ('done' in json) {
+          for (const frame of bstream.flush()) {
+            sendLastFrame(segmentId, false);
+            lastFrame = frame;
+          }
+          sendLastFrame(segmentId, true);
+          this.queue.put(SynthesizeStream.END_OF_STREAM);
+
+          if (segmentId === requestId) {
+            closing = true;
+            ws.close();
+            return;
+          }
+        }
+      });
+      ws.on('close', (code, reason) => {
+        if (!closing) {
+          this.#logger.error(`WebSocket closed with code ${code}: ${reason}`);
+        }
+        ws.removeAllListeners();
+      });
+    };
+
+    const url = `wss://api.cartesia.ai/tts/websocket?api_key=${this.#opts.apiKey}&cartesia_version=${VERSION}`;
+    const ws = new WebSocket(url);
+
+    try {
+      await new Promise((resolve, reject) => {
+        ws.on('open', resolve);
+        ws.on('error', (error) => reject(error));
+        ws.on('close', (code) => reject(`WebSocket returned ${code}`));
+      });
+
+      await Promise.all([inputTask(), sentenceStreamTask(ws), recvTask(ws)]);
+    } catch (e) {
+      throw new Error(`failed to connect to Cartesia: ${e}`);
+    }
+  }
+}
+
+const toCartesiaOptions = (opts: TTSOptions): { [id: string]: unknown } => {
+  const voice: { [id: string]: unknown } = {};
+  if (typeof opts.voice === 'string') {
+    voice.mode = 'id';
+    voice.id = opts.voice;
+  } else {
+    voice.mode = 'embedding';
+    voice.embedding = opts.voice;
+  }
+
+  const voiceControls: { [id: string]: unknown } = {};
+  if (opts.speed) {
+    voiceControls.speed = opts.speed;
+  }
+  if (opts.emotion) {
+    voiceControls.emotion = opts.emotion;
+  }
+
+  if (Object.keys({}).length) {
+    voice.__experimental_controls = voiceControls;
+  }
+
+  return {
+    model_id: opts.model,
+    voice,
+    output_format: {
+      container: 'raw',
+      encoding: opts.encoding,
+      sample_rate: opts.sampleRate,
+    },
+    language: opts.language,
+  };
+};