@cartesia/cartesia-js 0.0.3 → 1.0.0-alpha.1

package/src/react/index.ts

@@ -1,103 +1,152 @@
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
-import CartesiaAudio, { type Chunk, type StreamEventData } from "../audio";
-import { base64ToArray, bufferToWav } from "../audio/utils";
-import { SAMPLE_RATE } from "../lib/constants";
+import { Cartesia } from "../lib";
+import Player from "../tts/player";
+import type Source from "../tts/source";
+import type WebSocket from "../tts/websocket";
+import { pingServer } from "./utils";
 
-export type UseAudioOptions = {
+export type UseTTSOptions = {
 	apiKey: string | null;
 	baseUrl?: string;
+	sampleRate: number;
 };
 
-interface UseAudioReturn {
-	stream: (options: object) => void;
+export type PlaybackStatus = "inactive" | "playing" | "paused" | "finished";
+export type BufferStatus = "inactive" | "buffering" | "buffered";
+
+export type Metrics = {
+	modelLatency: number | null;
+};
+
+export interface UseTTSReturn {
+	buffer: (options: object) => Promise<void>;
 	play: (bufferDuration?: number) => Promise<void>;
-	download: () => Blob | null;
-	isPlaying: boolean;
+	pause: () => Promise<void>;
+	resume: () => Promise<void>;
+	toggle: () => Promise<void>;
+	source: Source | null;
+	playbackStatus: PlaybackStatus;
+	bufferStatus: BufferStatus;
+	isWaiting: boolean;
 	isConnected: boolean;
-	isStreamed: boolean;
-	chunks: Chunk[];
-	messages: StreamEventData["message"][];
+	metrics: Metrics;
 }
+
+const PING_INTERVAL = 5000;
+const DEFAULT_BUFFER_DURATION = 0.01;
+
+type Message = {
+	step_time: number;
+};
+
 /**
  * React hook to use the Cartesia audio API.
  */
-export function useAudio({ apiKey, baseUrl }: UseAudioOptions): UseAudioReturn {
+export function useTTS({
+	apiKey,
+	baseUrl,
+	sampleRate,
+}: UseTTSOptions): UseTTSReturn {
 	if (typeof window === "undefined") {
 		return {
-			stream: () => {},
+			buffer: async () => {},
 			play: async () => {},
-			download: () => null,
+			pause: async () => {},
+			resume: async () => {},
+			toggle: async () => {},
+			playbackStatus: "inactive",
+			bufferStatus: "inactive",
+			isWaiting: false,
+			source: null,
 			isConnected: false,
-			isPlaying: false,
-			isStreamed: false,
-			chunks: [],
-			messages: [],
+			metrics: {
+				modelLatency: null,
+			},
 		};
 	}
 
-	const audio = useMemo(() => {
+	const websocket = useMemo(() => {
 		if (!apiKey) {
 			return null;
 		}
-		const audio = new CartesiaAudio({ apiKey, baseUrl });
-		return audio;
-	}, [apiKey, baseUrl]);
-	const streamReturn = useRef<ReturnType<CartesiaAudio["stream"]> | null>(null);
-	const [isStreamed, setIsStreamed] = useState(false);
-	const [isPlaying, setIsPlaying] = useState(false);
+		const cartesia = new Cartesia({ apiKey, baseUrl });
+		baseUrl = baseUrl ?? cartesia.baseUrl;
+		return cartesia.tts.websocket({ sampleRate });
+	}, [apiKey, baseUrl, sampleRate]);
+	const websocketReturn = useRef<ReturnType<WebSocket["send"]> | null>(null);
+	const player = useRef<Player | null>(null);
+	const [playbackStatus, setPlaybackStatus] =
+		useState<PlaybackStatus>("inactive");
+	const [bufferStatus, setBufferStatus] = useState<BufferStatus>("inactive");
+	const [isWaiting, setIsWaiting] = useState(false);
 	const [isConnected, setIsConnected] = useState(false);
-	const [chunks, setChunks] = useState<Chunk[]>([]);
-	const [messages, setMessages] = useState<StreamEventData["message"][]>([]);
+	const [bufferDuration, setBufferDuration] = useState<number | null>(null);
+	const [messages, setMessages] = useState<Message[]>([]);
 
-	const stream = useCallback(
+	const buffer = useCallback(
 		async (options: object) => {
-			streamReturn.current = audio?.stream(options) ?? null;
-			if (!streamReturn.current) {
+			setMessages([]);
+			setBufferStatus("buffering");
+			websocketReturn.current = websocket?.send(options) ?? null;
+			if (!websocketReturn.current) {
 				return;
 			}
-			setMessages([]);
-			streamReturn.current.on(
-				"chunk",
-				({ chunks }: StreamEventData["chunk"]) => {
-					setChunks(chunks);
-				},
-			);
-			streamReturn.current.on(
-				"message",
-				(message: StreamEventData["message"]) => {
-					setMessages((messages) => [...messages, message]);
-				},
-			);
-			const { chunks } = await streamReturn.current.once("streamed");
-			setChunks(chunks);
-			setIsStreamed(true);
+			websocketReturn.current.on("message", (message) => {
+				setMessages((messages) => [...messages, JSON.parse(message)]);
+			});
+			await websocketReturn.current.source.once("close");
+			setBufferStatus("buffered");
 		},
-		[audio],
+		[websocket],
 	);
 
-	const download = useCallback(() => {
-		if (!isStreamed) {
-			return null;
+	const metrics = useMemo(() => {
+		// Model Latency is the first step time
+		if (messages.length === 0) {
+			return {
+				modelLatency: null,
+			};
 		}
-		const audio = bufferToWav(SAMPLE_RATE, [base64ToArray(chunks)]);
-		return new Blob([audio], { type: "audio/wav" });
-	}, [isStreamed, chunks]);
+		const modelLatency = messages[0].step_time ?? null;
+		return {
+			modelLatency: Math.trunc(modelLatency),
+		};
+	}, [messages]);
 
 	useEffect(() => {
 		let cleanup: (() => void) | undefined = () => {};
 		async function setupConnection() {
 			try {
-				const connection = await audio?.connect();
+				const connection = await websocket?.connect();
 				if (!connection) {
 					return;
 				}
 				setIsConnected(true);
+				connection.on("open", () => {
+					setIsConnected(true);
+				});
 				const unsubscribe = connection.on("close", () => {
 					setIsConnected(false);
 				});
+				const intervalId = setInterval(() => {
+					if (baseUrl) {
+						pingServer(new URL(baseUrl).origin).then((ping) => {
+							let bufferDuration: number;
+							if (ping < 300) {
+								bufferDuration = 0.01; // No buffering for very low latency
+							} else if (ping > 1500) {
+								bufferDuration = 6; // Max buffering for very high latency (6 seconds)
+							} else {
+								bufferDuration = (ping / 1000) * 4; // Adjust buffer duration based on ping
+							}
+							setBufferDuration(bufferDuration);
+						});
+					}
+				}, PING_INTERVAL);
 				return () => {
 					unsubscribe();
-					audio?.disconnect();
+					clearInterval(intervalId);
+					websocket?.disconnect();
 				};
 			} catch (e) {
 				console.error(e);
@@ -107,34 +156,77 @@ export function useAudio({ apiKey, baseUrl }: UseAudioOptions): UseAudioReturn {
 			cleanup = cleanupConnection;
 		});
 		return () => cleanup?.();
-	}, [audio]);
+	}, [websocket, baseUrl]);
 
-	const play = useCallback(
-		async (bufferDuration = 0) => {
-			if (isPlaying || !streamReturn.current) {
-				return;
+	const play = useCallback(async () => {
+		if (playbackStatus === "playing" || !websocketReturn.current) {
+			return;
+		}
+		setPlaybackStatus("playing");
+
+		const unsubscribes = [];
+		unsubscribes.push(
+			websocketReturn.current.source.on("wait", () => {
+				setIsWaiting(true);
+			}),
+		);
+		unsubscribes.push(
+			websocketReturn.current.source.on("read", () => {
+				setIsWaiting(false);
+			}),
+		);
+
+		player.current = new Player({
+			bufferDuration: bufferDuration ?? DEFAULT_BUFFER_DURATION,
+		});
+		// Wait for the playback to finish before setting isPlaying to false.
+		await player.current.play(websocketReturn.current.source);
+
+		for (const unsubscribe of unsubscribes) {
+			// Deregister the event listeners (.on()) that we registered above to avoid memory leaks.
+			unsubscribe();
+		}
+
+		setPlaybackStatus("finished");
+	}, [playbackStatus, bufferDuration]);
+
+	const pause = useCallback(async () => {
+		await player.current?.pause();
+		setPlaybackStatus("paused");
+	}, []);
+
+	const resume = useCallback(async () => {
+		await player.current?.resume();
+		setPlaybackStatus("playing");
+	}, []);
+
+	const toggle = useCallback(async () => {
+		await player.current?.toggle();
+		setPlaybackStatus((status) => {
+			if (status === "playing") {
+				return "paused";
 			}
-			setIsPlaying(true);
-			await streamReturn.current?.play({ bufferDuration });
-			setIsPlaying(false);
-		},
-		[isPlaying],
-	);
+			if (status === "paused") {
+				return "playing";
+			}
+			return status;
+		});
+	}, []);
 
 	// TODO:
-	// - [] Pause and stop playback.
 	// - [] Access the play and buffer cursors.
 	// - [] Seek to a specific time.
-	// These are probably best implemented by adding event listener
-	// functionality to the base library.
 	return {
-		stream,
+		buffer,
 		play,
-		download,
-		isPlaying,
+		pause,
+		source: websocketReturn.current?.source ?? null,
+		resume,
+		toggle,
+		playbackStatus,
+		bufferStatus,
+		isWaiting,
 		isConnected,
-		isStreamed,
-		chunks,
-		messages,
+		metrics,
 	};
 }

package/src/react/utils.ts

@@ -0,0 +1,11 @@
+/**
+ * Ping the server to calculate the round-trip time. This is useful for buffering audio in high-latency environments.
+ * @param url The URL to ping.
+ */
+
+export async function pingServer(url: string): Promise<number> {
+	const start = new Date().getTime();
+	await fetch(url);
+	const end = new Date().getTime();
+	return end - start;
+}

package/src/tts/index.ts

@@ -0,0 +1,17 @@
+import { Client } from "../lib/client";
+import type { WebSocketOptions } from "../types";
+import WebSocket from "./websocket";
+
+export default class TTS extends Client {
+	/**
+	 * Get a WebSocket client for streaming audio from the TTS API.
+	 *
+	 * @returns {WebSocket} A Cartesia WebSocket client.
+	 */
+	websocket(options: WebSocketOptions): WebSocket {
+		return new WebSocket(options, {
+			apiKey: this.apiKey,
+			baseUrl: this.baseUrl,
+		});
+	}
+}

package/src/tts/player.ts

@@ -0,0 +1,109 @@
+import Emittery from "emittery";
+import type Source from "./source";
+import { playAudioBuffer } from "./utils";
+
+type PlayEventData = {
+	finish: never;
+};
+
+export default class Player {
+	#context: AudioContext | null = null;
+	#startNextPlaybackAt = 0;
+	#bufferDuration: number;
+	#emitter = new Emittery<PlayEventData>();
+
+	/**
+	 * Create a new Player.
+	 *
+	 * @param options - Options for the Player.
+	 * @param options.bufferDuration - The duration of the audio buffer to play.
+	 */
+	constructor({ bufferDuration }: { bufferDuration: number }) {
+		this.#bufferDuration = bufferDuration;
+	}
+
+	async #playBuffer(buf: Float32Array, sampleRate: number) {
+		if (!this.#context) {
+			throw new Error("AudioContext not initialized.");
+		}
+
+		const startAt = this.#startNextPlaybackAt;
+		const duration = buf.length / sampleRate;
+		this.#startNextPlaybackAt =
+			duration + Math.max(this.#context.currentTime, this.#startNextPlaybackAt);
+
+		await playAudioBuffer(buf, this.#context, startAt, sampleRate);
+	}
+
+	/**
+	 * Play audio from a source.
+	 *
+	 * @param source The source to play audio from.
+	 * @returns A promise that resolves when the audio has finished playing.
+	 */
+	async play(source: Source) {
+		this.#startNextPlaybackAt = 0;
+		this.#context = new AudioContext({ sampleRate: source.sampleRate });
+		const buffer = new Float32Array(
+			source.durationToSampleCount(this.#bufferDuration),
+		);
+
+		const plays: Promise<void>[] = [];
+		while (true) {
+			const read = await source.read(buffer);
+			// If we've reached the end of the source, then read < buffer.length.
+			// In that case, we don't want to play the entire buffer, as that
+			// will cause repeated audio.
+			const playableAudio = buffer.slice(0, read);
+			plays.push(this.#playBuffer(playableAudio, source.sampleRate));
+
+			if (read < buffer.length) {
+				// No more audio to read.
+				await this.#emitter.emit("finish");
+				break;
+			}
+		}
+
+		await Promise.all(plays);
+	}
+
+	/**
+	 * Pause the audio.
+	 *
+	 * @returns A promise that resolves when the audio has been paused.
+	 */
+	async pause() {
+		if (!this.#context) {
+			throw new Error("AudioContext not initialized.");
+		}
+		await this.#context.suspend();
+	}
+
+	/**
+	 * Resume the audio.
+	 *
+	 * @returns A promise that resolves when the audio has been resumed.
+	 */
+	async resume() {
+		if (!this.#context) {
+			throw new Error("AudioContext not initialized.");
+		}
+		await this.#context.resume();
+	}
+
+	/**
+	 * Toggle the audio.
+	 *
+	 * @returns A promise that resolves when the audio has been toggled.
+	 */
+	async toggle() {
+		if (!this.#context) {
+			throw new Error("AudioContext not initialized.");
+		}
+		if (this.#context.state === "running") {
+			await this.pause();
+		} else {
+			await this.resume();
+		}
+	}
+}

package/src/tts/source.ts

@@ -0,0 +1,98 @@
+import Emittery from "emittery";
+import type { SourceEventData } from "../types";
+
+export default class Source {
+	#emitter = new Emittery<SourceEventData>();
+	#buffer = new Float32Array();
+	#readIndex = 0;
+	#closed = false;
+	#sampleRate: number;
+
+	on = this.#emitter.on.bind(this.#emitter);
+	once = this.#emitter.once.bind(this.#emitter);
+	events = this.#emitter.events.bind(this.#emitter);
+	off = this.#emitter.off.bind(this.#emitter);
+
+	/**
+	 * Create a new Source.
+	 *
+	 * @param options - Options for the Source.
+	 * @param options.sampleRate - The sample rate of the audio.
+	 */
+	constructor({ sampleRate }: { sampleRate: number }) {
+		this.#sampleRate = sampleRate;
+	}
+
+	get sampleRate() {
+		return this.#sampleRate;
+	}
+
+	/**
+	 * Append audio to the buffer.
+	 *
+	 * @param src The audio to append.
+	 */
+	async enqueue(src: Float32Array) {
+		// Append the audio to the buffer.
+		this.#buffer = new Float32Array([...this.#buffer, ...src]);
+		await this.#emitter.emit("enqueue");
+	}
+
+	/**
+	 * Read audio from the buffer.
+	 *
+	 * @param dst The buffer to read the audio into.
+	 * @returns The number of samples read. If the source is closed, this will be
+	 * less than the length of the provided buffer.
+	 */
+	async read(dst: Float32Array): Promise<number> {
+		// Read the buffer into the provided buffer.
+		const targetReadIndex = this.#readIndex + dst.length;
+
+		while (!this.#closed && targetReadIndex > this.#buffer.length) {
+			// Wait for more audio to be enqueued.
+			await this.#emitter.emit("wait");
+			await Promise.race([
+				this.#emitter.once("enqueue"),
+				this.#emitter.once("close"),
+			]);
+			await this.#emitter.emit("read");
+		}
+
+		const read = Math.min(dst.length, this.#buffer.length - this.#readIndex);
+		dst.set(this.#buffer.slice(this.#readIndex, this.#readIndex + read));
+		this.#readIndex += read;
+		return read;
+	}
+
+	/**
+	 * Get the number of samples in a given duration.
+	 *
+	 * @param durationSecs The duration in seconds.
+	 * @returns The number of samples.
+	 */
+	durationToSampleCount(durationSecs: number) {
+		return Math.trunc(durationSecs * this.#sampleRate);
+	}
+
+	get buffer() {
+		return this.#buffer;
+	}
+
+	get readIndex() {
+		return this.#readIndex;
+	}
+
+	/**
+	 * Close the source. This signals that no more audio will be enqueued.
+	 *
+	 * This will emit a "close" event.
+	 *
+	 * @returns A promise that resolves when the source is closed.
+	 */
+	async close() {
+		this.#closed = true;
+		await this.#emitter.emit("close");
+		this.#emitter.clearListeners();
+	}
+}