@cartesia/cartesia-js 0.0.1

package/src/audio/index.ts

@@ -0,0 +1,242 @@
+import Emittery from "emittery";
+import { humanId } from "human-id";
+import { Client } from "../lib/client";
+import { SAMPLE_RATE, constructWebsocketUrl } from "../lib/constants";
+import {
+	type Sentinel,
+	createMessageHandlerForContextId,
+	getBufferDuration,
+	isComplete,
+	isSentinel,
+	playAudioBuffer,
+} from "./utils";
+
+export type Chunk = string | Sentinel;
+export type StreamEventData = {
+	chunk: {
+		chunk: Chunk;
+		chunks: Chunk[];
+	};
+	message: unknown;
+};
+export default class extends Client {
+	socket?: WebSocket;
+	isConnected = false;
+
+	/**
+	 * Stream audio from a model.
+	 *
+	 * @param inputs - Stream options. Includes a `model` key and some `parameters`, which
+	 * are model-specific and can be found in the model's documentation.
+	 * @param options - Options for the stream.
+	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
+	 * If `0`, the stream will not time out.
+	 * @returns An object with a method `play` of type `(bufferDuration: number) => Promise<void>`
+	 * that plays the audio as it arrives, with `bufferDuration` seconds of audio buffered before
+	 * starting playback.
+	 */
+	stream(inputs: object, { timeout = 0 }: { timeout?: number } = {}) {
+		if (!this.isConnected) {
+			throw new Error("Not connected to WebSocket. Call .connect() first.");
+		}
+
+		// Send audio request.
+		const contextId = this.generateId();
+		this.socket?.send(
+			JSON.stringify({
+				data: inputs,
+				context_id: contextId,
+			}),
+		);
+
+		// Used to signal that the stream is complete, either because the
+		// WebSocket has closed, or because the stream has finished.
+		const streamCompleteController = new AbortController();
+		// Set a timeout.
+		let timeoutId: ReturnType<typeof setTimeout> | null = null;
+		if (timeout > 0) {
+			timeoutId = setTimeout(streamCompleteController.abort, timeout);
+		}
+		// Array of base64-encoded audio chunks, representing directly sampled
+		// audio data, i.e. floats in the range [-1, 1].
+		const chunks: Chunk[] = [];
+		// Used to dispatch events.
+		const emitter = new Emittery<StreamEventData>();
+		const handleMessage = createMessageHandlerForContextId(
+			contextId,
+			async ({ chunk, message }) => {
+				chunks.push(chunk);
+				await emitter.emit("chunk", {
+					chunk,
+					chunks,
+				});
+				await emitter.emit("message", message);
+				if (isSentinel(chunk)) {
+					streamCompleteController.abort();
+				} else if (timeoutId) {
+					clearTimeout(timeoutId);
+					timeoutId = setTimeout(streamCompleteController.abort, timeout);
+				}
+			},
+		);
+		this.socket?.addEventListener("message", handleMessage, {
+			signal: streamCompleteController.signal,
+		});
+		this.socket?.addEventListener("close", streamCompleteController.abort, {
+			once: true,
+		});
+		this.socket?.addEventListener("error", streamCompleteController.abort, {
+			once: true,
+		});
+		streamCompleteController.signal.addEventListener("abort", () => {
+			emitter.clearListeners();
+		});
+
+		const play = async ({ bufferDuration }: { bufferDuration: number }) => {
+			const context = new AudioContext({
+				sampleRate: SAMPLE_RATE,
+			});
+
+			let startNextPlaybackAt = 0;
+			const playLatestChunk = (chunk: Chunk) => {
+				if (isSentinel(chunk)) {
+					return true; // Indicates that playback has finished.
+				}
+				startNextPlaybackAt =
+					playAudioBuffer([chunk], context, startNextPlaybackAt) +
+					Math.max(context.currentTime, startNextPlaybackAt);
+				return false; // Indicates that playback has not finished.
+			};
+
+			const playChunks = (chunks: Chunk[]) => {
+				startNextPlaybackAt += playAudioBuffer(
+					chunks,
+					context,
+					startNextPlaybackAt,
+				);
+
+				if (isComplete(chunks)) {
+					return;
+				}
+			};
+
+			// tryStart tries to start playback if the buffer duration is
+			// already satisfied or if all the chunks have arrived. If it is
+			// not, it returns false, indicating that the caller should call
+			// it again when more chunks arrive.
+			const tryStart = async (chunks: Chunk[]) => {
+				startNextPlaybackAt = context.currentTime;
+
+				if (isComplete(chunks) || streamCompleteController.signal.aborted) {
+					playChunks(chunks);
+					return true; // Done playing.
+				}
+
+				if (getBufferDuration(chunks) > bufferDuration) {
+					// Play the initial chunks that we already have.
+					playChunks(chunks);
+					// If the stream is not complete, play new chunks as they
+					// arrive.
+					for await (const { chunk } of emitter.events("chunk")) {
+						if (playLatestChunk(chunk)) {
+							break;
+						}
+					}
+					return true; // Done playing.
+				}
+				return false; // Need to buffer more audio.
+			};
+
+			if (!(await tryStart(chunks))) {
+				for await (const { chunks } of emitter.events("chunk")) {
+					if (await tryStart(chunks)) {
+						break;
+					}
+				}
+			}
+		};
+
+		return {
+			play,
+			on: emitter.on.bind(emitter),
+			off: emitter.off.bind(emitter),
+			once: emitter.once.bind(emitter),
+			events: emitter.events.bind(emitter),
+		};
+	}
+
+	/**
+	 * Generate a unique ID suitable for a streaming context.
+	 *
+	 * Not suitable for security purposes or as a primary key, since
+	 * it lacks the amount of entropy required for those use cases.
+	 *
+	 * @returns A unique ID.
+	 */
+	generateId() {
+		return humanId({
+			separator: "-",
+			capitalize: false,
+		});
+	}
+
+	/**
+	 * Authenticate and connect to a Cartesia streaming WebSocket.
+	 *
+	 * @returns A promise that resolves when the WebSocket is connected.
+	 * @throws {Error} If the WebSocket fails to connect.
+	 */
+	connect() {
+		const url = constructWebsocketUrl(this.baseUrl);
+		url.searchParams.set("api_key", this.apiKey);
+		this.socket = new WebSocket(url);
+		this.socket.onopen = () => {
+			this.isConnected = true;
+		};
+		this.socket.onclose = () => {
+			this.isConnected = false;
+		};
+
+		return new Promise<void>((resolve, reject) => {
+			this.socket?.addEventListener(
+				"open",
+				() => {
+					resolve();
+				},
+				{
+					once: true,
+				},
+			);
+
+			const aborter = new AbortController();
+			this.socket?.addEventListener(
+				"error",
+				() => {
+					aborter.abort();
+					reject(new Error("WebSocket failed to connect."));
+				},
+				{
+					signal: aborter.signal,
+				},
+			);
+
+			this.socket?.addEventListener(
+				"close",
+				() => {
+					aborter.abort();
+					reject(new Error("WebSocket closed before it could connect."));
+				},
+				{
+					signal: aborter.signal,
+				},
+			);
+		});
+	}
+
+	/**
+	 * Disconnect from the Cartesia streaming WebSocket.
+	 */
+	disconnect() {
+		this.socket?.close();
+	}
+}

package/src/audio/utils.ts

@@ -0,0 +1,138 @@
+import base64 from "base64-js";
+import type { Chunk, StreamEventData } from ".";
+import { SAMPLE_RATE } from "../lib/constants";
+
+/**
+ * Get the duration of base64-encoded audio buffer(s) in seconds.
+ *
+ * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
+ * audio buffers.
+ * @returns The duration of the buffer(s) in seconds.
+ */
+export function getBufferDuration(b64: Chunk[]) {
+	const floats = base64ToArray(b64);
+	return floats.length / SAMPLE_RATE;
+}
+
+/**
+ * Convert base64-encoded audio buffer(s) to a Float32Array.
+ *
+ * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
+ * audio buffers.
+ * @returns The audio buffer(s) as a Float32Array.
+ */
+export function base64ToArray(b64: Chunk[]): Float32Array {
+	return filterSentinel(b64).reduce((acc, b) => {
+		const floats = new Float32Array(base64.toByteArray(b).buffer);
+		const newAcc = new Float32Array(acc.length + floats.length);
+		newAcc.set(acc, 0);
+		newAcc.set(floats, acc.length);
+		return newAcc;
+	}, new Float32Array(0));
+}
+
+/**
+ * Schedule an audio buffer to play at a given time in the passed context.
+ *
+ * @param b64 The base64-encoded audio buffer to play.
+ * @param context The audio context to play the buffer in.
+ * @param maybeStartAt The time to start playing the buffer at, or null to play
+ * immediately.
+ * @param onEnded The callback to call when the buffer has finished playing.
+ * @returns The duration of the buffer in seconds.
+ */
+export function playAudioBuffer(
+	b64: Chunk[],
+	context: AudioContext,
+	maybeStartAt: number | null = null,
+	onEnded: AudioScheduledSourceNode["onended"] = null,
+) {
+	const startAt = maybeStartAt ?? context.currentTime;
+
+	const floats = base64ToArray(b64);
+	const source = context.createBufferSource();
+	const buffer = context.createBuffer(1, floats.length, SAMPLE_RATE);
+	buffer.getChannelData(0).set(floats);
+	source.buffer = buffer;
+	source.connect(context.destination);
+	source.start(startAt);
+	source.onended = onEnded;
+
+	return buffer.duration;
+}
+
+/**
+ * Unwraps a chunk of audio data from a message event and calls the
+ * handler with it if the context ID matches.
+ *
+ * @param contextId The context ID to listen for.
+ * @param handler The handler to call with the chunk of audio data.
+ * @returns A message event handler.
+ */
+export function createMessageHandlerForContextId(
+	contextId: string,
+	handler: ({
+		chunk,
+		message,
+	}: {
+		chunk: Chunk;
+		message: StreamEventData["message"];
+	}) => void,
+) {
+	return (event: MessageEvent) => {
+		const message = JSON.parse(event.data);
+		if (message.context_id !== contextId) {
+			return; // Ignore messages for other contexts.
+		}
+		let chunk: Chunk;
+		if (message.done) {
+			// Convert the done message to a sentinel value.
+			chunk = getSentinel();
+		} else {
+			chunk = message.data;
+		}
+		handler({ chunk, message });
+	};
+}
+
+export type Sentinel = null;
+
+/**
+ * Get a sentinel value that indicates the end of a stream.
+ * @returns A sentinel value to indicate the end of a stream.
+ */
+export function getSentinel(): Sentinel {
+	return null;
+}
+
+/**
+ * Check if a chunk is a sentinel value (i.e. null).
+ *
+ * @param chunk
+ * @returns Whether the chunk is a sentinel value.
+ */
+export function isSentinel(x: unknown): x is Sentinel {
+	return x === getSentinel();
+}
+
+/**
+ * Filter out null values from a collection.
+ *
+ * @param collection The collection to filter.
+ * @returns The collection with null values removed.
+ */
+export function filterSentinel<T>(collection: T[]): Exclude<T, Sentinel>[] {
+	return collection.filter(
+		(x): x is Exclude<T, ReturnType<typeof getSentinel>> => !isSentinel(x),
+	);
+}
+
+/**
+ * Check if an array of chunks is complete by testing if the last chunk is a sentinel
+ * value (i.e. null).
+ * @param chunk
+ * @returns Whether the array of chunks is complete.
+ */
+export function isComplete(chunks: Chunk[]) {
+	return isSentinel(chunks[chunks.length - 1]);
+}

package/src/lib/client.ts

@@ -0,0 +1,17 @@
+import type { ClientOptions } from "../types";
+import { BASE_URL } from "./constants";
+
+export class Client {
+	apiKey: string;
+	baseUrl: string;
+
+	constructor(options: ClientOptions = {}) {
+		if (!(options.apiKey || process.env.CARTESIA_API_KEY)) {
+			throw new Error("Missing Cartesia API key.");
+		}
+
+		// biome-ignore lint/style/noNonNullAssertion: Guaranteed to be defined by the check above.
+		this.apiKey = (options.apiKey || process.env.CARTESIA_API_KEY)!;
+		this.baseUrl = options.baseUrl || BASE_URL;
+	}
+}

package/src/lib/constants.ts

@@ -0,0 +1,6 @@
+export const BASE_URL = "https://api.cartesia.ai/v0";
+export const SAMPLE_RATE = 44100;
+
+export const constructWebsocketUrl = (baseUrl: string) => {
+	return new URL(`${baseUrl.replace(/^http/, "ws")}/ws`);
+};

package/src/lib/index.ts

@@ -0,0 +1,13 @@
+import Audio from "../audio";
+import type { ClientOptions } from "../types";
+import { Client } from "./client";
+
+export class Cartesia extends Client {
+	audio: Audio;
+
+	constructor(options: ClientOptions = {}) {
+		super(options);
+
+		this.audio = new Audio(options);
+	}
+}

package/src/react/index.ts

@@ -0,0 +1,91 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import CartesiaAudio, { type Chunk, type StreamEventData } from "../audio";
+
+interface UseAudioOptions {
+	apiKey: string;
+	baseUrl?: string;
+}
+
+interface UseAudioReturn {
+	stream: (options: object) => void;
+	play: (bufferDuration?: number) => Promise<void>;
+	isPlaying: boolean;
+	chunks: Chunk[];
+	messages: StreamEventData["message"][];
+}
+/**
+ * React hook to use the Cartesia audio API.
+ */
+export function useAudio({ apiKey, baseUrl }: UseAudioOptions): UseAudioReturn {
+	if (typeof window === "undefined" || !apiKey) {
+		return {
+			stream: () => {},
+			play: async () => {},
+			isPlaying: false,
+			chunks: [],
+			messages: [],
+		};
+	}
+
+	const audio = useMemo(() => {
+		const audio = new CartesiaAudio({ apiKey, baseUrl });
+		return audio;
+	}, [apiKey, baseUrl]);
+	const streamReturn = useRef<ReturnType<CartesiaAudio["stream"]> | null>(null);
+	const [isPlaying, setIsPlaying] = useState(false);
+	const [chunks, setChunks] = useState<Chunk[]>([]);
+	const [messages, setMessages] = useState<StreamEventData["message"][]>([]);
+
+	const stream = useCallback(
+		(options: object) => {
+			streamReturn.current = audio?.stream(options) ?? null;
+			streamReturn.current.on(
+				"chunk",
+				({ chunks }: StreamEventData["chunk"]) => {
+					setChunks(chunks);
+				},
+			);
+			streamReturn.current.on(
+				"message",
+				(message: StreamEventData["message"]) => {
+					setMessages((messages) => [...messages, message]);
+				},
+			);
+		},
+		[audio],
+	);
+
+	useEffect(() => {
+		async function initialize() {
+			try {
+				await audio?.connect();
+			} catch (e) {
+				console.error(e);
+			}
+			return () => {
+				audio?.disconnect();
+			};
+		}
+		initialize();
+	}, [audio]);
+
+	const play = useCallback(
+		async (bufferDuration = 0) => {
+			if (isPlaying || !streamReturn.current) {
+				return;
+			}
+			setIsPlaying(true);
+			await streamReturn.current?.play({ bufferDuration });
+			setIsPlaying(false);
+		},
+		[isPlaying],
+	);
+
+	// TODO:
+	// - [] Pause and stop playback.
+	// - [] Access the play and buffer cursors.
+	// - [] Seek to a specific time.
+	// These are probably best implemented by adding event listener
+	// functionality to the base library.
+	return { stream, play, isPlaying, chunks, messages };
+}

package/src/types/index.ts

@@ -0,0 +1,4 @@
+export interface ClientOptions {
+	apiKey?: string;
+	baseUrl?: string;
+}

package/tsconfig.json

@@ -0,0 +1,3 @@
+{
+	"extends": "@repo/config-typescript/react-library.json"
+}