@cartesia/cartesia-js 0.0.4-alpha.0 → 1.0.0-alpha.2

package/src/tts/utils.ts

@@ -0,0 +1,150 @@
+import base64 from "base64-js";
+import type Emittery from "emittery";
+import type { Chunk, EmitteryCallbacks, Sentinel } from "../types";
+
+/**
+ * Convert base64-encoded audio buffer(s) to a Float32Array.
+ *
+ * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
+ * audio buffers.
+ * @returns The audio buffer(s) as a Float32Array.
+ */
+export function base64ToArray(b64: Chunk[]): Float32Array {
+	const byteArrays = filterSentinel(b64).map((b) => base64.toByteArray(b));
+	const totalLength = byteArrays.reduce(
+		(acc, arr) => acc + arr.length / Float32Array.BYTES_PER_ELEMENT,
+		0,
+	);
+	const result = new Float32Array(totalLength);
+
+	let offset = 0;
+	for (const arr of byteArrays) {
+		const floats = new Float32Array(arr.buffer);
+		result.set(floats, offset);
+		offset += floats.length;
+	}
+
+	return result;
+}
+
+/**
+ * Schedule an audio buffer to play at a given time in the passed context.
+ *
+ * @param floats The audio buffer to play.
+ * @param context The audio context to play the buffer in.
+ * @param startAt The time to start playing the buffer at.
+ * @param sampleRate The sample rate of the audio.
+ * @returns A promise that resolves when the audio has finished playing.
+ */
+export function playAudioBuffer(
+	floats: Float32Array,
+	context: AudioContext,
+	startAt: number,
+	sampleRate: number,
+) {
+	const source = context.createBufferSource();
+	const buffer = context.createBuffer(1, floats.length, sampleRate);
+	buffer.getChannelData(0).set(floats);
+	source.buffer = buffer;
+	source.connect(context.destination);
+	source.start(startAt);
+
+	return new Promise<void>((resolve) => {
+		source.onended = () => {
+			resolve();
+		};
+	});
+}
+
+/**
+ * Unwraps a chunk of audio data from a message event and calls the
+ * handler with it if the context ID matches.
+ *
+ * @param contextId The context ID to listen for.
+ * @param handler The handler to call with the chunk of audio data.
+ * @returns A message event handler.
+ */
+export function createMessageHandlerForContextId(
+	contextId: string,
+	handler: ({
+		chunk,
+		message,
+	}: {
+		chunk: Chunk;
+		message: string;
+	}) => void,
+) {
+	return (event: MessageEvent) => {
+		if (typeof event.data !== "string") {
+			return; // Ignore non-string messages.
+		}
+		const message = JSON.parse(event.data);
+		if (message.context_id !== contextId) {
+			return; // Ignore messages for other contexts.
+		}
+		let chunk: Chunk;
+		if (message.done) {
+			// Convert the done message to a sentinel value.
+			chunk = getSentinel();
+		} else {
+			chunk = message.data;
+		}
+		handler({ chunk, message: event.data });
+	};
+}
+
+/**
+ * Get a sentinel value that indicates the end of a stream.
+ * @returns A sentinel value to indicate the end of a stream.
+ */
+export function getSentinel(): Sentinel {
+	return null;
+}
+
+/**
+ * Check if a chunk is a sentinel value (i.e. null).
+ *
+ * @param chunk
+ * @returns Whether the chunk is a sentinel value.
+ */
+export function isSentinel(x: unknown): x is Sentinel {
+	return x === getSentinel();
+}
+
+/**
+ * Filter out null values from a collection.
+ *
+ * @param collection The collection to filter.
+ * @returns The collection with null values removed.
+ */
+export function filterSentinel<T>(collection: T[]): Exclude<T, Sentinel>[] {
+	return collection.filter(
+		(x): x is Exclude<T, ReturnType<typeof getSentinel>> => !isSentinel(x),
+	);
+}
+
+/**
+ * Check if an array of chunks is complete by testing if the last chunk is a sentinel
+ * value (i.e. null).
+ * @param chunk
+ * @returns Whether the array of chunks is complete.
+ */
+export function isComplete(chunks: Chunk[]) {
+	return isSentinel(chunks[chunks.length - 1]);
+}
+
+/**
+ * Get user-facing emitter callbacks for an Emittery instance.
+ * @param emitter The Emittery instance to get callbacks for.
+ * @returns User-facing emitter callbacks.
+ */
+export function getEmitteryCallbacks<T>(
+	emitter: Emittery<T>,
+): EmitteryCallbacks<T> {
+	return {
+		on: emitter.on.bind(emitter),
+		off: emitter.off.bind(emitter),
+		once: emitter.once.bind(emitter),
+		events: emitter.events.bind(emitter),
+	};
+}

package/src/tts/websocket.ts

@@ -0,0 +1,214 @@
+import Emittery from "emittery";
+import { humanId } from "human-id";
+import { WebSocket as PartySocketWebSocket } from "partysocket";
+import { Client } from "../lib/client";
+import { CARTESIA_VERSION, constructApiUrl } from "../lib/constants";
+import type {
+	ConnectionEventData,
+	EmitteryCallbacks,
+	StreamRequest,
+	WebSocketOptions,
+} from "../types";
+import Source from "./source";
+import {
+	base64ToArray,
+	createMessageHandlerForContextId,
+	getEmitteryCallbacks,
+	isSentinel,
+} from "./utils";
+
+export default class WebSocket extends Client {
+	socket?: PartySocketWebSocket;
+	#isConnected = false;
+	#sampleRate: number;
+
+	/**
+	 * Create a new WebSocket client.
+	 *
+	 * @param args - Arguments to pass to the Client constructor.
+	 */
+	constructor(
+		{ sampleRate }: WebSocketOptions,
+		...args: ConstructorParameters<typeof Client>
+	) {
+		super(...args);
+
+		this.#sampleRate = sampleRate;
+	}
+
+	/**
+	 * Send a message over the WebSocket in order to start a stream.
+	 *
+	 * @param inputs - Stream options.
+	 * @param options - Options for the stream.
+	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
+	 * If `0`, the stream will not time out.
+	 * @returns A Source object that can be passed to a Player to play the audio.
+	 */
+	send(
+		inputs: StreamRequest["inputs"],
+		{ timeout = 0 }: StreamRequest["options"] = {},
+	) {
+		if (!this.#isConnected) {
+			throw new Error("Not connected to WebSocket. Call .connect() first.");
+		}
+
+		// Send audio request.
+		const contextId = this.#generateId();
+		this.socket?.send(
+			JSON.stringify({
+				context_id: contextId,
+				...inputs,
+				output_format: {
+					container: "raw",
+					encoding: "pcm_f32le",
+					sample_rate: this.#sampleRate,
+				},
+			}),
+		);
+
+		const emitter = new Emittery<{
+			message: string;
+		}>();
+		const source = new Source({
+			sampleRate: this.#sampleRate,
+		});
+		// Used to signal that the stream is complete, either because the
+		// WebSocket has closed, or because the stream has finished.
+		const streamCompleteController = new AbortController();
+		// Set a timeout.
+		let timeoutId: ReturnType<typeof setTimeout> | null = null;
+		if (timeout > 0) {
+			timeoutId = setTimeout(streamCompleteController.abort, timeout);
+		}
+		const handleMessage = createMessageHandlerForContextId(
+			contextId,
+			async ({ chunk, message }) => {
+				emitter.emit("message", message);
+				if (isSentinel(chunk)) {
+					await source.close();
+					streamCompleteController.abort();
+					return;
+				}
+				await source.enqueue(base64ToArray([chunk]));
+				if (timeoutId) {
+					clearTimeout(timeoutId);
+					timeoutId = setTimeout(streamCompleteController.abort, timeout);
+				}
+			},
+		);
+		this.socket?.addEventListener("message", handleMessage, {
+			signal: streamCompleteController.signal,
+		});
+		this.socket?.addEventListener(
+			"close",
+			() => {
+				streamCompleteController.abort();
+			},
+			{
+				once: true,
+			},
+		);
+		this.socket?.addEventListener(
+			"error",
+			() => {
+				streamCompleteController.abort();
+			},
+			{
+				once: true,
+			},
+		);
+		streamCompleteController.signal.addEventListener("abort", () => {
+			source.close();
+			if (timeoutId) {
+				clearTimeout(timeoutId);
+			}
+		});
+
+		return { source, ...getEmitteryCallbacks(emitter) };
+	}
+
+	/**
+	 * Generate a unique ID suitable for a streaming context.
+	 *
+	 * Not suitable for security purposes or as a primary key, since
+	 * it lacks the amount of entropy required for those use cases.
+	 *
+	 * @returns A unique ID.
+	 */
+	#generateId() {
+		return humanId({
+			separator: "-",
+			capitalize: false,
+		});
+	}
+
+	/**
+	 * Authenticate and connect to a Cartesia streaming WebSocket.
+	 *
+	 * @returns A promise that resolves when the WebSocket is connected.
+	 * @throws {Error} If the WebSocket fails to connect.
+	 */
+	connect() {
+		const url = constructApiUrl(
+			this.baseUrl,
+			`/tts/websocket?cartesia_version=${CARTESIA_VERSION}`,
+			"ws",
+		);
+		url.searchParams.set("api_key", this.apiKey);
+		const emitter = new Emittery<ConnectionEventData>();
+		this.socket = new PartySocketWebSocket(url.toString());
+		this.socket.onopen = () => {
+			this.#isConnected = true;
+			emitter.emit("open");
+		};
+		this.socket.onclose = () => {
+			this.#isConnected = false;
+			emitter.emit("close");
+		};
+
+		return new Promise<EmitteryCallbacks<ConnectionEventData>>(
+			(resolve, reject) => {
+				this.socket?.addEventListener(
+					"open",
+					() => {
+						resolve(getEmitteryCallbacks(emitter));
+					},
+					{
+						once: true,
+					},
+				);
+
+				const aborter = new AbortController();
+				this.socket?.addEventListener(
+					"error",
+					() => {
+						aborter.abort();
+						reject(new Error("WebSocket failed to connect."));
+					},
+					{
+						signal: aborter.signal,
+					},
+				);
+
+				this.socket?.addEventListener(
+					"close",
+					() => {
+						aborter.abort();
+						reject(new Error("WebSocket closed before it could connect."));
+					},
+					{
+						signal: aborter.signal,
+					},
+				);
+			},
+		);
+	}
+
+	/**
+	 * Disconnect from the Cartesia streaming WebSocket.
+	 */
+	disconnect() {
+		this.socket?.close();
+	}
+}

package/src/types/index.ts

@@ -1,4 +1,67 @@
+import type Emittery from "emittery";
+
 export interface ClientOptions {
 	apiKey?: string;
 	baseUrl?: string;
 }
+
+export type Sentinel = null;
+
+export type Chunk = string | Sentinel;
+
+export type ConnectionEventData = {
+	open: never;
+	close: never;
+};
+
+export type StreamRequest = {
+	inputs: object;
+	options: {
+		timeout?: number;
+	};
+};
+
+export type EmitteryCallbacks<T> = {
+	on: Emittery<T>["on"];
+	off: Emittery<T>["off"];
+	once: Emittery<T>["once"];
+	events: Emittery<T>["events"];
+};
+
+export type CloneOptions =
+	| {
+			mode: "url";
+			link: string;
+	  }
+	| {
+			mode: "clip";
+			clip: Blob;
+	  };
+
+export type Voice = {
+	id: string;
+	name: string;
+	description: string;
+	embedding: number[];
+	is_public: boolean;
+	user_id: string;
+	created_at: string;
+};
+
+export type CreateVoice = Pick<Voice, "name" | "description" | "embedding"> &
+	Partial<Omit<Voice, "name" | "description" | "embedding">>;
+
+export type CloneResponse = {
+	embedding: number[];
+};
+
+export type WebSocketOptions = {
+	sampleRate: number;
+};
+
+export type SourceEventData = {
+	enqueue: never;
+	close: never;
+	wait: never;
+	read: never;
+};

package/src/voices/index.ts

@@ -0,0 +1,47 @@
+import { Client } from "../lib/client";
+import type { CloneOptions, CloneResponse, CreateVoice, Voice } from "../types";
+
+export default class Voices extends Client {
+	async list(): Promise<Voice[]> {
+		const response = await this.fetch("/voices");
+		return response.json();
+	}
+
+	async get(voiceId: string): Promise<Voice> {
+		const response = await this.fetch(`/voices/${voiceId}`);
+		return response.json();
+	}
+
+	async create(voice: CreateVoice): Promise<Voice> {
+		const response = await this.fetch("/voices", {
+			method: "POST",
+			body: JSON.stringify(voice),
+		});
+		return response.json() as Promise<Voice>;
+	}
+
+	async clone(options: CloneOptions): Promise<CloneResponse> {
+		if (options.mode === "url") {
+			const response = await this.fetch(
+				`/voices/clone/url?link=${options.link}`,
+				{
+					method: "POST",
+				},
+			);
+			return response.json();
+		}
+
+		if (options.mode === "clip") {
+			const formData = new FormData();
+			formData.append("clip", options.clip);
+
+			const response = await this.fetch("/voices/clone/clip", {
+				method: "POST",
+				body: formData,
+			});
+			return response.json();
+		}
+
+		throw new Error("Invalid mode for clone()");
+	}
+}