@cartesia/cartesia-js 0.0.3 → 1.0.0-alpha.1

package/src/{audio → tts}/utils.ts

@@ -1,19 +1,6 @@
 import base64 from "base64-js";
 import type Emittery from "emittery";
-import type { Chunk, StreamEventData } from ".";
-import { SAMPLE_RATE } from "../lib/constants";
-
-/**
- * Get the duration of base64-encoded audio buffer(s) in seconds.
- *
- * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
- * audio buffers.
- * @returns The duration of the buffer(s) in seconds.
- */
-export function getBufferDuration(b64: Chunk[]) {
-	const floats = base64ToArray(b64);
-	return floats.length / SAMPLE_RATE;
-}
+import type { Chunk, EmitteryCallbacks, Sentinel } from "../types";
 
 /**
  * Convert base64-encoded audio buffer(s) to a Float32Array.
@@ -35,31 +22,30 @@ export function base64ToArray(b64: Chunk[]): Float32Array {
 /**
  * Schedule an audio buffer to play at a given time in the passed context.
  *
- * @param b64 The base64-encoded audio buffer to play.
+ * @param floats The audio buffer to play.
  * @param context The audio context to play the buffer in.
- * @param maybeStartAt The time to start playing the buffer at, or null to play
- * immediately.
- * @param onEnded The callback to call when the buffer has finished playing.
- * @returns The duration of the buffer in seconds.
+ * @param startAt The time to start playing the buffer at.
+ * @param sampleRate The sample rate of the audio.
+ * @returns A promise that resolves when the audio has finished playing.
  */
 export function playAudioBuffer(
-	b64: Chunk[],
+	floats: Float32Array,
 	context: AudioContext,
-	maybeStartAt: number | null = null,
-	onEnded: AudioScheduledSourceNode["onended"] = null,
+	startAt: number,
+	sampleRate: number,
 ) {
-	const startAt = maybeStartAt ?? context.currentTime;
-
-	const floats = base64ToArray(b64);
 	const source = context.createBufferSource();
-	const buffer = context.createBuffer(1, floats.length, SAMPLE_RATE);
+	const buffer = context.createBuffer(1, floats.length, sampleRate);
 	buffer.getChannelData(0).set(floats);
 	source.buffer = buffer;
 	source.connect(context.destination);
 	source.start(startAt);
-	source.onended = onEnded;
 
-	return buffer.duration;
+	return new Promise<void>((resolve) => {
+		source.onended = () => {
+			resolve();
+		};
+	});
 }
 
 /**
@@ -77,10 +63,13 @@ export function createMessageHandlerForContextId(
 		message,
 	}: {
 		chunk: Chunk;
-		message: StreamEventData["message"];
+		message: string;
 	}) => void,
 ) {
 	return (event: MessageEvent) => {
+		if (typeof event.data !== "string") {
+			return; // Ignore non-string messages.
+		}
 		const message = JSON.parse(event.data);
 		if (message.context_id !== contextId) {
 			return; // Ignore messages for other contexts.
@@ -92,12 +81,10 @@ export function createMessageHandlerForContextId(
 		} else {
 			chunk = message.data;
 		}
-		handler({ chunk, message });
+		handler({ chunk, message: event.data });
 	};
 }
 
-export type Sentinel = null;
-
 /**
  * Get a sentinel value that indicates the end of a stream.
  * @returns A sentinel value to indicate the end of a stream.
@@ -138,12 +125,6 @@ export function isComplete(chunks: Chunk[]) {
 	return isSentinel(chunks[chunks.length - 1]);
 }
 
-export type EmitteryCallbacks<T> = {
-	on: Emittery<T>["on"];
-	off: Emittery<T>["off"];
-	once: Emittery<T>["once"];
-	events: Emittery<T>["events"];
-};
 /**
  * Get user-facing emitter callbacks for an Emittery instance.
  * @param emitter The Emittery instance to get callbacks for.
@@ -159,62 +140,3 @@ export function getEmitteryCallbacks<T>(
 		events: emitter.events.bind(emitter),
 	};
 }
-
-/**
- * Converts a base64-encoded audio buffer to a WAV file.
- * Source: https://gist.github.com/Daninet/22edc59cf2aee0b9a90c18e553e49297
- * @param b64 The base64-encoded audio buffer to convert to a WAV file.
- */
-export function bufferToWav(
-	sampleRate: number,
-	channelBuffers: Float32Array[],
-) {
-	const totalSamples = channelBuffers[0].length * channelBuffers.length;
-
-	const buffer = new ArrayBuffer(44 + totalSamples * 2);
-	const view = new DataView(buffer);
-
-	const writeString = (view: DataView, offset: number, string: string) => {
-		for (let i = 0; i < string.length; i++) {
-			view.setUint8(offset + i, string.charCodeAt(i));
-		}
-	};
-
-	/* RIFF identifier */
-	writeString(view, 0, "RIFF");
-	/* RIFF chunk length */
-	view.setUint32(4, 36 + totalSamples * 2, true);
-	/* RIFF type */
-	writeString(view, 8, "WAVE");
-	/* format chunk identifier */
-	writeString(view, 12, "fmt ");
-	/* format chunk length */
-	view.setUint32(16, 16, true);
-	/* sample format (raw) */
-	view.setUint16(20, 1, true);
-	/* channel count */
-	view.setUint16(22, channelBuffers.length, true);
-	/* sample rate */
-	view.setUint32(24, sampleRate, true);
-	/* byte rate (sample rate * block align) */
-	view.setUint32(28, sampleRate * 4, true);
-	/* block align (channel count * bytes per sample) */
-	view.setUint16(32, channelBuffers.length * 2, true);
-	/* bits per sample */
-	view.setUint16(34, 16, true);
-	/* data chunk identifier */
-	writeString(view, 36, "data");
-	/* data chunk length */
-	view.setUint32(40, totalSamples * 2, true);
-
-	let offset = 44;
-	for (let i = 0; i < channelBuffers[0].length; i++) {
-		for (let channel = 0; channel < channelBuffers.length; channel++) {
-			const s = Math.max(-1, Math.min(1, channelBuffers[channel][i]));
-			view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
-			offset += 2;
-		}
-	}
-
-	return buffer;
-}

package/src/tts/websocket.ts

@@ -0,0 +1,210 @@
+import Emittery from "emittery";
+import { humanId } from "human-id";
+import { WebSocket as PartySocketWebSocket } from "partysocket";
+import { Client } from "../lib/client";
+import { constructApiUrl } from "../lib/constants";
+import type {
+	ConnectionEventData,
+	EmitteryCallbacks,
+	StreamRequest,
+	WebSocketOptions,
+} from "../types";
+import Source from "./source";
+import {
+	base64ToArray,
+	createMessageHandlerForContextId,
+	getEmitteryCallbacks,
+	isSentinel,
+} from "./utils";
+
+export default class WebSocket extends Client {
+	socket?: PartySocketWebSocket;
+	#isConnected = false;
+	#sampleRate: number;
+
+	/**
+	 * Create a new WebSocket client.
+	 *
+	 * @param args - Arguments to pass to the Client constructor.
+	 */
+	constructor(
+		{ sampleRate }: WebSocketOptions,
+		...args: ConstructorParameters<typeof Client>
+	) {
+		super(...args);
+
+		this.#sampleRate = sampleRate;
+	}
+
+	/**
+	 * Send a message over the WebSocket in order to start a stream.
+	 *
+	 * @param inputs - Stream options.
+	 * @param options - Options for the stream.
+	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
+	 * If `0`, the stream will not time out.
+	 * @returns A Source object that can be passed to a Player to play the audio.
+	 */
+	send(
+		inputs: StreamRequest["inputs"],
+		{ timeout = 0 }: StreamRequest["options"] = {},
+	) {
+		if (!this.#isConnected) {
+			throw new Error("Not connected to WebSocket. Call .connect() first.");
+		}
+
+		// Send audio request.
+		const contextId = this.#generateId();
+		this.socket?.send(
+			JSON.stringify({
+				context_id: contextId,
+				...inputs,
+				output_format: {
+					container: "raw",
+					encoding: "pcm_f32le",
+					sample_rate: this.#sampleRate,
+				},
+			}),
+		);
+
+		const emitter = new Emittery<{
+			message: string;
+		}>();
+		const source = new Source({
+			sampleRate: this.#sampleRate,
+		});
+		// Used to signal that the stream is complete, either because the
+		// WebSocket has closed, or because the stream has finished.
+		const streamCompleteController = new AbortController();
+		// Set a timeout.
+		let timeoutId: ReturnType<typeof setTimeout> | null = null;
+		if (timeout > 0) {
+			timeoutId = setTimeout(streamCompleteController.abort, timeout);
+		}
+		const handleMessage = createMessageHandlerForContextId(
+			contextId,
+			async ({ chunk, message }) => {
+				emitter.emit("message", message);
+				if (isSentinel(chunk)) {
+					await source.close();
+					streamCompleteController.abort();
+					return;
+				}
+				await source.enqueue(base64ToArray([chunk]));
+				if (timeoutId) {
+					clearTimeout(timeoutId);
+					timeoutId = setTimeout(streamCompleteController.abort, timeout);
+				}
+			},
+		);
+		this.socket?.addEventListener("message", handleMessage, {
+			signal: streamCompleteController.signal,
+		});
+		this.socket?.addEventListener(
+			"close",
+			() => {
+				streamCompleteController.abort();
+			},
+			{
+				once: true,
+			},
+		);
+		this.socket?.addEventListener(
+			"error",
+			() => {
+				streamCompleteController.abort();
+			},
+			{
+				once: true,
+			},
+		);
+		streamCompleteController.signal.addEventListener("abort", () => {
+			source.close();
+			if (timeoutId) {
+				clearTimeout(timeoutId);
+			}
+		});
+
+		return { source, ...getEmitteryCallbacks(emitter) };
+	}
+
+	/**
+	 * Generate a unique ID suitable for a streaming context.
+	 *
+	 * Not suitable for security purposes or as a primary key, since
+	 * it lacks the amount of entropy required for those use cases.
+	 *
+	 * @returns A unique ID.
+	 */
+	#generateId() {
+		return humanId({
+			separator: "-",
+			capitalize: false,
+		});
+	}
+
+	/**
+	 * Authenticate and connect to a Cartesia streaming WebSocket.
+	 *
+	 * @returns A promise that resolves when the WebSocket is connected.
+	 * @throws {Error} If the WebSocket fails to connect.
+	 */
+	connect() {
+		const url = constructApiUrl(this.baseUrl, "/tts/websocket", "ws");
+		url.searchParams.set("api_key", this.apiKey);
+		const emitter = new Emittery<ConnectionEventData>();
+		this.socket = new PartySocketWebSocket(url.toString());
+		this.socket.onopen = () => {
+			this.#isConnected = true;
+			emitter.emit("open");
+		};
+		this.socket.onclose = () => {
+			this.#isConnected = false;
+			emitter.emit("close");
+		};
+
+		return new Promise<EmitteryCallbacks<ConnectionEventData>>(
+			(resolve, reject) => {
+				this.socket?.addEventListener(
+					"open",
+					() => {
+						resolve(getEmitteryCallbacks(emitter));
+					},
+					{
+						once: true,
+					},
+				);
+
+				const aborter = new AbortController();
+				this.socket?.addEventListener(
+					"error",
+					() => {
+						aborter.abort();
+						reject(new Error("WebSocket failed to connect."));
+					},
+					{
+						signal: aborter.signal,
+					},
+				);
+
+				this.socket?.addEventListener(
+					"close",
+					() => {
+						aborter.abort();
+						reject(new Error("WebSocket closed before it could connect."));
+					},
+					{
+						signal: aborter.signal,
+					},
+				);
+			},
+		);
+	}
+
+	/**
+	 * Disconnect from the Cartesia streaming WebSocket.
+	 */
+	disconnect() {
+		this.socket?.close();
+	}
+}

package/src/types/index.ts

@@ -1,4 +1,67 @@
+import type Emittery from "emittery";
+
 export interface ClientOptions {
 	apiKey?: string;
 	baseUrl?: string;
 }
+
+export type Sentinel = null;
+
+export type Chunk = string | Sentinel;
+
+export type ConnectionEventData = {
+	open: never;
+	close: never;
+};
+
+export type StreamRequest = {
+	inputs: object;
+	options: {
+		timeout?: number;
+	};
+};
+
+export type EmitteryCallbacks<T> = {
+	on: Emittery<T>["on"];
+	off: Emittery<T>["off"];
+	once: Emittery<T>["once"];
+	events: Emittery<T>["events"];
+};
+
+export type CloneOptions =
+	| {
+			mode: "url";
+			link: string;
+	  }
+	| {
+			mode: "clip";
+			clip: Blob;
+	  };
+
+export type Voice = {
+	id: string;
+	name: string;
+	description: string;
+	embedding: number[];
+	is_public: boolean;
+	user_id: string;
+	created_at: string;
+};
+
+export type CreateVoice = Pick<Voice, "name" | "description" | "embedding"> &
+	Partial<Omit<Voice, "name" | "description" | "embedding">>;
+
+export type CloneResponse = {
+	embedding: number[];
+};
+
+export type WebSocketOptions = {
+	sampleRate: number;
+};
+
+export type SourceEventData = {
+	enqueue: never;
+	close: never;
+	wait: never;
+	read: never;
+};

package/src/voices/index.ts

@@ -0,0 +1,47 @@
+import { Client } from "../lib/client";
+import type { CloneOptions, CloneResponse, CreateVoice, Voice } from "../types";
+
+export default class Voices extends Client {
+	async list(): Promise<Voice[]> {
+		const response = await this.fetch("/voices");
+		return response.json();
+	}
+
+	async get(voiceId: string): Promise<Voice> {
+		const response = await this.fetch(`/voices/${voiceId}`);
+		return response.json();
+	}
+
+	async create(voice: CreateVoice): Promise<Voice> {
+		const response = await this.fetch("/voices", {
+			method: "POST",
+			body: JSON.stringify(voice),
+		});
+		return response.json() as Promise<Voice>;
+	}
+
+	async clone(options: CloneOptions): Promise<CloneResponse> {
+		if (options.mode === "url") {
+			const response = await this.fetch(
+				`/voices/clone/url?link=${options.link}`,
+				{
+					method: "POST",
+				},
+			);
+			return response.json();
+		}
+
+		if (options.mode === "clip") {
+			const formData = new FormData();
+			formData.append("clip", options.clip);
+
+			const response = await this.fetch("/voices/clone/clip", {
+				method: "POST",
+				body: formData,
+			});
+			return response.json();
+		}
+
+		throw new Error("Invalid mode for clone()");
+	}
+}

package/dist/audio/index.d.mts

@@ -1,5 +0,0 @@
-import 'emittery';
-import 'partysocket';
-import '../lib/client.mjs';
-export { C as Chunk, l as ConnectionEventData, S as StreamEventData, m as StreamRequest, e as default } from '../index-DSBmfK9-.mjs';
-import '../types/index.mjs';

package/dist/audio/index.d.ts

@@ -1,5 +0,0 @@
-import 'emittery';
-import 'partysocket';
-import '../lib/client.js';
-export { C as Chunk, l as ConnectionEventData, S as StreamEventData, m as StreamRequest, e as default } from '../index-qwAyxV5I.js';
-import '../types/index.js';