@cartesia/cartesia-js 1.0.0 → 1.0.1

package/dist/types/index.d.cts

@@ -10,12 +10,64 @@ type ConnectionEventData = {
     open: never;
     close: never;
 };
+type VoiceSpecifier = {
+    mode: "id";
+    id: string;
+} | {
+    mode: "embedding";
+    embedding: number[];
+};
+type Emotion = "anger" | "sadness" | "positivity" | "curiosity" | "surprise";
+type Intensity = "lowest" | "low" | "high" | "highest";
+type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+type VoiceOptions = VoiceSpecifier & {
+    __experimental_controls?: {
+        speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+        emotion?: EmotionControl[];
+    };
+};
 type StreamRequest = {
-    inputs: object;
-    options: {
-        timeout?: number;
+    model_id: string;
+    transcript: string;
+    voice: VoiceOptions;
+    output_format?: {
+        container: string;
+        encoding: string;
+        sample_rate: number;
     };
+    context_id?: string;
+    continue?: boolean;
+    duration?: number;
+    language?: string;
+    add_timestamps?: boolean;
+};
+type StreamOptions = {
+    timeout?: number;
+};
+type WebSocketBaseResponse = {
+    context_id: string;
+    status_code: number;
+    done: boolean;
+};
+type WordTimestamps = {
+    words: string[];
+    start: number[];
+    end: number[];
+};
+type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+    type: "timestamps";
+    word_timestamps: WordTimestamps;
+};
+type WebSocketChunkResponse = WebSocketBaseResponse & {
+    type: "chunk";
+    data: string;
+    step_time: number;
+};
+type WebSocketErrorResponse = WebSocketBaseResponse & {
+    type: "error";
+    error: string;
 };
+type WebSocketResponse = WebSocketTimestampsResponse | WebSocketChunkResponse | WebSocketErrorResponse;
 type EmitteryCallbacks<T> = {
     on: emittery__default<T>["on"];
     off: emittery__default<T>["off"];
@@ -43,6 +95,8 @@ type CloneResponse = {
     embedding: number[];
 };
 type WebSocketOptions = {
+    container?: string;
+    encoding?: string;
     sampleRate: number;
 };
 type SourceEventData = {
@@ -51,5 +105,7 @@ type SourceEventData = {
     wait: never;
     read: never;
 };
+type TypedArray = Float32Array | Int16Array | Uint8Array;
+type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";
 
-export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Sentinel, SourceEventData, StreamRequest, Voice, WebSocketOptions };
+export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Emotion, EmotionControl, Encoding, Intensity, Sentinel, SourceEventData, StreamOptions, StreamRequest, TypedArray, Voice, VoiceOptions, VoiceSpecifier, WebSocketBaseResponse, WebSocketChunkResponse, WebSocketErrorResponse, WebSocketOptions, WebSocketResponse, WebSocketTimestampsResponse, WordTimestamps };

package/dist/types/index.d.ts

@@ -10,12 +10,64 @@ type ConnectionEventData = {
     open: never;
     close: never;
 };
+type VoiceSpecifier = {
+    mode: "id";
+    id: string;
+} | {
+    mode: "embedding";
+    embedding: number[];
+};
+type Emotion = "anger" | "sadness" | "positivity" | "curiosity" | "surprise";
+type Intensity = "lowest" | "low" | "high" | "highest";
+type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+type VoiceOptions = VoiceSpecifier & {
+    __experimental_controls?: {
+        speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+        emotion?: EmotionControl[];
+    };
+};
 type StreamRequest = {
-    inputs: object;
-    options: {
-        timeout?: number;
+    model_id: string;
+    transcript: string;
+    voice: VoiceOptions;
+    output_format?: {
+        container: string;
+        encoding: string;
+        sample_rate: number;
     };
+    context_id?: string;
+    continue?: boolean;
+    duration?: number;
+    language?: string;
+    add_timestamps?: boolean;
+};
+type StreamOptions = {
+    timeout?: number;
+};
+type WebSocketBaseResponse = {
+    context_id: string;
+    status_code: number;
+    done: boolean;
+};
+type WordTimestamps = {
+    words: string[];
+    start: number[];
+    end: number[];
+};
+type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+    type: "timestamps";
+    word_timestamps: WordTimestamps;
+};
+type WebSocketChunkResponse = WebSocketBaseResponse & {
+    type: "chunk";
+    data: string;
+    step_time: number;
+};
+type WebSocketErrorResponse = WebSocketBaseResponse & {
+    type: "error";
+    error: string;
 };
+type WebSocketResponse = WebSocketTimestampsResponse | WebSocketChunkResponse | WebSocketErrorResponse;
 type EmitteryCallbacks<T> = {
     on: emittery__default<T>["on"];
     off: emittery__default<T>["off"];
@@ -43,6 +95,8 @@ type CloneResponse = {
     embedding: number[];
 };
 type WebSocketOptions = {
+    container?: string;
+    encoding?: string;
     sampleRate: number;
 };
 type SourceEventData = {
@@ -51,5 +105,7 @@ type SourceEventData = {
     wait: never;
     read: never;
 };
+type TypedArray = Float32Array | Int16Array | Uint8Array;
+type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";
 
-export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Sentinel, SourceEventData, StreamRequest, Voice, WebSocketOptions };
+export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Emotion, EmotionControl, Encoding, Intensity, Sentinel, SourceEventData, StreamOptions, StreamRequest, TypedArray, Voice, VoiceOptions, VoiceSpecifier, WebSocketBaseResponse, WebSocketChunkResponse, WebSocketErrorResponse, WebSocketOptions, WebSocketResponse, WebSocketTimestampsResponse, WordTimestamps };

package/dist/voices/index.js

@@ -1,9 +1,9 @@
 import {
   Voices
-} from "../chunk-SGXUEFII.js";
-import "../chunk-PQ5EVEEH.js";
+} from "../chunk-KUSVZXDT.js";
+import "../chunk-5M33ZF3Y.js";
 import "../chunk-2BFEKY3F.js";
-import "../chunk-WIFMLPT5.js";
+import "../chunk-GHY2WEOK.js";
 export {
   Voices as default
 };

package/package.json

@@ -4,7 +4,7 @@
 		"name": "Cartesia",
 		"url": "https://cartesia.ai"
 	},
-	"version": "1.0.0",
+	"version": "1.0.1",
 	"description": "Client for the Cartesia API.",
 	"type": "module",
 	"module": "./dist/index.js",

package/src/index.ts

@@ -2,3 +2,5 @@ export { Cartesia as default } from "./lib";
 export * from "./lib";
 export * from "./types";
 export { default as WebPlayer } from "./tts/player";
+export { default as Source } from "./tts/source";
+export { default as WebSocket } from "./tts/websocket";

package/src/react/index.ts

@@ -4,12 +4,14 @@ import { Cartesia } from "../lib";
 import Player from "../tts/player";
 import type Source from "../tts/source";
 import type WebSocket from "../tts/websocket";
+import type { StreamRequest } from "../types";
 import { pingServer } from "./utils";
 
 export type UseTTSOptions = {
 	apiKey: string | null;
 	baseUrl?: string;
 	sampleRate: number;
+	onError?: (error: Error) => void;
 };
 
 export type PlaybackStatus = "inactive" | "playing" | "paused" | "finished";
@@ -20,7 +22,7 @@ export type Metrics = {
 };
 
 export interface UseTTSReturn {
-	buffer: (options: object) => Promise<void>;
+	buffer: (options: StreamRequest) => Promise<void>;
 	play: (bufferDuration?: number) => Promise<void>;
 	pause: () => Promise<void>;
 	resume: () => Promise<void>;
@@ -47,6 +49,7 @@ export function useTTS({
 	apiKey,
 	baseUrl,
 	sampleRate,
+	onError,
 }: UseTTSOptions): UseTTSReturn {
 	if (typeof window === "undefined") {
 		return {
@@ -72,7 +75,11 @@ export function useTTS({
 		}
 		const cartesia = new Cartesia({ apiKey, baseUrl });
 		baseUrl = baseUrl ?? cartesia.baseUrl;
-		return cartesia.tts.websocket({ sampleRate });
+		return cartesia.tts.websocket({
+			container: "raw",
+			encoding: "pcm_f32le",
+			sampleRate,
+		});
 	}, [apiKey, baseUrl, sampleRate]);
 	const websocketReturn = useRef<ReturnType<WebSocket["send"]> | null>(null);
 	const player = useRef<Player | null>(null);
@@ -85,23 +92,35 @@ export function useTTS({
 	const [messages, setMessages] = useState<Message[]>([]);
 
 	const buffer = useCallback(
-		async (options: object) => {
+		async (options: StreamRequest) => {
 			websocketReturn.current?.stop(); // Abort the previous request if it exists.
 
-			setMessages([]);
-			setBufferStatus("buffering");
-			websocketReturn.current = websocket?.send(options) ?? null;
-			if (!websocketReturn.current) {
-				return;
+			try {
+				setMessages([]);
+				setBufferStatus("buffering");
+				websocketReturn.current = websocket?.send(options) ?? null;
+				if (!websocketReturn.current) {
+					return;
+				}
+				const unsubscribe = websocketReturn.current.on("message", (message) => {
+					const parsedMessage = JSON.parse(message);
+					setMessages((messages) => [...messages, parsedMessage]);
+					if (parsedMessage.error) {
+						onError?.(new Error(parsedMessage.error));
+					}
+				});
+				await websocketReturn.current.source.once("close");
+				setBufferStatus("buffered");
+				unsubscribe();
+			} catch (error) {
+				if (error instanceof Error) {
+					onError?.(error);
+				} else {
+					console.error(error);
+				}
 			}
-			const unsubscribe = websocketReturn.current.on("message", (message) => {
-				setMessages((messages) => [...messages, JSON.parse(message)]);
-			});
-			await websocketReturn.current.source.once("close");
-			setBufferStatus("buffered");
-			unsubscribe();
 		},
-		[websocket],
+		[websocket, onError],
 	);
 
 	const metrics = useMemo(() => {
@@ -173,64 +192,96 @@ export function useTTS({
 	}, [websocket, baseUrl]);
 
 	const play = useCallback(async () => {
-		if (playbackStatus === "playing" || !websocketReturn.current) {
-			return;
-		}
-		if (player.current) {
-			// Stop the current player if it exists.
-			await player.current.stop();
-		}
+		try {
+			if (playbackStatus === "playing" || !websocketReturn.current) {
+				return;
+			}
+			if (player.current) {
+				// Stop the current player if it exists.
+				await player.current.stop();
+			}
 
-		setPlaybackStatus("playing");
-
-		const unsubscribes = [];
-		unsubscribes.push(
-			websocketReturn.current.source.on("wait", () => {
-				setIsWaiting(true);
-			}),
-		);
-		unsubscribes.push(
-			websocketReturn.current.source.on("read", () => {
-				setIsWaiting(false);
-			}),
-		);
-
-		player.current = new Player({
-			bufferDuration: bufferDuration ?? DEFAULT_BUFFER_DURATION,
-		});
-		// Wait for the playback to finish before setting isPlaying to false.
-		await player.current.play(websocketReturn.current.source);
+			setPlaybackStatus("playing");
 
-		for (const unsubscribe of unsubscribes) {
-			// Deregister the event listeners (.on()) that we registered above to avoid memory leaks.
-			unsubscribe();
-		}
+			const unsubscribes = [];
+			unsubscribes.push(
+				websocketReturn.current.source.on("wait", () => {
+					setIsWaiting(true);
+				}),
+			);
+			unsubscribes.push(
+				websocketReturn.current.source.on("read", () => {
+					setIsWaiting(false);
+				}),
+			);
 
-		setPlaybackStatus("finished");
-	}, [playbackStatus, bufferDuration]);
+			player.current = new Player({
+				bufferDuration: bufferDuration ?? DEFAULT_BUFFER_DURATION,
+			});
+			// Wait for the playback to finish before setting isPlaying to false.
+			await player.current.play(websocketReturn.current.source);
+
+			for (const unsubscribe of unsubscribes) {
+				// Deregister the event listeners (.on()) that we registered above to avoid memory leaks.
+				unsubscribe();
+			}
+
+			setPlaybackStatus("finished");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [playbackStatus, bufferDuration, onError]);
 
 	const pause = useCallback(async () => {
-		await player.current?.pause();
-		setPlaybackStatus("paused");
-	}, []);
+		try {
+			await player.current?.pause();
+			setPlaybackStatus("paused");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [onError]);
 
 	const resume = useCallback(async () => {
-		await player.current?.resume();
-		setPlaybackStatus("playing");
-	}, []);
+		try {
+			await player.current?.resume();
+			setPlaybackStatus("playing");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [onError]);
 
 	const toggle = useCallback(async () => {
-		await player.current?.toggle();
-		setPlaybackStatus((status) => {
-			if (status === "playing") {
-				return "paused";
-			}
-			if (status === "paused") {
-				return "playing";
+		try {
+			await player.current?.toggle();
+			setPlaybackStatus((status) => {
+				if (status === "playing") {
+					return "paused";
+				}
+				if (status === "paused") {
+					return "playing";
+				}
+				return status;
+			});
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
 			}
-			return status;
-		});
-	}, []);
+		}
+	}, [onError]);
 
 	return {
 		buffer,

package/src/tts/source.ts

@@ -1,13 +1,30 @@
 import Emittery from "emittery";
-import type { SourceEventData } from "../types";
+import type { Encoding, SourceEventData, TypedArray } from "../types";
+
+type EncodingInfo = {
+	arrayType:
+		| Float32ArrayConstructor
+		| Int16ArrayConstructor
+		| Uint8ArrayConstructor;
+	bytesPerElement: number;
+};
+
+export const ENCODING_MAP: Record<Encoding, EncodingInfo> = {
+	pcm_f32le: { arrayType: Float32Array, bytesPerElement: 4 },
+	pcm_s16le: { arrayType: Int16Array, bytesPerElement: 2 },
+	pcm_alaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+	pcm_mulaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+};
 
 export default class Source {
 	#emitter = new Emittery<SourceEventData>();
-	#buffer: Float32Array;
+	#buffer: TypedArray;
 	#readIndex = 0;
 	#writeIndex = 0;
 	#closed = false;
 	#sampleRate: number;
+	#encoding: Encoding;
+	#container: string;
 
 	on = this.#emitter.on.bind(this.#emitter);
 	once = this.#emitter.once.bind(this.#emitter);
@@ -20,21 +37,46 @@ export default class Source {
 	 * @param options - Options for the Source.
 	 * @param options.sampleRate - The sample rate of the audio.
 	 */
-	constructor({ sampleRate }: { sampleRate: number }) {
+	constructor({
+		sampleRate,
+		encoding,
+		container,
+	}: { sampleRate: number; encoding: string; container: string }) {
 		this.#sampleRate = sampleRate;
-		this.#buffer = new Float32Array(1024); // Initial size, can be adjusted
+		this.#encoding = encoding as Encoding;
+		this.#container = container;
+		this.#buffer = this.#createBuffer(1024); // Initial size, can be adjusted
 	}
 
 	get sampleRate() {
 		return this.#sampleRate;
 	}
 
+	get encoding() {
+		return this.#encoding;
+	}
+
+	get container() {
+		return this.#container;
+	}
+
+	/**
+	 * Create a new buffer for the source.
+	 *
+	 * @param size - The size of the buffer to create.
+	 * @returns The new buffer as a TypedArray based on the encoding.
+	 */
+	#createBuffer(size: number): TypedArray {
+		const { arrayType: ArrayType } = ENCODING_MAP[this.#encoding];
+		return new ArrayType(size);
+	}
+
 	/**
 	 * Append audio to the buffer.
 	 *
 	 * @param src The audio to append.
 	 */
-	async enqueue(src: Float32Array) {
+	async enqueue(src: TypedArray) {
 		const requiredCapacity = this.#writeIndex + src.length;
 
 		// Resize buffer if necessary
@@ -44,7 +86,7 @@ export default class Source {
 				newCapacity *= 2; // Double the buffer size
 			}
 
-			const newBuffer = new Float32Array(newCapacity);
+			const newBuffer = this.#createBuffer(newCapacity);
 			newBuffer.set(this.#buffer);
 			this.#buffer = newBuffer;
 		}
@@ -62,7 +104,7 @@ export default class Source {
 	 * @returns The number of samples read. If the source is closed, this will be
 	 * less than the length of the provided buffer.
 	 */
-	async read(dst: Float32Array): Promise<number> {
+	async read(dst: TypedArray): Promise<number> {
 		// Read the buffer into the provided buffer.
 		const targetReadIndex = this.#readIndex + dst.length;
 
@@ -100,6 +142,10 @@ export default class Source {
 		return this.#readIndex;
 	}
 
+	get writeIndex() {
+		return this.#writeIndex;
+	}
+
 	/**
 	 * Close the source. This signals that no more audio will be enqueued.
 	 *

package/src/tts/utils.ts

@@ -1,25 +1,38 @@
 import base64 from "base64-js";
 import type Emittery from "emittery";
-import type { Chunk, EmitteryCallbacks, Sentinel } from "../types";
+import type {
+	Chunk,
+	EmitteryCallbacks,
+	Encoding,
+	Sentinel,
+	TypedArray,
+	WebSocketResponse,
+} from "../types";
+import { ENCODING_MAP } from "./source";
 
 /**
- * Convert base64-encoded audio buffer(s) to a Float32Array.
+ * Convert base64-encoded audio buffer(s) to a TypedArray.
  *
  * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
  * audio buffers.
- * @returns The audio buffer(s) as a Float32Array.
+ * @param encoding The encoding of the audio buffer(s).
+ * @returns The audio buffer(s) as a TypedArray.
  */
-export function base64ToArray(b64: Chunk[]): Float32Array {
+export function base64ToArray(b64: Chunk[], encoding: string): TypedArray {
 	const byteArrays = filterSentinel(b64).map((b) => base64.toByteArray(b));
+
+	const { arrayType: ArrayType, bytesPerElement } =
+		ENCODING_MAP[encoding as Encoding];
+
 	const totalLength = byteArrays.reduce(
-		(acc, arr) => acc + arr.length / Float32Array.BYTES_PER_ELEMENT,
+		(acc, arr) => acc + arr.length / bytesPerElement,
 		0,
 	);
-	const result = new Float32Array(totalLength);
+	const result = new ArrayType(totalLength);
 
 	let offset = 0;
 	for (const arr of byteArrays) {
-		const floats = new Float32Array(arr.buffer);
+		const floats = new ArrayType(arr.buffer);
 		result.set(floats, offset);
 		offset += floats.length;
 	}
@@ -70,26 +83,27 @@ export function createMessageHandlerForContextId(
 		chunk,
 		message,
 	}: {
-		chunk: Chunk;
+		chunk?: Chunk;
 		message: string;
+		data: WebSocketResponse;
 	}) => void,
 ) {
 	return (event: MessageEvent) => {
 		if (typeof event.data !== "string") {
 			return; // Ignore non-string messages.
 		}
-		const message = JSON.parse(event.data);
+		const message: WebSocketResponse = JSON.parse(event.data);
 		if (message.context_id !== contextId) {
 			return; // Ignore messages for other contexts.
 		}
-		let chunk: Chunk;
+		let chunk: Chunk | undefined;
 		if (message.done) {
 			// Convert the done message to a sentinel value.
 			chunk = getSentinel();
-		} else {
+		} else if (message.type === "chunk") {
 			chunk = message.data;
 		}
-		handler({ chunk, message: event.data });
+		handler({ chunk, message: event.data, data: message });
 	};
 }