@cartesia/cartesia-js 1.0.0-alpha.4 → 1.0.1

package/dist/tts/websocket.d.cts

@@ -1,7 +1,7 @@
 import * as emittery from 'emittery';
 import { WebSocket as WebSocket$1 } from 'partysocket';
 import { Client } from '../lib/client.cjs';
-import { WebSocketOptions, StreamRequest, EmitteryCallbacks, ConnectionEventData } from '../types/index.cjs';
+import { WebSocketOptions, StreamRequest, StreamOptions, WordTimestamps, EmitteryCallbacks, ConnectionEventData } from '../types/index.cjs';
 import Source from './source.cjs';
 
 declare class WebSocket extends Client {
@@ -12,28 +12,38 @@ declare class WebSocket extends Client {
      *
      * @param args - Arguments to pass to the Client constructor.
      */
-    constructor({ sampleRate }: WebSocketOptions, ...args: ConstructorParameters<typeof Client>);
+    constructor({ sampleRate, container, encoding }: WebSocketOptions, ...args: ConstructorParameters<typeof Client>);
     /**
-     * Send a message over the WebSocket in order to start a stream.
+     * Send a message over the WebSocket to start a stream.
      *
-     * @param inputs - Stream options.
+     * @param inputs - Stream options. Defined in the StreamRequest type.
      * @param options - Options for the stream.
      * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
-     * If `0`, the stream will not time out.
+     *                          If set to `0`, the stream will not time out.
      * @returns A Source object that can be passed to a Player to play the audio.
+     * @returns An Emittery instance that emits messages from the WebSocket.
+     * @returns An abort function that can be called to cancel the stream.
      */
-    send(inputs: StreamRequest["inputs"], { timeout }?: StreamRequest["options"]): {
-        on: <Name extends keyof emittery.OmnipresentEventData | "message">(eventName: Name | readonly Name[], listener: (eventData: ({
+    send({ ...inputs }: StreamRequest, { timeout }?: StreamOptions): {
+        stop: {
+            (reason?: any): void;
+            (reason?: any): void;
+        };
+        on: <Name extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name | readonly Name[], listener: (eventData: ({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name]) => void | Promise<void>) => emittery.UnsubscribeFunction;
-        off: <Name_1 extends keyof emittery.OmnipresentEventData | "message">(eventName: Name_1 | readonly Name_1[], listener: (eventData: ({
+        off: <Name_1 extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name_1 | readonly Name_1[], listener: (eventData: ({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name_1]) => void | Promise<void>) => void;
-        once: <Name_2 extends keyof emittery.OmnipresentEventData | "message">(eventName: Name_2 | readonly Name_2[]) => emittery.EmitteryOncePromise<({
+        once: <Name_2 extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name_2 | readonly Name_2[]) => emittery.EmitteryOncePromise<({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name_2]>;
-        events: <Name_3 extends "message">(eventName: Name_3 | readonly Name_3[]) => AsyncIterableIterator<{
+        events: <Name_3 extends "timestamps" | "message">(eventName: Name_3 | readonly Name_3[]) => AsyncIterableIterator<{
             message: string;
+            timestamps: WordTimestamps;
         }[Name_3]>;
         source: Source;
     };

package/dist/tts/websocket.d.ts

@@ -1,7 +1,7 @@
 import * as emittery from 'emittery';
 import { WebSocket as WebSocket$1 } from 'partysocket';
 import { Client } from '../lib/client.js';
-import { WebSocketOptions, StreamRequest, EmitteryCallbacks, ConnectionEventData } from '../types/index.js';
+import { WebSocketOptions, StreamRequest, StreamOptions, WordTimestamps, EmitteryCallbacks, ConnectionEventData } from '../types/index.js';
 import Source from './source.js';
 
 declare class WebSocket extends Client {
@@ -12,28 +12,38 @@ declare class WebSocket extends Client {
      *
      * @param args - Arguments to pass to the Client constructor.
      */
-    constructor({ sampleRate }: WebSocketOptions, ...args: ConstructorParameters<typeof Client>);
+    constructor({ sampleRate, container, encoding }: WebSocketOptions, ...args: ConstructorParameters<typeof Client>);
     /**
-     * Send a message over the WebSocket in order to start a stream.
+     * Send a message over the WebSocket to start a stream.
      *
-     * @param inputs - Stream options.
+     * @param inputs - Stream options. Defined in the StreamRequest type.
      * @param options - Options for the stream.
      * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
-     * If `0`, the stream will not time out.
+     *                          If set to `0`, the stream will not time out.
      * @returns A Source object that can be passed to a Player to play the audio.
+     * @returns An Emittery instance that emits messages from the WebSocket.
+     * @returns An abort function that can be called to cancel the stream.
      */
-    send(inputs: StreamRequest["inputs"], { timeout }?: StreamRequest["options"]): {
-        on: <Name extends keyof emittery.OmnipresentEventData | "message">(eventName: Name | readonly Name[], listener: (eventData: ({
+    send({ ...inputs }: StreamRequest, { timeout }?: StreamOptions): {
+        stop: {
+            (reason?: any): void;
+            (reason?: any): void;
+        };
+        on: <Name extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name | readonly Name[], listener: (eventData: ({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name]) => void | Promise<void>) => emittery.UnsubscribeFunction;
-        off: <Name_1 extends keyof emittery.OmnipresentEventData | "message">(eventName: Name_1 | readonly Name_1[], listener: (eventData: ({
+        off: <Name_1 extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name_1 | readonly Name_1[], listener: (eventData: ({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name_1]) => void | Promise<void>) => void;
-        once: <Name_2 extends keyof emittery.OmnipresentEventData | "message">(eventName: Name_2 | readonly Name_2[]) => emittery.EmitteryOncePromise<({
+        once: <Name_2 extends "timestamps" | keyof emittery.OmnipresentEventData | "message">(eventName: Name_2 | readonly Name_2[]) => emittery.EmitteryOncePromise<({
             message: string;
+            timestamps: WordTimestamps;
         } & emittery.OmnipresentEventData)[Name_2]>;
-        events: <Name_3 extends "message">(eventName: Name_3 | readonly Name_3[]) => AsyncIterableIterator<{
+        events: <Name_3 extends "timestamps" | "message">(eventName: Name_3 | readonly Name_3[]) => AsyncIterableIterator<{
             message: string;
+            timestamps: WordTimestamps;
         }[Name_3]>;
         source: Source;
     };

package/dist/tts/websocket.js

@@ -1,11 +1,11 @@
 import {
   WebSocket
-} from "../chunk-IQAXBRHU.js";
-import "../chunk-PQ6CIPFW.js";
-import "../chunk-PQ5EVEEH.js";
+} from "../chunk-ASZKHN7Q.js";
+import "../chunk-5M33ZF3Y.js";
 import "../chunk-2BFEKY3F.js";
-import "../chunk-RO7TY474.js";
-import "../chunk-WIFMLPT5.js";
+import "../chunk-BHY7MNGT.js";
+import "../chunk-6YQ6KDIQ.js";
+import "../chunk-GHY2WEOK.js";
 export {
   WebSocket as default
 };

package/dist/types/index.d.cts

@@ -10,12 +10,64 @@ type ConnectionEventData = {
     open: never;
     close: never;
 };
+type VoiceSpecifier = {
+    mode: "id";
+    id: string;
+} | {
+    mode: "embedding";
+    embedding: number[];
+};
+type Emotion = "anger" | "sadness" | "positivity" | "curiosity" | "surprise";
+type Intensity = "lowest" | "low" | "high" | "highest";
+type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+type VoiceOptions = VoiceSpecifier & {
+    __experimental_controls?: {
+        speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+        emotion?: EmotionControl[];
+    };
+};
 type StreamRequest = {
-    inputs: object;
-    options: {
-        timeout?: number;
+    model_id: string;
+    transcript: string;
+    voice: VoiceOptions;
+    output_format?: {
+        container: string;
+        encoding: string;
+        sample_rate: number;
     };
+    context_id?: string;
+    continue?: boolean;
+    duration?: number;
+    language?: string;
+    add_timestamps?: boolean;
+};
+type StreamOptions = {
+    timeout?: number;
+};
+type WebSocketBaseResponse = {
+    context_id: string;
+    status_code: number;
+    done: boolean;
+};
+type WordTimestamps = {
+    words: string[];
+    start: number[];
+    end: number[];
+};
+type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+    type: "timestamps";
+    word_timestamps: WordTimestamps;
+};
+type WebSocketChunkResponse = WebSocketBaseResponse & {
+    type: "chunk";
+    data: string;
+    step_time: number;
+};
+type WebSocketErrorResponse = WebSocketBaseResponse & {
+    type: "error";
+    error: string;
 };
+type WebSocketResponse = WebSocketTimestampsResponse | WebSocketChunkResponse | WebSocketErrorResponse;
 type EmitteryCallbacks<T> = {
     on: emittery__default<T>["on"];
     off: emittery__default<T>["off"];
@@ -43,6 +95,8 @@ type CloneResponse = {
     embedding: number[];
 };
 type WebSocketOptions = {
+    container?: string;
+    encoding?: string;
     sampleRate: number;
 };
 type SourceEventData = {
@@ -51,5 +105,7 @@ type SourceEventData = {
     wait: never;
     read: never;
 };
+type TypedArray = Float32Array | Int16Array | Uint8Array;
+type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";
 
-export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Sentinel, SourceEventData, StreamRequest, Voice, WebSocketOptions };
+export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Emotion, EmotionControl, Encoding, Intensity, Sentinel, SourceEventData, StreamOptions, StreamRequest, TypedArray, Voice, VoiceOptions, VoiceSpecifier, WebSocketBaseResponse, WebSocketChunkResponse, WebSocketErrorResponse, WebSocketOptions, WebSocketResponse, WebSocketTimestampsResponse, WordTimestamps };

package/dist/types/index.d.ts

@@ -10,12 +10,64 @@ type ConnectionEventData = {
     open: never;
     close: never;
 };
+type VoiceSpecifier = {
+    mode: "id";
+    id: string;
+} | {
+    mode: "embedding";
+    embedding: number[];
+};
+type Emotion = "anger" | "sadness" | "positivity" | "curiosity" | "surprise";
+type Intensity = "lowest" | "low" | "high" | "highest";
+type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+type VoiceOptions = VoiceSpecifier & {
+    __experimental_controls?: {
+        speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+        emotion?: EmotionControl[];
+    };
+};
 type StreamRequest = {
-    inputs: object;
-    options: {
-        timeout?: number;
+    model_id: string;
+    transcript: string;
+    voice: VoiceOptions;
+    output_format?: {
+        container: string;
+        encoding: string;
+        sample_rate: number;
     };
+    context_id?: string;
+    continue?: boolean;
+    duration?: number;
+    language?: string;
+    add_timestamps?: boolean;
+};
+type StreamOptions = {
+    timeout?: number;
+};
+type WebSocketBaseResponse = {
+    context_id: string;
+    status_code: number;
+    done: boolean;
+};
+type WordTimestamps = {
+    words: string[];
+    start: number[];
+    end: number[];
+};
+type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+    type: "timestamps";
+    word_timestamps: WordTimestamps;
+};
+type WebSocketChunkResponse = WebSocketBaseResponse & {
+    type: "chunk";
+    data: string;
+    step_time: number;
+};
+type WebSocketErrorResponse = WebSocketBaseResponse & {
+    type: "error";
+    error: string;
 };
+type WebSocketResponse = WebSocketTimestampsResponse | WebSocketChunkResponse | WebSocketErrorResponse;
 type EmitteryCallbacks<T> = {
     on: emittery__default<T>["on"];
     off: emittery__default<T>["off"];
@@ -43,6 +95,8 @@ type CloneResponse = {
     embedding: number[];
 };
 type WebSocketOptions = {
+    container?: string;
+    encoding?: string;
     sampleRate: number;
 };
 type SourceEventData = {
@@ -51,5 +105,7 @@ type SourceEventData = {
     wait: never;
     read: never;
 };
+type TypedArray = Float32Array | Int16Array | Uint8Array;
+type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";
 
-export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Sentinel, SourceEventData, StreamRequest, Voice, WebSocketOptions };
+export type { Chunk, ClientOptions, CloneOptions, CloneResponse, ConnectionEventData, CreateVoice, EmitteryCallbacks, Emotion, EmotionControl, Encoding, Intensity, Sentinel, SourceEventData, StreamOptions, StreamRequest, TypedArray, Voice, VoiceOptions, VoiceSpecifier, WebSocketBaseResponse, WebSocketChunkResponse, WebSocketErrorResponse, WebSocketOptions, WebSocketResponse, WebSocketTimestampsResponse, WordTimestamps };

package/dist/voices/index.js

@@ -1,9 +1,9 @@
 import {
   Voices
-} from "../chunk-SGXUEFII.js";
-import "../chunk-PQ5EVEEH.js";
+} from "../chunk-KUSVZXDT.js";
+import "../chunk-5M33ZF3Y.js";
 import "../chunk-2BFEKY3F.js";
-import "../chunk-WIFMLPT5.js";
+import "../chunk-GHY2WEOK.js";
 export {
   Voices as default
 };

package/package.json

@@ -4,7 +4,7 @@
 		"name": "Cartesia",
 		"url": "https://cartesia.ai"
 	},
-	"version": "1.0.0-alpha.4",
+	"version": "1.0.1",
 	"description": "Client for the Cartesia API.",
 	"type": "module",
 	"module": "./dist/index.js",

package/src/index.ts

@@ -2,3 +2,5 @@ export { Cartesia as default } from "./lib";
 export * from "./lib";
 export * from "./types";
 export { default as WebPlayer } from "./tts/player";
+export { default as Source } from "./tts/source";
+export { default as WebSocket } from "./tts/websocket";

package/src/react/index.ts

@@ -4,12 +4,14 @@ import { Cartesia } from "../lib";
 import Player from "../tts/player";
 import type Source from "../tts/source";
 import type WebSocket from "../tts/websocket";
+import type { StreamRequest } from "../types";
 import { pingServer } from "./utils";
 
 export type UseTTSOptions = {
 	apiKey: string | null;
 	baseUrl?: string;
 	sampleRate: number;
+	onError?: (error: Error) => void;
 };
 
 export type PlaybackStatus = "inactive" | "playing" | "paused" | "finished";
@@ -20,7 +22,7 @@ export type Metrics = {
 };
 
 export interface UseTTSReturn {
-	buffer: (options: object) => Promise<void>;
+	buffer: (options: StreamRequest) => Promise<void>;
 	play: (bufferDuration?: number) => Promise<void>;
 	pause: () => Promise<void>;
 	resume: () => Promise<void>;
@@ -47,6 +49,7 @@ export function useTTS({
 	apiKey,
 	baseUrl,
 	sampleRate,
+	onError,
 }: UseTTSOptions): UseTTSReturn {
 	if (typeof window === "undefined") {
 		return {
@@ -72,7 +75,11 @@ export function useTTS({
 		}
 		const cartesia = new Cartesia({ apiKey, baseUrl });
 		baseUrl = baseUrl ?? cartesia.baseUrl;
-		return cartesia.tts.websocket({ sampleRate });
+		return cartesia.tts.websocket({
+			container: "raw",
+			encoding: "pcm_f32le",
+			sampleRate,
+		});
 	}, [apiKey, baseUrl, sampleRate]);
 	const websocketReturn = useRef<ReturnType<WebSocket["send"]> | null>(null);
 	const player = useRef<Player | null>(null);
@@ -85,21 +92,35 @@ export function useTTS({
 	const [messages, setMessages] = useState<Message[]>([]);
 
 	const buffer = useCallback(
-		async (options: object) => {
-			setMessages([]);
-			setBufferStatus("buffering");
-			websocketReturn.current = websocket?.send(options) ?? null;
-			if (!websocketReturn.current) {
-				return;
+		async (options: StreamRequest) => {
+			websocketReturn.current?.stop(); // Abort the previous request if it exists.
+
+			try {
+				setMessages([]);
+				setBufferStatus("buffering");
+				websocketReturn.current = websocket?.send(options) ?? null;
+				if (!websocketReturn.current) {
+					return;
+				}
+				const unsubscribe = websocketReturn.current.on("message", (message) => {
+					const parsedMessage = JSON.parse(message);
+					setMessages((messages) => [...messages, parsedMessage]);
+					if (parsedMessage.error) {
+						onError?.(new Error(parsedMessage.error));
+					}
+				});
+				await websocketReturn.current.source.once("close");
+				setBufferStatus("buffered");
+				unsubscribe();
+			} catch (error) {
+				if (error instanceof Error) {
+					onError?.(error);
+				} else {
+					console.error(error);
+				}
 			}
-			const unsubscribe = websocketReturn.current.on("message", (message) => {
-				setMessages((messages) => [...messages, JSON.parse(message)]);
-			});
-			await websocketReturn.current.source.once("close");
-			setBufferStatus("buffered");
-			unsubscribe();
 		},
-		[websocket],
+		[websocket, onError],
 	);
 
 	const metrics = useMemo(() => {
@@ -171,63 +192,97 @@ export function useTTS({
 	}, [websocket, baseUrl]);
 
 	const play = useCallback(async () => {
-		if (playbackStatus === "playing" || !websocketReturn.current) {
-			return;
-		}
-		setPlaybackStatus("playing");
-
-		const unsubscribes = [];
-		unsubscribes.push(
-			websocketReturn.current.source.on("wait", () => {
-				setIsWaiting(true);
-			}),
-		);
-		unsubscribes.push(
-			websocketReturn.current.source.on("read", () => {
-				setIsWaiting(false);
-			}),
-		);
-
-		player.current = new Player({
-			bufferDuration: bufferDuration ?? DEFAULT_BUFFER_DURATION,
-		});
-		// Wait for the playback to finish before setting isPlaying to false.
-		await player.current.play(websocketReturn.current.source);
+		try {
+			if (playbackStatus === "playing" || !websocketReturn.current) {
+				return;
+			}
+			if (player.current) {
+				// Stop the current player if it exists.
+				await player.current.stop();
+			}
 
-		for (const unsubscribe of unsubscribes) {
-			// Deregister the event listeners (.on()) that we registered above to avoid memory leaks.
-			unsubscribe();
-		}
+			setPlaybackStatus("playing");
+
+			const unsubscribes = [];
+			unsubscribes.push(
+				websocketReturn.current.source.on("wait", () => {
+					setIsWaiting(true);
+				}),
+			);
+			unsubscribes.push(
+				websocketReturn.current.source.on("read", () => {
+					setIsWaiting(false);
+				}),
+			);
 
-		setPlaybackStatus("finished");
-	}, [playbackStatus, bufferDuration]);
+			player.current = new Player({
+				bufferDuration: bufferDuration ?? DEFAULT_BUFFER_DURATION,
+			});
+			// Wait for the playback to finish before setting isPlaying to false.
+			await player.current.play(websocketReturn.current.source);
+
+			for (const unsubscribe of unsubscribes) {
+				// Deregister the event listeners (.on()) that we registered above to avoid memory leaks.
+				unsubscribe();
+			}
+
+			setPlaybackStatus("finished");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [playbackStatus, bufferDuration, onError]);
 
 	const pause = useCallback(async () => {
-		await player.current?.pause();
-		setPlaybackStatus("paused");
-	}, []);
+		try {
+			await player.current?.pause();
+			setPlaybackStatus("paused");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [onError]);
 
 	const resume = useCallback(async () => {
-		await player.current?.resume();
-		setPlaybackStatus("playing");
-	}, []);
+		try {
+			await player.current?.resume();
+			setPlaybackStatus("playing");
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
+			}
+		}
+	}, [onError]);
 
 	const toggle = useCallback(async () => {
-		await player.current?.toggle();
-		setPlaybackStatus((status) => {
-			if (status === "playing") {
-				return "paused";
-			}
-			if (status === "paused") {
-				return "playing";
+		try {
+			await player.current?.toggle();
+			setPlaybackStatus((status) => {
+				if (status === "playing") {
+					return "paused";
+				}
+				if (status === "paused") {
+					return "playing";
+				}
+				return status;
+			});
+		} catch (error) {
+			if (error instanceof Error) {
+				onError?.(error);
+			} else {
+				console.error(error);
 			}
-			return status;
-		});
-	}, []);
+		}
+	}, [onError]);
 
-	// TODO:
-	// - [] Access the play and buffer cursors.
-	// - [] Seek to a specific time.
 	return {
 		buffer,
 		play,

package/src/tts/player.ts

@@ -1,16 +1,10 @@
-import Emittery from "emittery";
 import type Source from "./source";
 import { playAudioBuffer } from "./utils";
 
-type PlayEventData = {
-	finish: never;
-};
-
 export default class Player {
 	#context: AudioContext | null = null;
 	#startNextPlaybackAt = 0;
 	#bufferDuration: number;
-	#emitter = new Emittery<PlayEventData>();
 
 	/**
 	 * Create a new Player.
@@ -26,6 +20,9 @@ export default class Player {
 		if (!this.#context) {
 			throw new Error("AudioContext not initialized.");
 		}
+		if (buf.length === 0) {
+			return;
+		}
 
 		const startAt = this.#startNextPlaybackAt;
 		const duration = buf.length / sampleRate;
@@ -60,11 +57,9 @@ export default class Player {
 
 			if (read < buffer.length) {
 				// No more audio to read.
-				await this.#emitter.emit("finish");
 				break;
 			}
 		}
-
 		await Promise.all(plays);
 	}
 
@@ -107,4 +102,16 @@ export default class Player {
 			await this.resume();
 		}
 	}
+
+	/**
+	 * Stop the audio.
+	 *
+	 * @returns A promise that resolves when the audio has been stopped.
+	 */
+	async stop() {
+		if (!this.#context) {
+			throw new Error("AudioContext not initialized.");
+		}
+		await this.#context?.close();
+	}
 }