@cartesia/cartesia-js 1.0.0 → 1.0.1

package/src/tts/websocket.ts

@@ -6,8 +6,10 @@ import { CARTESIA_VERSION, constructApiUrl } from "../lib/constants";
 import type {
 	ConnectionEventData,
 	EmitteryCallbacks,
+	StreamOptions,
 	StreamRequest,
 	WebSocketOptions,
+	WordTimestamps,
 } from "../types";
 import Source from "./source";
 import {
@@ -21,6 +23,8 @@ export default class WebSocket extends Client {
 	socket?: PartySocketWebSocket;
 	#isConnected = false;
 	#sampleRate: number;
+	#container: string;
+	#encoding: string;
 
 	/**
 	 * Create a new WebSocket client.
@@ -28,18 +32,20 @@ export default class WebSocket extends Client {
 	 * @param args - Arguments to pass to the Client constructor.
 	 */
 	constructor(
-		{ sampleRate }: WebSocketOptions,
+		{ sampleRate, container, encoding }: WebSocketOptions,
 		...args: ConstructorParameters<typeof Client>
 	) {
 		super(...args);
 
 		this.#sampleRate = sampleRate;
+		this.#container = container ?? "raw"; // Default to raw audio for backwards compatibility.
+		this.#encoding = encoding ?? "pcm_f32le"; // Default to 32-bit floating point PCM for backwards compatibility.
 	}
 
 	/**
 	 * Send a message over the WebSocket to start a stream.
 	 *
-	 * @param inputs - Stream options.
+	 * @param inputs - Stream options. Defined in the StreamRequest type.
 	 * @param options - Options for the stream.
 	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
 	 *                          If set to `0`, the stream will not time out.
@@ -47,33 +53,37 @@ export default class WebSocket extends Client {
 	 * @returns An Emittery instance that emits messages from the WebSocket.
 	 * @returns An abort function that can be called to cancel the stream.
 	 */
-	send(
-		inputs: StreamRequest["inputs"],
-		{ timeout = 0 }: StreamRequest["options"] = {},
-	) {
+	send({ ...inputs }: StreamRequest, { timeout = 0 }: StreamOptions = {}) {
 		if (!this.#isConnected) {
 			throw new Error("Not connected to WebSocket. Call .connect() first.");
 		}
 
+		if (!inputs.context_id) {
+			inputs.context_id = this.#generateId();
+		}
+		if (!inputs.output_format) {
+			inputs.output_format = {
+				container: this.#container,
+				encoding: this.#encoding,
+				sample_rate: this.#sampleRate,
+			};
+		}
+
 		// Send audio request.
-		const contextId = this.#generateId();
 		this.socket?.send(
 			JSON.stringify({
-				context_id: contextId,
 				...inputs,
-				output_format: {
-					container: "raw",
-					encoding: "pcm_f32le",
-					sample_rate: this.#sampleRate,
-				},
 			}),
 		);
 
 		const emitter = new Emittery<{
 			message: string;
+			timestamps: WordTimestamps;
 		}>();
 		const source = new Source({
 			sampleRate: this.#sampleRate,
+			encoding: this.#encoding,
+			container: this.#container,
 		});
 		// Used to signal that the stream is complete, either because the
 		// WebSocket has closed, or because the stream has finished.
@@ -84,19 +94,26 @@ export default class WebSocket extends Client {
 			timeoutId = setTimeout(streamCompleteController.abort, timeout);
 		}
 		const handleMessage = createMessageHandlerForContextId(
-			contextId,
-			async ({ chunk, message }) => {
+			inputs.context_id,
+			async ({ chunk, message, data }) => {
 				emitter.emit("message", message);
+				if (data.type === "timestamps") {
+					emitter.emit("timestamps", data.word_timestamps);
+					return;
+				}
 				if (isSentinel(chunk)) {
 					await source.close();
 					streamCompleteController.abort();
 					return;
 				}
-				await source.enqueue(base64ToArray([chunk]));
 				if (timeoutId) {
 					clearTimeout(timeoutId);
 					timeoutId = setTimeout(streamCompleteController.abort, timeout);
 				}
+				if (!chunk) {
+					return;
+				}
+				await source.enqueue(base64ToArray([chunk], this.#encoding));
 			},
 		);
 		this.socket?.addEventListener("message", handleMessage, {

package/src/types/index.ts

@@ -14,13 +14,85 @@ export type ConnectionEventData = {
 	close: never;
 };
 
+export type VoiceSpecifier =
+	| {
+			mode: "id";
+			id: string;
+	  }
+	| {
+			mode: "embedding";
+			embedding: number[];
+	  };
+
+export type Emotion =
+	| "anger"
+	| "sadness"
+	| "positivity"
+	| "curiosity"
+	| "surprise";
+export type Intensity = "lowest" | "low" | "high" | "highest";
+export type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+
+export type VoiceOptions = VoiceSpecifier & {
+	__experimental_controls?: {
+		speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+		emotion?: EmotionControl[];
+	};
+};
+
 export type StreamRequest = {
-	inputs: object;
-	options: {
-		timeout?: number;
+	model_id: string;
+	transcript: string;
+	voice: VoiceOptions;
+	output_format?: {
+		container: string;
+		encoding: string;
+		sample_rate: number;
 	};
+	context_id?: string;
+	continue?: boolean;
+	duration?: number;
+	language?: string;
+	add_timestamps?: boolean;
+};
+
+export type StreamOptions = {
+	timeout?: number;
+};
+
+export type WebSocketBaseResponse = {
+	context_id: string;
+	status_code: number;
+	done: boolean;
+};
+
+export type WordTimestamps = {
+	words: string[];
+	start: number[];
+	end: number[];
+};
+
+export type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+	type: "timestamps";
+	word_timestamps: WordTimestamps;
+};
+
+export type WebSocketChunkResponse = WebSocketBaseResponse & {
+	type: "chunk";
+	data: string;
+	step_time: number;
+};
+
+export type WebSocketErrorResponse = WebSocketBaseResponse & {
+	type: "error";
+	error: string;
 };
 
+export type WebSocketResponse =
+	| WebSocketTimestampsResponse
+	| WebSocketChunkResponse
+	| WebSocketErrorResponse;
+
 export type EmitteryCallbacks<T> = {
 	on: Emittery<T>["on"];
 	off: Emittery<T>["off"];
@@ -56,6 +128,8 @@ export type CloneResponse = {
 };
 
 export type WebSocketOptions = {
+	container?: string;
+	encoding?: string;
 	sampleRate: number;
 };
 
@@ -65,3 +139,7 @@ export type SourceEventData = {
 	wait: never;
 	read: never;
 };
+
+export type TypedArray = Float32Array | Int16Array | Uint8Array;
+
+export type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";