@cartesia/cartesia-js 1.0.0-alpha.4 → 1.0.1

package/src/tts/source.ts

@@ -1,13 +1,30 @@
 import Emittery from "emittery";
-import type { SourceEventData } from "../types";
+import type { Encoding, SourceEventData, TypedArray } from "../types";
+
+type EncodingInfo = {
+	arrayType:
+		| Float32ArrayConstructor
+		| Int16ArrayConstructor
+		| Uint8ArrayConstructor;
+	bytesPerElement: number;
+};
+
+export const ENCODING_MAP: Record<Encoding, EncodingInfo> = {
+	pcm_f32le: { arrayType: Float32Array, bytesPerElement: 4 },
+	pcm_s16le: { arrayType: Int16Array, bytesPerElement: 2 },
+	pcm_alaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+	pcm_mulaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+};
 
 export default class Source {
 	#emitter = new Emittery<SourceEventData>();
-	#buffer: Float32Array;
+	#buffer: TypedArray;
 	#readIndex = 0;
 	#writeIndex = 0;
 	#closed = false;
 	#sampleRate: number;
+	#encoding: Encoding;
+	#container: string;
 
 	on = this.#emitter.on.bind(this.#emitter);
 	once = this.#emitter.once.bind(this.#emitter);
@@ -20,21 +37,46 @@ export default class Source {
 	 * @param options - Options for the Source.
 	 * @param options.sampleRate - The sample rate of the audio.
 	 */
-	constructor({ sampleRate }: { sampleRate: number }) {
+	constructor({
+		sampleRate,
+		encoding,
+		container,
+	}: { sampleRate: number; encoding: string; container: string }) {
 		this.#sampleRate = sampleRate;
-		this.#buffer = new Float32Array(1024); // Initial size, can be adjusted
+		this.#encoding = encoding as Encoding;
+		this.#container = container;
+		this.#buffer = this.#createBuffer(1024); // Initial size, can be adjusted
 	}
 
 	get sampleRate() {
 		return this.#sampleRate;
 	}
 
+	get encoding() {
+		return this.#encoding;
+	}
+
+	get container() {
+		return this.#container;
+	}
+
+	/**
+	 * Create a new buffer for the source.
+	 *
+	 * @param size - The size of the buffer to create.
+	 * @returns The new buffer as a TypedArray based on the encoding.
+	 */
+	#createBuffer(size: number): TypedArray {
+		const { arrayType: ArrayType } = ENCODING_MAP[this.#encoding];
+		return new ArrayType(size);
+	}
+
 	/**
 	 * Append audio to the buffer.
 	 *
 	 * @param src The audio to append.
 	 */
-	async enqueue(src: Float32Array) {
+	async enqueue(src: TypedArray) {
 		const requiredCapacity = this.#writeIndex + src.length;
 
 		// Resize buffer if necessary
@@ -44,7 +86,7 @@ export default class Source {
 				newCapacity *= 2; // Double the buffer size
 			}
 
-			const newBuffer = new Float32Array(newCapacity);
+			const newBuffer = this.#createBuffer(newCapacity);
 			newBuffer.set(this.#buffer);
 			this.#buffer = newBuffer;
 		}
@@ -62,7 +104,7 @@ export default class Source {
 	 * @returns The number of samples read. If the source is closed, this will be
 	 * less than the length of the provided buffer.
 	 */
-	async read(dst: Float32Array): Promise<number> {
+	async read(dst: TypedArray): Promise<number> {
 		// Read the buffer into the provided buffer.
 		const targetReadIndex = this.#readIndex + dst.length;
 
@@ -100,6 +142,10 @@ export default class Source {
 		return this.#readIndex;
 	}
 
+	get writeIndex() {
+		return this.#writeIndex;
+	}
+
 	/**
 	 * Close the source. This signals that no more audio will be enqueued.
 	 *

package/src/tts/utils.ts

@@ -1,25 +1,38 @@
 import base64 from "base64-js";
 import type Emittery from "emittery";
-import type { Chunk, EmitteryCallbacks, Sentinel } from "../types";
+import type {
+	Chunk,
+	EmitteryCallbacks,
+	Encoding,
+	Sentinel,
+	TypedArray,
+	WebSocketResponse,
+} from "../types";
+import { ENCODING_MAP } from "./source";
 
 /**
- * Convert base64-encoded audio buffer(s) to a Float32Array.
+ * Convert base64-encoded audio buffer(s) to a TypedArray.
  *
  * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
  * audio buffers.
- * @returns The audio buffer(s) as a Float32Array.
+ * @param encoding The encoding of the audio buffer(s).
+ * @returns The audio buffer(s) as a TypedArray.
  */
-export function base64ToArray(b64: Chunk[]): Float32Array {
+export function base64ToArray(b64: Chunk[], encoding: string): TypedArray {
 	const byteArrays = filterSentinel(b64).map((b) => base64.toByteArray(b));
+
+	const { arrayType: ArrayType, bytesPerElement } =
+		ENCODING_MAP[encoding as Encoding];
+
 	const totalLength = byteArrays.reduce(
-		(acc, arr) => acc + arr.length / Float32Array.BYTES_PER_ELEMENT,
+		(acc, arr) => acc + arr.length / bytesPerElement,
 		0,
 	);
-	const result = new Float32Array(totalLength);
+	const result = new ArrayType(totalLength);
 
 	let offset = 0;
 	for (const arr of byteArrays) {
-		const floats = new Float32Array(arr.buffer);
+		const floats = new ArrayType(arr.buffer);
 		result.set(floats, offset);
 		offset += floats.length;
 	}
@@ -70,26 +83,27 @@ export function createMessageHandlerForContextId(
 		chunk,
 		message,
 	}: {
-		chunk: Chunk;
+		chunk?: Chunk;
 		message: string;
+		data: WebSocketResponse;
 	}) => void,
 ) {
 	return (event: MessageEvent) => {
 		if (typeof event.data !== "string") {
 			return; // Ignore non-string messages.
 		}
-		const message = JSON.parse(event.data);
+		const message: WebSocketResponse = JSON.parse(event.data);
 		if (message.context_id !== contextId) {
 			return; // Ignore messages for other contexts.
 		}
-		let chunk: Chunk;
+		let chunk: Chunk | undefined;
 		if (message.done) {
 			// Convert the done message to a sentinel value.
 			chunk = getSentinel();
-		} else {
+		} else if (message.type === "chunk") {
 			chunk = message.data;
 		}
-		handler({ chunk, message: event.data });
+		handler({ chunk, message: event.data, data: message });
 	};
 }
 

package/src/tts/websocket.ts

@@ -6,8 +6,10 @@ import { CARTESIA_VERSION, constructApiUrl } from "../lib/constants";
 import type {
 	ConnectionEventData,
 	EmitteryCallbacks,
+	StreamOptions,
 	StreamRequest,
 	WebSocketOptions,
+	WordTimestamps,
 } from "../types";
 import Source from "./source";
 import {
@@ -21,6 +23,8 @@ export default class WebSocket extends Client {
 	socket?: PartySocketWebSocket;
 	#isConnected = false;
 	#sampleRate: number;
+	#container: string;
+	#encoding: string;
 
 	/**
 	 * Create a new WebSocket client.
@@ -28,50 +32,58 @@ export default class WebSocket extends Client {
 	 * @param args - Arguments to pass to the Client constructor.
 	 */
 	constructor(
-		{ sampleRate }: WebSocketOptions,
+		{ sampleRate, container, encoding }: WebSocketOptions,
 		...args: ConstructorParameters<typeof Client>
 	) {
 		super(...args);
 
 		this.#sampleRate = sampleRate;
+		this.#container = container ?? "raw"; // Default to raw audio for backwards compatibility.
+		this.#encoding = encoding ?? "pcm_f32le"; // Default to 32-bit floating point PCM for backwards compatibility.
 	}
 
 	/**
-	 * Send a message over the WebSocket in order to start a stream.
+	 * Send a message over the WebSocket to start a stream.
 	 *
-	 * @param inputs - Stream options.
+	 * @param inputs - Stream options. Defined in the StreamRequest type.
 	 * @param options - Options for the stream.
 	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
-	 * If `0`, the stream will not time out.
+	 *                          If set to `0`, the stream will not time out.
 	 * @returns A Source object that can be passed to a Player to play the audio.
+	 * @returns An Emittery instance that emits messages from the WebSocket.
+	 * @returns An abort function that can be called to cancel the stream.
 	 */
-	send(
-		inputs: StreamRequest["inputs"],
-		{ timeout = 0 }: StreamRequest["options"] = {},
-	) {
+	send({ ...inputs }: StreamRequest, { timeout = 0 }: StreamOptions = {}) {
 		if (!this.#isConnected) {
 			throw new Error("Not connected to WebSocket. Call .connect() first.");
 		}
 
+		if (!inputs.context_id) {
+			inputs.context_id = this.#generateId();
+		}
+		if (!inputs.output_format) {
+			inputs.output_format = {
+				container: this.#container,
+				encoding: this.#encoding,
+				sample_rate: this.#sampleRate,
+			};
+		}
+
 		// Send audio request.
-		const contextId = this.#generateId();
 		this.socket?.send(
 			JSON.stringify({
-				context_id: contextId,
 				...inputs,
-				output_format: {
-					container: "raw",
-					encoding: "pcm_f32le",
-					sample_rate: this.#sampleRate,
-				},
 			}),
 		);
 
 		const emitter = new Emittery<{
 			message: string;
+			timestamps: WordTimestamps;
 		}>();
 		const source = new Source({
 			sampleRate: this.#sampleRate,
+			encoding: this.#encoding,
+			container: this.#container,
 		});
 		// Used to signal that the stream is complete, either because the
 		// WebSocket has closed, or because the stream has finished.
@@ -82,19 +94,26 @@ export default class WebSocket extends Client {
 			timeoutId = setTimeout(streamCompleteController.abort, timeout);
 		}
 		const handleMessage = createMessageHandlerForContextId(
-			contextId,
-			async ({ chunk, message }) => {
+			inputs.context_id,
+			async ({ chunk, message, data }) => {
 				emitter.emit("message", message);
+				if (data.type === "timestamps") {
+					emitter.emit("timestamps", data.word_timestamps);
+					return;
+				}
 				if (isSentinel(chunk)) {
 					await source.close();
 					streamCompleteController.abort();
 					return;
 				}
-				await source.enqueue(base64ToArray([chunk]));
 				if (timeoutId) {
 					clearTimeout(timeoutId);
 					timeoutId = setTimeout(streamCompleteController.abort, timeout);
 				}
+				if (!chunk) {
+					return;
+				}
+				await source.enqueue(base64ToArray([chunk], this.#encoding));
 			},
 		);
 		this.socket?.addEventListener("message", handleMessage, {
@@ -125,7 +144,11 @@ export default class WebSocket extends Client {
 			}
 		});
 
-		return { source, ...getEmitteryCallbacks(emitter) };
+		return {
+			source,
+			...getEmitteryCallbacks(emitter),
+			stop: streamCompleteController.abort.bind(streamCompleteController),
+		};
 	}
 
 	/**

package/src/types/index.ts

@@ -14,13 +14,85 @@ export type ConnectionEventData = {
 	close: never;
 };
 
+export type VoiceSpecifier =
+	| {
+			mode: "id";
+			id: string;
+	  }
+	| {
+			mode: "embedding";
+			embedding: number[];
+	  };
+
+export type Emotion =
+	| "anger"
+	| "sadness"
+	| "positivity"
+	| "curiosity"
+	| "surprise";
+export type Intensity = "lowest" | "low" | "high" | "highest";
+export type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+
+export type VoiceOptions = VoiceSpecifier & {
+	__experimental_controls?: {
+		speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+		emotion?: EmotionControl[];
+	};
+};
+
 export type StreamRequest = {
-	inputs: object;
-	options: {
-		timeout?: number;
+	model_id: string;
+	transcript: string;
+	voice: VoiceOptions;
+	output_format?: {
+		container: string;
+		encoding: string;
+		sample_rate: number;
 	};
+	context_id?: string;
+	continue?: boolean;
+	duration?: number;
+	language?: string;
+	add_timestamps?: boolean;
+};
+
+export type StreamOptions = {
+	timeout?: number;
+};
+
+export type WebSocketBaseResponse = {
+	context_id: string;
+	status_code: number;
+	done: boolean;
+};
+
+export type WordTimestamps = {
+	words: string[];
+	start: number[];
+	end: number[];
+};
+
+export type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+	type: "timestamps";
+	word_timestamps: WordTimestamps;
+};
+
+export type WebSocketChunkResponse = WebSocketBaseResponse & {
+	type: "chunk";
+	data: string;
+	step_time: number;
+};
+
+export type WebSocketErrorResponse = WebSocketBaseResponse & {
+	type: "error";
+	error: string;
 };
 
+export type WebSocketResponse =
+	| WebSocketTimestampsResponse
+	| WebSocketChunkResponse
+	| WebSocketErrorResponse;
+
 export type EmitteryCallbacks<T> = {
 	on: Emittery<T>["on"];
 	off: Emittery<T>["off"];
@@ -56,6 +128,8 @@ export type CloneResponse = {
 };
 
 export type WebSocketOptions = {
+	container?: string;
+	encoding?: string;
 	sampleRate: number;
 };
 
@@ -65,3 +139,7 @@ export type SourceEventData = {
 	wait: never;
 	read: never;
 };
+
+export type TypedArray = Float32Array | Int16Array | Uint8Array;
+
+export type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";