@cartesia/cartesia-js 1.0.0 → 1.0.2

package/src/tts/source.ts

@@ -1,13 +1,30 @@
 import Emittery from "emittery";
-import type { SourceEventData } from "../types";
+import type { Encoding, SourceEventData, TypedArray } from "../types";
+
+type EncodingInfo = {
+	arrayType:
+		| Float32ArrayConstructor
+		| Int16ArrayConstructor
+		| Uint8ArrayConstructor;
+	bytesPerElement: number;
+};
+
+export const ENCODING_MAP: Record<Encoding, EncodingInfo> = {
+	pcm_f32le: { arrayType: Float32Array, bytesPerElement: 4 },
+	pcm_s16le: { arrayType: Int16Array, bytesPerElement: 2 },
+	pcm_alaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+	pcm_mulaw: { arrayType: Uint8Array, bytesPerElement: 1 },
+};
 
 export default class Source {
 	#emitter = new Emittery<SourceEventData>();
-	#buffer: Float32Array;
+	#buffer: TypedArray;
 	#readIndex = 0;
 	#writeIndex = 0;
 	#closed = false;
 	#sampleRate: number;
+	#encoding: Encoding;
+	#container: string;
 
 	on = this.#emitter.on.bind(this.#emitter);
 	once = this.#emitter.once.bind(this.#emitter);
@@ -20,21 +37,46 @@ export default class Source {
 	 * @param options - Options for the Source.
 	 * @param options.sampleRate - The sample rate of the audio.
 	 */
-	constructor({ sampleRate }: { sampleRate: number }) {
+	constructor({
+		sampleRate,
+		encoding,
+		container,
+	}: { sampleRate: number; encoding: string; container: string }) {
 		this.#sampleRate = sampleRate;
-		this.#buffer = new Float32Array(1024); // Initial size, can be adjusted
+		this.#encoding = encoding as Encoding;
+		this.#container = container;
+		this.#buffer = this.#createBuffer(1024); // Initial size, can be adjusted
 	}
 
 	get sampleRate() {
 		return this.#sampleRate;
 	}
 
+	get encoding() {
+		return this.#encoding;
+	}
+
+	get container() {
+		return this.#container;
+	}
+
+	/**
+	 * Create a new buffer for the source.
+	 *
+	 * @param size - The size of the buffer to create.
+	 * @returns The new buffer as a TypedArray based on the encoding.
+	 */
+	#createBuffer(size: number): TypedArray {
+		const { arrayType: ArrayType } = ENCODING_MAP[this.#encoding];
+		return new ArrayType(size);
+	}
+
 	/**
 	 * Append audio to the buffer.
 	 *
 	 * @param src The audio to append.
 	 */
-	async enqueue(src: Float32Array) {
+	async enqueue(src: TypedArray) {
 		const requiredCapacity = this.#writeIndex + src.length;
 
 		// Resize buffer if necessary
@@ -44,7 +86,7 @@ export default class Source {
 				newCapacity *= 2; // Double the buffer size
 			}
 
-			const newBuffer = new Float32Array(newCapacity);
+			const newBuffer = this.#createBuffer(newCapacity);
 			newBuffer.set(this.#buffer);
 			this.#buffer = newBuffer;
 		}
@@ -62,7 +104,7 @@ export default class Source {
 	 * @returns The number of samples read. If the source is closed, this will be
 	 * less than the length of the provided buffer.
 	 */
-	async read(dst: Float32Array): Promise<number> {
+	async read(dst: TypedArray): Promise<number> {
 		// Read the buffer into the provided buffer.
 		const targetReadIndex = this.#readIndex + dst.length;
 
@@ -100,6 +142,10 @@ export default class Source {
 		return this.#readIndex;
 	}
 
+	get writeIndex() {
+		return this.#writeIndex;
+	}
+
 	/**
 	 * Close the source. This signals that no more audio will be enqueued.
 	 *

package/src/tts/utils.ts

@@ -1,25 +1,38 @@
 import base64 from "base64-js";
 import type Emittery from "emittery";
-import type { Chunk, EmitteryCallbacks, Sentinel } from "../types";
+import type {
+	Chunk,
+	EmitteryCallbacks,
+	Encoding,
+	Sentinel,
+	TypedArray,
+	WebSocketResponse,
+} from "../types";
+import { ENCODING_MAP } from "./source";
 
 /**
- * Convert base64-encoded audio buffer(s) to a Float32Array.
+ * Convert base64-encoded audio buffer(s) to a TypedArray.
  *
  * @param b64 The base64-encoded audio buffer, or an array of base64-encoded
  * audio buffers.
- * @returns The audio buffer(s) as a Float32Array.
+ * @param encoding The encoding of the audio buffer(s).
+ * @returns The audio buffer(s) as a TypedArray.
  */
-export function base64ToArray(b64: Chunk[]): Float32Array {
+export function base64ToArray(b64: Chunk[], encoding: string): TypedArray {
 	const byteArrays = filterSentinel(b64).map((b) => base64.toByteArray(b));
+
+	const { arrayType: ArrayType, bytesPerElement } =
+		ENCODING_MAP[encoding as Encoding];
+
 	const totalLength = byteArrays.reduce(
-		(acc, arr) => acc + arr.length / Float32Array.BYTES_PER_ELEMENT,
+		(acc, arr) => acc + arr.length / bytesPerElement,
 		0,
 	);
-	const result = new Float32Array(totalLength);
+	const result = new ArrayType(totalLength);
 
 	let offset = 0;
 	for (const arr of byteArrays) {
-		const floats = new Float32Array(arr.buffer);
+		const floats = new ArrayType(arr.buffer);
 		result.set(floats, offset);
 		offset += floats.length;
 	}
@@ -70,26 +83,27 @@ export function createMessageHandlerForContextId(
 		chunk,
 		message,
 	}: {
-		chunk: Chunk;
+		chunk?: Chunk;
 		message: string;
+		data: WebSocketResponse;
 	}) => void,
 ) {
 	return (event: MessageEvent) => {
 		if (typeof event.data !== "string") {
 			return; // Ignore non-string messages.
 		}
-		const message = JSON.parse(event.data);
+		const message: WebSocketResponse = JSON.parse(event.data);
 		if (message.context_id !== contextId) {
 			return; // Ignore messages for other contexts.
 		}
-		let chunk: Chunk;
+		let chunk: Chunk | undefined;
 		if (message.done) {
 			// Convert the done message to a sentinel value.
 			chunk = getSentinel();
-		} else {
+		} else if (message.type === "chunk") {
 			chunk = message.data;
 		}
-		handler({ chunk, message: event.data });
+		handler({ chunk, message: event.data, data: message });
 	};
 }
 

package/src/tts/websocket.ts

@@ -6,8 +6,10 @@ import { CARTESIA_VERSION, constructApiUrl } from "../lib/constants";
 import type {
 	ConnectionEventData,
 	EmitteryCallbacks,
+	StreamOptions,
 	StreamRequest,
 	WebSocketOptions,
+	WordTimestamps,
 } from "../types";
 import Source from "./source";
 import {
@@ -21,6 +23,8 @@ export default class WebSocket extends Client {
 	socket?: PartySocketWebSocket;
 	#isConnected = false;
 	#sampleRate: number;
+	#container: string;
+	#encoding: string;
 
 	/**
 	 * Create a new WebSocket client.
@@ -28,18 +32,20 @@ export default class WebSocket extends Client {
 	 * @param args - Arguments to pass to the Client constructor.
 	 */
 	constructor(
-		{ sampleRate }: WebSocketOptions,
+		{ sampleRate, container, encoding }: WebSocketOptions,
 		...args: ConstructorParameters<typeof Client>
 	) {
 		super(...args);
 
 		this.#sampleRate = sampleRate;
+		this.#container = container ?? "raw"; // Default to raw audio for backwards compatibility.
+		this.#encoding = encoding ?? "pcm_f32le"; // Default to 32-bit floating point PCM for backwards compatibility.
 	}
 
 	/**
 	 * Send a message over the WebSocket to start a stream.
 	 *
-	 * @param inputs - Stream options.
+	 * @param inputs - Stream options. Defined in the StreamRequest type.
 	 * @param options - Options for the stream.
 	 * @param options.timeout - The maximum time to wait for a chunk before cancelling the stream.
 	 *                          If set to `0`, the stream will not time out.
@@ -47,33 +53,37 @@ export default class WebSocket extends Client {
 	 * @returns An Emittery instance that emits messages from the WebSocket.
 	 * @returns An abort function that can be called to cancel the stream.
 	 */
-	send(
-		inputs: StreamRequest["inputs"],
-		{ timeout = 0 }: StreamRequest["options"] = {},
-	) {
+	send({ ...inputs }: StreamRequest, { timeout = 0 }: StreamOptions = {}) {
 		if (!this.#isConnected) {
 			throw new Error("Not connected to WebSocket. Call .connect() first.");
 		}
 
+		if (!inputs.context_id) {
+			inputs.context_id = this.#generateId();
+		}
+		if (!inputs.output_format) {
+			inputs.output_format = {
+				container: this.#container,
+				encoding: this.#encoding,
+				sample_rate: this.#sampleRate,
+			};
+		}
+
 		// Send audio request.
-		const contextId = this.#generateId();
 		this.socket?.send(
 			JSON.stringify({
-				context_id: contextId,
 				...inputs,
-				output_format: {
-					container: "raw",
-					encoding: "pcm_f32le",
-					sample_rate: this.#sampleRate,
-				},
 			}),
 		);
 
 		const emitter = new Emittery<{
 			message: string;
+			timestamps: WordTimestamps;
 		}>();
 		const source = new Source({
 			sampleRate: this.#sampleRate,
+			encoding: this.#encoding,
+			container: this.#container,
 		});
 		// Used to signal that the stream is complete, either because the
 		// WebSocket has closed, or because the stream has finished.
@@ -84,19 +94,26 @@ export default class WebSocket extends Client {
 			timeoutId = setTimeout(streamCompleteController.abort, timeout);
 		}
 		const handleMessage = createMessageHandlerForContextId(
-			contextId,
-			async ({ chunk, message }) => {
+			inputs.context_id,
+			async ({ chunk, message, data }) => {
 				emitter.emit("message", message);
+				if (data.type === "timestamps") {
+					emitter.emit("timestamps", data.word_timestamps);
+					return;
+				}
 				if (isSentinel(chunk)) {
 					await source.close();
 					streamCompleteController.abort();
 					return;
 				}
-				await source.enqueue(base64ToArray([chunk]));
 				if (timeoutId) {
 					clearTimeout(timeoutId);
 					timeoutId = setTimeout(streamCompleteController.abort, timeout);
 				}
+				if (!chunk) {
+					return;
+				}
+				await source.enqueue(base64ToArray([chunk], this.#encoding));
 			},
 		);
 		this.socket?.addEventListener("message", handleMessage, {
@@ -155,14 +172,16 @@ export default class WebSocket extends Client {
 	 * @returns A promise that resolves when the WebSocket is connected.
 	 * @throws {Error} If the WebSocket fails to connect.
 	 */
-	connect() {
-		const url = constructApiUrl(this.baseUrl, "/tts/websocket", {
-			websocket: true,
-		});
-		url.searchParams.set("api_key", this.apiKey);
-		url.searchParams.set("cartesia_version", CARTESIA_VERSION);
+	async connect() {
 		const emitter = new Emittery<ConnectionEventData>();
-		this.socket = new PartySocketWebSocket(url.toString());
+		this.socket = new PartySocketWebSocket(async () => {
+			const url = constructApiUrl(this.baseUrl, "/tts/websocket", {
+				websocket: true,
+			});
+			url.searchParams.set("api_key", await this.apiKey());
+			url.searchParams.set("cartesia_version", CARTESIA_VERSION);
+			return url.toString();
+		});
 		this.socket.onopen = () => {
 			this.#isConnected = true;
 			emitter.emit("open");

package/src/types/index.ts

@@ -1,7 +1,7 @@
 import type Emittery from "emittery";
 
 export interface ClientOptions {
-	apiKey?: string;
+	apiKey?: string | (() => Promise<string>);
 	baseUrl?: string;
 }
 
@@ -14,13 +14,85 @@ export type ConnectionEventData = {
 	close: never;
 };
 
+export type VoiceSpecifier =
+	| {
+			mode: "id";
+			id: string;
+	  }
+	| {
+			mode: "embedding";
+			embedding: number[];
+	  };
+
+export type Emotion =
+	| "anger"
+	| "sadness"
+	| "positivity"
+	| "curiosity"
+	| "surprise";
+export type Intensity = "lowest" | "low" | "high" | "highest";
+export type EmotionControl = Emotion | `${Emotion}:${Intensity}`;
+
+export type VoiceOptions = VoiceSpecifier & {
+	__experimental_controls?: {
+		speed?: "slowest" | "slow" | "normal" | "fast" | "fastest";
+		emotion?: EmotionControl[];
+	};
+};
+
 export type StreamRequest = {
-	inputs: object;
-	options: {
-		timeout?: number;
+	model_id: string;
+	transcript: string;
+	voice: VoiceOptions;
+	output_format?: {
+		container: string;
+		encoding: string;
+		sample_rate: number;
 	};
+	context_id?: string;
+	continue?: boolean;
+	duration?: number;
+	language?: string;
+	add_timestamps?: boolean;
+};
+
+export type StreamOptions = {
+	timeout?: number;
+};
+
+export type WebSocketBaseResponse = {
+	context_id: string;
+	status_code: number;
+	done: boolean;
 };
 
+export type WordTimestamps = {
+	words: string[];
+	start: number[];
+	end: number[];
+};
+
+export type WebSocketTimestampsResponse = WebSocketBaseResponse & {
+	type: "timestamps";
+	word_timestamps: WordTimestamps;
+};
+
+export type WebSocketChunkResponse = WebSocketBaseResponse & {
+	type: "chunk";
+	data: string;
+	step_time: number;
+};
+
+export type WebSocketErrorResponse = WebSocketBaseResponse & {
+	type: "error";
+	error: string;
+};
+
+export type WebSocketResponse =
+	| WebSocketTimestampsResponse
+	| WebSocketChunkResponse
+	| WebSocketErrorResponse;
+
 export type EmitteryCallbacks<T> = {
 	on: Emittery<T>["on"];
 	off: Emittery<T>["off"];
@@ -32,10 +104,12 @@ export type CloneOptions =
 	| {
 			mode: "url";
 			link: string;
+			enhance?: boolean;
 	  }
 	| {
 			mode: "clip";
 			clip: Blob;
+			enhance?: boolean;
 	  };
 
 export type Voice = {
@@ -46,16 +120,23 @@ export type Voice = {
 	is_public: boolean;
 	user_id: string;
 	created_at: string;
+	language: string;
 };
 
 export type CreateVoice = Pick<Voice, "name" | "description" | "embedding"> &
 	Partial<Omit<Voice, "name" | "description" | "embedding">>;
 
+export type UpdateVoice = Partial<
+	Pick<Voice, "name" | "description" | "embedding">
+>;
+
 export type CloneResponse = {
 	embedding: number[];
 };
 
 export type WebSocketOptions = {
+	container?: string;
+	encoding?: string;
 	sampleRate: number;
 };
 
@@ -65,3 +146,7 @@ export type SourceEventData = {
 	wait: never;
 	read: never;
 };
+
+export type TypedArray = Float32Array | Int16Array | Uint8Array;
+
+export type Encoding = "pcm_f32le" | "pcm_s16le" | "pcm_alaw" | "pcm_mulaw";

package/src/voices/index.ts

@@ -1,41 +1,48 @@
 import { Client } from "../lib/client";
-import type { CloneOptions, CloneResponse, CreateVoice, Voice } from "../types";
+import type {
+	CloneOptions,
+	CloneResponse,
+	CreateVoice,
+	UpdateVoice,
+	Voice,
+} from "../types";
 
 export default class Voices extends Client {
 	async list(): Promise<Voice[]> {
-		const response = await this.fetch("/voices");
+		const response = await this._fetch("/voices");
 		return response.json();
 	}
 
 	async get(voiceId: string): Promise<Voice> {
-		const response = await this.fetch(`/voices/${voiceId}`);
+		const response = await this._fetch(`/voices/${voiceId}`);
 		return response.json();
 	}
 
 	async create(voice: CreateVoice): Promise<Voice> {
-		const response = await this.fetch("/voices", {
+		const response = await this._fetch("/voices", {
 			method: "POST",
 			body: JSON.stringify(voice),
 		});
 		return response.json() as Promise<Voice>;
 	}
 
-	async clone(options: CloneOptions): Promise<CloneResponse> {
-		if (options.mode === "url") {
-			const response = await this.fetch(
-				`/voices/clone/url?link=${options.link}`,
-				{
-					method: "POST",
-				},
-			);
-			return response.json();
-		}
+	async update(id: string, voice: UpdateVoice): Promise<Voice> {
+		const response = await this._fetch(`/voices/${id}`, {
+			method: "PATCH",
+			body: JSON.stringify(voice),
+		});
+		return response.json() as Promise<Voice>;
+	}
 
+	async clone(options: CloneOptions): Promise<CloneResponse> {
 		if (options.mode === "clip") {
 			const formData = new FormData();
 			formData.append("clip", options.clip);
+			if (options.enhance !== undefined) {
+				formData.append("enhance", options.enhance.toString());
+			}
 
-			const response = await this.fetch("/voices/clone/clip", {
+			const response = await this._fetch("/voices/clone/clip", {
 				method: "POST",
 				body: formData,
 			});

package/dist/chunk-PQ5EVEEH.js

@@ -1,34 +0,0 @@
-import {
-  BASE_URL,
-  CARTESIA_VERSION,
-  constructApiUrl
-} from "./chunk-2BFEKY3F.js";
-import {
-  __spreadProps,
-  __spreadValues
-} from "./chunk-WIFMLPT5.js";
-
-// src/lib/client.ts
-import fetch from "cross-fetch";
-var Client = class {
-  constructor(options = {}) {
-    if (!(options.apiKey || process.env.CARTESIA_API_KEY)) {
-      throw new Error("Missing Cartesia API key.");
-    }
-    this.apiKey = options.apiKey || process.env.CARTESIA_API_KEY;
-    this.baseUrl = options.baseUrl || BASE_URL;
-  }
-  fetch(path, options = {}) {
-    const url = constructApiUrl(this.baseUrl, path);
-    return fetch(url.toString(), __spreadProps(__spreadValues({}, options), {
-      headers: __spreadValues({
-        "X-API-Key": this.apiKey,
-        "Cartesia-Version": CARTESIA_VERSION
-      }, options.headers)
-    }));
-  }
-};
-
-export {
-  Client
-};