@ai-sdk/cartesia 0.0.1 → 3.0.2

package/src/cartesia-provider.ts

@@ -0,0 +1,190 @@
+import {
+  type Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
+  type Experimental_RealtimeFactoryV4GetTokenOptions as RealtimeFactoryV4GetTokenOptions,
+  NoSuchModelError,
+  type TranscriptionModelV4,
+  type SpeechModelV4,
+  type ProviderV4,
+} from '@ai-sdk/provider';
+import {
+  loadApiKey,
+  withUserAgentSuffix,
+  type FetchFunction,
+} from '@ai-sdk/provider-utils';
+import { CartesiaTranscriptionModel } from './cartesia-transcription-model';
+import type { CartesiaTranscriptionModelId } from './cartesia-transcription-options';
+import { CartesiaSpeechModel } from './cartesia-speech-model';
+import type { CartesiaSpeechModelId } from './cartesia-speech-options';
+import { CartesiaRealtimeModel } from './cartesia-realtime-model';
+import type { CartesiaRealtimeModelId } from './cartesia-realtime-model-options';
+import { VERSION } from './version';
+
+/**
+ * The Cartesia API version sent with every request via the `Cartesia-Version`
+ * header. See https://docs.cartesia.ai/api-reference/api-versioning
+ */
+const CARTESIA_API_VERSION = '2026-03-01';
+
+export interface CartesiaProvider extends ProviderV4 {
+  (
+    modelId: CartesiaSpeechModelId,
+    settings?: {},
+  ): {
+    speech: CartesiaSpeechModel;
+  };
+
+  /**
+   * Creates a model for transcription.
+   */
+  transcription(modelId: CartesiaTranscriptionModelId): TranscriptionModelV4;
+
+  /**
+   * Creates a model for speech generation.
+   */
+  speech(modelId: CartesiaSpeechModelId): SpeechModelV4;
+
+  /**
+   * Creates a realtime Ink speech-to-text model.
+   */
+  experimental_realtime: RealtimeFactoryV4;
+
+  /**
+   * @deprecated Use `embeddingModel` instead.
+   */
+  textEmbeddingModel(modelId: string): never;
+}
+
+export interface CartesiaProviderSettings {
+  /**
+   * API key for authenticating requests.
+   */
+  apiKey?: string;
+
+  /**
+   * The Cartesia API version to use (sent via the `Cartesia-Version` header).
+   */
+  version?: string;
+
+  /**
+   * Custom headers to include in the requests.
+   */
+  headers?: Record<string, string>;
+
+  /**
+   * Custom fetch implementation. You can use it as a middleware to intercept requests,
+   * or to provide a custom fetch implementation for e.g. testing.
+   */
+  fetch?: FetchFunction;
+}
+
+/**
+ * Create a Cartesia provider instance.
+ */
+export function createCartesia(
+  options: CartesiaProviderSettings = {},
+): CartesiaProvider {
+  const getHeaders = () =>
+    withUserAgentSuffix(
+      {
+        Authorization: `Bearer ${loadApiKey({
+          apiKey: options.apiKey,
+          environmentVariableName: 'CARTESIA_API_KEY',
+          description: 'Cartesia',
+        })}`,
+        'Cartesia-Version': options.version ?? CARTESIA_API_VERSION,
+        ...options.headers,
+      },
+      `ai-sdk/cartesia/${VERSION}`,
+    );
+
+  const createTranscriptionModel = (modelId: CartesiaTranscriptionModelId) =>
+    new CartesiaTranscriptionModel(modelId, {
+      provider: `cartesia.transcription`,
+      url: ({ path }) => `https://api.cartesia.ai${path}`,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  const createSpeechModel = (modelId: CartesiaSpeechModelId) =>
+    new CartesiaSpeechModel(modelId, {
+      provider: `cartesia.speech`,
+      url: ({ path }) => `https://api.cartesia.ai${path}`,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  const createRealtimeModel = (modelId: CartesiaRealtimeModelId) =>
+    new CartesiaRealtimeModel(modelId, {
+      provider: `cartesia.realtime`,
+      baseURL: 'https://api.cartesia.ai',
+      version: options.version ?? CARTESIA_API_VERSION,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  const experimentalRealtimeFactory = Object.assign(
+    (modelId: CartesiaRealtimeModelId) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions: RealtimeFactoryV4GetTokenOptions) => {
+        const secret = await createRealtimeModel(
+          tokenOptions.model as CartesiaRealtimeModelId,
+        ).doCreateClientSecret({
+          sessionConfig: tokenOptions.sessionConfig,
+          expiresAfterSeconds: tokenOptions.expiresAfterSeconds,
+        });
+
+        return {
+          token: secret.token,
+          url: secret.url,
+          expiresAt: secret.expiresAt,
+        };
+      },
+    },
+  ) as RealtimeFactoryV4;
+
+  const provider = function (modelId: CartesiaSpeechModelId) {
+    return {
+      speech: createSpeechModel(modelId),
+    };
+  };
+
+  provider.specificationVersion = 'v4' as const;
+  provider.transcription = createTranscriptionModel;
+  provider.transcriptionModel = createTranscriptionModel;
+  provider.speech = createSpeechModel;
+  provider.speechModel = createSpeechModel;
+  provider.experimental_realtime = experimentalRealtimeFactory;
+
+  // Required ProviderV4 methods that are not supported
+  provider.languageModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'languageModel',
+      message: 'Cartesia does not provide language models',
+    });
+  };
+
+  provider.embeddingModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'embeddingModel',
+      message: 'Cartesia does not provide embedding models',
+    });
+  };
+  provider.textEmbeddingModel = provider.embeddingModel;
+
+  provider.imageModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'imageModel',
+      message: 'Cartesia does not provide image models',
+    });
+  };
+
+  return provider as CartesiaProvider;
+}
+
+/**
+ * Default Cartesia provider instance.
+ */
+export const cartesia = createCartesia();

package/src/cartesia-realtime-model-options.ts

@@ -0,0 +1,3 @@
+export type CartesiaRealtimeModelId = 'ink-2' | (string & {});
+
+export type CartesiaRealtimeModelOptions = Record<string, never>;

package/src/cartesia-realtime-model.ts

@@ -0,0 +1,326 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
+  Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import {
+  convertBase64ToUint8Array,
+  type FetchFunction,
+} from '@ai-sdk/provider-utils';
+import type { CartesiaRealtimeModelId } from './cartesia-realtime-model-options';
+
+export type CartesiaRealtimeModelConfig = {
+  provider: string;
+  baseURL: string;
+  version: string;
+  headers: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+  _internal?: {
+    currentDate?: () => Date;
+  };
+};
+
+type CartesiaRealtimeWireEvent = {
+  type: string;
+  request_id?: string;
+  transcript?: string;
+  text?: string;
+  is_final?: boolean;
+  duration?: number;
+  message?: string;
+  error_code?: string;
+};
+
+type ManualTranscriptState = {
+  duration: number;
+  events: CartesiaRealtimeWireEvent[];
+  text: string;
+};
+
+/**
+ * Realtime Ink model using Cartesia's STT WebSocket APIs.
+ */
+export class CartesiaRealtimeModel implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly provider: string;
+  readonly modelId: CartesiaRealtimeModelId;
+
+  private readonly config: CartesiaRealtimeModelConfig;
+  private manualTranscript: ManualTranscriptState = {
+    duration: 0,
+    events: [],
+    text: '',
+  };
+  private sessionCreated = false;
+  private useTurnDetection = true;
+
+  constructor(
+    modelId: CartesiaRealtimeModelId,
+    config: CartesiaRealtimeModelConfig,
+  ) {
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+
+  async doCreateClientSecret(
+    options: RealtimeModelV4ClientSecretOptions,
+  ): Promise<RealtimeModelV4ClientSecretResult> {
+    const expiresIn = options.expiresAfterSeconds;
+    if (
+      expiresIn != null &&
+      (!Number.isInteger(expiresIn) || expiresIn <= 0 || expiresIn > 3600)
+    ) {
+      throw new Error(
+        'Cartesia realtime client secrets must expire between 1 and 3600 seconds.',
+      );
+    }
+
+    const sessionConfig = options.sessionConfig;
+    const language = sessionConfig?.inputAudioTranscription?.language;
+    if (language != null && language !== 'en') {
+      throw new Error('Cartesia Ink 2 currently supports English only.');
+    }
+
+    const fetchFn = this.config.fetch ?? fetch;
+    const response = await fetchFn(`${this.config.baseURL}/access-token`, {
+      method: 'POST',
+      headers: {
+        ...this.config.headers(),
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        grants: { stt: true },
+        ...(expiresIn != null ? { expires_in: expiresIn } : {}),
+      }),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `Cartesia realtime client secret request failed: ${response.status} ${text}`,
+      );
+    }
+
+    const data = (await response.json()) as { token?: unknown };
+    if (typeof data.token !== 'string' || data.token.length === 0) {
+      throw new Error(
+        'Cartesia realtime client secret response did not include a token.',
+      );
+    }
+
+    const turnDetection = sessionConfig?.turnDetection;
+    const useTurnDetection =
+      turnDetection === undefined ||
+      (turnDetection !== null && turnDetection.type !== 'disabled');
+    const inputAudioFormat = sessionConfig?.inputAudioFormat;
+    const url = new URL(
+      useTurnDetection
+        ? `${this.config.baseURL}/stt/turns/websocket`
+        : `${this.config.baseURL}/stt/websocket`,
+    );
+    url.protocol = url.protocol === 'http:' ? 'ws:' : 'wss:';
+    url.searchParams.set('model', this.modelId);
+    url.searchParams.set(
+      'encoding',
+      resolveCartesiaEncoding(inputAudioFormat?.type),
+    );
+    url.searchParams.set(
+      'sample_rate',
+      String(inputAudioFormat?.rate ?? 24000),
+    );
+    url.searchParams.set('cartesia_version', this.config.version);
+    if (!useTurnDetection && language != null) {
+      url.searchParams.set('language', language);
+    }
+
+    return {
+      token: data.token,
+      url: url.toString(),
+      ...(expiresIn != null
+        ? {
+            expiresAt:
+              Math.floor(
+                (
+                  this.config._internal?.currentDate?.() ?? new Date()
+                ).getTime() / 1000,
+              ) + expiresIn,
+          }
+        : {}),
+    };
+  }
+
+  getWebSocketConfig(options: { token: string; url: string }): {
+    url: string;
+  } {
+    const url = new URL(options.url);
+    this.useTurnDetection = url.pathname.includes('/stt/turns/websocket');
+    url.searchParams.set('access_token', options.token);
+    return { url: url.toString() };
+  }
+
+  parseServerEvent(
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const event = raw as CartesiaRealtimeWireEvent;
+    const requestId = event.request_id ?? 'cartesia-realtime';
+    let mapped: RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[];
+
+    switch (event.type) {
+      case 'connected':
+        this.sessionCreated = true;
+        return {
+          type: 'session-created',
+          sessionId: event.request_id,
+          raw,
+        };
+
+      case 'turn.start':
+        mapped = { type: 'speech-started', itemId: requestId, raw };
+        break;
+
+      case 'turn.resume':
+        mapped = [
+          { type: 'speech-started', itemId: requestId, raw },
+          { type: 'custom', rawType: event.type, raw },
+        ];
+        break;
+
+      case 'turn.end':
+        mapped = [
+          { type: 'speech-stopped', itemId: requestId, raw },
+          {
+            type: 'input-transcription-completed',
+            itemId: requestId,
+            transcript: event.transcript ?? '',
+            raw,
+          },
+        ];
+        break;
+
+      case 'transcript':
+        if (event.is_final === true) {
+          this.manualTranscript.text += event.text ?? '';
+          this.manualTranscript.duration += event.duration ?? 0;
+          this.manualTranscript.events.push(event);
+        }
+        mapped = { type: 'custom', rawType: event.type, raw };
+        break;
+
+      case 'flush_done':
+        mapped = this.finishManualTranscript(event, raw);
+        break;
+
+      case 'done': {
+        if (this.manualTranscript.text.length > 0) {
+          const completed = this.finishManualTranscript(event, raw);
+          mapped = Array.isArray(completed)
+            ? [...completed, { type: 'custom', rawType: event.type, raw }]
+            : [completed, { type: 'custom', rawType: event.type, raw }];
+        } else {
+          mapped = { type: 'custom', rawType: event.type, raw };
+        }
+        break;
+      }
+
+      case 'error':
+        mapped = {
+          type: 'error',
+          message: event.message ?? 'Unknown Cartesia realtime error',
+          code: event.error_code,
+          raw,
+        };
+        break;
+
+      default:
+        mapped = { type: 'custom', rawType: event.type, raw };
+        break;
+    }
+
+    return this.withSessionCreated(mapped, requestId, raw);
+  }
+
+  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
+    switch (event.type) {
+      case 'session-update':
+      case 'input-audio-clear':
+      case 'response-create':
+        return null;
+      case 'input-audio-append':
+        return convertBase64ToUint8Array(event.audio);
+      case 'input-audio-commit':
+        return this.useTurnDetection ? null : 'finalize';
+      default:
+        throw new Error(
+          `Cartesia Ink 2 does not support realtime client event "${event.type}".`,
+        );
+    }
+  }
+
+  buildSessionConfig(_config: RealtimeModelV4SessionConfig): null {
+    return null;
+  }
+
+  private finishManualTranscript(
+    event: CartesiaRealtimeWireEvent,
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    const state = this.manualTranscript;
+    this.manualTranscript = { duration: 0, events: [], text: '' };
+    if (state.text.length === 0) {
+      return { type: 'custom', rawType: event.type, raw };
+    }
+
+    return [
+      {
+        type: 'audio-committed',
+        itemId: event.request_id,
+        raw,
+      },
+      {
+        type: 'input-transcription-completed',
+        itemId: event.request_id ?? 'cartesia-realtime',
+        transcript: state.text,
+        raw: {
+          event,
+          transcriptEvents: state.events,
+          duration: state.duration,
+        },
+      },
+    ];
+  }
+
+  private withSessionCreated(
+    event: RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[],
+    sessionId: string,
+    raw: unknown,
+  ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[] {
+    if (this.sessionCreated) {
+      return event;
+    }
+    this.sessionCreated = true;
+    const created: RealtimeModelV4ServerEvent = {
+      type: 'session-created',
+      sessionId,
+      raw,
+    };
+    return Array.isArray(event) ? [created, ...event] : [created, event];
+  }
+}
+
+function resolveCartesiaEncoding(type: string | undefined): string {
+  switch (type) {
+    case undefined:
+    case 'audio/pcm':
+      return 'pcm_s16le';
+    case 'audio/pcmu':
+      return 'pcm_mulaw';
+    case 'audio/pcma':
+      return 'pcm_alaw';
+    default:
+      throw new Error(`Unsupported Cartesia realtime audio format: ${type}`);
+  }
+}

package/src/cartesia-speech-api-types.ts

@@ -0,0 +1,64 @@
+export type CartesiaSpeechOutputFormat =
+  | {
+      container: 'mp3';
+      sample_rate: CartesiaSpeechSampleRate;
+      bit_rate: CartesiaSpeechBitRate;
+    }
+  | {
+      container: 'raw' | 'wav';
+      encoding: CartesiaSpeechEncoding;
+      sample_rate: CartesiaSpeechSampleRate;
+    };
+
+export type CartesiaSpeechEncoding =
+  | 'pcm_f32le'
+  | 'pcm_s16le'
+  | 'pcm_mulaw'
+  | 'pcm_alaw';
+
+export type CartesiaSpeechSampleRate =
+  | 8000
+  | 16000
+  | 22050
+  | 24000
+  | 44100
+  | 48000;
+
+export type CartesiaSpeechBitRate = 32000 | 64000 | 96000 | 128000 | 192000;
+
+export type CartesiaSpeechAPITypes = {
+  /**
+   * The ID of the model to use for the generation.
+   */
+  model_id: string;
+
+  /**
+   * The transcript to generate speech for.
+   */
+  transcript: string;
+
+  /**
+   * The voice specifier. Cartesia uses id mode with a voice id.
+   */
+  voice: {
+    mode: 'id';
+    id: string;
+  };
+
+  /**
+   * The language to generate speech in (ISO 639-1 code).
+   */
+  language?: string;
+
+  /**
+   * The output audio format.
+   */
+  output_format: CartesiaSpeechOutputFormat;
+
+  /**
+   * Controls supported generation attributes.
+   */
+  generation_config?: {
+    speed?: number;
+  };
+};

package/src/cartesia-speech-model-options.ts

@@ -0,0 +1,40 @@
+import { z } from 'zod/v4';
+
+// https://docs.cartesia.ai/api-reference/tts/bytes
+export const cartesiaSpeechModelOptionsSchema = z.object({
+  /** Container format for the output audio (raw, wav, mp3). */
+  container: z.enum(['raw', 'wav', 'mp3']).nullish(),
+  /** Encoding type for the audio output (pcm_f32le, pcm_s16le, pcm_mulaw, pcm_alaw). */
+  encoding: z
+    .enum(['pcm_f32le', 'pcm_s16le', 'pcm_mulaw', 'pcm_alaw'])
+    .nullish(),
+  /** Sample rate for the output audio in Hz (e.g. 8000, 16000, 22050, 24000, 44100, 48000). */
+  sampleRate: z
+    .union([
+      z.literal(8000),
+      z.literal(16000),
+      z.literal(22050),
+      z.literal(24000),
+      z.literal(44100),
+      z.literal(48000),
+    ])
+    .nullish(),
+  /** Bitrate for mp3 output in bits per second (e.g. 32000, 64000, 128000, 192000). */
+  bitRate: z
+    .union([
+      z.literal(32000),
+      z.literal(64000),
+      z.literal(96000),
+      z.literal(128000),
+      z.literal(192000),
+    ])
+    .nullish(),
+  /** Controls the speed of the generated speech. */
+  speed: z.number().min(0.6).max(1.5).nullish(),
+  /** The language to generate speech in (ISO 639-1 code). */
+  language: z.string().nullish(),
+});
+
+export type CartesiaSpeechModelOptions = z.infer<
+  typeof cartesiaSpeechModelOptionsSchema
+>;