@ai-sdk/cartesia 0.0.1 → 3.0.2

package/src/cartesia-speech-model.ts

@@ -0,0 +1,285 @@
+import type { SpeechModelV4, SharedV4Warning } from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createBinaryResponseHandler,
+  parseProviderOptions,
+  postJsonToApi,
+  serializeModelOptions,
+  WORKFLOW_SERIALIZE,
+  WORKFLOW_DESERIALIZE,
+} from '@ai-sdk/provider-utils';
+import type { CartesiaConfig } from './cartesia-config';
+import { cartesiaFailedResponseHandler } from './cartesia-error';
+import { cartesiaSpeechModelOptionsSchema } from './cartesia-speech-model-options';
+import type {
+  CartesiaSpeechAPITypes,
+  CartesiaSpeechBitRate,
+  CartesiaSpeechEncoding,
+  CartesiaSpeechOutputFormat,
+  CartesiaSpeechSampleRate,
+} from './cartesia-speech-api-types';
+import type {
+  CartesiaSpeechModelId,
+  CartesiaSpeechVoiceId,
+} from './cartesia-speech-options';
+
+interface CartesiaSpeechModelConfig extends CartesiaConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+// Default output format used when no outputFormat / provider options are set.
+const DEFAULT_OUTPUT_FORMAT: CartesiaSpeechAPITypes['output_format'] = {
+  container: 'mp3',
+  sample_rate: 44100,
+  bit_rate: 128000,
+};
+
+const OUTPUT_FORMATS: Record<string, CartesiaSpeechOutputFormat> = {
+  alaw: { container: 'raw', encoding: 'pcm_alaw', sample_rate: 8000 },
+  mp3: DEFAULT_OUTPUT_FORMAT,
+  mulaw: { container: 'raw', encoding: 'pcm_mulaw', sample_rate: 8000 },
+  pcm: { container: 'raw', encoding: 'pcm_f32le', sample_rate: 44100 },
+  raw: { container: 'raw', encoding: 'pcm_f32le', sample_rate: 44100 },
+  wav: { container: 'wav', encoding: 'pcm_s16le', sample_rate: 44100 },
+};
+
+const SAMPLE_RATES = [8000, 16000, 22050, 24000, 44100, 48000] as const;
+
+function isSampleRate(value: number): value is CartesiaSpeechSampleRate {
+  return SAMPLE_RATES.includes(value as CartesiaSpeechSampleRate);
+}
+
+function resolveOutputFormat({
+  outputFormat,
+  providerOptions,
+  warnings,
+}: {
+  outputFormat: string;
+  providerOptions:
+    | {
+        container?: 'raw' | 'wav' | 'mp3' | null;
+        encoding?: CartesiaSpeechEncoding | null;
+        sampleRate?: CartesiaSpeechSampleRate | null;
+        bitRate?: CartesiaSpeechBitRate | null;
+      }
+    | undefined;
+  warnings: SharedV4Warning[];
+}): CartesiaSpeechOutputFormat {
+  const [formatName, sampleRateText, ...extraParts] = outputFormat
+    .toLowerCase()
+    .split('_');
+  const mapped = OUTPUT_FORMATS[formatName];
+  let resolved = mapped ? { ...mapped } : { ...DEFAULT_OUTPUT_FORMAT };
+
+  if (!mapped) {
+    warnings.push({
+      type: 'unsupported',
+      feature: 'outputFormat',
+      details: `Unknown output format "${outputFormat}". Falling back to mp3. Use providerOptions.cartesia to configure container, encoding, and sampleRate directly.`,
+    });
+  } else if (sampleRateText != null) {
+    const parsedRate = Number(sampleRateText);
+    if (
+      extraParts.length === 0 &&
+      Number.isInteger(parsedRate) &&
+      isSampleRate(parsedRate)
+    ) {
+      resolved.sample_rate = parsedRate;
+    } else {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'outputFormat',
+        details: `Unsupported Cartesia sample rate in output format "${outputFormat}". Using ${resolved.sample_rate} Hz instead.`,
+      });
+    }
+  }
+
+  const container = providerOptions?.container ?? resolved.container;
+  const sampleRate = providerOptions?.sampleRate ?? resolved.sample_rate;
+
+  if (container === 'mp3') {
+    if (providerOptions?.encoding != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'providerOptions.cartesia.encoding',
+        details:
+          'Cartesia MP3 output does not accept an encoding. The encoding option was ignored.',
+      });
+    }
+
+    return {
+      container,
+      sample_rate: sampleRate,
+      bit_rate:
+        providerOptions?.bitRate ??
+        (resolved.container === 'mp3' ? resolved.bit_rate : 128000),
+    };
+  }
+
+  if (providerOptions?.bitRate != null) {
+    warnings.push({
+      type: 'unsupported',
+      feature: 'providerOptions.cartesia.bitRate',
+      details:
+        'Cartesia raw and WAV output do not accept a bit rate. The bitRate option was ignored.',
+    });
+  }
+
+  return {
+    container,
+    encoding:
+      providerOptions?.encoding ??
+      (resolved.container === 'mp3'
+        ? container === 'wav'
+          ? 'pcm_s16le'
+          : 'pcm_f32le'
+        : resolved.encoding),
+    sample_rate: sampleRate,
+  };
+}
+
+export class CartesiaSpeechModel implements SpeechModelV4 {
+  readonly specificationVersion = 'v4';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  static [WORKFLOW_SERIALIZE](model: CartesiaSpeechModel) {
+    return serializeModelOptions({
+      modelId: model.modelId,
+      config: model.config,
+    });
+  }
+
+  static [WORKFLOW_DESERIALIZE](options: {
+    modelId: CartesiaSpeechModelId;
+    config: CartesiaSpeechModelConfig;
+  }) {
+    return new CartesiaSpeechModel(options.modelId, options.config);
+  }
+
+  constructor(
+    readonly modelId: CartesiaSpeechModelId,
+    private readonly config: CartesiaSpeechModelConfig,
+  ) {}
+
+  private async getArgs({
+    text,
+    voice,
+    outputFormat = 'mp3',
+    instructions,
+    language,
+    speed,
+    providerOptions,
+  }: Parameters<SpeechModelV4['doGenerate']>[0]) {
+    const warnings: SharedV4Warning[] = [];
+
+    // Parse provider options
+    const cartesiaOptions = await parseProviderOptions({
+      provider: 'cartesia',
+      providerOptions,
+      schema: cartesiaSpeechModelOptionsSchema,
+    });
+
+    if (!voice) {
+      throw new Error('Cartesia speech models require a `voice` to be set.');
+    }
+
+    const outputFormatObject = resolveOutputFormat({
+      outputFormat,
+      providerOptions: cartesiaOptions,
+      warnings,
+    });
+
+    // Create request body
+    const requestBody: CartesiaSpeechAPITypes = {
+      model_id: this.modelId,
+      transcript: text,
+      voice: {
+        mode: 'id',
+        id: voice as CartesiaSpeechVoiceId,
+      },
+      output_format: outputFormatObject,
+    };
+
+    // Map generic language
+    if (language) {
+      requestBody.language = language;
+    }
+
+    // Provider-specific options override the corresponding generic options.
+    if (cartesiaOptions) {
+      if (cartesiaOptions.language != null) {
+        requestBody.language = cartesiaOptions.language;
+      }
+    }
+
+    const resolvedSpeed = cartesiaOptions?.speed ?? speed;
+    if (resolvedSpeed != null) {
+      if (resolvedSpeed >= 0.6 && resolvedSpeed <= 1.5) {
+        requestBody.generation_config = { speed: resolvedSpeed };
+      } else {
+        warnings.push({
+          type: 'unsupported',
+          feature: 'speed',
+          details:
+            'Cartesia speed must be between 0.6 and 1.5. The speed option was ignored.',
+        });
+      }
+    }
+
+    if (instructions) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'instructions',
+        details: `Cartesia speech models do not support instructions. Instructions parameter was ignored.`,
+      });
+    }
+
+    return {
+      requestBody,
+      warnings,
+    };
+  }
+
+  async doGenerate(
+    options: Parameters<SpeechModelV4['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<SpeechModelV4['doGenerate']>>> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const { requestBody, warnings } = await this.getArgs(options);
+
+    const {
+      value: audio,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postJsonToApi({
+      url: this.config.url({
+        path: '/tts/bytes',
+        modelId: this.modelId,
+      }),
+      headers: combineHeaders(this.config.headers?.(), options.headers),
+      body: requestBody,
+      failedResponseHandler: cartesiaFailedResponseHandler,
+      successfulResponseHandler: createBinaryResponseHandler(),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    return {
+      audio,
+      warnings,
+      request: {
+        body: JSON.stringify(requestBody),
+      },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+    };
+  }
+}

package/src/cartesia-speech-options.ts

@@ -0,0 +1,9 @@
+export type CartesiaSpeechModelId =
+  | 'sonic-3.5'
+  | 'sonic-3'
+  | 'sonic-2'
+  | 'sonic-turbo'
+  | 'sonic-latest'
+  | (string & {});
+
+export type CartesiaSpeechVoiceId = string;

package/src/cartesia-transcription-model-options.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod/v4';
+
+// https://docs.cartesia.ai/api-reference/stt/transcribe
+export const cartesiaTranscriptionModelOptionsSchema = z.object({
+  /** The language of the audio (ISO 639-1 code). Defaults to English. */
+  language: z.string().nullish(),
+  /** The timestamp granularities to populate. Currently only `word` is supported. */
+  timestampGranularities: z.array(z.enum(['word'])).nullish(),
+});
+
+export type CartesiaTranscriptionModelOptions = z.infer<
+  typeof cartesiaTranscriptionModelOptionsSchema
+>;

package/src/cartesia-transcription-model.ts

@@ -0,0 +1,157 @@
+import type { TranscriptionModelV4, SharedV4Warning } from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  convertBase64ToUint8Array,
+  createJsonResponseHandler,
+  mediaTypeToExtension,
+  parseProviderOptions,
+  postFormDataToApi,
+  serializeModelOptions,
+  WORKFLOW_SERIALIZE,
+  WORKFLOW_DESERIALIZE,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import type { CartesiaConfig } from './cartesia-config';
+import { cartesiaFailedResponseHandler } from './cartesia-error';
+import { cartesiaTranscriptionModelOptionsSchema } from './cartesia-transcription-model-options';
+import type { CartesiaTranscriptionModelId } from './cartesia-transcription-options';
+
+interface CartesiaTranscriptionModelConfig extends CartesiaConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+export class CartesiaTranscriptionModel implements TranscriptionModelV4 {
+  readonly specificationVersion = 'v4';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  static [WORKFLOW_SERIALIZE](model: CartesiaTranscriptionModel) {
+    return serializeModelOptions({
+      modelId: model.modelId,
+      config: model.config,
+    });
+  }
+
+  static [WORKFLOW_DESERIALIZE](options: {
+    modelId: CartesiaTranscriptionModelId;
+    config: CartesiaTranscriptionModelConfig;
+  }) {
+    return new CartesiaTranscriptionModel(options.modelId, options.config);
+  }
+
+  constructor(
+    readonly modelId: CartesiaTranscriptionModelId,
+    private readonly config: CartesiaTranscriptionModelConfig,
+  ) {}
+
+  private async getArgs({
+    audio,
+    mediaType,
+    providerOptions,
+  }: Parameters<TranscriptionModelV4['doGenerate']>[0]) {
+    const warnings: SharedV4Warning[] = [];
+
+    // Parse provider options
+    const cartesiaOptions = await parseProviderOptions({
+      provider: 'cartesia',
+      providerOptions,
+      schema: cartesiaTranscriptionModelOptionsSchema,
+    });
+
+    // Create form data with base fields
+    const formData = new FormData();
+    const blob =
+      audio instanceof Uint8Array
+        ? new Blob([audio])
+        : new Blob([convertBase64ToUint8Array(audio)]);
+
+    const fileExtension = mediaTypeToExtension(mediaType);
+    formData.append('model', this.modelId);
+    formData.append(
+      'file',
+      new File([blob], 'audio', { type: mediaType }),
+      `audio.${fileExtension}`,
+    );
+
+    // Add provider-specific options
+    if (cartesiaOptions) {
+      if (cartesiaOptions.language != null) {
+        formData.set('language', cartesiaOptions.language);
+      }
+      if (cartesiaOptions.timestampGranularities != null) {
+        for (const granularity of cartesiaOptions.timestampGranularities) {
+          formData.append('timestamp_granularities[]', granularity);
+        }
+      }
+    }
+
+    return {
+      formData,
+      warnings,
+    };
+  }
+
+  async doGenerate(
+    options: Parameters<TranscriptionModelV4['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<TranscriptionModelV4['doGenerate']>>> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const { formData, warnings } = await this.getArgs(options);
+
+    const {
+      value: response,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postFormDataToApi({
+      url: this.config.url({
+        path: '/stt',
+        modelId: this.modelId,
+      }),
+      headers: combineHeaders(this.config.headers?.(), options.headers),
+      formData,
+      failedResponseHandler: cartesiaFailedResponseHandler,
+      successfulResponseHandler: createJsonResponseHandler(
+        cartesiaTranscriptionResponseSchema,
+      ),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    return {
+      text: response.text,
+      segments:
+        response.words?.map(word => ({
+          text: word.word,
+          startSecond: word.start,
+          endSecond: word.end,
+        })) ?? [],
+      language: response.language ?? undefined,
+      durationInSeconds: response.duration ?? undefined,
+      warnings,
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+    };
+  }
+}
+
+const cartesiaTranscriptionResponseSchema = z.object({
+  text: z.string(),
+  language: z.string().nullish(),
+  duration: z.number().nullish(),
+  words: z
+    .array(
+      z.object({
+        word: z.string(),
+        start: z.number(),
+        end: z.number(),
+      }),
+    )
+    .nullish(),
+});

package/src/cartesia-transcription-options.ts

@@ -0,0 +1 @@
+export type CartesiaTranscriptionModelId = 'ink-whisper' | (string & {});

package/src/index.ts

@@ -0,0 +1,20 @@
+export { createCartesia, cartesia } from './cartesia-provider';
+export type {
+  CartesiaProvider,
+  CartesiaProviderSettings,
+} from './cartesia-provider';
+export { CartesiaSpeechModel } from './cartesia-speech-model';
+export { CartesiaTranscriptionModel } from './cartesia-transcription-model';
+export { CartesiaRealtimeModel } from './cartesia-realtime-model';
+export type {
+  CartesiaRealtimeModelId,
+  CartesiaRealtimeModelOptions,
+} from './cartesia-realtime-model-options';
+export type {
+  CartesiaSpeechModelId,
+  CartesiaSpeechVoiceId,
+} from './cartesia-speech-options';
+export type { CartesiaTranscriptionModelId } from './cartesia-transcription-options';
+export type { CartesiaSpeechModelOptions } from './cartesia-speech-model-options';
+export type { CartesiaTranscriptionModelOptions } from './cartesia-transcription-model-options';
+export { VERSION } from './version';

package/src/version.ts

@@ -0,0 +1,6 @@
+// Version string of this package injected at build time.
+declare const __PACKAGE_VERSION__: string | undefined;
+export const VERSION: string =
+  typeof __PACKAGE_VERSION__ !== 'undefined'
+    ? __PACKAGE_VERSION__
+    : '0.0.0-test';