@ai-sdk/fish-audio 0.0.0 → 2.0.1

package/src/fish-audio-speech-api-types.ts

@@ -0,0 +1,126 @@
+export type FishAudioSpeechFormat = 'wav' | 'pcm' | 'mp3' | 'opus';
+
+export type FishAudioSpeechMp3Bitrate = 64 | 128 | 192;
+
+export type FishAudioSpeechOpusBitrate = -1000 | 24000 | 32000 | 48000 | 64000;
+
+export type FishAudioSpeechLatency = 'low' | 'normal' | 'balanced';
+
+export type FishAudioProsodyControl = {
+  /**
+   * Speech rate multiplier, 0.5 to 2.0.
+   */
+  speed?: number;
+
+  /**
+   * Volume offset in dB.
+   */
+  volume?: number;
+
+  /**
+   * Loudness normalization. S2-Pro only.
+   */
+  normalize_loudness?: boolean;
+};
+
+/**
+ * Request body for `POST /v1/tts`. The model is selected via the `model` HTTP
+ * header rather than a body field.
+ *
+ * Inline `references` (zero-shot voice cloning) are intentionally omitted:
+ * they require a MessagePack request body, and this provider sends JSON only.
+ * Pre-upload reference audio and pass `reference_id` instead.
+ *
+ * https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
+ */
+export type FishAudioSpeechAPITypes = {
+  /**
+   * The text to synthesize.
+   */
+  text: string;
+
+  /**
+   * Voice model ID, or an array of IDs for multi-speaker dialogue.
+   */
+  reference_id?: string | string[];
+
+  /**
+   * Speed and volume adjustments.
+   */
+  prosody?: FishAudioProsodyControl;
+
+  /**
+   * Output audio container format.
+   */
+  format?: FishAudioSpeechFormat;
+
+  /**
+   * Output sample rate in Hz. Falls back to the format default when unset.
+   */
+  sample_rate?: number;
+
+  /**
+   * Bitrate in kbps for mp3 output.
+   */
+  mp3_bitrate?: FishAudioSpeechMp3Bitrate;
+
+  /**
+   * Bitrate in bps for opus output. `-1000` selects automatic.
+   */
+  opus_bitrate?: FishAudioSpeechOpusBitrate;
+
+  /**
+   * Latency/quality tradeoff.
+   */
+  latency?: FishAudioSpeechLatency;
+
+  /**
+   * Expressiveness, 0 to 1.
+   */
+  temperature?: number;
+
+  /**
+   * Nucleus sampling, 0 to 1.
+   */
+  top_p?: number;
+
+  /**
+   * Text segment size for processing, 100 to 300.
+   */
+  chunk_length?: number;
+
+  /**
+   * Minimum characters before splitting into a new chunk, 0 to 100.
+   */
+  min_chunk_length?: number;
+
+  /**
+   * Text normalization for English and Chinese.
+   */
+  normalize?: boolean;
+
+  /**
+   * Maximum audio tokens to generate per text chunk.
+   */
+  max_new_tokens?: number;
+
+  /**
+   * Values above 1.0 discourage repeated audio patterns.
+   */
+  repetition_penalty?: number;
+
+  /**
+   * Reuse prior audio as context for voice consistency.
+   */
+  condition_on_previous_chunks?: boolean;
+
+  /**
+   * Early-stop threshold used in batch processing, 0 to 1.
+   */
+  early_stop_threshold?: number;
+
+  /**
+   * Request-scoped backend flags, e.g. `['quality-guard']`.
+   */
+  features?: string[];
+};

package/src/fish-audio-speech-model-options.ts

@@ -0,0 +1,115 @@
+import { z } from 'zod/v4';
+
+// https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
+export const fishAudioSpeechModelOptionsSchema = z.object({
+  /**
+   * Voice model ID(s). A single ID selects one speaker; an array enables
+   * multi-speaker dialogue (S2-Pro models), in which case the text must mark
+   * turns with speaker tokens such as `<|speaker:0|>`. The speaker index maps
+   * to the position in this array.
+   *
+   * Takes precedence over the top-level `voice` option.
+   */
+  referenceId: z.union([z.string(), z.array(z.string())]).optional(),
+
+  /**
+   * Output sample rate in Hz. Falls back to the format default when unset
+   * (44100 Hz for wav/pcm/mp3, 48000 Hz for opus).
+   */
+  sampleRate: z.number().int().positive().optional(),
+
+  /**
+   * Bitrate in kbps for mp3 output. Ignored for other formats.
+   */
+  mp3Bitrate: z
+    .union([z.literal(64), z.literal(128), z.literal(192)])
+    .optional(),
+
+  /**
+   * Bitrate in bps for opus output, where `-1000` selects automatic. Ignored
+   * for other formats.
+   */
+  opusBitrate: z
+    .union([
+      z.literal(-1000),
+      z.literal(24_000),
+      z.literal(32_000),
+      z.literal(48_000),
+      z.literal(64_000),
+    ])
+    .optional(),
+
+  /**
+   * Latency/quality tradeoff. `normal` gives the best quality, `balanced`
+   * reduces latency, and `low` is the fastest.
+   */
+  latency: z.enum(['low', 'normal', 'balanced']).optional(),
+
+  /**
+   * Volume offset in dB. Negative values are quieter.
+   */
+  volume: z.number().optional(),
+
+  /**
+   * Loudness normalization. Supported by the S2 family (`s2-pro` and
+   * `s2.1-pro`). Fish Audio accepts it on `s1` but ignores it, so the provider
+   * emits a warning in that case.
+   */
+  normalizeLoudness: z.boolean().optional(),
+
+  /**
+   * Governs expressiveness. Higher values are more varied, lower values more
+   * consistent.
+   */
+  temperature: z.number().min(0).max(1).optional(),
+
+  /**
+   * Controls diversity via nucleus sampling.
+   */
+  topP: z.number().min(0).max(1).optional(),
+
+  /**
+   * Text segment size for processing.
+   */
+  chunkLength: z.number().int().min(100).max(300).optional(),
+
+  /**
+   * Minimum characters before splitting into a new chunk.
+   */
+  minChunkLength: z.number().int().min(0).max(100).optional(),
+
+  /**
+   * Text normalization for English and Chinese. Helps stability with numbers.
+   */
+  normalize: z.boolean().optional(),
+
+  /**
+   * Maximum audio tokens to generate per text chunk.
+   */
+  maxNewTokens: z.number().int().positive().optional(),
+
+  /**
+   * Values above 1.0 discourage repeated audio patterns.
+   */
+  repetitionPenalty: z.number().optional(),
+
+  /**
+   * Reuse prior audio as context for voice consistency across chunks.
+   */
+  conditionOnPreviousChunks: z.boolean().optional(),
+
+  /**
+   * Early-stop threshold used in batch processing.
+   */
+  earlyStopThreshold: z.number().min(0).max(1).optional(),
+
+  /**
+   * Request-scoped flags passed through to the inference backend, e.g.
+   * `['quality-guard']`.
+   */
+  features: z.array(z.string()).optional(),
+});
+
+export type FishAudioSpeechModelOptions = z.infer<
+  typeof fishAudioSpeechModelOptionsSchema
+>;

package/src/fish-audio-speech-model.ts

@@ -0,0 +1,280 @@
+import type { SharedV3Warning, SpeechModelV3 } from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createBinaryResponseHandler,
+  parseProviderOptions,
+  postJsonToApi,
+} from '@ai-sdk/provider-utils';
+import type { FishAudioConfig } from './fish-audio-config';
+import { fishAudioFailedResponseHandler } from './fish-audio-error';
+import type {
+  FishAudioSpeechAPITypes,
+  FishAudioSpeechFormat,
+  FishAudioProsodyControl,
+} from './fish-audio-speech-api-types';
+import { fishAudioSpeechModelOptionsSchema } from './fish-audio-speech-model-options';
+import type { FishAudioSpeechModelId } from './fish-audio-speech-options';
+
+interface FishAudioSpeechModelConfig extends FishAudioConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+const SUPPORTED_FORMATS: FishAudioSpeechFormat[] = [
+  'wav',
+  'pcm',
+  'mp3',
+  'opus',
+];
+
+const DEFAULT_FORMAT: FishAudioSpeechFormat = 'mp3';
+
+// Fish Audio accepts `prosody.speed` between 0.5 and 2.0.
+const MIN_SPEED = 0.5;
+const MAX_SPEED = 2;
+
+function resolveFormat({
+  outputFormat,
+  warnings,
+}: {
+  outputFormat: string | undefined;
+  warnings: SharedV3Warning[];
+}): FishAudioSpeechFormat {
+  if (outputFormat == null) {
+    return DEFAULT_FORMAT;
+  }
+
+  const normalized = outputFormat.toLowerCase();
+  const matched = SUPPORTED_FORMATS.find(format => format === normalized);
+
+  if (matched == null) {
+    warnings.push({
+      type: 'unsupported',
+      feature: 'outputFormat',
+      details: `Fish Audio does not support the output format "${outputFormat}". Falling back to ${DEFAULT_FORMAT}. Supported formats are ${SUPPORTED_FORMATS.join(', ')}.`,
+    });
+    return DEFAULT_FORMAT;
+  }
+
+  return matched;
+}
+
+export class FishAudioSpeechModel implements SpeechModelV3 {
+  readonly specificationVersion = 'v3';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: FishAudioSpeechModelId,
+    private readonly config: FishAudioSpeechModelConfig,
+  ) {}
+
+  private async getArgs({
+    text,
+    voice,
+    outputFormat,
+    instructions,
+    language,
+    speed,
+    providerOptions,
+  }: Parameters<SpeechModelV3['doGenerate']>[0]) {
+    const warnings: SharedV3Warning[] = [];
+
+    const fishAudioOptions = await parseProviderOptions({
+      provider: 'fishAudio',
+      providerOptions,
+      schema: fishAudioSpeechModelOptionsSchema,
+    });
+
+    const format = resolveFormat({ outputFormat, warnings });
+
+    const requestBody: FishAudioSpeechAPITypes = {
+      text,
+      format,
+    };
+
+    // `providerOptions.fishAudio.referenceId` wins over the generic `voice` so
+    // that multi-speaker arrays are expressible.
+    const referenceId = fishAudioOptions?.referenceId ?? voice;
+    if (referenceId != null) {
+      requestBody.reference_id = referenceId;
+    }
+
+    const prosody: FishAudioProsodyControl = {};
+
+    if (speed != null) {
+      if (speed >= MIN_SPEED && speed <= MAX_SPEED) {
+        prosody.speed = speed;
+      } else {
+        warnings.push({
+          type: 'unsupported',
+          feature: 'speed',
+          details: `Fish Audio speed must be between ${MIN_SPEED} and ${MAX_SPEED}. The speed option was ignored.`,
+        });
+      }
+    }
+
+    if (fishAudioOptions?.volume != null) {
+      prosody.volume = fishAudioOptions.volume;
+    }
+
+    if (fishAudioOptions?.normalizeLoudness != null) {
+      // Fish Audio accepts `normalize_loudness` on s1 but ignores it, so warn
+      // rather than let it silently do nothing. Only s1 is known to ignore it;
+      // unrecognized model IDs are left alone.
+      if (this.modelId === 's1') {
+        warnings.push({
+          type: 'unsupported',
+          feature: 'providerOptions.fishAudio.normalizeLoudness',
+          details:
+            'Fish Audio ignores normalizeLoudness on s1. It is supported by the S2 family (s2-pro, s2.1-pro).',
+        });
+      } else {
+        prosody.normalize_loudness = fishAudioOptions.normalizeLoudness;
+      }
+    }
+
+    if (Object.keys(prosody).length > 0) {
+      requestBody.prosody = prosody;
+    }
+
+    if (language != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'language',
+        details:
+          'Fish Audio infers the language from the input text and the selected voice, and has no language parameter. The language option was ignored.',
+      });
+    }
+
+    if (instructions != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'instructions',
+        details:
+          'Fish Audio does not support instructions. The instructions option was ignored.',
+      });
+    }
+
+    if (fishAudioOptions != null) {
+      if (fishAudioOptions.sampleRate != null) {
+        requestBody.sample_rate = fishAudioOptions.sampleRate;
+      }
+
+      if (fishAudioOptions.mp3Bitrate != null) {
+        if (format === 'mp3') {
+          requestBody.mp3_bitrate = fishAudioOptions.mp3Bitrate;
+        } else {
+          warnings.push({
+            type: 'unsupported',
+            feature: 'providerOptions.fishAudio.mp3Bitrate',
+            details: `mp3Bitrate only applies to mp3 output. The option was ignored for ${format} output.`,
+          });
+        }
+      }
+
+      if (fishAudioOptions.opusBitrate != null) {
+        if (format === 'opus') {
+          requestBody.opus_bitrate = fishAudioOptions.opusBitrate;
+        } else {
+          warnings.push({
+            type: 'unsupported',
+            feature: 'providerOptions.fishAudio.opusBitrate',
+            details: `opusBitrate only applies to opus output. The option was ignored for ${format} output.`,
+          });
+        }
+      }
+
+      if (fishAudioOptions.latency != null) {
+        requestBody.latency = fishAudioOptions.latency;
+      }
+
+      if (fishAudioOptions.temperature != null) {
+        requestBody.temperature = fishAudioOptions.temperature;
+      }
+
+      if (fishAudioOptions.topP != null) {
+        requestBody.top_p = fishAudioOptions.topP;
+      }
+
+      if (fishAudioOptions.chunkLength != null) {
+        requestBody.chunk_length = fishAudioOptions.chunkLength;
+      }
+
+      if (fishAudioOptions.minChunkLength != null) {
+        requestBody.min_chunk_length = fishAudioOptions.minChunkLength;
+      }
+
+      if (fishAudioOptions.normalize != null) {
+        requestBody.normalize = fishAudioOptions.normalize;
+      }
+
+      if (fishAudioOptions.maxNewTokens != null) {
+        requestBody.max_new_tokens = fishAudioOptions.maxNewTokens;
+      }
+
+      if (fishAudioOptions.repetitionPenalty != null) {
+        requestBody.repetition_penalty = fishAudioOptions.repetitionPenalty;
+      }
+
+      if (fishAudioOptions.conditionOnPreviousChunks != null) {
+        requestBody.condition_on_previous_chunks =
+          fishAudioOptions.conditionOnPreviousChunks;
+      }
+
+      if (fishAudioOptions.earlyStopThreshold != null) {
+        requestBody.early_stop_threshold = fishAudioOptions.earlyStopThreshold;
+      }
+
+      if (fishAudioOptions.features != null) {
+        requestBody.features = fishAudioOptions.features;
+      }
+    }
+
+    return { requestBody, warnings };
+  }
+
+  async doGenerate(
+    options: Parameters<SpeechModelV3['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<SpeechModelV3['doGenerate']>>> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const { requestBody, warnings } = await this.getArgs(options);
+
+    const {
+      value: audio,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postJsonToApi({
+      url: this.config.url({ path: '/v1/tts', modelId: this.modelId }),
+      // Fish Audio selects the TTS model with a `model` HTTP header rather
+      // than a request body field.
+      headers: combineHeaders(
+        this.config.headers?.(),
+        { model: this.modelId },
+        options.headers,
+      ),
+      body: requestBody,
+      failedResponseHandler: fishAudioFailedResponseHandler,
+      successfulResponseHandler: createBinaryResponseHandler(),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    return {
+      audio,
+      warnings,
+      request: {
+        body: JSON.stringify(requestBody),
+      },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+    };
+  }
+}

package/src/fish-audio-speech-options.ts

@@ -0,0 +1,21 @@
+/**
+ * Fish Audio TTS model IDs, sent via the `model` HTTP header.
+ *
+ * `s2.1-pro` is the model Fish Audio recommends by default. `s2.1-pro-free` is
+ * a free developer tier with no time-to-first-audio or data-processing
+ * guarantees, so prefer `s2.1-pro` for production use.
+ *
+ * https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
+ */
+export type FishAudioSpeechModelId =
+  | 's1'
+  | 's2-pro'
+  | 's2.1-pro'
+  | 's2.1-pro-free'
+  | (string & {});
+
+/**
+ * A Fish Audio voice model ID (`reference_id`), either from the Fish Audio
+ * voice library or one of your own uploaded models.
+ */
+export type FishAudioSpeechVoiceId = string;

package/src/fish-audio-transcription-model-options.ts

@@ -0,0 +1,28 @@
+import { z } from 'zod/v4';
+
+// https://docs.fish.audio/api-reference/endpoint/openapi-v1/speech-to-text
+export const fishAudioTranscriptionModelOptionsSchema = z.object({
+  /**
+   * Language of the audio.
+   *
+   * A hint only. Fish Audio passes it to the model, but auto-detection is
+   * authoritative and overrides it, so this changes neither the transcript nor
+   * the reported language. The detected language is reported as
+   * `result.language`.
+   */
+  language: z.string().optional(),
+
+  /**
+   * Whether to skip precise timestamps. Mirrors the Fish Audio
+   * `ignore_timestamps` parameter, whose API default is `true`.
+   *
+   * This provider defaults it to `false` so that `result.segments` is
+   * populated. Fish Audio documents an added latency cost for audio shorter
+   * than 30 seconds; set this to `true` to trade segments for that latency.
+   */
+  ignoreTimestamps: z.boolean().optional(),
+});
+
+export type FishAudioTranscriptionModelOptions = z.infer<
+  typeof fishAudioTranscriptionModelOptionsSchema
+>;