@ai-sdk/fish-audio 0.0.0 → 2.0.1

package/docs/190-fish-audio.mdx

@@ -0,0 +1,346 @@
+---
+title: Fish Audio
+description: Learn how to use the Fish Audio provider for the AI SDK.
+---
+
+# Fish Audio Provider
+
+The [Fish Audio](https://fish.audio/) provider contains speech generation (S1 and
+S2 models) and speech-to-text transcription support.
+
+## Setup
+
+The Fish Audio provider is available in the `@ai-sdk/fish-audio` module. You can install it with
+
+<Tabs items={['pnpm', 'npm', 'yarn', 'bun']}>
+  <Tab>
+    <Snippet text="pnpm add @ai-sdk/fish-audio" dark />
+  </Tab>
+  <Tab>
+    <Snippet text="npm install @ai-sdk/fish-audio" dark />
+  </Tab>
+  <Tab>
+    <Snippet text="yarn add @ai-sdk/fish-audio" dark />
+  </Tab>
+
+  <Tab>
+    <Snippet text="bun add @ai-sdk/fish-audio" dark />
+  </Tab>
+</Tabs>
+
+## Provider Instance
+
+You can import the default provider instance `fishAudio` from `@ai-sdk/fish-audio`:
+
+```ts
+import { fishAudio } from '@ai-sdk/fish-audio';
+```
+
+If you need a customized setup, you can import `createFishAudio` from `@ai-sdk/fish-audio` and create a provider instance with your settings:
+
+```ts
+import { createFishAudio } from '@ai-sdk/fish-audio';
+
+const fishAudio = createFishAudio({
+  // custom settings, e.g.
+  fetch: customFetch,
+});
+```
+
+You can use the following optional settings to customize the Fish Audio provider instance:
+
+- **apiKey** _string_
+
+  API key that is being sent using the `Authorization` header.
+  It defaults to the `FISH_AUDIO_API_KEY` environment variable.
+
+- **baseURL** _string_
+
+  Base URL for the API calls.
+  Defaults to `https://api.fish.audio`.
+
+- **headers** _Record&lt;string,string&gt;_
+
+  Custom headers to include in the requests.
+
+- **fetch** _(input: RequestInfo, init?: RequestInit) => Promise&lt;Response&gt;_
+
+  Custom [fetch](https://developer.mozilla.org/en-US/docs/Web/API/fetch) implementation.
+  Defaults to the global `fetch` function.
+  You can use it as a middleware to intercept requests,
+  or to provide a custom fetch implementation for e.g. testing.
+
+## Speech Models
+
+You can create models that call the [Fish Audio text-to-speech API](https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech)
+using the `.speech()` factory method.
+
+```ts
+import { fishAudio } from '@ai-sdk/fish-audio';
+import { generateSpeech } from 'ai';
+
+const { audio } = await generateSpeech({
+  model: fishAudio.speech('s1'),
+  text: 'Hello from Fish Audio!',
+});
+```
+
+The `voice` option selects a Fish Audio voice model ID (`reference_id`), either
+from the [Fish Audio voice library](https://fish.audio/) or one of your own
+uploaded models. Omit it to use the default voice.
+
+```ts
+const { audio } = await generateSpeech({
+  model: fishAudio.speech('s1'),
+  text: 'Hello from Fish Audio!',
+  voice: '933563129e564b19a115bedd57b7406a',
+  outputFormat: 'opus',
+  speed: 1.1,
+});
+```
+
+Voice listing is not part of the AI SDK speech model specification, so browse
+voices with the [Fish Audio list-models
+endpoint](https://docs.fish.audio/api-reference/endpoint/openapi-v1/list-models)
+directly:
+
+```ts
+const response = await fetch(
+  'https://api.fish.audio/model?page_size=20&sort_by=task_count',
+  { headers: { Authorization: `Bearer ${process.env.FISH_AUDIO_API_KEY}` } },
+);
+const { items } = await response.json();
+// Each item's `_id` is a value you can pass as `voice`.
+```
+
+Add `self=true` to list only your own uploaded models, or `language=en` / `tag=narration` to filter.
+
+### Provider Options
+
+The following provider options are available:
+
+- **referenceId** _string | array of strings_
+
+  Voice model ID(s). A single ID selects one speaker; an array enables
+  multi-speaker dialogue (S2-Pro models). Takes precedence over the top-level
+  `voice` option.
+  Optional.
+
+- **sampleRate** _number_
+
+  Output sample rate in Hz. Falls back to the format default when unset
+  (44100 Hz for `wav`/`pcm`/`mp3`, 48000 Hz for `opus`).
+  Optional.
+
+- **mp3Bitrate** _64 | 128 | 192_
+
+  Bitrate in kbps for `mp3` output. Ignored for other formats.
+  Optional.
+
+- **opusBitrate** _-1000 | 24000 | 32000 | 48000 | 64000_
+
+  Bitrate in bps for `opus` output, where `-1000` selects automatic. Ignored for
+  other formats.
+  Optional.
+
+- **latency** _'low' | 'normal' | 'balanced'_
+
+  Latency/quality tradeoff. `normal` gives the best quality, `balanced` reduces
+  latency, and `low` is the fastest.
+  Optional.
+
+- **volume** _number_
+
+  Volume offset in dB. Negative values are quieter.
+  Optional.
+
+- **normalizeLoudness** _boolean_
+
+  Loudness normalization. Supported by the S2 family (`s2-pro` and
+  `s2.1-pro`). Fish Audio accepts it on `s1` but ignores it, so the provider
+  drops it and emits a warning in that case.
+  Optional.
+
+- **temperature** _number_
+
+  Governs expressiveness (0 to 1). Higher values are more varied.
+  Optional.
+
+- **topP** _number_
+
+  Controls diversity via nucleus sampling (0 to 1).
+  Optional.
+
+- **chunkLength** _number_
+
+  Text segment size for processing (100 to 300).
+  Optional.
+
+- **minChunkLength** _number_
+
+  Minimum characters before splitting into a new chunk (0 to 100).
+  Optional.
+
+- **normalize** _boolean_
+
+  Text normalization for English and Chinese. Helps stability with numbers.
+  Optional.
+
+- **maxNewTokens** _number_
+
+  Maximum audio tokens to generate per text chunk.
+  Optional.
+
+- **repetitionPenalty** _number_
+
+  Values above 1.0 discourage repeated audio patterns.
+  Optional.
+
+- **conditionOnPreviousChunks** _boolean_
+
+  Reuse prior audio as context for voice consistency across chunks.
+  Optional.
+
+- **earlyStopThreshold** _number_
+
+  Early-stop threshold used in batch processing (0 to 1).
+  Optional.
+
+- **features** _array of strings_
+
+  Request-scoped flags passed through to the inference backend, e.g.
+  `['quality-guard']`.
+  Optional.
+
+### Multi-Speaker Dialogue
+
+S2-Pro models support multi-speaker dialogue. Pass an array of voice model IDs
+via `referenceId` and mark turns in the text with `<|speaker:N|>`, where `N`
+indexes into that array.
+
+```ts
+const { audio } = await generateSpeech({
+  model: fishAudio.speech('s2-pro'),
+  text: '<|speaker:0|>Hello!<|speaker:1|>Hi there!',
+  providerOptions: {
+    fishAudio: {
+      referenceId: [
+        '933563129e564b19a115bedd57b7406a',
+        'bf322df2096a46f18c579d0baa36f41d',
+      ],
+    },
+  },
+});
+```
+
+### Output Formats
+
+Fish Audio supports the `wav`, `pcm`, `mp3`, and `opus` output formats. Any other
+value falls back to `mp3` and produces a warning.
+
+<Note>
+  Fish Audio infers the language from the input text and the selected voice, and
+  has no language parameter. The `language` and `instructions` options are not
+  supported and produce warnings.
+</Note>
+
+### Model Capabilities
+
+| Model           | Multi-Speaker       | Notes                                                                     |
+| --------------- | ------------------- | ------------------------------------------------------------------------- |
+| `s1`            |                     | Ignores `normalizeLoudness`                                               |
+| `s2-pro`        | <Check size={18} /> | Supports `normalizeLoudness`                                              |
+| `s2.1-pro`      | <Check size={18} /> | Recommended default; supports `normalizeLoudness`                         |
+| `s2.1-pro-free` |                     | Free developer tier; no time-to-first-audio or data-processing guarantees |
+
+<Note>
+  Streaming text-to-speech (Fish Audio's TTS-live WebSocket and timestamped
+  streaming endpoints) is not currently supported, and neither is inline
+  zero-shot voice cloning via `references`, which requires a MessagePack request
+  body. Upload reference audio to Fish Audio and pass its `reference_id` via
+  `voice` or `referenceId` instead.
+</Note>
+
+## Transcription Models
+
+You can create models that call the [Fish Audio speech-to-text API](https://docs.fish.audio/api-reference/endpoint/openapi-v1/speech-to-text)
+using the `.transcription()` factory method.
+
+```ts
+import { fishAudio } from '@ai-sdk/fish-audio';
+import { transcribe } from 'ai';
+import { readFile } from 'node:fs/promises';
+
+const result = await transcribe({
+  model: fishAudio.transcription(),
+  audio: await readFile('audio.mp3'),
+});
+```
+
+The Fish Audio speech-to-text endpoint currently exposes no model selector and
+serves a single model, so the model ID is optional and defaults to
+`'transcribe-1'`. It is a routing label and is not sent to the API. Fish Audio
+expects to add more ASR models and to select them with the `model` HTTP header,
+matching the text-to-speech endpoint.
+
+### Provider Options
+
+The following provider options are available:
+
+- **language** _string_
+
+  Language of the audio. A hint only: Fish Audio passes it to the model, but
+  auto-detection is authoritative and overrides it, so it changes neither the
+  transcript nor the reported language.
+  Optional.
+
+- **ignoreTimestamps** _boolean_
+
+  Whether to skip precise timestamps. Mirrors the Fish Audio
+  `ignore_timestamps` parameter, whose API default is `true`. This provider
+  defaults it to `false` so that `segments` is populated. Fish Audio documents
+  an added latency cost for audio shorter than 30 seconds; set this to `true` to
+  trade segments for that latency.
+  Optional.
+
+```ts
+const result = await transcribe({
+  model: fishAudio.transcription(),
+  audio: await readFile('audio.mp3'),
+  providerOptions: {
+    fishAudio: {
+      language: 'en',
+      ignoreTimestamps: false,
+    },
+  },
+});
+```
+
+`result.language` reports the detected language as an ISO-639-1 code (e.g. `en`).
+It is always a two-letter code, never a locale such as `en-US`, and is
+`undefined` when Fish Audio detects no language.
+
+The human-readable language name (e.g. `English`) is available as provider
+metadata:
+
+```ts
+console.log(result.language); // 'en'
+console.log(result.providerMetadata?.fishAudio?.language); // 'English'
+```
+
+<Note>
+  The provider metadata `language` is a display name intended for presentation.
+  Its exact form is not guaranteed, so avoid matching or branching on it — use
+  `result.language` for anything programmatic.
+</Note>
+
+<Note>
+  Setting `ignoreTimestamps` to `true` makes Fish Audio return an empty
+  `segments` array. The provider therefore requests timestamps by default.
+</Note>
+
+### Model Capabilities
+
+| Model          | Transcription       | Duration            | Segments            | Language            |
+| -------------- | ------------------- | ------------------- | ------------------- | ------------------- |
+| `transcribe-1` | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> | <Check size={18} /> |

package/package.json

@@ -1,20 +1,76 @@
 {
   "name": "@ai-sdk/fish-audio",
-  "version": "0.0.0",
-  "description": "AI SDK provider for Fish Audio",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "Apache-2.0",
+  "version": "2.0.1",
   "type": "module",
-  "publishConfig": {
-    "access": "public"
-  },
+  "license": "Apache-2.0",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "files": [
-    "index.js",
+    "dist/**/*",
+    "docs/**/*",
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.test-d.ts",
+    "!src/**/__snapshots__",
+    "!src/**/__fixtures__",
+    "CHANGELOG.md",
     "README.md"
-  ]
-}
+  ],
+  "directories": {
+    "doc": "./docs"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@ai-sdk/provider": "3.0.14",
+    "@ai-sdk/provider-utils": "4.0.42"
+  },
+  "devDependencies": {
+    "@types/node": "20.17.24",
+    "tsup": "^8",
+    "typescript": "5.6.3",
+    "zod": "3.25.76",
+    "@ai-sdk/test-server": "1.0.6",
+    "@vercel/ai-tsconfig": "0.0.0"
+  },
+  "peerDependencies": {
+    "zod": "^3.25.76 || ^4.1.8"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "homepage": "https://ai-sdk.dev/docs",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vercel/ai",
+    "directory": "packages/fish-audio"
+  },
+  "bugs": {
+    "url": "https://github.com/vercel/ai/issues"
+  },
+  "keywords": [
+    "ai"
+  ],
+  "scripts": {
+    "build": "tsup --tsconfig tsconfig.build.json",
+    "build:watch": "tsup --tsconfig tsconfig.build.json --watch",
+    "clean": "del-cli dist docs",
+    "type-check": "tsc --noEmit",
+    "test": "pnpm test:node && pnpm test:edge",
+    "test:update": "pnpm test:node -u",
+    "test:watch": "vitest --config vitest.node.config.js",
+    "test:edge": "vitest --config vitest.edge.config.js --run",
+    "test:node": "vitest --config vitest.node.config.js --run"
+  }
+}

package/src/fish-audio-config.ts

@@ -0,0 +1,8 @@
+import type { FetchFunction } from '@ai-sdk/provider-utils';
+
+export type FishAudioConfig = {
+  provider: string;
+  url: (options: { modelId: string; path: string }) => string;
+  headers?: () => Record<string, string | undefined>;
+  fetch?: FetchFunction;
+};

package/src/fish-audio-error.ts

@@ -0,0 +1,17 @@
+import { createJsonErrorResponseHandler } from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+
+// Fish Audio returns `{ status, message }` for documented error responses
+// (401 no permission, 402 no payment).
+// https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
+export const fishAudioErrorDataSchema = z.object({
+  status: z.number().nullish(),
+  message: z.string().nullish(),
+});
+
+export type FishAudioErrorData = z.infer<typeof fishAudioErrorDataSchema>;
+
+export const fishAudioFailedResponseHandler = createJsonErrorResponseHandler({
+  errorSchema: fishAudioErrorDataSchema,
+  errorToMessage: data => data.message ?? 'Unknown Fish Audio error',
+});

package/src/fish-audio-provider.ts

@@ -0,0 +1,163 @@
+import {
+  NoSuchModelError,
+  type ProviderV3,
+  type SpeechModelV3,
+  type TranscriptionModelV3,
+} from '@ai-sdk/provider';
+import {
+  loadApiKey,
+  withUserAgentSuffix,
+  type FetchFunction,
+} from '@ai-sdk/provider-utils';
+import { FishAudioSpeechModel } from './fish-audio-speech-model';
+import type { FishAudioSpeechModelId } from './fish-audio-speech-options';
+import { FishAudioTranscriptionModel } from './fish-audio-transcription-model';
+import type { FishAudioTranscriptionModelId } from './fish-audio-transcription-options';
+import { VERSION } from './version';
+
+export interface FishAudioProvider extends ProviderV3 {
+  (
+    modelId: FishAudioSpeechModelId,
+    settings?: {},
+  ): {
+    speech: FishAudioSpeechModel;
+  };
+
+  /**
+   * Creates a model for speech generation.
+   */
+  speech(modelId: FishAudioSpeechModelId): SpeechModelV3;
+
+  /**
+   * Creates a model for speech generation.
+   *
+   * Narrowed to required: Fish Audio always provides speech models.
+   */
+  speechModel(modelId: FishAudioSpeechModelId): SpeechModelV3;
+
+  /**
+   * Creates a model for transcription.
+   */
+  transcription(modelId?: FishAudioTranscriptionModelId): TranscriptionModelV3;
+
+  /**
+   * Creates a model for transcription.
+   *
+   * Narrowed to required: Fish Audio always provides a transcription model.
+   */
+  transcriptionModel(
+    modelId?: FishAudioTranscriptionModelId,
+  ): TranscriptionModelV3;
+}
+
+export interface FishAudioProviderSettings {
+  /**
+   * API key for authenticating requests.
+   */
+  apiKey?: string;
+
+  /**
+   * Base URL for the API calls.
+   */
+  baseURL?: string;
+
+  /**
+   * Custom headers to include in the requests.
+   */
+  headers?: Record<string, string>;
+
+  /**
+   * Custom fetch implementation. You can use it as a middleware to intercept requests,
+   * or to provide a custom fetch implementation for e.g. testing.
+   */
+  fetch?: FetchFunction;
+}
+
+const DEFAULT_BASE_URL = 'https://api.fish.audio';
+
+/**
+ * Create a Fish Audio provider instance.
+ */
+export function createFishAudio(
+  options: FishAudioProviderSettings = {},
+): FishAudioProvider {
+  const baseURL = options.baseURL?.replace(/\/$/, '') ?? DEFAULT_BASE_URL;
+
+  const getHeaders = () =>
+    withUserAgentSuffix(
+      {
+        Authorization: `Bearer ${loadApiKey({
+          apiKey: options.apiKey,
+          environmentVariableName: 'FISH_AUDIO_API_KEY',
+          description: 'Fish Audio',
+        })}`,
+        ...options.headers,
+      },
+      `ai-sdk/fish-audio/${VERSION}`,
+    );
+
+  const createSpeechModel = (modelId: FishAudioSpeechModelId) =>
+    new FishAudioSpeechModel(modelId, {
+      provider: 'fish-audio.speech',
+      url: ({ path }) => `${baseURL}${path}`,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  // `/v1/asr` has no model selector, so the model ID is a routing label and
+  // defaults to `transcribe-1`.
+  const createTranscriptionModel = (
+    modelId: FishAudioTranscriptionModelId = 'transcribe-1',
+  ) =>
+    new FishAudioTranscriptionModel(modelId, {
+      provider: 'fish-audio.transcription',
+      url: ({ path }) => `${baseURL}${path}`,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  const provider = function (modelId: FishAudioSpeechModelId) {
+    return {
+      speech: createSpeechModel(modelId),
+    };
+  };
+
+  provider.specificationVersion = 'v3' as const;
+  provider.speech = createSpeechModel;
+  provider.speechModel = createSpeechModel;
+  provider.transcription = createTranscriptionModel;
+  provider.transcriptionModel = createTranscriptionModel;
+
+  // Required ProviderV3 methods that are not supported
+  provider.languageModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'languageModel',
+      message: 'Fish Audio does not provide language models',
+    });
+  };
+
+  provider.embeddingModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'embeddingModel',
+      message: 'Fish Audio does not provide embedding models',
+    });
+  };
+  provider.textEmbeddingModel = provider.embeddingModel;
+
+  provider.imageModel = (modelId: string) => {
+    throw new NoSuchModelError({
+      modelId,
+      modelType: 'imageModel',
+      message: 'Fish Audio does not provide image models',
+    });
+  };
+
+  return provider as FishAudioProvider;
+}
+
+/**
+ * Default Fish Audio provider instance.
+ */
+export const fishAudio = createFishAudio();