@tanstack/ai-byteplus 0.0.0

package/src/audio/transcription-provider-options.ts

@@ -0,0 +1,46 @@
+/**
+ * Provider-specific options for BytePlus Seed Speech ASR
+ * (`POST /api/v3/auc/bigmodel/recognize/flash`).
+ *
+ * Field names mirror the wire format: everything except `uid` and
+ * `audio_format` is forwarded inside the request's `request` block.
+ */
+export interface BytePlusTranscriptionProviderOptions {
+  /**
+   * Recognition model family. Defaults to `bigmodel`; the specific model is
+   * selected by the `X-Api-Resource-Id` header rather than by this value.
+   */
+  model_name?: string
+  /**
+   * Container hint for the submitted audio (`mp3`, `wav`, `ogg`, …). Only
+   * needed when the format can't be inferred from the input — a `File`'s name
+   * or MIME type, or a data URL's MIME type, is used automatically.
+   */
+  audio_format?: string
+  /** Inverse text normalisation: render spoken numbers, dates etc. as digits. */
+  enable_itn?: boolean
+  /** Insert punctuation into the transcript. */
+  enable_punc?: boolean
+  /** Disfluency removal — drop fillers and stutters. */
+  enable_ddc?: boolean
+  /**
+   * Attach speaker labels to each utterance. When present they are surfaced
+   * as `segment.speaker` on the result.
+   */
+  enable_speaker_info?: boolean
+  /**
+   * Return the per-utterance breakdown as well as the flat transcript.
+   * Defaults to `true` so `segments` and `words` are populated.
+   */
+  show_utterances?: boolean
+  /**
+   * Spoken-language hint (e.g. `en-US`). Overrides the cross-provider
+   * `TranscriptionOptions.language`.
+   */
+  language?: string
+  /**
+   * Caller identifier echoed back in BytePlus' request logs. Useful for
+   * correlating usage; defaults to `tanstack-ai`.
+   */
+  uid?: string
+}

package/src/audio/tts-provider-options.ts

@@ -0,0 +1,122 @@
+import type {
+  BytePlusTTSAudioFormat,
+  BytePlusTTSReference,
+  BytePlusTTSSampleRate,
+  BytePlusTTSSubtitle,
+} from './wire-types'
+import type { TTSResult } from '@tanstack/ai'
+
+/**
+ * Seed Speech voice identifier (`speaker` on the wire).
+ *
+ * Voice ids encode language, gender, character name and model generation:
+ * `en_female_stokie_uranus_bigtts` is the English female "Stokie" voice on
+ * TTS 2.0. The generation suffix matters when picking one:
+ *
+ * - `_uranus_bigtts` — TTS 2.0 voices (the current generation).
+ * - `_mars_bigtts` / `_moon_bigtts` — TTS 1.0 voices.
+ * - `*_emo_v2_*` — TTS 1.0 voices that additionally accept emotion tags.
+ *
+ * The full roster lives at
+ * https://docs.byteplus.com/en/docs/byteplusvoice/voicelist and changes far
+ * more often than this package ships, so the union stays open: any string is
+ * accepted, and the one listed id is the adapter's default.
+ */
+export type BytePlusTTSVoice = 'en_female_stokie_uranus_bigtts' | (string & {})
+
+/**
+ * Provider-specific options for BytePlus Seed Speech TTS
+ * (`POST /api/v3/tts/create`).
+ *
+ * These map 1:1 onto the wire fields so the BytePlus documentation stays
+ * useful; where a cross-provider `TTSOptions` field covers the same ground
+ * (`voice`, `format`, `speed`), the option here wins.
+ */
+export interface BytePlusTTSProviderOptions {
+  /**
+   * Voice id. Overrides `TTSOptions.voice` when both are set. It is sent as
+   * the `speaker` of a single `references` entry.
+   */
+  speaker?: BytePlusTTSVoice
+  /**
+   * Full `references` array, for voice cloning or image-referenced delivery.
+   * When set this replaces the entry the adapter would otherwise build from
+   * `speaker` / `TTSOptions.voice`, so include a `speaker` entry yourself if
+   * you still want a stock voice. Address audio references from the text with
+   * `@Audio1`..`@Audio3`.
+   */
+  references?: Array<BytePlusTTSReference>
+  /**
+   * Output format. Overrides the mapping applied to `TTSOptions.format`,
+   * which is useful for `ogg_opus` and for pinning `pcm` explicitly.
+   */
+  format?: BytePlusTTSAudioFormat
+  /**
+   * Output sample rate in Hz. Defaults to 24000 — the adapter always sends an
+   * explicit rate because the documented server default (40000) is not one of
+   * the values the endpoint accepts. For `pcm` output this is also what the
+   * returned `contentType` (`audio/L16;rate=…`) reports.
+   */
+  sample_rate?: BytePlusTTSSampleRate
+  /**
+   * Pitch adjustment in the range `-12`..`12`, where `0` is the voice's
+   * natural pitch.
+   */
+  pitch_rate?: number
+  /**
+   * Speaking rate in the range `-50`..`100` (`-50` = 0.5×, `0` = 1×,
+   * `100` = 2×). Overrides the value derived from `TTSOptions.speed`.
+   */
+  speech_rate?: number
+  /**
+   * Loudness adjustment in the range `-50`..`100`, where `0` is the voice's
+   * natural level.
+   */
+  loudness_rate?: number
+  /**
+   * Ask for sentence- and word-level timings alongside the audio. They are
+   * surfaced on {@link BytePlusTTSResult.subtitle}.
+   */
+  enable_subtitle?: boolean
+  /**
+   * Watermark the generated audio. The field name is confirmed against the
+   * endpoint schema; the boolean type is assumed and unprobed.
+   */
+  watermark?: boolean
+}
+
+/**
+ * BytePlus-specific extension of `TTSResult`.
+ *
+ * The cross-provider `TTSResult` has nowhere to put the subtitle timings or
+ * the temporary download URL, so callers who want them narrow the result:
+ *
+ * ```ts
+ * const result: BytePlusTTSResult = await generateSpeech({ adapter, text })
+ * for (const sentence of result.subtitle?.sentences ?? []) {
+ *   console.log(sentence.text, sentence.start_time)
+ * }
+ * ```
+ */
+export interface BytePlusTTSResult extends TTSResult {
+  /**
+   * Sentence and word timings, present only when
+   * `modelOptions.enable_subtitle` was set. Their `start_time` / `end_time`
+   * are **milliseconds**, even though `duration` and
+   * {@link BytePlusTTSResult.originalDuration} are seconds.
+   */
+  subtitle?: BytePlusTTSSubtitle
+  /**
+   * Length of the audio in seconds *before* `speech_rate` was applied. This
+   * is what BytePlus bills on and what the 120 s cap applies to, so it is the
+   * number to meter against — `duration` reflects the delivered clip and can
+   * exceed 120 s when the speech is slowed down.
+   */
+  originalDuration?: number
+  /**
+   * Temporary download URL for the same audio that `audio` carries as base64.
+   * **Expires roughly 2 hours after generation** — persist the bytes, not the
+   * link.
+   */
+  url?: string
+}

package/src/audio/wire-types.ts

@@ -0,0 +1,290 @@
+/**
+ * Minimal wire types for the BytePlus **Seed Speech** HTTP API (TTS + ASR).
+ *
+ * Seed Speech is a separate product from Ark: it lives on
+ * `voice.ap-southeast-1.bytepluses.com`, authenticates with `X-Api-Key`
+ * (a different key from `ARK_API_KEY`), and returns a flat numeric error
+ * envelope instead of Ark's OpenAI-shaped one.
+ *
+ * Only the fields the adapters read or write are modelled here — this is a
+ * hand-written subset, not a generated schema.
+ *
+ * Provenance:
+ * - Endpoints, auth header, format/rate ranges and the 120 s TTS output cap:
+ *   BytePlus Seed Speech docs (`docs.byteplus.com/en/docs/byteplusvoice`),
+ *   captured in the Phase 0 research notes.
+ * - Error envelope `{code, message}`: verified live — an Ark key sent as
+ *   `X-Api-Key` returns HTTP 401 `{"code":45000010,"message":"Invalid X-Api-Key"}`.
+ * - ASR request/response shape (`user`/`audio`/`request` in, `audio_info` +
+ *   `result.utterances` out, all timings in **milliseconds**): the Volcengine
+ *   flash-recognition reference the BytePlus endpoint is derived from
+ *   (`docs.volcengine.com/docs/6561/1631584`).
+ *
+ * No Seed Speech API key was available when these were written, so the TTS
+ * response fields are documented-but-unverified; the adapters parse them
+ * defensively rather than assuming they are always present.
+ */
+
+// ============================================================================
+// TTS — POST /api/v3/tts/create
+// ============================================================================
+
+/** Output container/codec accepted by `audio_config.format`. */
+export type BytePlusTTSAudioFormat = 'wav' | 'mp3' | 'pcm' | 'ogg_opus'
+
+/**
+ * Sample rates `audio_config.sample_rate` accepts.
+ *
+ * The docs also state a *default* of 40000, which is not one of the valid
+ * values — a documentation bug. The adapter therefore always sends an
+ * explicit rate rather than relying on the server default.
+ */
+export const BYTEPLUS_TTS_SAMPLE_RATES = [
+  8000, 16000, 24000, 32000, 44100, 48000,
+] as const
+
+/** A sample rate `audio_config.sample_rate` accepts. */
+export type BytePlusTTSSampleRate = (typeof BYTEPLUS_TTS_SAMPLE_RATES)[number]
+
+/**
+ * One entry of the request's `references` array.
+ *
+ * This is where the voice lives: `speaker` names a stock voice, while
+ * `audio_url` / `audio_data` supply reference clips to clone (the `@Audio1`..
+ * `@Audio3` markers in `text_prompt` address them positionally). Exactly one
+ * of `speaker`, `audio_url` or `audio_data` may be set per entry; at most 3
+ * audio references (30 s / 10 MB each) and 1 image reference (10 MB), and
+ * image references are mutually exclusive with audio ones.
+ *
+ * **Member object shape is unresolved — must live-probe when the Seed Speech
+ * key lands.** The docs list the member fields flat (`speaker | audio_data |
+ * audio_url | image_data | image_url`) without a worked example, so whether
+ * the server wants a flat `{ speaker }` or a discriminated `{ type, ... }`
+ * could not be settled. The adapter sends the flat reading — see
+ * `buildTTSRequestBody` in `../adapters/tts`.
+ */
+export interface BytePlusTTSReference {
+  /** Stock voice id, e.g. `en_female_stokie_uranus_bigtts`. */
+  speaker?: string
+  /** URL of a reference clip to clone (≤30 s, ≤10 MB). */
+  audio_url?: string
+  /** Base64 reference clip to clone (≤30 s, ≤10 MB). */
+  audio_data?: string
+  /** URL of a reference image (≤10 MB). Mutually exclusive with audio refs. */
+  image_url?: string
+  /** Base64 reference image (≤10 MB). Mutually exclusive with audio refs. */
+  image_data?: string
+}
+
+/**
+ * `audio_config` block of a TTS request — exactly six fields.
+ *
+ * The three `*_rate` fields are integer percentages relative to the voice's
+ * neutral delivery, not multipliers.
+ */
+export interface BytePlusTTSAudioConfig {
+  /** Output format. Defaults to `wav` server-side. */
+  format?: BytePlusTTSAudioFormat
+  /**
+   * Output sample rate in Hz. See {@link BYTEPLUS_TTS_SAMPLE_RATES} for the
+   * valid values and for why the adapter always sends one.
+   */
+  sample_rate?: number
+  /** Speaking rate, `-50`..`100`. `-50` = 0.5×, `0` = 1×, `100` = 2×. */
+  speech_rate?: number
+  /** Loudness adjustment, `-50`..`100`. `0` is the voice's natural level. */
+  loudness_rate?: number
+  /** Pitch adjustment, `-12`..`12`. `0` is the voice's natural pitch. */
+  pitch_rate?: number
+  /** Emit sentence and word timings in the response. Defaults to `false`. */
+  enable_subtitle?: boolean
+}
+
+/** Request body for `POST /api/v3/tts/create` — exactly five fields. */
+export interface BytePlusTTSCreateRequest {
+  /** Seed Speech synthesis model, e.g. `seed-audio-1.0`. */
+  model: string
+  /**
+   * The text to speak (≤3000 chars). Dual-purpose: either literal text or a
+   * natural-language description of the delivery, and the place the
+   * `@Audio1`..`@Audio3` reference markers go.
+   *
+   * **`text_prompt` is correct — do not "fix" this to `text`.** This endpoint
+   * has no request-side `text` field. The `text` spelling belongs to the
+   * *other* endpoint, `/tts/unidirectional` (TTS 2.0), where it sits under
+   * `req_params.text`. Confirmed against
+   * docs.byteplus.com/en/docs/byteplusvoice/seedaudio-01, which lists this
+   * body as exactly `model`, `text_prompt`, `references`, `audio_config`,
+   * `watermark`.
+   */
+  text_prompt: string
+  /** Voice selection and cloning references. See {@link BytePlusTTSReference}. */
+  references?: Array<BytePlusTTSReference>
+  audio_config?: BytePlusTTSAudioConfig
+  /**
+   * Watermark the generated audio. The field name is confirmed; the boolean
+   * type is assumed by analogy with Seedream's `watermark` and unprobed.
+   */
+  watermark?: boolean
+}
+
+/**
+ * One timed entry of a TTS subtitle track.
+ *
+ * **Times are milliseconds** — unlike the response's `duration` fields, which
+ * are seconds. The endpoint genuinely mixes units.
+ */
+export interface BytePlusTTSSubtitleEntry {
+  text?: string
+  start_time?: number
+  end_time?: number
+}
+
+/** Sentence- and word-level timings returned when `enable_subtitle` is set. */
+export interface BytePlusTTSSubtitle {
+  sentences?: Array<BytePlusTTSSubtitleEntry>
+  words?: Array<BytePlusTTSSubtitleEntry>
+}
+
+/** Response body for `POST /api/v3/tts/create`. */
+export interface BytePlusTTSCreateResponse {
+  /**
+   * Status code — `0` on success, a flat error code otherwise.
+   *
+   * Typed as `number | string` because only the *error* envelope was verified
+   * live (HTTP 401 `{"code":45000010,…}`); the success envelope's shape is
+   * docs-derived and no voice key was available to confirm it. Treating a
+   * string code as "not a failure" would return a failed 200 as success, so
+   * the adapter coerces before comparing — see `isZeroCode` in
+   * `adapters/tts.ts`.
+   */
+  code?: number | string
+  message?: string
+  /** Base64-encoded audio in the requested `audio_config.format`. */
+  audio?: string
+  /**
+   * Length of the delivered audio in **seconds** (float), after `speech_rate`
+   * is applied. This can legitimately exceed 120 when the clip is slowed
+   * down — the 120 s cap applies to {@link BytePlusTTSCreateResponse.original_duration}.
+   */
+  duration?: number | string
+  /**
+   * Length in **seconds** (float) before rate adjustment. This is the billing
+   * basis and is capped at 120.
+   */
+  original_duration?: number | string
+  /** Temporary download URL for the same audio. Expires after ~2 hours. */
+  url?: string
+  /** Sentence and word timings, present when `enable_subtitle` was set. */
+  subtitle?: BytePlusTTSSubtitle
+}
+
+// ============================================================================
+// ASR — POST /api/v3/auc/bigmodel/recognize/flash
+// ============================================================================
+
+/**
+ * Value of the `X-Api-Resource-Id` header that selects the Seed ASR turbo
+ * model. The flash endpoint takes no `model` field in its body — the model is
+ * chosen entirely by this header.
+ */
+export const BYTEPLUS_ASR_RESOURCE_ID = 'volc.seedasr.auc_turbo'
+
+/** Header name carrying {@link BYTEPLUS_ASR_RESOURCE_ID}. */
+export const BYTEPLUS_ASR_RESOURCE_HEADER = 'X-Api-Resource-Id'
+
+/**
+ * Audio input. Exactly one of `url` or `data` is sent — the endpoint accepts
+ * files up to 2 hours long / 100 MB.
+ */
+export interface BytePlusASRAudio {
+  /** Publicly reachable URL of the audio file. */
+  url?: string
+  /** Base64-encoded audio bytes. */
+  data?: string
+  /** Container hint, e.g. `mp3`, `wav`, `ogg`. */
+  format?: string
+}
+
+/** `request` block of a recognition call. */
+export interface BytePlusASRRequestOptions {
+  /** Recognition model family. Defaults to `bigmodel`. */
+  model_name?: string
+  /** Inverse text normalisation (spoken numbers → digits). */
+  enable_itn?: boolean
+  /** Insert punctuation. */
+  enable_punc?: boolean
+  /** Disfluency removal ("um", repeated words). */
+  enable_ddc?: boolean
+  /** Attach per-utterance speaker labels. */
+  enable_speaker_info?: boolean
+  /** Return the `utterances` breakdown as well as the flat transcript. */
+  show_utterances?: boolean
+  /** Spoken language hint, e.g. `en-US`. */
+  language?: string
+}
+
+/** Request body for `POST /api/v3/auc/bigmodel/recognize/flash`. */
+export interface BytePlusASRRecognizeRequest {
+  user?: { uid?: string }
+  audio: BytePlusASRAudio
+  request?: BytePlusASRRequestOptions
+}
+
+/** One recognised word. `start_time` / `end_time` are milliseconds. */
+export interface BytePlusASRWord {
+  text?: string
+  start_time?: number
+  end_time?: number
+  confidence?: number
+}
+
+/** One recognised utterance. `start_time` / `end_time` are milliseconds. */
+export interface BytePlusASRUtterance {
+  text?: string
+  start_time?: number
+  end_time?: number
+  words?: Array<BytePlusASRWord>
+  /**
+   * Extra per-utterance annotations. Speaker labels arrive here when
+   * `enable_speaker_info` is set; the exact key is read defensively because it
+   * could not be confirmed against a live response.
+   */
+  additions?: Record<string, string>
+}
+
+export interface BytePlusASRResult {
+  text?: string
+  utterances?: Array<BytePlusASRUtterance>
+}
+
+/**
+ * Response body for `POST /api/v3/auc/bigmodel/recognize/flash`.
+ *
+ * The Volcengine-lineage wire shape nests everything under `result`; BytePlus'
+ * prose docs describe the same payload as "transcript + utterances", so the
+ * flat spelling is tolerated as a fallback.
+ */
+export interface BytePlusASRRecognizeResponse {
+  /** `duration` is the audio length in **milliseconds**. */
+  audio_info?: { duration?: number }
+  result?: BytePlusASRResult
+  /** Flat alias for `result.text`. */
+  transcript?: string
+  /** Flat alias for `result.utterances`. */
+  utterances?: Array<BytePlusASRUtterance>
+}
+
+// ============================================================================
+// Errors
+// ============================================================================
+
+/**
+ * Seed Speech error envelope: a flat numeric `code` plus a `message`, e.g.
+ * `{"code": 45000010, "message": "Invalid X-Api-Key"}` (verified live on a
+ * 401). Format it with `bytePlusVoiceError` from `../utils/client`.
+ */
+export interface BytePlusVoiceErrorBody {
+  code?: number
+  message?: string
+}