@tanstack/ai 0.55.0 → 0.57.0

package/skills/ai-core/media-generation/SKILL.md

@@ -6,7 +6,9 @@ description: >
   generateAudio() with geminiAudio/falAudio, generateVideo() with async
   polling (openaiVideo/geminiVideo/grokVideo/falVideo/byteplusVideo/openRouterVideo,
   per-model typed durations), generateSpeech() with openaiSpeech/byteplusSpeech/elevenlabsSpeech,
-  generateTranscription() with openaiTranscription/byteplusTranscription. React hooks:
+  generateTranscription() with openaiTranscription/byteplusTranscription,
+  generateVoice() with elevenlabsVoiceDesign (create a voice, then speak with it).
+  React hooks:
   useGenerateImage, useGenerateAudio,
   useGenerateSpeech, useTranscription, useGenerateVideo.
   TanStack Start server function integration with toServerSentEventsResponse.
@@ -21,6 +23,7 @@ sources:
   - 'TanStack/ai:docs/media/video-generation.md'
   - 'TanStack/ai:docs/media/text-to-speech.md'
   - 'TanStack/ai:docs/adapters/elevenlabs.md'
+  - 'TanStack/ai:docs/media/voice-creation.md'
   - 'TanStack/ai:docs/media/transcription.md'
   - 'TanStack/ai:docs/advanced/debug-logging.md'
 ---
@@ -406,7 +409,126 @@ const { generate, result, isLoading } = useGenerateSpeech({
 // Play:   <audio src={`data:audio/${result.format};base64,${result.audio}`} controls />
 ```
 
-### 4. Audio Transcription
+**Dialogue (`turns`) and timings (`timestamps`).** `text` + `voice` is one
+speaker. For a multi-voice script pass `turns` instead of `text` (they are
+mutually exclusive), and set `timestamps: true` to get `result.alignment`
+(per character or per word, `alignment.unit` says which) and `result.segments`
+(one per turn or per sentence). All times are seconds.
+
+```typescript
+import { generateSpeech } from '@tanstack/ai'
+import { byteplusSpeech } from '@tanstack/ai-byteplus'
+
+// Second voice id comes from the BytePlus voice list.
+const SECOND_VOICE = 'your-second-voice-id'
+
+const result = await generateSpeech({
+  adapter: byteplusSpeech('seed-audio-1.0'),
+  turns: [
+    { text: 'Do you sell picks?', voice: 'en_female_stokie_uranus_bigtts' },
+    { text: 'By the till.', voice: SECOND_VOICE },
+  ],
+  timestamps: true,
+})
+
+result.alignment?.endSeconds.at(-1) // where speech stops, not where the file does
+result.segments?.[0] // { startSeconds, endSeconds, turnIndex?, voice?, text? }
+```
+
+Both are adapter capabilities, not universal. The activity rejects the request
+before it reaches the provider when the adapter cannot do it, so read
+`adapter.capabilities` rather than guessing:
+
+| Adapter                 | `maxSpeakers` | `timestamps`                                     |
+| ----------------------- | ------------- | ------------------------------------------------ |
+| `byteplusSpeech`        | 3             | yes (`enable_subtitle`, word + sentence)         |
+| `elevenlabsSpeech`      | 10            | yes (character, plus voice segments on dialogue) |
+| `geminiSpeech`          | 2             | no                                               |
+| every other TTS adapter | not supported | no                                               |
+
+### 4. Voice Creation
+
+Adapter: `elevenlabsVoiceDesign` (`eleven_ttv_v3`, `eleven_multilingual_ttv_v2`).
+
+`generateVoice()` makes a voice that does not exist in any catalog, either
+from a text description or from a clip of a real speaker. It returns voice ids
+you pass straight back to `generateSpeech()` as `voice`.
+
+> Pass `prompt`, or `referenceAudio`, or both — the activity throws when
+> neither is given. ElevenLabs always needs `prompt`, because its design
+> endpoint requires a description, and only `eleven_ttv_v3` accepts
+> `referenceAudio`. Without a `name` you get **previews**, which expire;
+> with a `name` the best candidate is kept in the provider's voice library.
+> Check `saved` on each returned voice rather than assuming. Remote audio
+> URLs are rejected: read the file and pass bytes.
+
+```typescript
+import { generateSpeech, generateVoice } from '@tanstack/ai'
+import {
+  elevenlabsSpeech,
+  elevenlabsVoiceDesign,
+} from '@tanstack/ai-elevenlabs'
+
+const designed = await generateVoice({
+  adapter: elevenlabsVoiceDesign('eleven_ttv_v3'),
+  prompt: 'A warm, gravelly narrator in his sixties with a slight Irish lilt',
+  name: 'Irish Narrator', // omit to audition previews instead
+})
+
+const [voice] = designed.voices
+if (!voice) throw new Error('The provider returned no voices.')
+
+// voice.voiceId  -> pass to generateSpeech()
+// voice.audio    -> base64 preview, when the provider returns one
+// voice.saved    -> true only when it is in the provider's library
+// voice.status   -> 'ready' on every adapter today
+
+const speech = await generateSpeech({
+  adapter: elevenlabsSpeech('eleven_v3'),
+  text: 'Once upon a time...',
+  voice: voice.voiceId,
+})
+```
+
+**Status.** Every returned voice carries `status`. It is `'ready'` on every
+adapter today, because they all finish the voice before returning. The
+`'training'` and `'failed'` members exist for providers that build a voice
+asynchronously; no adapter returns them yet, so do not write polling code
+against them.
+
+**Finding voices again.** `generateVoice()` hands back an id you are expected
+to store. `listVoices({ adapter: <a TTS adapter>, origins })` reads the
+account catalog back when you did not.
+
+```typescript
+import { listVoices } from '@tanstack/ai'
+import { elevenlabsSpeech } from '@tanstack/ai-elevenlabs'
+
+const { voices } = await listVoices({
+  adapter: elevenlabsSpeech('eleven_v3'),
+  origins: ['generated', 'cloned'],
+})
+```
+
+`listVoices` hangs off the **TTS** adapter, not the voice adapter, because
+`voice` is a `generateSpeech()` option — that is where the id gets consumed.
+It is OPTIONAL, and only providers with a per-account catalog implement it.
+Where the catalog is fixed the package publishes it instead — `GeminiTTSVoices`
+from `@tanstack/ai-gemini`, or the `OpenAITTSVoice` union from
+`@tanstack/ai-openai`. Prefer those: a type union beats a network call.
+Calling `listVoices()` on such an adapter throws and points at them.
+
+There is no React hook for this activity. Call it from a server route or
+server function and return the result as JSON.
+
+`elevenlabsVoiceDesign` is the only `generateVoice()` adapter in this repo.
+xAI, BytePlus, and fal.ai each publish a voice-cloning API and are the
+candidates for the next one, but none is implemented — do not write code
+against them from this file.
+
+OpenAI, Gemini, and Cloudflare have fixed voice catalogs and will not get one.
+
+### 5. Audio Transcription
 
 Adapters: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
 gpt-4o-mini-transcribe, gpt-4o-transcribe-diarize) and `byteplusTranscription`
@@ -486,7 +608,7 @@ const { generate, result, isLoading } = useTranscription({
 // Trigger: generate({ audio: dataUrl, language: 'en' })
 ```
 
-### 5. Video Generation (Experimental -- async polling)
+### 6. Video Generation (Experimental -- async polling)
 
 Video generation uses a jobs/polling architecture. The server creates a job,
 polls for status, and streams updates to the client. Adapters: `openaiVideo`
@@ -662,7 +784,7 @@ const { generate, result, jobId, videoStatus, isLoading } = useGenerateVideo({
 // result (on completion): { url }
 ```
 
-### 6. Cost tracking (fal billable units)
+### 7. Cost tracking (fal billable units)
 
 fal bills media generation by usage-based units, not tokens. Every fal media
 adapter (`falImage`, `falAudio`, `falSpeech`, `falTranscription`, `falVideo`)
@@ -692,7 +814,7 @@ if (result.usage?.billed) {
 For video, the units arrive with the completed result: `getVideoJobStatus()`
 returns `usage` and emits a `video:usage` devtools event when fal reports it.
 
-### 7. Durable persistence (job lifecycle + artifact bytes)
+### 8. Durable persistence (job lifecycle + artifact bytes)
 
 To make generations survive a server restart and be re-served later, add
 `withGenerationPersistence` from `@tanstack/ai-persistence` as generation
@@ -1014,7 +1136,11 @@ generateAudio({
 
 ### g. MEDIUM: Gemini TTS multi-speaker with 0 or 3+ speakers
 
-`multiSpeakerVoiceConfig.speakerVoiceConfigs` is validated to be length 1 or 2. Passing an empty array or three+ entries throws at the adapter boundary
+Prefer `turns` for new code: it builds `multiSpeakerVoiceConfig` and the
+labelled prompt for you, and the two-speaker cap is enforced by the activity
+from `capabilities.maxSpeakers`.
+
+The hand-rolled form below still works. `multiSpeakerVoiceConfig.speakerVoiceConfigs` is validated to be length 1 or 2. Passing an empty array or three+ entries throws at the adapter boundary
 (not at Gemini's API) with a clear error. Don't try to work around it with
 `as any`.
 

package/src/activities/chat/tools/tool-calls.ts

@@ -233,6 +233,15 @@ export class ToolCallManager<
    * Add a TOOL_CALL_START event to begin tracking a tool call (AG-UI)
    */
   addToolCallStartEvent(event: ToolCallStartEvent): void {
+    // AG-UI's TOOL_CALL_START carries no index, and a non-first-party or
+    // malformed producer can send a second START for a toolCallId that is
+    // already tracked. Without this guard, a repeat with the same index
+    // overwrites the slot (wiping any TOOL_CALL_ARGS already accumulated),
+    // and a repeat with a missing/different index inserts a duplicate row
+    // that getToolCalls() returns twice, running the tool twice.
+    for (const toolCall of this.toolCallsMap.values()) {
+      if (toolCall.id === event.toolCallId) return
+    }
     const index = (event as AdapterYieldChunk).index ?? this.toolCallsMap.size
     const name = event.toolCallName ?? event.toolName
     this.toolCallsMap.set(index, {

package/src/activities/evaluate/adapter.ts

@@ -0,0 +1,212 @@
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { TokenUsage } from '../../types'
+
+/**
+ * Configuration for evaluate adapter instances.
+ */
+export interface EvaluateAdapterConfig {
+  apiKey?: string
+  baseUrl?: string
+  timeout?: number
+  headers?: Record<string, string>
+}
+
+/**
+ * Shared JSON value for `state` and question `instructions`.
+ * A JSON array is one value, not a batch.
+ */
+export type EvaluateJsonValue = string | object | Array<unknown>
+
+/** Content the model judges. A JSON array is one state, not a batch. */
+export type EvaluateState = EvaluateJsonValue
+
+/** Question text. Matches TypeSafe: string, object, or array. */
+export type EvaluateInstructions = EvaluateJsonValue
+
+/**
+ * TypeSafe choice question on the adapter wire.
+ *
+ * Generic parameters:
+ * - TOptions: option key to description (or `null` when the key is enough)
+ */
+export interface WireChoiceQuestion<
+  TOptions extends Record<string, string | null> = Record<
+    string,
+    string | null
+  >,
+> {
+  type: 'choice'
+  instructions: EvaluateInstructions
+  criteria: TOptions
+}
+
+/**
+ * TypeSafe score question on the adapter wire.
+ *
+ * Generic parameters:
+ * - TLevels: ordered level labels, at least two
+ */
+export interface WireScoreQuestion<
+  TLevels extends ReadonlyArray<string> = ReadonlyArray<string>,
+> {
+  type: 'score'
+  instructions: EvaluateInstructions
+  criteria: TLevels
+}
+
+/**
+ * TypeSafe yes/no question on the adapter wire.
+ * Public helpers call this `boolean`. The wire type is `noul`.
+ */
+export interface WireNoulQuestion {
+  type: 'noul'
+  instructions: EvaluateInstructions
+  criteria?: {
+    true?: string
+    false?: string
+  }
+}
+
+/** Question payload adapters send to the provider. */
+export type WireQuestion =
+  | WireChoiceQuestion
+  | WireScoreQuestion
+  | WireNoulQuestion
+
+/** TypeSafe choice answer. Adapters do not invent a public `.value`. */
+export interface WireChoiceAnswer {
+  type: 'choice'
+  choice: string
+  probabilities: Record<string, number>
+  confidence: number
+}
+
+/** TypeSafe score answer. `score` is the raw fraction. */
+export interface WireScoreAnswer {
+  type: 'score'
+  score: number
+  legend: Record<string, string>
+  probabilities: Record<string, number>
+  confidence: number
+}
+
+/** TypeSafe yes/no answer. `noul` is P(true). */
+export interface WireNoulAnswer {
+  type: 'noul'
+  noul: number
+}
+
+/** Provider payload for one question. The activity maps this to a unified answer. */
+export type WireAnswer = WireChoiceAnswer | WireScoreAnswer | WireNoulAnswer
+
+/**
+ * Options passed to {@link EvaluateAdapter.evaluate}.
+ */
+export interface EvaluateOptions<
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  model: string
+  /** Shared state every question judges. A JSON array is one state, not a batch. */
+  state: EvaluateState
+  /** TypeSafe wire questions, keyed by the caller's question ids. */
+  questions: Record<string, WireQuestion>
+  /** Provider-specific options forwarded by `decide()`. */
+  modelOptions?: TProviderOptions
+  /** Forwarded to the provider request for cancellation. */
+  abortSignal?: AbortSignal
+  /**
+   * Internal logger threaded from `decide()`. Adapters must call
+   * `logger.request()` before the provider call and `logger.errors()` in catch
+   * blocks.
+   */
+  logger: InternalLogger
+}
+
+/**
+ * Provider-level evaluate result. Adapters return the wire payload plus usage.
+ * The activity maps answers to the unified public shape.
+ */
+export interface EvaluateAdapterResult {
+  /** Resolved model id from the provider. */
+  model: string
+  answers: Record<string, WireAnswer>
+  usage: TokenUsage
+}
+
+/**
+ * Evaluate adapter interface with pre-resolved generics.
+ *
+ * An adapter is created by a provider function: `provider('model')` → `adapter`.
+ * All type resolution happens at the provider call site, not in this interface.
+ *
+ * Generic parameters:
+ * - TModel: The specific model name (e.g. `'jev-latest'`)
+ * - TProviderOptions: Provider-specific options (already resolved)
+ */
+export interface EvaluateAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  /** Discriminator for adapter kind */
+  readonly kind: 'evaluate'
+  /** Adapter name identifier */
+  readonly name: string
+  /** The model this adapter is configured for */
+  readonly model: TModel
+
+  /**
+   * @internal Type-only properties for inference. Not assigned at runtime.
+   */
+  '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  /**
+   * Evaluate typed questions against `state`. Return the provider payload.
+   * Do not invent unified `.value` fields. The activity maps wire answers.
+   */
+  evaluate: (
+    options: EvaluateOptions<TProviderOptions>,
+  ) => Promise<EvaluateAdapterResult>
+}
+
+/**
+ * An EvaluateAdapter with any/unknown type parameters.
+ * Useful as a constraint in generic functions and interfaces.
+ */
+export type AnyEvaluateAdapter = EvaluateAdapter<any, any>
+
+/**
+ * Abstract base class for evaluate adapters.
+ * Extend this class to implement an evaluate adapter for a specific provider.
+ *
+ * Generic parameters match EvaluateAdapter. The provider function resolves them.
+ */
+export abstract class BaseEvaluateAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> implements EvaluateAdapter<TModel, TProviderOptions> {
+  readonly kind = 'evaluate' as const
+  abstract readonly name: string
+  readonly model: TModel
+
+  // Type-only property - never assigned at runtime
+  declare '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  protected config: EvaluateAdapterConfig
+
+  constructor(config: EvaluateAdapterConfig = {}, model: TModel) {
+    this.config = config
+    this.model = model
+  }
+
+  abstract evaluate(
+    options: EvaluateOptions<TProviderOptions>,
+  ): Promise<EvaluateAdapterResult>
+
+  protected generateId(): string {
+    return `${this.name}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+  }
+}