@tanstack/ai-gemini 0.29.4 → 0.31.1

package/src/adapters/tts.ts

@@ -7,7 +7,7 @@ import {
 import { GEMINI_TTS_VOICES } from '../model-meta'
 import { buildGeminiUsage } from '../usage'
 import type { GEMINI_TTS_MODELS, GeminiTTSVoice } from '../model-meta'
-import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type { TTSCapabilities, TTSOptions, TTSResult } from '@tanstack/ai'
 import type { GoogleGenAI, SpeechConfig } from '@google/genai'
 import type { GeminiClientConfig } from '../utils/client'
 
@@ -122,6 +122,11 @@ export class GeminiTTSAdapter<
 > extends BaseTTSAdapter<TModel, GeminiTTSProviderOptions> {
   readonly name = 'gemini' as const
 
+  /**
+   * Gemini multi-speaker TTS tops out at 2 voices, and reports no timings.
+   */
+  override readonly capabilities: TTSCapabilities = { maxSpeakers: 2 }
+
   private readonly client: GoogleGenAI
 
   constructor(config: GeminiTTSConfig, model: TModel) {
@@ -138,7 +143,7 @@ export class GeminiTTSAdapter<
   async generateSpeech(
     options: TTSOptions<GeminiTTSProviderOptions>,
   ): Promise<TTSResult> {
-    const { model, text, modelOptions, voice, logger } = options
+    const { model, modelOptions, voice, logger, turns } = options
 
     logger.request(`activity=generateSpeech provider=gemini model=${model}`, {
       provider: 'gemini',
@@ -146,8 +151,24 @@ export class GeminiTTSAdapter<
     })
 
     const speechConfig: SpeechConfig = {}
-
-    if (modelOptions?.multiSpeakerVoiceConfig) {
+    // Gemini has no turn structure on the wire: speakers are named in the
+    // prompt and mapped to voices in the config. `turns` carries voice names,
+    // so the voice name doubles as the speaker label.
+    const text = turns
+      ? turns.map((turn) => `${turn.voice}: ${turn.text}`).join('\n')
+      : options.text
+
+    if (turns) {
+      const voiceNames = [...new Set(turns.map((turn) => turn.voice))].map(
+        toGeminiVoice,
+      )
+      speechConfig.multiSpeakerVoiceConfig = {
+        speakerVoiceConfigs: voiceNames.map((voiceName) => ({
+          speaker: voiceName,
+          voiceConfig: { prebuiltVoiceConfig: { voiceName } },
+        })),
+      }
+    } else if (modelOptions?.multiSpeakerVoiceConfig) {
       // Validate multi-speaker config: 1 or 2 speakers allowed.
       const speakerConfigs =
         modelOptions.multiSpeakerVoiceConfig.speakerVoiceConfigs
@@ -168,15 +189,8 @@ export class GeminiTTSAdapter<
       // modelOptions.voiceConfig is supplied its values win — but we still
       // fall back to `voice` / 'Kore' if the supplied voiceConfig is missing
       // prebuiltVoiceConfig.voiceName.
-      if (
-        voice !== undefined &&
-        !(GEMINI_TTS_VOICES as ReadonlyArray<string>).includes(voice)
-      ) {
-        throw new Error(
-          `Invalid Gemini TTS voice "${voice}". Valid voices are: ${GEMINI_TTS_VOICES.join(', ')}.`,
-        )
-      }
-      const defaultVoiceName = (voice as GeminiTTSVoice | undefined) ?? 'Kore'
+      const defaultVoiceName =
+        voice !== undefined ? toGeminiVoice(voice) : 'Kore'
       const supplied = modelOptions?.voiceConfig
       const resolvedVoiceName =
         supplied?.prebuiltVoiceConfig?.voiceName ?? defaultVoiceName
@@ -282,6 +296,17 @@ export class GeminiTTSAdapter<
   }
 }
 
+/** Narrow a caller-supplied voice string to a known Gemini voice, or throw. */
+function toGeminiVoice(voice: string): GeminiTTSVoice {
+  const match = GEMINI_TTS_VOICES.find((known) => known === voice)
+  if (!match) {
+    throw new Error(
+      `Invalid Gemini TTS voice "${voice}". Valid voices are: ${GEMINI_TTS_VOICES.join(', ')}.`,
+    )
+  }
+  return match
+}
+
 function parsePcmMimeType(
   mimeType: string,
 ): { sampleRate: number; channels: number; bitsPerSample: number } | undefined {

package/src/experimental/text-interactions/adapter.ts

@@ -279,7 +279,7 @@ export class GeminiTextInteractionsAdapter<
       const stream = (await this.client.interactions.create(
         { ...request, stream: true } as GeminiInteractionsRequestBody &
           Parameters<typeof this.client.interactions.create>[0],
-        { signal: options.abortController?.signal },
+        { signal: abortSignalFromOptions(options) },
       )) as AsyncIterable<InteractionSSEEvent>
 
       for await (const chunk of translateInteractionEvents(
@@ -427,7 +427,7 @@ export class GeminiTextInteractionsAdapter<
       )
       const result = (await this.client.interactions.create(
         request as Parameters<typeof this.client.interactions.create>[0],
-        { signal: chatOptions.abortController?.signal },
+        { signal: abortSignalFromOptions(chatOptions) },
       )) as Interaction
 
       const rawText = extractTextFromInteraction(result)
@@ -442,9 +442,7 @@ export class GeminiTextInteractionsAdapter<
       try {
         parsed = JSON.parse(rawText)
       } catch {
-        throw new Error(
-          `Failed to parse structured output as JSON. Content: ${rawText.slice(0, 200)}${rawText.length > 200 ? '...' : ''}`,
-        )
+        throw new Error(jsonContentParseError(rawText, 'structured output'))
       }
 
       return { data: parsed, rawText }
@@ -463,6 +461,166 @@ export class GeminiTextInteractionsAdapter<
       )
     }
   }
+
+  async *structuredOutputStream(
+    options: StructuredOutputOptions<GeminiTextInteractionsProviderOptions>,
+  ): AsyncIterable<AdapterYieldChunk> {
+    const { chatOptions, outputSchema } = options
+    const runId = chatOptions.runId ?? generateId(this.name)
+    const threadId = chatOptions.threadId ?? generateId(this.name)
+    const timestamp = Date.now()
+    const effectivePreviousInteractionId =
+      chatOptions.modelOptions?.previous_interaction_id ??
+      this.interactionIdByThread.get(threadId)
+    const baseRequest = buildInteractionsRequest({
+      ...chatOptions,
+      modelOptions: {
+        ...chatOptions.modelOptions,
+        previous_interaction_id: effectivePreviousInteractionId,
+      },
+    })
+    const request: GeminiInteractionsRequestBody = {
+      ...baseRequest,
+      stream: true,
+      response_format: {
+        type: 'text',
+        mime_type: 'application/json',
+        schema: outputSchema,
+      },
+    }
+
+    try {
+      chatOptions.logger.request(
+        `activity=structuredOutputStream provider=gemini-text-interactions model=${this.model} messages=${chatOptions.messages.length}`,
+        { provider: this.name, model: this.model, request },
+      )
+      const stream = (await this.client.interactions.create(
+        request as GeminiInteractionsRequestBody &
+          Parameters<typeof this.client.interactions.create>[0],
+        { signal: abortSignalFromOptions(chatOptions) },
+      )) as AsyncIterable<InteractionSSEEvent>
+
+      let rawText = ''
+      let finished:
+        | Extract<AdapterYieldChunk, { type: typeof EventType.RUN_FINISHED }>
+        | undefined
+      let failed = false
+      for await (const chunk of translateInteractionEvents(
+        stream,
+        chatOptions.model,
+        runId,
+        threadId,
+        chatOptions.parentRunId,
+        timestamp,
+        this.name,
+        chatOptions.logger,
+      )) {
+        if (chunk.type === EventType.TEXT_MESSAGE_CONTENT)
+          rawText += chunk.delta
+        if (
+          chunk.type === EventType.CUSTOM &&
+          chunk.name === 'gemini.interactionId'
+        ) {
+          const value =
+            chunk.value as GeminiInteractionsCustomEventValue<'gemini.interactionId'>
+          this.interactionIdByThread.set(threadId, value.interactionId)
+        }
+        if (chunk.type === EventType.RUN_ERROR) failed = true
+        if (chunk.type === EventType.RUN_FINISHED) finished = chunk
+        else yield chunk
+      }
+
+      if (failed) return
+      if (!finished) {
+        yield interactionsStructuredStreamError(
+          chatOptions,
+          runId,
+          'Gemini Interactions structured-output stream ended without a terminal event',
+          'truncated-stream',
+        )
+        return
+      }
+      if (!rawText) {
+        yield interactionsStructuredStreamError(
+          chatOptions,
+          runId,
+          'Gemini Interactions structured-output stream contained no content',
+          'empty-response',
+        )
+        return
+      }
+      let object: unknown
+      try {
+        object = JSON.parse(rawText)
+      } catch {
+        yield interactionsStructuredStreamError(
+          chatOptions,
+          runId,
+          jsonContentParseError(
+            rawText,
+            'Gemini Interactions structured-output stream',
+          ),
+          'parse-error',
+        )
+        return
+      }
+      yield {
+        type: EventType.CUSTOM,
+        name: 'structured-output.complete',
+        value: { object, raw: rawText },
+        model: chatOptions.model,
+        timestamp: Date.now(),
+      }
+      yield { ...finished, timestamp: Date.now() }
+    } catch (error) {
+      const message =
+        error instanceof Error
+          ? error.message
+          : 'An unknown error occurred during structured output streaming.'
+      chatOptions.logger.errors(
+        'gemini-text-interactions.structuredOutputStream fatal',
+        { error, source: 'gemini-text-interactions.structuredOutputStream' },
+      )
+      yield interactionsStructuredStreamError(
+        chatOptions,
+        runId,
+        message,
+        'provider-error',
+      )
+    }
+  }
+}
+
+function abortSignalFromOptions(
+  options: Pick<
+    TextOptions<GeminiTextInteractionsProviderOptions>,
+    'request' | 'abortController'
+  >,
+) {
+  return options.request?.signal ?? options.abortController?.signal
+}
+
+function jsonContentParseError(rawText: string, label: string) {
+  const snippet = rawText.slice(0, 200)
+  const ellipsis = rawText.length > 200 ? '...' : ''
+  return `Failed to parse ${label} as JSON. Content: ${snippet}${ellipsis}`
+}
+
+function interactionsStructuredStreamError(
+  options: TextOptions<GeminiTextInteractionsProviderOptions>,
+  runId: string,
+  message: string,
+  code: string,
+): Extract<AdapterYieldChunk, { type: typeof EventType.RUN_ERROR }> {
+  return {
+    type: EventType.RUN_ERROR,
+    runId,
+    model: options.model,
+    timestamp: Date.now(),
+    message,
+    code,
+    error: { message, code },
+  }
 }
 
 /** @experimental Interactions API is in Beta. */