@ai-sdk/google 4.0.82 → 4.0.84

package/docs/15-google.mdx

@@ -1147,6 +1147,18 @@ emitted as custom events. Inspect `rawType` for `goAway`,
 `sessionResumptionUpdate`, and `generationComplete` and use the event's `raw`
 payload for provider-specific details.
 
+With input audio transcription enabled, `parseServerEvent` emits
+`input-transcription-completed` events with a stable `itemId` for each user
+utterance. Each `transcript` contains the accumulated text so far. If you consume
+these events directly, replace the text for that `itemId` with `transcript` on
+each event. The realtime session handles this automatically.
+
+Google's `finished` transcription marker ends an utterance even when the marker
+contains no text. For payloads without this marker, response completion and
+interruption provide fallback boundaries. These fallback boundaries cannot
+unambiguously associate all delayed transcription fragments. Input item IDs use
+their own sequence and need not match the model's response IDs.
+
 ### Live Translation
 
 Use `gemini-3.5-live-translate-preview` for low-latency speech-to-speech

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/google",
-  "version": "4.0.82",
+  "version": "4.0.84",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -35,8 +35,8 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "4.0.18",
-    "@ai-sdk/provider-utils": "5.0.49"
+    "@ai-sdk/provider": "4.0.19",
+    "@ai-sdk/provider-utils": "5.0.50"
   },
   "devDependencies": {
     "@ai-sdk/test-server": "2.0.2",

package/src/google-image-model.ts

@@ -32,6 +32,12 @@ interface GoogleImageModelConfig {
   };
 }
 
+const googleImageModelsWithFileInputSupport = new Set<string>([
+  'gemini-2.5-flash-image',
+  'gemini-3-pro-image-preview',
+  'gemini-3.1-flash-image-preview',
+]);
+
 export class GoogleImageModel implements ImageModelV4 {
   readonly specificationVersion = 'v4';
 
@@ -56,6 +62,16 @@ export class GoogleImageModel implements ImageModelV4 {
     return 1;
   }
 
+  get supportsFileInputs(): boolean | undefined {
+    return googleImageModelsWithFileInputSupport.has(this.modelId)
+      ? true
+      : undefined;
+  }
+
+  get supportsMaskInputs(): boolean | undefined {
+    return this.supportsFileInputs === true ? false : undefined;
+  }
+
   get provider(): string {
     return this.config.provider;
   }

package/src/realtime/google-realtime-event-mapper.ts

@@ -25,8 +25,8 @@ type GoogleRealtimeServerContent = {
       text?: string;
     }>;
   };
-  outputTranscription?: { text?: string };
-  inputTranscription?: { text?: string };
+  outputTranscription?: { text?: string; finished?: boolean };
+  inputTranscription?: { text?: string; finished?: boolean };
   turnComplete?: boolean;
   waitingForInput?: boolean;
 };
@@ -38,7 +38,7 @@ type GoogleRealtimeWireEvent = {
   };
   toolCallCancellation?: unknown;
   serverContent?: GoogleRealtimeServerContent;
-  inputTranscription?: { text?: string };
+  inputTranscription?: { text?: string; finished?: boolean };
   goAway?: { timeLeft?: string };
   sessionResumptionUpdate?: {
     newHandle?: string;
@@ -56,6 +56,11 @@ type GoogleRealtimeWireEvent = {
  */
 export class GoogleRealtimeEventMapper {
   private turnCounter = 0;
+  private inputTranscriptionCounter = 0;
+  private inputTranscriptionBuffer = '';
+  private inputTranscriptionBoundary: 'none' | 'interrupted' | 'turn-complete' =
+    'none';
+  private preserveInputTranscriptionAcrossTurnComplete = false;
   private hasAudio = false;
   private hasText = false;
   private hasTranscript = false;
@@ -71,12 +76,11 @@ export class GoogleRealtimeEventMapper {
   }
 
   /**
-   * Rolls over to the next turn lazily, only once new model content actually
-   * arrives. `turnComplete` merely marks the current turn closed; the counter
-   * is not advanced until the next response begins. This keeps a transcript
-   * that arrives shortly after `turnComplete` attached to the turn it belongs
-   * to, since Google delivers transcription independently with no guaranteed
-   * ordering relative to `turnComplete`.
+   * Rolls over to the next turn lazily, once model content for the next turn
+   * arrives. `turnComplete` merely marks the current turn closed.
+   * This keeps an output transcript that arrives shortly after `turnComplete`
+   * attached to the turn it belongs to, since Google delivers transcription
+   * independently with no guaranteed ordering relative to `turnComplete`.
    */
   private beginTurnIfClosed(): void {
     if (!this.turnClosed) return;
@@ -85,6 +89,63 @@ export class GoogleRealtimeEventMapper {
     this.hasText = false;
     this.hasTranscript = false;
     this.turnClosed = false;
+    // Once the next response starts, its completion must delimit its own
+    // input even if the interrupted response never sent a turnComplete.
+    this.preserveInputTranscriptionAcrossTurnComplete = false;
+  }
+
+  // Google streams input transcription as a sequence of non-accumulating
+  // fragments for a single user utterance (e.g. "The quick brown fox",
+  // " jumps over the", " lazy dog."). Concatenate consecutive fragments into
+  // one running transcript under a stable synthetic id so the realtime reducer
+  // (which overwrites the message for a given id) surfaces them as a single
+  // coherent user message rather than one message per fragment. Google's
+  // `finished` signal is the authoritative utterance boundary. For older
+  // payloads without that signal, response boundaries remain a fallback.
+  //
+  // Input transcriptions can arrive independently of response turn events, so
+  // their synthetic IDs use a sequence independent of `turnCounter`.
+  private accumulateInputTranscription({
+    text,
+    finished,
+  }: {
+    text?: string;
+    finished?: boolean;
+  }): { itemId: string; transcript: string } | undefined {
+    // Google can send `finished` separately from text. Process it even when
+    // text is absent or empty, without emitting an empty user message.
+    if (!text) {
+      if (finished === true) {
+        this.finishInputTranscription();
+      }
+      return undefined;
+    }
+
+    const hasFinishedSignal = finished != null;
+    if (
+      this.inputTranscriptionBuffer !== '' &&
+      (this.inputTranscriptionBoundary === 'interrupted' ||
+        (this.inputTranscriptionBoundary === 'turn-complete' &&
+          !hasFinishedSignal))
+    ) {
+      this.finishInputTranscription();
+    }
+    this.inputTranscriptionBoundary = 'none';
+    this.inputTranscriptionBuffer += text;
+    const result = {
+      itemId: `google-input-${this.inputTranscriptionCounter}`,
+      transcript: this.inputTranscriptionBuffer,
+    };
+    if (finished === true) {
+      this.finishInputTranscription();
+    }
+    return result;
+  }
+
+  private finishInputTranscription(): void {
+    if (this.inputTranscriptionBuffer === '') return;
+    this.inputTranscriptionCounter++;
+    this.inputTranscriptionBuffer = '';
   }
 
   parseServerEvent(
@@ -151,13 +212,17 @@ export class GoogleRealtimeEventMapper {
       return this.parseServerContent(data.serverContent, raw);
     }
 
-    if (data.inputTranscription?.text != null) {
-      return {
-        type: 'input-transcription-completed',
-        itemId: `google-input-${this.turnCounter}`,
-        transcript: data.inputTranscription.text,
-        raw,
-      };
+    if (data.inputTranscription != null) {
+      const transcription = this.accumulateInputTranscription(
+        data.inputTranscription,
+      );
+      if (transcription != null) {
+        return {
+          type: 'input-transcription-completed',
+          ...transcription,
+          raw,
+        };
+      }
     }
 
     return { type: 'custom', rawType: String(Object.keys(data)[0]), raw };
@@ -170,6 +235,12 @@ export class GoogleRealtimeEventMapper {
     const events: RealtimeModelV4ServerEvent[] = [];
 
     if (serverContent.interrupted) {
+      // A barge-in ends the current user utterance; the next input
+      // transcription fragment belongs to a fresh utterance. The interrupted
+      // response's trailing turnComplete must not split that new utterance.
+      this.inputTranscriptionBoundary = 'interrupted';
+      this.preserveInputTranscriptionAcrossTurnComplete = true;
+      this.turnClosed = true;
       events.push({
         type: 'speech-started',
         raw,
@@ -214,13 +285,17 @@ export class GoogleRealtimeEventMapper {
       });
     }
 
-    if (serverContent.inputTranscription?.text) {
-      events.push({
-        type: 'input-transcription-completed',
-        itemId: `google-input-${this.turnCounter}`,
-        transcript: serverContent.inputTranscription.text,
-        raw,
-      });
+    if (serverContent.inputTranscription != null) {
+      const transcription = this.accumulateInputTranscription(
+        serverContent.inputTranscription,
+      );
+      if (transcription != null) {
+        events.push({
+          type: 'input-transcription-completed',
+          ...transcription,
+          raw,
+        });
+      }
     }
 
     // `generationComplete` means generation has stopped, but playback and the
@@ -292,6 +367,17 @@ export class GoogleRealtimeEventMapper {
       // Mark the turn closed but defer advancing the counter until the next
       // response actually begins (see `beginTurnIfClosed`).
       this.turnClosed = true;
+      if (this.preserveInputTranscriptionAcrossTurnComplete) {
+        this.preserveInputTranscriptionAcrossTurnComplete = false;
+      } else {
+        // For payloads without `finished`, the completed response remains the
+        // fallback signal that the next transcription is a new utterance.
+        // A transcription carrying `finished` can still arrive late and close
+        // the preceding utterance after this response boundary.
+        if (this.inputTranscriptionBoundary !== 'interrupted') {
+          this.inputTranscriptionBoundary = 'turn-complete';
+        }
+      }
     }
 
     if (events.length === 0) {
@@ -481,14 +567,24 @@ export function buildGoogleSessionConfig(
     setup.outputAudioTranscription = {};
   }
 
-  // Default to the lowest-latency thinking level so a session on a
-  // background-reasoning model works without provider options. Merged last so
-  // it survives a raw `providerOptions.generationConfig`.
-  const thinkingConfig =
+  // Background-reasoning models require a `thinkingLevel` or `thinkingBudget`.
+  // Default to the lowest-latency level when the effective thinking config (the
+  // typed option, else a raw `providerOptions.generationConfig.thinkingConfig`)
+  // sets neither. Merged last so it survives a raw
+  // `providerOptions.generationConfig`.
+  const rawGenerationConfig = restProviderOptions.generationConfig;
+  const explicitThinkingConfig =
     googleOptions?.thinkingConfig ??
-    (isThinkingLiveModel(modelId)
-      ? { thinkingLevel: 'low' as const }
+    (isRecord(rawGenerationConfig) &&
+    isRecord(rawGenerationConfig.thinkingConfig)
+      ? rawGenerationConfig.thinkingConfig
       : undefined);
+  const thinkingConfig =
+    isThinkingLiveModel(modelId) &&
+    explicitThinkingConfig?.thinkingLevel == null &&
+    explicitThinkingConfig?.thinkingBudget == null
+      ? { ...explicitThinkingConfig, thinkingLevel: 'low' as const }
+      : googleOptions?.thinkingConfig;
   const applyThinkingConfig = () => {
     if (thinkingConfig == null) return;
     const target = isRecord(setup.generationConfig)