@tanstack/ai-gemini 0.32.1 → 0.34.1

package/src/adapters/files.ts

@@ -0,0 +1,84 @@
+import {
+  BaseFilesAdapter,
+  normalizeFileUploadInput,
+} from '@tanstack/ai/adapters'
+import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils/client'
+import type { File as GeminiFile, GoogleGenAI } from '@google/genai'
+import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters'
+import type { GeminiClientConfig } from '../utils/client'
+
+export interface GeminiFilesConfig extends GeminiClientConfig {}
+
+/**
+ * Gemini Files adapter — uploads media to the Gemini Files API and references
+ * it by its file URI. Pair with `geminiText()` / `geminiImage()`: reference the
+ * returned handle via `fileSourceFromHandle(handle)`, which uses the handle URI
+ * (Gemini fetches it server-side as `fileData.fileUri`).
+ */
+export class GeminiFilesAdapter extends BaseFilesAdapter<'gemini'> {
+  readonly name = 'gemini' as const
+  private readonly client: GoogleGenAI
+
+  constructor(config: GeminiFilesConfig) {
+    super()
+    this.client = createGeminiClient(config)
+  }
+
+  async upload(input: FileUploadInput): Promise<FileHandle<'gemini'>> {
+    const { blob, mimeType } = normalizeFileUploadInput(input)
+    const file = await this.client.files.upload({
+      file: blob,
+      ...(mimeType ? { config: { mimeType } } : {}),
+    })
+    return toFileHandle(file)
+  }
+
+  async get(id: string): Promise<FileHandle<'gemini'>> {
+    return toFileHandle(await this.client.files.get({ name: id }))
+  }
+
+  async delete(id: string): Promise<void> {
+    await this.client.files.delete({ name: id })
+  }
+}
+
+function toFileHandle(file: GeminiFile): FileHandle<'gemini'> {
+  // `name` (e.g. "files/abc-123") is the lifecycle id; `uri` is the URL Gemini
+  // fetches when the handle is referenced in a message.
+  if (!file.name) {
+    throw new Error('gemini: files.upload returned a file without a name')
+  }
+  const expiresAt = file.expirationTime
+    ? Date.parse(file.expirationTime)
+    : undefined
+  return {
+    id: file.name,
+    provider: 'gemini',
+    ...(file.uri ? { uri: file.uri } : {}),
+    ...(file.mimeType ? { mimeType: file.mimeType } : {}),
+    ...(file.sizeBytes ? { sizeBytes: Number(file.sizeBytes) } : {}),
+    ...(expiresAt !== undefined && !Number.isNaN(expiresAt)
+      ? { expiresAt }
+      : {}),
+  }
+}
+
+/**
+ * Create a Gemini Files adapter with an explicit API key.
+ */
+export function createGeminiFiles(
+  apiKey: string,
+  config?: Omit<GeminiFilesConfig, 'apiKey'>,
+): GeminiFilesAdapter {
+  return new GeminiFilesAdapter({ ...config, apiKey })
+}
+
+/**
+ * Create a Gemini Files adapter, reading the API key from `GOOGLE_API_KEY` /
+ * `GEMINI_API_KEY`.
+ */
+export function geminiFiles(
+  config?: Omit<GeminiFilesConfig, 'apiKey'>,
+): GeminiFilesAdapter {
+  return createGeminiFiles(getGeminiApiKeyFromEnv(), config)
+}

package/src/adapters/image.ts

@@ -1,4 +1,8 @@
-import { resolveMediaPrompt } from '@tanstack/ai'
+import {
+  fileReferenceFor,
+  isFileSource,
+  resolveMediaPrompt,
+} from '@tanstack/ai'
 import { BaseImageAdapter } from '@tanstack/ai/adapters'
 import {
   createGeminiClient,
@@ -75,6 +79,8 @@ export class GeminiImageAdapter<
 > {
   override readonly kind = 'image' as const
   readonly name = 'gemini' as const
+  // Consumes Gemini Files API references (geminiFiles()) as fileData.fileUri.
+  override readonly supportsFileSources = true
 
   // Type-only property - never assigned at runtime
   declare '~types': {
@@ -270,10 +276,14 @@ export class GeminiImageAdapter<
     // `fileData` and Gemini fetches them server-side — same as the chat
     // adapter. Fetching locally and inlining as base64 double-buffers the
     // image and OOMs on memory-constrained runtimes (e.g. Cloudflare
-    // Workers).
+    // Workers). A file source's handle is the file URI (throws when another
+    // provider issued it).
+    const fileUri = isFileSource(part.source)
+      ? fileReferenceFor(part.source, this.name)
+      : part.source.value
     return {
       fileData: {
-        fileUri: part.source.value,
+        fileUri,
         mimeType: part.source.mimeType ?? 'image/jpeg',
       },
     }

package/src/adapters/text.ts

@@ -1,5 +1,10 @@
 import { FinishReason } from '@google/genai'
-import { EventType, normalizeSystemPrompts } from '@tanstack/ai'
+import {
+  EventType,
+  fileReferenceFor,
+  isFileSource,
+  normalizeSystemPrompts,
+} from '@tanstack/ai'
 import { toRunErrorRawEvent } from '@tanstack/ai/adapter-internals'
 import { BaseTextAdapter } from '@tanstack/ai/adapters'
 import { convertToolsToProviderFormat } from '../tools/tool-converter'
@@ -26,6 +31,7 @@ import type {
   GenerateContentParameters,
   GenerateContentResponse,
   GoogleGenAI,
+  GroundingMetadata,
   Part,
   ThinkingLevel,
   VideoMetadata,
@@ -35,6 +41,7 @@ import type {
   Modality,
   ModelMessage,
   AdapterYieldChunk,
+  ProviderExecutedToolSource,
   TextOptions,
 } from '@tanstack/ai'
 import type { ExternalTextProviderOptions } from '../text/text-provider-options'
@@ -56,6 +63,26 @@ const DEFAULT_MEDIA_MIME_TYPES = {
   document: 'application/pdf',
 } as const
 
+function getGroundingSources(
+  metadata: GroundingMetadata,
+): Array<ProviderExecutedToolSource> {
+  const sources = new Map<string, ProviderExecutedToolSource>()
+  for (const chunk of metadata.groundingChunks ?? []) {
+    const url = chunk.web?.uri
+    if (!url) continue
+    const existing = sources.get(url)
+    if (existing) {
+      if (!existing.title && chunk.web?.title) existing.title = chunk.web.title
+      continue
+    }
+    sources.set(url, {
+      url,
+      ...(chunk.web?.title ? { title: chunk.web.title } : {}),
+    })
+  }
+  return [...sources.values()]
+}
+
 /**
  * Content block shape for an Interactions API `input` step. The installed
  * @google/genai types predate the video `processing` field, so we model the
@@ -107,8 +134,11 @@ function contentPartToInteraction(part: ContentPart): InteractionContent {
     source.type === 'data'
       ? source.mimeType
       : (source.mimeType ?? DEFAULT_MEDIA_MIME_TYPES[part.type])
-  const base =
-    source.type === 'data'
+  // A Gemini Files API handle maps to the `uri` field, same as a public URL;
+  // `fileReferenceFor` throws when another provider issued it.
+  const base = isFileSource(source)
+    ? { uri: fileReferenceFor(source, 'gemini'), mime_type: mimeType }
+    : source.type === 'data'
       ? { data: source.value, mime_type: mimeType }
       : { uri: source.value, mime_type: mimeType }
 
@@ -216,6 +246,8 @@ export class GeminiTextAdapter<
 > {
   override readonly kind = 'text' as const
   readonly name = 'gemini' as const
+  // Consumes Gemini Files API references (geminiFiles()) as fileData.fileUri.
+  override readonly supportsFileSources = true
 
   private readonly client: GoogleGenAI
 
@@ -594,6 +626,51 @@ export class GeminiTextAdapter<
     let hasEmittedRunStarted = false
     let hasEmittedTextMessageStart = false
     let hasEmittedStepStarted = false
+    let groundingMetadata: GroundingMetadata | undefined
+    let groundingCallEmitted = false
+    const adapterName = this.name
+
+    const emitGroundingToolCall = function* (): Generator<AdapterYieldChunk> {
+      if (!groundingMetadata || groundingCallEmitted) return
+      const sources = getGroundingSources(groundingMetadata)
+      const hasGoogleSearchEvidence =
+        sources.length > 0 ||
+        (groundingMetadata.webSearchQueries?.length ?? 0) > 0 ||
+        groundingMetadata.searchEntryPoint !== undefined
+      if (!hasGoogleSearchEvidence) return
+
+      groundingCallEmitted = true
+      const toolCallId = generateId(adapterName)
+      const metadata: GeminiToolCallMetadata = {
+        providerExecuted: true,
+        sources,
+        gemini: { groundingMetadata },
+      }
+      yield {
+        type: EventType.TOOL_CALL_START,
+        toolCallId,
+        toolCallName: 'google_search',
+        toolName: 'google_search',
+        parentMessageId: messageId,
+        model,
+        timestamp: Date.now(),
+        index: toolCallMap.size,
+        metadata,
+      }
+      yield {
+        type: EventType.TOOL_CALL_END,
+        toolCallId,
+        toolCallName: 'google_search',
+        toolName: 'google_search',
+        model,
+        timestamp: Date.now(),
+        input: {
+          ...(groundingMetadata.webSearchQueries && {
+            queries: groundingMetadata.webSearchQueries,
+          }),
+        },
+      }
+    }
 
     for await (const chunk of result) {
       logger.provider(`provider=gemini`, { chunk })
@@ -610,6 +687,10 @@ export class GeminiTextAdapter<
         }
       }
 
+      if (chunk.candidates?.[0]?.groundingMetadata) {
+        groundingMetadata = chunk.candidates[0].groundingMetadata
+      }
+
       if (chunk.candidates?.[0]?.content?.parts) {
         const parts = chunk.candidates[0].content.parts
 
@@ -820,6 +901,7 @@ export class GeminiTextAdapter<
       }
 
       if (chunk.candidates?.[0]?.finishReason) {
+        yield* emitGroundingToolCall()
         const finishReason = chunk.candidates[0].finishReason
 
         // Emit TOOL_CALL_END for all tracked tool calls. functionCall parts on
@@ -910,6 +992,8 @@ export class GeminiTextAdapter<
         }
       }
     }
+
+    yield* emitGroundingToolCall()
   }
 
   private convertContentPartToGemini(part: ContentPart): Part {
@@ -920,6 +1004,13 @@ export class GeminiTextAdapter<
       case 'audio':
       case 'video':
       case 'document': {
+        // File references (Gemini Files API) and public URLs both pass
+        // through as `fileData`; Gemini fetches the URI server-side. A
+        // file source's handle is the file URI (throws when another provider
+        // issued it).
+        const fileUri = isFileSource(part.source)
+          ? fileReferenceFor(part.source, this.name)
+          : part.source.value
         const geminiPart: Part =
           part.source.type === 'data'
             ? {
@@ -930,7 +1021,7 @@ export class GeminiTextAdapter<
               }
             : {
                 fileData: {
-                  fileUri: part.source.value,
+                  fileUri,
                   // For URL sources, use provided mimeType or fall back to
                   // reasonable defaults.
                   mimeType:
@@ -994,6 +1085,11 @@ export class GeminiTextAdapter<
 
       if (msg.role === 'assistant' && msg.toolCalls?.length) {
         for (const toolCall of msg.toolCalls) {
+          const metadata = toolCall.metadata as
+            | GeminiToolCallMetadata
+            | undefined
+          if (metadata?.providerExecuted) continue
+
           let parsedArgs: Record<string, unknown> = {}
           try {
             parsedArgs = toolCall.function.arguments
@@ -1006,9 +1102,7 @@ export class GeminiTextAdapter<
             parsedArgs = {}
           }
 
-          const thoughtSignature = (
-            toolCall.metadata as GeminiToolCallMetadata | undefined
-          )?.thoughtSignature
+          const thoughtSignature = metadata?.thoughtSignature
           // Gemini requires thoughtSignature at the Part level (sibling of
           // functionCall), not nested inside functionCall. Nesting it causes
           // the API to reject the next turn with
@@ -1045,6 +1139,9 @@ export class GeminiTextAdapter<
                 },
               })
             } else {
+              const fileUri = isFileSource(part.source)
+                ? fileReferenceFor(part.source, this.name)
+                : part.source.value
               const defaultMimeType = {
                 image: 'image/jpeg',
                 audio: 'audio/mp3',
@@ -1053,7 +1150,7 @@ export class GeminiTextAdapter<
               }[part.type]
               mediaParts.push({
                 fileData: {
-                  fileUri: part.source.value,
+                  fileUri,
                   mimeType: part.source.mimeType ?? defaultMimeType,
                 },
               })

package/src/adapters/video.ts

@@ -2,7 +2,12 @@ import {
   GenerateVideosOperation,
   VideoGenerationReferenceType,
 } from '@google/genai'
-import { resolveMediaPrompt } from '@tanstack/ai'
+import {
+  fileReferenceFor,
+  isFileSource,
+  resolveMediaPrompt,
+  unsupportedFileSourceError,
+} from '@tanstack/ai'
 import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters'
 import { arrayBufferToBase64 } from '@tanstack/ai-utils'
 import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils'
@@ -93,6 +98,14 @@ async function imagePartToVeoImage(
       mimeType: part.source.mimeType || 'image/png',
     }
   }
+  if (isFileSource(part.source)) {
+    // Veo's predict API accepts only inline bytes or a gs:// reference — there's
+    // no way to reference a Files API handle here.
+    throw unsupportedFileSourceError(
+      'gemini',
+      'for Veo video generation, which needs inline image bytes or a gs:// reference — pass a data: URI or gs:// URL',
+    )
+  }
   const url = part.source.value
   if (url.startsWith('gs://')) {
     return {
@@ -143,15 +156,21 @@ async function imagePartToVeoImage(
 function mediaPartToInteractionsContent(
   part: ImagePart<MediaInputMetadata> | VideoPart<MediaInputMetadata>,
 ): InteractionContent {
+  // A Gemini Files API reference maps to the `uri` field, same as a public
+  // URL (mirrors the Interactions text adapter). `fileReferenceFor` throws
+  // when another provider issued the handle.
+  const sourceValue = isFileSource(part.source)
+    ? fileReferenceFor(part.source, 'gemini')
+    : part.source.value
   const mimeType = part.source.mimeType
   if (part.type === 'image') {
     return part.source.type === 'data'
-      ? { type: 'image', data: part.source.value, mime_type: mimeType }
-      : { type: 'image', uri: part.source.value, mime_type: mimeType }
+      ? { type: 'image', data: sourceValue, mime_type: mimeType }
+      : { type: 'image', uri: sourceValue, mime_type: mimeType }
   }
   return part.source.type === 'data'
-    ? { type: 'video', data: part.source.value, mime_type: mimeType }
-    : { type: 'video', uri: part.source.value, mime_type: mimeType }
+    ? { type: 'video', data: sourceValue, mime_type: mimeType }
+    : { type: 'video', uri: sourceValue, mime_type: mimeType }
 }
 
 /**
@@ -255,6 +274,9 @@ export class GeminiVideoAdapter<
   GeminiVideoModelDurationByName
 > {
   readonly name = 'gemini' as const
+  // The Interactions path consumes Gemini Files API references as content
+  // `uri`s; the Veo path still rejects them (raw bytes / gs:// only).
+  override readonly supportsFileSources = true
 
   protected client: GoogleGenAI
   private readonly allowUrlFetch: boolean

package/src/experimental/text-interactions/adapter.ts

@@ -1,4 +1,4 @@
-import { EventType } from '@tanstack/ai'
+import { EventType, fileReferenceFor, isFileSource } from '@tanstack/ai'
 import { BaseTextAdapter } from '@tanstack/ai/adapters'
 import { parse as parsePartialJSON } from 'partial-json'
 import {
@@ -200,6 +200,8 @@ export class GeminiTextInteractionsAdapter<
 > {
   override readonly kind = 'text' as const
   override readonly name = 'gemini-text-interactions' as const
+  // Consumes Gemini Files API references (geminiFiles()) as content `uri`s.
+  override readonly supportsFileSources = true
 
   private readonly client: GoogleGenAI
   // Tracks the most recent server-assigned interaction id per threadId
@@ -911,6 +913,13 @@ function contentPartToBlock(part: ContentPart): ContentBlock {
   if (part.type === 'text') {
     return { type: 'text', text: part.content }
   }
+  // A Gemini Files API handle maps to the `uri` field (isData stays false),
+  // same as a public URL. `fileReferenceFor` checks the issuer against
+  // 'gemini', the name geminiFiles() stamps on its handles, not this
+  // adapter's own name.
+  const sourceValue = isFileSource(part.source)
+    ? fileReferenceFor(part.source, 'gemini')
+    : part.source.value
   const isData = part.source.type === 'data'
   switch (part.type) {
     case 'image': {
@@ -920,8 +929,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock {
         'image',
       )
       return isData
-        ? { type: 'image', data: part.source.value, mime_type }
-        : { type: 'image', uri: part.source.value, mime_type }
+        ? { type: 'image', data: sourceValue, mime_type }
+        : { type: 'image', uri: sourceValue, mime_type }
     }
     case 'audio': {
       const mime_type = validateMime(
@@ -930,8 +939,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock {
         'audio',
       )
       return isData
-        ? { type: 'audio', data: part.source.value, mime_type }
-        : { type: 'audio', uri: part.source.value, mime_type }
+        ? { type: 'audio', data: sourceValue, mime_type }
+        : { type: 'audio', uri: sourceValue, mime_type }
     }
     case 'video': {
       const mime_type = validateMime(
@@ -940,8 +949,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock {
         'video',
       )
       return isData
-        ? { type: 'video', data: part.source.value, mime_type }
-        : { type: 'video', uri: part.source.value, mime_type }
+        ? { type: 'video', data: sourceValue, mime_type }
+        : { type: 'video', uri: sourceValue, mime_type }
     }
     case 'document': {
       const mime_type = validateMime(
@@ -950,8 +959,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock {
         'document',
       )
       return isData
-        ? { type: 'document', data: part.source.value, mime_type }
-        : { type: 'document', uri: part.source.value, mime_type }
+        ? { type: 'document', data: sourceValue, mime_type }
+        : { type: 'document', uri: sourceValue, mime_type }
     }
   }
 }

package/src/index.ts

@@ -19,6 +19,14 @@ export {
   type GeminiSummarizeModel,
 } from './adapters/summarize'
 
+// Files adapter - upload media to the Gemini Files API and reference by file URI
+export {
+  GeminiFilesAdapter,
+  createGeminiFiles,
+  geminiFiles,
+  type GeminiFilesConfig,
+} from './adapters/files'
+
 // Image adapter
 export {
   GeminiImageAdapter,

package/src/message-types.ts

@@ -1,3 +1,6 @@
+import type { ProviderExecutedToolMetadata } from '@tanstack/ai'
+import type { GroundingMetadata } from '@google/genai'
+
 /**
  * Gemini-specific metadata types for multimodal content parts.
  * These types extend the base ContentPart metadata with Gemini-specific options.
@@ -178,6 +181,9 @@ export interface GeminiMessageMetadataByModality {
  *
  * @see https://ai.google.dev/gemini-api/docs/thinking
  */
-export interface GeminiToolCallMetadata {
+export interface GeminiToolCallMetadata extends ProviderExecutedToolMetadata {
   thoughtSignature?: string
+  gemini?: {
+    groundingMetadata: GroundingMetadata
+  }
 }