@tanstack/ai-gemini 0.26.5 → 0.27.0

package/src/adapters/text.ts

@@ -28,6 +28,7 @@ import type {
   GoogleGenAI,
   Part,
   ThinkingLevel,
+  VideoMetadata,
 } from '@google/genai'
 import type {
   ContentPart,
@@ -40,9 +41,113 @@ import type { ExternalTextProviderOptions } from '../text/text-provider-options'
 import type {
   GeminiMessageMetadataByModality,
   GeminiToolCallMetadata,
+  GeminiVideoMetadata,
+  GeminiVideoProcessing,
 } from '../message-types'
 import type { GeminiClientConfig } from '../utils/client'
 
+/**
+ * Fallback MIME types for URL-sourced media parts that don't specify one.
+ */
+const DEFAULT_MEDIA_MIME_TYPES = {
+  image: 'image/jpeg',
+  audio: 'audio/mp3',
+  video: 'video/mp4',
+  document: 'application/pdf',
+} as const
+
+/**
+ * Content block shape for an Interactions API `input` step. The installed
+ * @google/genai types predate the video `processing` field, so we model the
+ * subset we emit and cast at the call site.
+ */
+type InteractionContent =
+  | { type: 'text'; text: string }
+  | {
+      type: 'video'
+      uri?: string
+      data?: string
+      mime_type?: string
+      processing?: GeminiVideoProcessing
+    }
+  | {
+      type: 'image' | 'audio' | 'document'
+      uri?: string
+      data?: string
+      mime_type?: string
+    }
+
+interface InteractionStep {
+  type: 'user_input' | 'model_output'
+  content: Array<InteractionContent>
+}
+
+/** True when any message carries a video part requesting agentic processing. */
+function hasAgenticVideo(messages: Array<ModelMessage>): boolean {
+  return messages.some(
+    (msg) =>
+      Array.isArray(msg.content) &&
+      msg.content.some(
+        (part) =>
+          part.type === 'video' &&
+          (part.metadata as GeminiVideoMetadata | undefined)?.processing ===
+            'agentic',
+      ),
+  )
+}
+
+/** Convert a single content part to an Interactions API content block. */
+function contentPartToInteraction(part: ContentPart): InteractionContent {
+  if (part.type === 'text') {
+    return { type: 'text', text: part.content }
+  }
+
+  const source = part.source
+  const mimeType =
+    source.type === 'data'
+      ? source.mimeType
+      : (source.mimeType ?? DEFAULT_MEDIA_MIME_TYPES[part.type])
+  const base =
+    source.type === 'data'
+      ? { data: source.value, mime_type: mimeType }
+      : { uri: source.value, mime_type: mimeType }
+
+  if (part.type === 'video') {
+    const processing = (part.metadata as GeminiVideoMetadata | undefined)
+      ?.processing
+    return { type: 'video', ...base, ...(processing && { processing }) }
+  }
+  return { type: part.type, ...base }
+}
+
+/**
+ * Build the Interactions API `input` from chat messages. Each user/assistant
+ * message becomes a `user_input` / `model_output` step wrapping its content
+ * blocks — the wrapping the Python SDK performs implicitly but the JS SDK
+ * does not. Tool messages are skipped (unsupported on this path).
+ */
+function buildInteractionsInput(
+  messages: Array<ModelMessage>,
+): Array<InteractionStep> {
+  const steps: Array<InteractionStep> = []
+  for (const msg of messages) {
+    if (msg.role === 'tool') continue
+    const stepType = msg.role === 'assistant' ? 'model_output' : 'user_input'
+    const content: Array<InteractionContent> = []
+    if (Array.isArray(msg.content)) {
+      for (const part of msg.content) {
+        content.push(contentPartToInteraction(part))
+      }
+    } else if (msg.content) {
+      content.push({ type: 'text', text: msg.content })
+    }
+    if (content.length > 0) {
+      steps.push({ type: stepType, content })
+    }
+  }
+  return steps
+}
+
 /**
  * Configuration for Gemini text adapter
  */
@@ -122,6 +227,13 @@ export class GeminiTextAdapter<
   async *chatStream(
     options: TextOptions<GeminiTextProviderOptions>,
   ): AsyncIterable<AdapterYieldChunk> {
+    // Agentic video understanding is only exposed through the Interactions API,
+    // not generateContent. Detect it and take that path instead.
+    if (hasAgenticVideo(options.messages)) {
+      yield* this.interactionsStream(options)
+      return
+    }
+
     const mappedOptions = this.mapCommonOptionsToGemini(options)
     const { logger } = options
 
@@ -161,6 +273,114 @@ export class GeminiTextAdapter<
     }
   }
 
+  /**
+   * Agentic video-understanding path via the Interactions API.
+   *
+   * The Interactions API (unlike `generateContent`) requires message parts to
+   * be wrapped in `user_input` / `model_output` steps, and it accepts the
+   * `processing: 'agentic'` video flag. This is a non-streaming call whose
+   * single text result is re-emitted as AG-UI stream chunks.
+   */
+  private async *interactionsStream(
+    options: TextOptions<GeminiTextProviderOptions>,
+  ): AsyncIterable<AdapterYieldChunk> {
+    const model = options.model
+    const { logger } = options
+    const runId = options.runId ?? generateId(this.name)
+    const threadId = options.threadId ?? generateId(this.name)
+    const messageId = generateId(this.name)
+
+    try {
+      logger.request(
+        `activity=chat provider=gemini model=${model} messages=${options.messages.length} mode=interactions-agentic-video`,
+        { provider: 'gemini', model },
+      )
+
+      const normalizedPrompts = normalizeSystemPrompts(options.systemPrompts)
+      const systemInstruction =
+        normalizedPrompts.length > 0
+          ? normalizedPrompts.map((p) => p.content).join('\n')
+          : undefined
+
+      const input = buildInteractionsInput(options.messages)
+
+      // The installed @google/genai (2.10.0) Interactions `VideoContent` type
+      // predates the `processing` field, so the structurally-built input is
+      // cast at the call boundary. The SDK forwards it to the wire unchanged.
+      const interaction = await this.client.interactions.create({
+        model,
+        ...(systemInstruction !== undefined && {
+          system_instruction: systemInstruction,
+        }),
+        input: input as never,
+      })
+
+      const text = interaction.output_text ?? ''
+
+      yield {
+        type: EventType.RUN_STARTED,
+        runId,
+        threadId,
+        model,
+        timestamp: Date.now(),
+        parentRunId: options.parentRunId,
+      }
+      yield {
+        type: EventType.TEXT_MESSAGE_START,
+        messageId,
+        model,
+        timestamp: Date.now(),
+        role: 'assistant',
+      }
+      if (text) {
+        yield {
+          type: EventType.TEXT_MESSAGE_CONTENT,
+          messageId,
+          model,
+          timestamp: Date.now(),
+          delta: text,
+          content: text,
+        }
+      }
+      yield {
+        type: EventType.TEXT_MESSAGE_END,
+        messageId,
+        model,
+        timestamp: Date.now(),
+      }
+      yield {
+        type: EventType.RUN_FINISHED,
+        runId,
+        threadId,
+        model,
+        timestamp: Date.now(),
+        finishReason: 'stop',
+      }
+    } catch (error) {
+      const rawEvent = toRunErrorRawEvent(error)
+      logger.errors('gemini.interactionsStream fatal', {
+        error,
+        source: 'gemini.interactionsStream',
+      })
+      yield {
+        type: EventType.RUN_ERROR,
+        model,
+        timestamp: Date.now(),
+        message:
+          error instanceof Error
+            ? error.message
+            : 'An unknown error occurred during the chat stream.',
+        ...(rawEvent !== undefined && { rawEvent }),
+        error: {
+          message:
+            error instanceof Error
+              ? error.message
+              : 'An unknown error occurred during the chat stream.',
+        },
+      }
+    }
+  }
+
   /**
    * Generate structured output using Gemini's native JSON response format.
    * Uses responseMimeType: 'application/json' and responseSchema for structured output.
@@ -595,29 +815,42 @@ export class GeminiTextAdapter<
       case 'audio':
       case 'video':
       case 'document': {
-        if (part.source.type === 'data') {
-          return {
-            inlineData: {
-              data: part.source.value,
-              mimeType: part.source.mimeType,
-            },
+        const geminiPart: Part =
+          part.source.type === 'data'
+            ? {
+                inlineData: {
+                  data: part.source.value,
+                  mimeType: part.source.mimeType,
+                },
+              }
+            : {
+                fileData: {
+                  fileUri: part.source.value,
+                  // For URL sources, use provided mimeType or fall back to
+                  // reasonable defaults.
+                  mimeType:
+                    part.source.mimeType ?? DEFAULT_MEDIA_MIME_TYPES[part.type],
+                },
+              }
+
+        // Apply single-pass video sampling controls (fps / clip offsets) from
+        // the part metadata. `processing: 'agentic'` is handled separately via
+        // the Interactions API and never reaches this generateContent path.
+        if (part.type === 'video') {
+          const meta = part.metadata as GeminiVideoMetadata | undefined
+          const videoMetadata: VideoMetadata = {
+            ...(meta?.fps !== undefined && { fps: meta.fps }),
+            ...(meta?.startOffset !== undefined && {
+              startOffset: meta.startOffset,
+            }),
+            ...(meta?.endOffset !== undefined && { endOffset: meta.endOffset }),
           }
-        } else {
-          // For URL sources, use provided mimeType or fall back to reasonable defaults
-          const defaultMimeType = {
-            image: 'image/jpeg',
-            audio: 'audio/mp3',
-            video: 'video/mp4',
-            document: 'application/pdf',
-          }[part.type]
-
-          return {
-            fileData: {
-              fileUri: part.source.value,
-              mimeType: part.source.mimeType ?? defaultMimeType,
-            },
+          if (Object.keys(videoMetadata).length > 0) {
+            geminiPart.videoMetadata = videoMetadata
           }
         }
+
+        return geminiPart
       }
       default: {
         const _exhaustiveCheck: never = part

package/src/files/index.ts

@@ -0,0 +1,114 @@
+import { FileState } from '@google/genai'
+import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils'
+import type { GeminiVideoMetadata } from '../message-types'
+import type { VideoPart } from '@tanstack/ai'
+
+/**
+ * A file uploaded to the Gemini Files API and ready to reference in a message.
+ */
+export interface GeminiUploadedFile {
+  /** Resource name, e.g. `"files/abc123"`. */
+  name: string
+  /** File URI to reference from message content (as a `url` source). */
+  uri: string
+  /** MIME type reported by the Files API. */
+  mimeType: string
+}
+
+/**
+ * Options for {@link uploadGeminiFile}.
+ */
+export interface GeminiUploadFileOptions {
+  /**
+   * API key. Falls back to `GOOGLE_API_KEY` / `GEMINI_API_KEY` from the
+   * environment when omitted.
+   */
+  apiKey?: string
+  /**
+   * MIME type of the file (e.g. `"video/mp4"`). Recommended so the Files API
+   * processes and serves the file with the correct type.
+   */
+  mimeType?: string
+  /** Poll interval while the file is `PROCESSING`, in ms. Default `5000`. */
+  pollIntervalMs?: number
+  /** Max time to wait for processing, in ms. Default `300000` (5 min). */
+  timeoutMs?: number
+}
+
+/**
+ * Upload a file via the Gemini Files API and wait until it is `ACTIVE`.
+ *
+ * Large media (notably video) must be uploaded rather than inlined as base64;
+ * the Files API processes uploads asynchronously. This wraps the upload +
+ * poll-until-ready loop and returns a reference you can drop into message
+ * content as a `url` source (see {@link geminiVideoPart}).
+ *
+ * @throws if the upload has no URI, or processing fails or times out.
+ */
+export async function uploadGeminiFile(
+  file: string | Blob,
+  options: GeminiUploadFileOptions = {},
+): Promise<GeminiUploadedFile> {
+  const {
+    apiKey = getGeminiApiKeyFromEnv(),
+    mimeType,
+    pollIntervalMs = 5000,
+    timeoutMs = 300_000,
+  } = options
+
+  const client = createGeminiClient({ apiKey })
+
+  let uploaded = await client.files.upload({
+    file,
+    ...(mimeType && { config: { mimeType } }),
+  })
+
+  const fileName = uploaded.name
+  if (!fileName) {
+    throw new Error('Gemini file upload did not return a file name.')
+  }
+
+  const deadline = Date.now() + timeoutMs
+  while (uploaded.state === FileState.PROCESSING) {
+    if (Date.now() > deadline) {
+      throw new Error(
+        `Gemini file processing timed out after ${timeoutMs}ms (${fileName}).`,
+      )
+    }
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
+    uploaded = await client.files.get({ name: fileName })
+  }
+
+  if (uploaded.state === FileState.FAILED) {
+    throw new Error(
+      `Gemini file processing failed: ${uploaded.error?.message ?? String(uploaded.state)}`,
+    )
+  }
+  if (!uploaded.uri) {
+    throw new Error('Gemini file upload did not return a URI.')
+  }
+
+  return {
+    name: fileName,
+    uri: uploaded.uri,
+    mimeType: uploaded.mimeType ?? mimeType ?? 'application/octet-stream',
+  }
+}
+
+/**
+ * Build a TanStack AI video content part from an uploaded Gemini file.
+ *
+ * Pass `metadata` to control understanding — e.g.
+ * `{ processing: 'agentic' }` to route through the agentic Interactions path,
+ * or `{ fps, startOffset, endOffset }` for single-pass sampling controls.
+ */
+export function geminiVideoPart(
+  file: GeminiUploadedFile,
+  metadata?: GeminiVideoMetadata,
+): VideoPart<GeminiVideoMetadata> {
+  return {
+    type: 'video',
+    source: { type: 'url', value: file.uri, mimeType: file.mimeType },
+    ...(metadata && { metadata }),
+  }
+}

package/src/index.ts

@@ -60,6 +60,14 @@ export type {
 // having to add `@google/genai` to their own dependencies.
 export { HarmBlockThreshold, HarmCategory } from '@google/genai'
 
+// Files API helpers — upload + poll a file (e.g. video) until it is ACTIVE
+export {
+  uploadGeminiFile,
+  geminiVideoPart,
+  type GeminiUploadedFile,
+  type GeminiUploadFileOptions,
+} from './files/index'
+
 // Embedding adapter - for embedding vectors
 export {
   GeminiEmbeddingAdapter,
@@ -168,6 +176,7 @@ export type {
   GeminiImageMetadata,
   GeminiAudioMetadata,
   GeminiVideoMetadata,
+  GeminiVideoProcessing,
   GeminiDocumentMetadata,
   GeminiMessageMetadataByModality,
 } from './message-types'

package/src/message-types.ts

@@ -87,6 +87,19 @@ export interface GeminiAudioMetadata {
   mimeType?: GeminiAudioMimeType
 }
 
+/**
+ * How Gemini processes a video for understanding.
+ *
+ * - `static` (default): single-pass frame sampling via `generateContent`.
+ * - `agentic`: multi-pass "agentic" video understanding, GA on the
+ *   `agentic_video`-capable flash models (`gemini-3.7-flash`,
+ *   `gemini-3.6-flash`, `gemini-3.5-flash-lite`). The text adapter routes the
+ *   request through the Interactions API instead of `generateContent`. With
+ *   `agentic`, the sampling rate is expressed in the text prompt (e.g. "watch
+ *   it at 0.5 fps"), not via `fps`.
+ */
+export type GeminiVideoProcessing = 'agentic' | 'static'
+
 /**
  * Metadata for Gemini video content parts.
  */
@@ -98,6 +111,30 @@ export interface GeminiVideoMetadata {
    * @see https://ai.google.dev/gemini-api/docs/vision#video-requirements
    */
   mimeType?: GeminiVideoMimeType
+  /**
+   * How the model processes this video for understanding. When set, the
+   * adapter routes the request through the Gemini Interactions API. Omit for
+   * the default single-pass `generateContent` behavior.
+   */
+  processing?: GeminiVideoProcessing
+  /**
+   * Frame-rate sampling density (frames per second) for single-pass
+   * (`generateContent`) understanding. Valid range (0, 24]; defaults to 1.0
+   * on the server. Ignored when `processing` is `agentic`.
+   *
+   * @see https://ai.google.dev/gemini-api/docs/vision#customize-frame-rate
+   */
+  fps?: number
+  /**
+   * Clip start offset, as a decimal number of seconds with an `s` suffix
+   * (e.g. `"10.5s"`). Restricts understanding to a segment of the video.
+   */
+  startOffset?: string
+  /**
+   * Clip end offset, as a decimal number of seconds with an `s` suffix
+   * (e.g. `"45s"`). Restricts understanding to a segment of the video.
+   */
+  endOffset?: string
 }
 
 /**

package/src/model-meta.ts

@@ -14,6 +14,7 @@ interface ModelMeta<TProviderOptions = unknown> {
     input: Array<'text' | 'image' | 'audio' | 'video' | 'document'>
     output: Array<'text' | 'image' | 'audio' | 'video'>
     capabilities?: Array<
+      | 'agentic_video'
       | 'audio_generation'
       | 'batch_api'
       | 'caching'
@@ -879,6 +880,7 @@ const GEMINI_3_7_FLASH = {
     input: ['text', 'image', 'video', 'audio', 'document'],
     output: ['text'],
     capabilities: [
+      'agentic_video',
       'batch_api',
       'caching',
       'function_calling',
@@ -923,6 +925,7 @@ const GEMINI_3_6_FLASH = {
     input: ['text', 'image', 'video', 'audio', 'document'],
     output: ['text'],
     capabilities: [
+      'agentic_video',
       'batch_api',
       'caching',
       'function_calling',
@@ -1006,6 +1009,7 @@ const GEMINI_3_5_FLASH_LITE = {
     input: ['text', 'image', 'video', 'audio', 'document'],
     output: ['text'],
     capabilities: [
+      'agentic_video',
       'batch_api',
       'caching',
       'function_calling',