@tanstack/ai-gemini 0.16.2 → 0.17.1

package/src/adapters/image.ts

@@ -1,4 +1,6 @@
+import { resolveMediaPrompt } from '@tanstack/ai'
 import { BaseImageAdapter } from '@tanstack/ai/adapters'
+import { arrayBufferToBase64 } from '@tanstack/ai-utils'
 import {
   createGeminiClient,
   generateId,
@@ -14,6 +16,7 @@ import {
 } from '../image/image-provider-options'
 import type { GEMINI_IMAGE_MODELS } from '../model-meta'
 import type {
+  GeminiImageModelInputModalitiesByName,
   GeminiImageModelProviderOptionsByName,
   GeminiImageModelSizeByName,
   GeminiImageProviderOptions,
@@ -22,13 +25,18 @@ import type {
   GeneratedImage,
   ImageGenerationOptions,
   ImageGenerationResult,
+  ImagePart,
+  MediaInputMetadata,
+  ResolvedMediaPrompt,
 } from '@tanstack/ai'
 import type {
+  Content,
   GenerateContentConfig,
   GenerateContentResponse,
   GenerateImagesConfig,
   GenerateImagesResponse,
   GoogleGenAI,
+  Part,
 } from '@google/genai'
 import type { GeminiClientConfig } from '../utils'
 
@@ -60,7 +68,8 @@ export class GeminiImageAdapter<
   TModel,
   GeminiImageProviderOptions,
   GeminiImageModelProviderOptionsByName,
-  GeminiImageModelSizeByName
+  GeminiImageModelSizeByName,
+  GeminiImageModelInputModalitiesByName
 > {
   override readonly kind = 'image' as const
   readonly name = 'gemini' as const
@@ -70,6 +79,7 @@ export class GeminiImageAdapter<
     providerOptions: GeminiImageProviderOptions
     modelProviderOptionsByName: GeminiImageModelProviderOptionsByName
     modelSizeByName: GeminiImageModelSizeByName
+    modelInputModalitiesByName: GeminiImageModelInputModalitiesByName
   }
 
   private readonly client: GoogleGenAI
@@ -82,7 +92,7 @@ export class GeminiImageAdapter<
   async generateImages(
     options: ImageGenerationOptions<GeminiImageProviderOptions>,
   ): Promise<ImageGenerationResult> {
-    const { model, prompt, logger } = options
+    const { model, logger } = options
 
     logger.request(
       `activity=generateImage provider=gemini model=${this.model}`,
@@ -93,10 +103,35 @@ export class GeminiImageAdapter<
     )
 
     try {
-      validatePrompt({ prompt, model })
+      const resolved = resolveMediaPrompt(options.prompt)
+
+      // Image-only prompts are allowed (the image inputs carry the intent);
+      // a prompt with neither text nor images is always an error.
+      if (resolved.images.length === 0) {
+        validatePrompt({ prompt: resolved.text, model })
+      }
+
+      if (resolved.videos.length > 0) {
+        throw new Error(
+          `${this.name}.generateImages does not support video prompt parts (model: ${model}).`,
+        )
+      }
+      if (resolved.audios.length > 0) {
+        throw new Error(
+          `${this.name}.generateImages does not support audio prompt parts (model: ${model}).`,
+        )
+      }
 
       if (this.isGeminiImageModel(model)) {
-        return await this.generateWithGeminiApi(options)
+        return await this.generateWithGeminiApi(options, resolved)
+      }
+
+      // Imagen does not accept image inputs — it's strictly text-to-image.
+      if (resolved.images.length > 0) {
+        throw new Error(
+          `${this.name}: model "${model}" (Imagen) does not support image prompt parts. ` +
+            `Use a Gemini-native image model (e.g. gemini-2.5-flash-image, "nano-banana") for image-conditioned generation.`,
+        )
       }
 
       // Imagen models path (generateImages API)
@@ -107,7 +142,7 @@ export class GeminiImageAdapter<
 
       const response = await this.client.models.generateImages({
         model,
-        prompt,
+        prompt: resolved.text,
         config,
       })
 
@@ -127,18 +162,12 @@ export class GeminiImageAdapter<
 
   private async generateWithGeminiApi(
     options: ImageGenerationOptions<GeminiImageProviderOptions>,
+    resolved: ResolvedMediaPrompt,
   ): Promise<ImageGenerationResult> {
-    const { model, prompt, size, numberOfImages, modelOptions } = options
+    const { model, size, numberOfImages, modelOptions } = options
 
     const parsedSize = size ? parseNativeImageSize(size) : undefined
 
-    // The generateContent API has no numberOfImages parameter.
-    // Instead, augment the prompt to request multiple images when needed.
-    const augmentedPrompt =
-      numberOfImages && numberOfImages > 1
-        ? `${prompt} Generate ${numberOfImages} distinct images.`
-        : prompt
-
     // GeminiImageProviderOptions is Imagen-shaped — most fields
     // (personGeneration, safetyFilterLevel, addWatermark, outputMimeType,
     // outputCompressionQuality, guidanceScale, enhancePrompt,
@@ -170,15 +199,105 @@ export class GeminiImageAdapter<
       }),
     }
 
+    const contents = await this.buildContents(resolved, numberOfImages)
+
     const response = await this.client.models.generateContent({
       model,
-      contents: augmentedPrompt,
+      contents,
       config,
     })
 
     return this.transformGeminiResponse(model, response)
   }
 
+  /**
+   * Build the multimodal `contents` payload. Text-only prompts pass through
+   * as a plain string (the SDK accepts it directly); prompts with image
+   * parts become a single user `Content` whose `parts` mirror the prompt's
+   * interleaved order — position is meaningful to Gemini ("not like this
+   * *(image)*, more like this *(image)*").
+   *
+   * The generateContent API has no numberOfImages parameter, so when more
+   * than one image is requested a trailing instruction is appended.
+   */
+  private async buildContents(
+    resolved: ResolvedMediaPrompt,
+    numberOfImages: number | undefined,
+  ): Promise<string | Array<Content>> {
+    const countInstruction =
+      numberOfImages && numberOfImages > 1
+        ? `Generate ${numberOfImages} distinct images.`
+        : undefined
+
+    if (resolved.images.length === 0) {
+      return countInstruction
+        ? `${resolved.text} ${countInstruction}`
+        : resolved.text
+    }
+
+    const parts: Array<Part> = await Promise.all(
+      resolved.parts.map((part) => {
+        if (part.type === 'text') {
+          return Promise.resolve<Part>({ text: part.content })
+        }
+        if (part.type === 'image') {
+          return this.imagePartToGeminiPart(part)
+        }
+        // Video / audio parts were rejected in generateImages above.
+        throw new Error(
+          `gemini: unsupported prompt part type "${part.type}" in image generation.`,
+        )
+      }),
+    )
+    if (countInstruction) {
+      parts.push({ text: countInstruction })
+    }
+    return [{ role: 'user', parts }]
+  }
+
+  private async imagePartToGeminiPart(
+    part: ImagePart<MediaInputMetadata>,
+  ): Promise<Part> {
+    if (part.source.type === 'data') {
+      return {
+        inlineData: {
+          mimeType: part.source.mimeType || 'image/png',
+          data: part.source.value,
+        },
+      }
+    }
+    // For URL sources, prefer passing the URL through as `fileData` when it
+    // looks like a Google Files API URI; otherwise fetch and inline as base64.
+    if (
+      part.source.value.startsWith('gs://') ||
+      /^https?:\/\/generativelanguage\.googleapis\.com\//.test(
+        part.source.value,
+      )
+    ) {
+      return {
+        fileData: {
+          fileUri: part.source.value,
+          ...(part.source.mimeType && { mimeType: part.source.mimeType }),
+        },
+      }
+    }
+    const response = await fetch(part.source.value)
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch image input (${response.status} ${response.statusText}): ${part.source.value}`,
+      )
+    }
+    const blob = await response.blob()
+    const buffer = await blob.arrayBuffer()
+    const base64 = arrayBufferToBase64(buffer)
+    return {
+      inlineData: {
+        mimeType: part.source.mimeType || blob.type || 'image/png',
+        data: base64,
+      },
+    }
+  }
+
   private transformGeminiResponse(
     model: string,
     response: GenerateContentResponse,

package/src/adapters/video.ts

@@ -0,0 +1,411 @@
+import {
+  GenerateVideosOperation,
+  VideoGenerationReferenceType,
+} from '@google/genai'
+import { resolveMediaPrompt } from '@tanstack/ai'
+import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters'
+import { arrayBufferToBase64 } from '@tanstack/ai-utils'
+import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils'
+import { getGeminiVideoDurationOptions } from '../video/video-provider-options'
+import type { DurationOptions } from '@tanstack/ai/adapters'
+import type {
+  ImagePart,
+  MediaInputMetadata,
+  VideoGenerationOptions,
+  VideoJobResult,
+  VideoStatusResult,
+  VideoUrlResult,
+} from '@tanstack/ai'
+import type {
+  GenerateVideosConfig,
+  GoogleGenAI,
+  Image,
+  VideoGenerationReferenceImage,
+} from '@google/genai'
+import type {
+  GeminiVideoModel,
+  GeminiVideoModelDurationByName,
+  GeminiVideoModelInputModalitiesByName,
+  GeminiVideoModelProviderOptionsByName,
+  GeminiVideoModelSizeByName,
+  GeminiVideoProviderOptions,
+  GeminiVideoSize,
+} from '../video/video-provider-options'
+import type { GeminiClientConfig } from '../utils'
+
+/**
+ * Configuration for Gemini video adapter.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export interface GeminiVideoConfig extends GeminiClientConfig {}
+
+/**
+ * Extract a human-readable message from a long-running operation's error,
+ * which the SDK types as `Record<string, unknown>` (a google.rpc.Status).
+ */
+function operationErrorMessage(error: Record<string, unknown>): string {
+  if (typeof error.message === 'string' && error.message.length > 0) {
+    return error.message
+  }
+  return JSON.stringify(error)
+}
+
+/**
+ * Convert a TanStack image prompt part into the genai `Image` shape Veo
+ * accepts: base64 `imageBytes` (data sources, data: URIs, fetched HTTP
+ * URLs) or a `gcsUri` passthrough for Cloud Storage references.
+ */
+async function imagePartToVeoImage(
+  part: ImagePart<MediaInputMetadata>,
+): Promise<Image> {
+  if (part.source.type === 'data') {
+    return {
+      imageBytes: part.source.value,
+      mimeType: part.source.mimeType || 'image/png',
+    }
+  }
+  const url = part.source.value
+  if (url.startsWith('gs://')) {
+    return {
+      gcsUri: url,
+      ...(part.source.mimeType && { mimeType: part.source.mimeType }),
+    }
+  }
+  if (url.startsWith('data:')) {
+    const match = url.match(/^data:([^;,]+)?(;base64)?,(.*)$/)
+    if (!match || !match[2]) {
+      throw new Error(
+        'gemini: only base64 data: URIs are supported for video image inputs.',
+      )
+    }
+    return {
+      imageBytes: match[3] ?? '',
+      mimeType: match[1] || part.source.mimeType || 'image/png',
+    }
+  }
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch image input (${response.status} ${response.statusText}): ${url}`,
+    )
+  }
+  const blob = await response.blob()
+  const buffer = await blob.arrayBuffer()
+  return {
+    imageBytes: arrayBufferToBase64(buffer),
+    mimeType: part.source.mimeType || blob.type || 'image/png',
+  }
+}
+
+/**
+ * Gemini Veo Video Generation Adapter
+ *
+ * Tree-shakeable adapter for Google Veo video generation. Veo runs as a
+ * long-running operation: `createVideoJob` starts the operation via the
+ * `:predictLongRunning` endpoint, `getVideoStatus` polls it, and
+ * `getVideoUrl` extracts the generated video's URI once it completes.
+ *
+ * Image prompt parts are routed by `metadata.role`:
+ * - `'start_frame'` (or the first un-roled image) → the input image the
+ *   video starts from
+ * - `'end_frame'` → `lastFrame` (the frame the video ends on)
+ * - `'reference'` / `'character'` → `referenceImages` (asset references,
+ *   Veo 3.1)
+ *
+ * Note: the returned video URI is served by the Gemini Files API and
+ * requires the API key (`x-goog-api-key` header or `?key=` query
+ * parameter) to download.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export class GeminiVideoAdapter<
+  TModel extends GeminiVideoModel,
+> extends BaseVideoAdapter<
+  TModel,
+  GeminiVideoProviderOptions,
+  GeminiVideoModelProviderOptionsByName,
+  GeminiVideoModelSizeByName,
+  GeminiVideoModelInputModalitiesByName,
+  GeminiVideoModelDurationByName
+> {
+  readonly name = 'gemini' as const
+
+  protected client: GoogleGenAI
+
+  constructor(config: GeminiVideoConfig, model: TModel) {
+    super({}, model)
+    this.client = createGeminiClient(config)
+  }
+
+  async createVideoJob(
+    options: VideoGenerationOptions<
+      GeminiVideoProviderOptions,
+      GeminiVideoSize,
+      GeminiVideoModelDurationByName[TModel]
+    >,
+  ): Promise<VideoJobResult> {
+    const { prompt, size, duration, modelOptions, logger } = options
+
+    logger.request(
+      `activity=video.create provider=${this.name} model=${this.model} size=${size ?? 'default'} duration=${duration ?? 'default'}`,
+      { provider: this.name, model: this.model },
+    )
+
+    try {
+      const resolved = resolveMediaPrompt(prompt)
+
+      if (resolved.videos.length > 0) {
+        throw new Error(
+          `${this.name}.createVideoJob does not support video prompt parts (model: ${this.model}).`,
+        )
+      }
+      if (resolved.audios.length > 0) {
+        throw new Error(
+          `${this.name}.createVideoJob does not support audio prompt parts (model: ${this.model}).`,
+        )
+      }
+
+      const { image, lastFrame, referenceImages } = await this.routeImageParts(
+        resolved.images,
+      )
+
+      const config: GenerateVideosConfig = {
+        ...modelOptions,
+        ...(size !== undefined && { aspectRatio: size }),
+        ...(duration !== undefined && { durationSeconds: duration }),
+        ...(lastFrame && { lastFrame }),
+        ...(referenceImages.length > 0 && { referenceImages }),
+      }
+
+      const operation = await this.client.models.generateVideos({
+        model: this.model,
+        prompt: resolved.text,
+        ...(image && { image }),
+        config,
+      })
+
+      if (!operation.name) {
+        throw new Error(
+          'Veo did not return an operation name for the video generation job.',
+        )
+      }
+
+      return { jobId: operation.name, model: this.model }
+    } catch (error) {
+      logger.errors(`${this.name}.createVideoJob fatal`, {
+        error,
+        source: `${this.name}.createVideoJob`,
+      })
+      throw error
+    }
+  }
+
+  /**
+   * Route image prompt parts onto Veo's request fields by `metadata.role`.
+   */
+  private async routeImageParts(
+    parts: Array<ImagePart<MediaInputMetadata>>,
+  ): Promise<{
+    image: Image | undefined
+    lastFrame: Image | undefined
+    referenceImages: Array<VideoGenerationReferenceImage>
+  }> {
+    let image: Image | undefined
+    let lastFrame: Image | undefined
+    const referenceImages: Array<VideoGenerationReferenceImage> = []
+
+    for (const part of parts) {
+      const role = part.metadata?.role
+      switch (role) {
+        case 'end_frame': {
+          if (lastFrame) {
+            throw new Error(
+              `${this.name}: Veo accepts at most one 'end_frame' image.`,
+            )
+          }
+          lastFrame = await imagePartToVeoImage(part)
+          break
+        }
+        case 'reference':
+        case 'character': {
+          referenceImages.push({
+            image: await imagePartToVeoImage(part),
+            referenceType: VideoGenerationReferenceType.ASSET,
+          })
+          break
+        }
+        case 'start_frame':
+        case undefined: {
+          if (image) {
+            throw new Error(
+              `${this.name}: Veo accepts at most one starting image; received multiple 'start_frame'/un-roled images. Use metadata.role ('end_frame', 'reference') to disambiguate the others.`,
+            )
+          }
+          image = await imagePartToVeoImage(part)
+          break
+        }
+        case 'mask':
+        case 'control':
+          throw new Error(
+            `${this.name}: unsupported image role "${role}" for Veo video generation.`,
+          )
+      }
+    }
+
+    return { image, lastFrame, referenceImages }
+  }
+
+  async getVideoStatus(jobId: string): Promise<VideoStatusResult> {
+    const operation = await this.getOperation(jobId)
+
+    if (!operation.done) {
+      return { jobId, status: 'processing' }
+    }
+
+    if (operation.error) {
+      return {
+        jobId,
+        status: 'failed',
+        error: operationErrorMessage(operation.error),
+      }
+    }
+
+    // The operation can finish "successfully" with every sample dropped by
+    // Responsible-AI filters — surface that as a failure instead of letting
+    // getVideoUrl() throw on an empty response.
+    const videos = operation.response?.generatedVideos ?? []
+    if (videos.length === 0) {
+      const reasons = operation.response?.raiMediaFilteredReasons
+      return {
+        jobId,
+        status: 'failed',
+        error: reasons?.length
+          ? `Video was filtered by Responsible-AI: ${reasons.join('; ')}`
+          : 'Veo returned no generated videos.',
+      }
+    }
+
+    return { jobId, status: 'completed' }
+  }
+
+  async getVideoUrl(jobId: string): Promise<VideoUrlResult> {
+    const operation = await this.getOperation(jobId)
+
+    if (!operation.done) {
+      throw new Error(
+        `Video is not ready yet. Check status first. Job ID: ${jobId}`,
+      )
+    }
+
+    if (operation.error) {
+      throw new Error(
+        `Video generation failed: ${operationErrorMessage(operation.error)}`,
+      )
+    }
+
+    const uri = operation.response?.generatedVideos?.[0]?.video?.uri
+    if (!uri) {
+      const reasons = operation.response?.raiMediaFilteredReasons
+      throw new Error(
+        reasons?.length
+          ? `Video was filtered by Responsible-AI: ${reasons.join('; ')}`
+          : `Video URL not found in operation response. Job ID: ${jobId}`,
+      )
+    }
+
+    return { jobId, url: uri }
+  }
+
+  override availableDurations(): DurationOptions<
+    GeminiVideoModelDurationByName[TModel]
+  > {
+    return getGeminiVideoDurationOptions(this.model)
+  }
+
+  override snapDuration(
+    seconds: number,
+  ): GeminiVideoModelDurationByName[TModel] | undefined {
+    return snapToDurationOption(seconds, this.availableDurations())
+  }
+
+  /**
+   * Fetch the long-running operation by name. The SDK's
+   * `operations.getVideosOperation` needs a real `GenerateVideosOperation`
+   * instance (it calls `_fromAPIResponse` on it), so reconstruct one from
+   * the job ID rather than passing an object literal.
+   */
+  private async getOperation(jobId: string): Promise<GenerateVideosOperation> {
+    const operation = new GenerateVideosOperation()
+    operation.name = jobId
+    return await this.client.operations.getVideosOperation({ operation })
+  }
+}
+
+/**
+ * Creates a Gemini video adapter with an explicit API key.
+ * Type resolution happens here at the call site.
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ *
+ * @param model - The model name (e.g., 'veo-3.1-generate-preview')
+ * @param apiKey - Your Google API key
+ * @param config - Optional additional configuration
+ * @returns Configured Gemini video adapter instance with resolved types
+ *
+ * @example
+ * ```typescript
+ * const adapter = createGeminiVideo('veo-3.1-generate-preview', 'your-api-key');
+ *
+ * const { jobId } = await generateVideo({
+ *   adapter,
+ *   prompt: 'A beautiful sunset over the ocean',
+ *   duration: adapter.snapDuration(7), // → 6
+ * });
+ * ```
+ */
+export function createGeminiVideo<TModel extends GeminiVideoModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<GeminiVideoConfig, 'apiKey'>,
+): GeminiVideoAdapter<TModel> {
+  return new GeminiVideoAdapter({ apiKey, ...config }, model)
+}
+
+/**
+ * Creates a Gemini video adapter with automatic API key detection from environment variables.
+ * Type resolution happens here at the call site.
+ *
+ * Looks for `GOOGLE_API_KEY` or `GEMINI_API_KEY` in:
+ * - `process.env` (Node.js)
+ * - `window.env` (Browser with injected env)
+ *
+ * @experimental Video generation is an experimental feature and may change.
+ *
+ * @param model - The model name (e.g., 'veo-3.1-generate-preview')
+ * @param config - Optional configuration (excluding apiKey which is auto-detected)
+ * @returns Configured Gemini video adapter instance with resolved types
+ * @throws Error if GOOGLE_API_KEY or GEMINI_API_KEY is not found in environment
+ *
+ * @example
+ * ```typescript
+ * // Automatically uses GOOGLE_API_KEY from environment
+ * const adapter = geminiVideo('veo-3.1-generate-preview');
+ *
+ * // Create a video generation job
+ * const { jobId } = await generateVideo({
+ *   adapter,
+ *   prompt: 'A cat playing piano'
+ * });
+ *
+ * // Poll for status
+ * const status = await getVideoJobStatus({ adapter, jobId });
+ * ```
+ */
+export function geminiVideo<TModel extends GeminiVideoModel>(
+  model: TModel,
+  config?: Omit<GeminiVideoConfig, 'apiKey'>,
+): GeminiVideoAdapter<TModel> {
+  const apiKey = getGeminiApiKeyFromEnv()
+  return createGeminiVideo(model, apiKey, config)
+}

package/src/image/image-provider-options.ts

@@ -189,6 +189,18 @@ export type GeminiImageModelSizeByName = {
   [K in Exclude<GeminiImageModels, GeminiNativeImageModels>]: GeminiImageSize
 }
 
+/**
+ * Per-model prompt input modalities. Gemini-native image models accept image
+ * parts in the multimodal prompt (image-conditioned generation via
+ * generateContent); Imagen models are strictly text-to-image, so their
+ * `prompt` is constrained to text at compile time.
+ */
+export type GeminiImageModelInputModalitiesByName = {
+  [K in GeminiNativeImageModels]: readonly ['image']
+} & {
+  [K in Exclude<GeminiImageModels, GeminiNativeImageModels>]: readonly []
+}
+
 /**
  * Valid sizes for Gemini Imagen models
  * Gemini uses aspect ratios, but we map common WIDTHxHEIGHT formats to aspect ratios

package/src/index.ts

@@ -61,6 +61,30 @@ export {
   type GeminiAudioProviderOptions,
 } from './adapters/audio'
 
+// Video / Veo generation adapter (experimental)
+/**
+ * @experimental Veo video generation is an experimental feature and may change.
+ */
+export {
+  GeminiVideoAdapter,
+  createGeminiVideo,
+  geminiVideo,
+  type GeminiVideoConfig,
+} from './adapters/video'
+export {
+  GEMINI_VIDEO_DURATIONS,
+  getGeminiVideoDurationOptions,
+} from './video/video-provider-options'
+export type {
+  GeminiVideoModel,
+  GeminiVideoModelDurationByName,
+  GeminiVideoModelInputModalitiesByName,
+  GeminiVideoModelProviderOptionsByName,
+  GeminiVideoModelSizeByName,
+  GeminiVideoProviderOptions,
+  GeminiVideoSize,
+} from './video/video-provider-options'
+
 // Re-export models from model-meta for convenience
 export {
   GEMINI_MODELS,
@@ -71,6 +95,7 @@ export { GEMINI_IMAGE_MODELS as GeminiImageModels } from './model-meta'
 export { GEMINI_TTS_MODELS as GeminiTTSModels } from './model-meta'
 export { GEMINI_TTS_VOICES as GeminiTTSVoices } from './model-meta'
 export { GEMINI_AUDIO_MODELS as GeminiAudioModels } from './model-meta'
+export { GEMINI_VIDEO_MODELS as GeminiVideoModels } from './model-meta'
 export type { GeminiModels as GeminiTextModel } from './model-meta'
 export type { GeminiImageModels as GeminiImageModel } from './model-meta'
 export type { GeminiTTSVoice } from './model-meta'