@tanstack/ai 0.29.0 → 0.32.0

package/dist/esm/types.d.ts

@@ -1189,6 +1189,76 @@ export interface SummarizationResult {
     summary: string;
     usage: TokenUsage;
 }
+/**
+ * Optional role hint on a media input part (image / video / audio). Adapters
+ * read `metadata.role` to route the part to the provider-specific request
+ * field — e.g. `'mask'` → OpenAI `mask` / fal `mask_url`, `'end_frame'` → fal
+ * `end_image_url`, `'reference'` → fal `reference_image_urls`. When omitted
+ * the adapter falls back to positional routing.
+ */
+export type MediaInputRole = 'reference' | 'mask' | 'control' | 'start_frame' | 'end_frame' | 'character';
+/**
+ * Metadata convention for image / video / audio inputs to media generation.
+ * Carried on `ImagePart.metadata` / `VideoPart.metadata` / `AudioPart.metadata`
+ * when used as conditioning inputs to `generateImage()` or `generateVideo()`.
+ */
+export interface MediaInputMetadata {
+    /** Optional role hint disambiguating the part's intent for the adapter */
+    role?: MediaInputRole;
+    /**
+     * Optional user-defined label for this input (e.g. `'woman-in-red-dress'`).
+     * **Informational only** — adapters never read it and the SDK never
+     * rewrites prompt text based on it. Use it to correlate parts with the
+     * references you write in your prompt using the provider's own syntax
+     * (fal's `@Image1`, OpenAI's "image 1", etc.), or for your own
+     * bookkeeping/logging.
+     */
+    tag?: string;
+}
+/**
+ * A single part of a multimodal media-generation prompt. Reuses the chat
+ * content-part shapes: text parts carry the instruction, image / video /
+ * audio parts carry conditioning inputs (with an optional
+ * `metadata.role` hint — see {@link MediaInputRole}).
+ */
+export type MediaPromptPart = TextPart | ImagePart<MediaInputMetadata> | VideoPart<MediaInputMetadata> | AudioPart<MediaInputMetadata>;
+/**
+ * Prompt accepted by `generateImage()` / `generateVideo()`: a plain string,
+ * or an ordered array of content parts for image-conditioned generation
+ * ("not like this *(image)*, more like this *(image)*"). Part order is
+ * meaningful — adapters with native multimodal prompts (Gemini, OpenRouter)
+ * preserve the interleaving; named-field providers (fal, OpenAI, xAI)
+ * extract the media parts and flatten the text. Text is always sent
+ * verbatim: to reference inputs from the prompt, write the provider's own
+ * syntax yourself (e.g. fal's `@Image1`, OpenAI's "image 1"). An array may
+ * be media-only (e.g. upscalers or pure img2img endpoints that take no
+ * instruction text).
+ */
+export type MediaPrompt = string | Array<MediaPromptPart>;
+/**
+ * Non-text modalities a media-generation model can accept in its prompt.
+ */
+export type MediaPromptModality = 'image' | 'video' | 'audio';
+/** Maps a prompt modality to its content-part type. @internal */
+interface MediaPartByModality {
+    image: ImagePart<MediaInputMetadata>;
+    video: VideoPart<MediaInputMetadata>;
+    audio: AudioPart<MediaInputMetadata>;
+}
+/**
+ * Prompt type narrowed to the modalities a specific model supports.
+ * `MediaPromptFor<never>` (a text-only model) is `string | Array<TextPart>`;
+ * `MediaPromptFor<'image'>` additionally admits image parts, etc. Used by
+ * the activity option types together with the adapter's per-model input
+ * modality map so unsupported parts fail at compile time.
+ */
+export type MediaPromptFor<TModalities extends MediaPromptModality = never> = string | Array<TextPart | MediaPartByModality[TModalities]>;
+/**
+ * Per-model map from model name to the prompt modalities it accepts, used as
+ * an adapter type parameter (`TModelInputModalitiesByName`). Models absent
+ * from the map fall back to the unconstrained {@link MediaPrompt}.
+ */
+export type ModelInputModalitiesByName = Record<string, ReadonlyArray<MediaPromptModality>>;
 /**
  * Options for image generation.
  * These are the common options supported across providers.
@@ -1196,8 +1266,16 @@ export interface SummarizationResult {
 export interface ImageGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string> {
     /** The model to use for image generation */
     model: string;
-    /** Text description of the desired image(s) */
-    prompt: string;
+    /**
+     * Description of the desired image(s): a plain string, or an ordered array
+     * of content parts for image-conditioned generation (image-to-image,
+     * reference-guided, edit, multi-reference). Media parts may carry
+     * `metadata.role` to disambiguate intent (mask, control, reference, …).
+     * Adapters map parts onto the provider-native request — e.g. Gemini
+     * multimodal `contents`, OpenAI `images.edit()`, fal `image_url` /
+     * `mask_url` — and throw a clear runtime error for unsupported modalities.
+     */
+    prompt: MediaPrompt;
     /** Number of images to generate (default: 1) */
     numberOfImages?: number;
     /** Image size in WIDTHxHEIGHT format (e.g., "1024x1024") */
@@ -1293,15 +1371,26 @@ export interface AudioGenerationResult {
  *
  * @experimental Video generation is an experimental feature and may change.
  */
-export interface VideoGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string> {
+export interface VideoGenerationOptions<TProviderOptions extends object = object, TSize extends string | undefined = string, TDuration extends string | number | undefined = number> {
     /** The model to use for video generation */
     model: string;
-    /** Text description of the desired video */
-    prompt: string;
+    /**
+     * Description of the desired video: a plain string, or an ordered array of
+     * content parts for image-conditioned generation. Image parts may carry
+     * `metadata.role` (`'start_frame' | 'end_frame' | 'reference' |
+     * 'character'`) to disambiguate intent; adapters route them onto the
+     * provider-native request (e.g. OpenAI Sora `input_reference`, fal
+     * `image_url` / `end_image_url`) and throw at runtime if unsupported.
+     */
+    prompt: MediaPrompt;
     /** Video size — format depends on the provider (e.g., "16:9", "1280x720") */
     size?: TSize;
-    /** Video duration in seconds */
-    duration?: number;
+    /**
+     * Video duration in seconds. Adapters that declare a per-model duration
+     * map narrow this to the model's valid union; use
+     * `adapter.snapDuration(seconds)` to coerce raw seconds to a valid value.
+     */
+    duration?: TDuration;
     /** Model-specific options for video generation */
     modelOptions?: TProviderOptions;
     /**

package/dist/esm/utilities/media-prompt.d.ts

@@ -0,0 +1,35 @@
+import { AudioPart, ImagePart, MediaInputMetadata, MediaPrompt, MediaPromptPart, VideoPart } from '../types.js';
+/**
+ * A {@link MediaPrompt} decomposed into the views adapters consume.
+ *
+ * Adapters with native multimodal prompts (Gemini `contents`, OpenRouter
+ * chat content parts) consume `parts` to preserve interleaving; named-field
+ * providers (fal, OpenAI) consume `text` plus the typed media buckets.
+ *
+ * Prompt text is **never rewritten**: text parts are concatenated verbatim.
+ * Providers that support referencing inputs from the prompt (e.g. fal's
+ * `@Image1`, OpenAI's "image 1" prose) expect the user to write that syntax
+ * themselves — the SDK does not inject or substitute markers.
+ */
+export interface ResolvedMediaPrompt {
+    /**
+     * Text parts concatenated verbatim (paragraph-separated). Empty string
+     * for media-only prompts.
+     */
+    text: string;
+    /** The prompt as ordered parts; a string prompt becomes one text part. */
+    parts: Array<MediaPromptPart>;
+    /** Image parts in prompt order. */
+    images: Array<ImagePart<MediaInputMetadata>>;
+    /** Video parts in prompt order. */
+    videos: Array<VideoPart<MediaInputMetadata>>;
+    /** Audio parts in prompt order. */
+    audios: Array<AudioPart<MediaInputMetadata>>;
+}
+/**
+ * Decompose a {@link MediaPrompt} into flattened text and per-modality part
+ * buckets, preserving prompt order everywhere. This is the single downrev
+ * point from the canonical interleaved prompt shape to the named-field
+ * request shapes most providers expose.
+ */
+export declare function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt;

package/dist/esm/utilities/media-prompt.js

@@ -0,0 +1,43 @@
+function resolveMediaPrompt(prompt) {
+  if (typeof prompt === "string") {
+    const textPart = { type: "text", content: prompt };
+    return {
+      text: prompt,
+      parts: [textPart],
+      images: [],
+      videos: [],
+      audios: []
+    };
+  }
+  const images = [];
+  const videos = [];
+  const audios = [];
+  const textSegments = [];
+  for (const part of prompt) {
+    switch (part.type) {
+      case "text":
+        if (part.content) textSegments.push(part.content);
+        break;
+      case "image":
+        images.push(part);
+        break;
+      case "video":
+        videos.push(part);
+        break;
+      case "audio":
+        audios.push(part);
+        break;
+    }
+  }
+  return {
+    text: textSegments.join("\n\n"),
+    parts: prompt,
+    images,
+    videos,
+    audios
+  };
+}
+export {
+  resolveMediaPrompt
+};
+//# sourceMappingURL=media-prompt.js.map

package/dist/esm/utilities/media-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"media-prompt.js","sources":["../../../src/utilities/media-prompt.ts"],"sourcesContent":["import type {\n  AudioPart,\n  ImagePart,\n  MediaInputMetadata,\n  MediaPrompt,\n  MediaPromptPart,\n  TextPart,\n  VideoPart,\n} from '../types'\n\n/**\n * A {@link MediaPrompt} decomposed into the views adapters consume.\n *\n * Adapters with native multimodal prompts (Gemini `contents`, OpenRouter\n * chat content parts) consume `parts` to preserve interleaving; named-field\n * providers (fal, OpenAI) consume `text` plus the typed media buckets.\n *\n * Prompt text is **never rewritten**: text parts are concatenated verbatim.\n * Providers that support referencing inputs from the prompt (e.g. fal's\n * `@Image1`, OpenAI's \"image 1\" prose) expect the user to write that syntax\n * themselves — the SDK does not inject or substitute markers.\n */\nexport interface ResolvedMediaPrompt {\n  /**\n   * Text parts concatenated verbatim (paragraph-separated). Empty string\n   * for media-only prompts.\n   */\n  text: string\n  /** The prompt as ordered parts; a string prompt becomes one text part. */\n  parts: Array<MediaPromptPart>\n  /** Image parts in prompt order. */\n  images: Array<ImagePart<MediaInputMetadata>>\n  /** Video parts in prompt order. */\n  videos: Array<VideoPart<MediaInputMetadata>>\n  /** Audio parts in prompt order. */\n  audios: Array<AudioPart<MediaInputMetadata>>\n}\n\n/**\n * Decompose a {@link MediaPrompt} into flattened text and per-modality part\n * buckets, preserving prompt order everywhere. This is the single downrev\n * point from the canonical interleaved prompt shape to the named-field\n * request shapes most providers expose.\n */\nexport function resolveMediaPrompt(prompt: MediaPrompt): ResolvedMediaPrompt {\n  if (typeof prompt === 'string') {\n    const textPart: TextPart = { type: 'text', content: prompt }\n    return {\n      text: prompt,\n      parts: [textPart],\n      images: [],\n      videos: [],\n      audios: [],\n    }\n  }\n\n  const images: Array<ImagePart<MediaInputMetadata>> = []\n  const videos: Array<VideoPart<MediaInputMetadata>> = []\n  const audios: Array<AudioPart<MediaInputMetadata>> = []\n  const textSegments: Array<string> = []\n\n  for (const part of prompt) {\n    switch (part.type) {\n      case 'text':\n        if (part.content) textSegments.push(part.content)\n        break\n      case 'image':\n        images.push(part)\n        break\n      case 'video':\n        videos.push(part)\n        break\n      case 'audio':\n        audios.push(part)\n        break\n    }\n  }\n\n  return {\n    text: textSegments.join('\\n\\n'),\n    parts: prompt,\n    images,\n    videos,\n    audios,\n  }\n}\n"],"names":[],"mappings":"AA4CO,SAAS,mBAAmB,QAA0C;AAC3E,MAAI,OAAO,WAAW,UAAU;AAC9B,UAAM,WAAqB,EAAE,MAAM,QAAQ,SAAS,OAAA;AACpD,WAAO;AAAA,MACL,MAAM;AAAA,MACN,OAAO,CAAC,QAAQ;AAAA,MAChB,QAAQ,CAAA;AAAA,MACR,QAAQ,CAAA;AAAA,MACR,QAAQ,CAAA;AAAA,IAAC;AAAA,EAEb;AAEA,QAAM,SAA+C,CAAA;AACrD,QAAM,SAA+C,CAAA;AACrD,QAAM,SAA+C,CAAA;AACrD,QAAM,eAA8B,CAAA;AAEpC,aAAW,QAAQ,QAAQ;AACzB,YAAQ,KAAK,MAAA;AAAA,MACX,KAAK;AACH,YAAI,KAAK,QAAS,cAAa,KAAK,KAAK,OAAO;AAChD;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,MACF,KAAK;AACH,eAAO,KAAK,IAAI;AAChB;AAAA,IAAA;AAAA,EAEN;AAEA,SAAO;AAAA,IACL,MAAM,aAAa,KAAK,MAAM;AAAA,IAC9B,OAAO;AAAA,IACP;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;"}

package/package.json

@@ -1,14 +1,22 @@
 {
   "name": "@tanstack/ai",
-  "version": "0.29.0",
+  "version": "0.32.0",
   "description": "Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.",
   "author": "Tanner Linsley",
   "license": "MIT",
+  "homepage": "https://tanstack.com/ai",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/TanStack/ai.git",
     "directory": "packages/ai"
   },
+  "bugs": {
+    "url": "https://github.com/TanStack/ai/issues"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/tannerlinsley"
+  },
   "type": "module",
   "module": "./dist/esm/index.js",
   "types": "./dist/esm/index.d.ts",
@@ -68,7 +76,7 @@
     "@ag-ui/core": "^0.0.52",
     "@standard-schema/spec": "^1.1.0",
     "partial-json": "^0.1.7",
-    "@tanstack/ai-event-client": "0.6.0"
+    "@tanstack/ai-event-client": "0.6.3"
   },
   "peerDependencies": {
     "@opentelemetry/api": ">=1.9.0"

package/skills/ai-core/media-generation/SKILL.md

@@ -3,8 +3,9 @@ name: ai-core/media-generation
 description: >
   Image, audio, video, speech (TTS), and transcription generation using
   activity-specific adapters: generateImage() with openaiImage/geminiImage,
-  generateAudio() with geminiAudio/falAudio, generateVideo() with async
-  polling, generateSpeech() with openaiSpeech, generateTranscription() with
+  generateAudio() with geminiAudio/falAudio, generateVideo() with
+  openaiVideo/geminiVideo (async polling, per-model typed durations),
+  generateSpeech() with openaiSpeech, generateTranscription() with
   openaiTranscription. React hooks: useGenerateImage, useGenerateAudio,
   useGenerateSpeech, useTranscription, useGenerateVideo.
   TanStack Start server function integration with toServerSentEventsResponse.
@@ -189,6 +190,103 @@ Result shape: `ImageGenerationResult` with `images` array where each entry
 has `b64Json?`, `url?`, and `revisedPrompt?`. OpenAI image URLs expire
 after 1 hour -- download or display immediately.
 
+#### Image-conditioned generation: multimodal `prompt` parts
+
+Both `generateImage()` and `generateVideo()` accept the `prompt` either as
+a plain string or as an ordered array of content parts (`TextPart` /
+`ImagePart` / `VideoPart` / `AudioPart` — the same shapes used elsewhere in
+TanStack AI). Part order is meaningful: natively multimodal providers
+(Gemini, OpenRouter) receive parts in order; named-field providers (OpenAI,
+fal, xAI) extract media parts and flatten the text. Prompt text is always
+sent verbatim — to reference inputs from the prompt, write the provider's
+own syntax (fal `@Image1`, OpenAI "image 1" prose); the SDK never injects
+or rewrites markers. Each media part may carry an optional
+`metadata.role` hint that adapters use to route the part to the
+provider-specific field. The accepted part types are narrowed per model at
+compile time via the adapter's input-modality map.
+
+```typescript
+import { generateImage } from '@tanstack/ai'
+import { openaiImage } from '@tanstack/ai-openai'
+
+// Image-to-image (OpenAI gpt-image-2 / gpt-image-1, dall-e-2)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Turn this into a cinematic product photo' },
+    { type: 'image', source: { type: 'url', value: 'https://…/product.png' } },
+  ],
+})
+
+// Multi-reference (up to 16 for gpt-image models; up to ~14 for Gemini native
+// — a provider limit, not enforced by the SDK)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Apply the second image as style to the first' },
+    { type: 'image', source: { type: 'url', value: 'https://…/product.png' } },
+    { type: 'image', source: { type: 'url', value: 'https://…/style.png' } },
+  ],
+})
+
+// Inpaint via metadata.role === 'mask' (OpenAI gpt-image models, dall-e-2; fal mask_url)
+await generateImage({
+  adapter: openaiImage('gpt-image-2'),
+  prompt: [
+    { type: 'text', content: 'Replace the masked region with a tree' },
+    { type: 'image', source: { type: 'url', value: photoUrl } },
+    {
+      type: 'image',
+      source: { type: 'url', value: maskUrl },
+      metadata: { role: 'mask' },
+    },
+  ],
+})
+
+// Image-to-video (OpenAI Sora: single input_reference; fal: image_url + optional end_image_url)
+import { generateVideo } from '@tanstack/ai'
+import { falVideo } from '@tanstack/ai-fal'
+
+await generateVideo({
+  adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'),
+  prompt: [
+    { type: 'image', source: { type: 'url', value: firstFrameUrl } },
+    { type: 'text', content: 'Slow cinematic push-in' },
+    {
+      type: 'image',
+      source: { type: 'url', value: lastFrameUrl },
+      metadata: { role: 'end_frame' },
+    },
+  ],
+})
+```
+
+**Role hints** (`metadata.role`):
+
+| Role            | Maps to                                                                                               |
+| --------------- | ----------------------------------------------------------------------------------------------------- |
+| `'reference'`   | fal `reference_image_urls`; Gemini multimodal part; positional otherwise                              |
+| `'character'`   | Same as `'reference'`; Veo `referenceImages` slot (planned — no Veo adapter yet)                      |
+| `'mask'`        | OpenAI `mask` (gpt-image-2, gpt-image-1, dall-e-2); fal `mask_url`                                    |
+| `'control'`     | fal `control_image_url` (ControlNet / depth / pose)                                                   |
+| `'start_frame'` | fal `start_image_url` (or the endpoint's field, e.g. `image_url` on Kling i2v); Veo `image` (planned) |
+| `'end_frame'`   | fal `end_image_url` (or e.g. `tail_image_url` / `last_frame_url`); Veo `lastFrame` (planned)          |
+
+**Provider support matrix:**
+
+| Provider   | `generateImage` image parts                                                                                                                                                                              | `generateVideo` image parts                                                                                                                                                                        |
+| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| OpenAI     | gpt-image-2 / gpt-image-1 / -mini → `images.edit()` (up to 16). dall-e-2 → edit (1). dall-e-3 throws.                                                                                                    | Sora-2 / -pro → `input_reference` (single). Throws if >1.                                                                                                                                          |
+| Gemini     | Native (gemini-\*-flash-image, "nano-banana") → multimodal `contents`. Imagen throws.                                                                                                                    | No native Veo adapter yet — deferred to a follow-up.                                                                                                                                               |
+| fal        | Per-endpoint field names from a generated map (`pnpm generate:fal-image-fields`). Defaults: 1 input → `image_url`; >1 → `image_urls`; roles → `mask_url` / `control_image_url` / `reference_image_urls`. | Per-endpoint map (e.g. Kling i2v start frame → `image_url`). Defaults: 1 input → `image_url`; `start_frame`/`end_frame` → `start_image_url`/`end_image_url`; `reference` → `reference_image_urls`. |
+| Grok       | grok-imagine models → `/v1/images/edits` JSON endpoint (≤3 sources, addressed by xAI in request order; prompt sent verbatim; mask/control throw). grok-2-image-1212 throws.                              | n/a                                                                                                                                                                                                |
+| OpenRouter | Prompt parts map 1:1 onto multimodal `text` / `image_url` content parts, preserving interleaved order.                                                                                                   | n/a                                                                                                                                                                                                |
+| Anthropic  | n/a (no image generation API).                                                                                                                                                                           | n/a                                                                                                                                                                                                |
+
+Video and audio prompt parts follow the same `metadata.role` convention
+for video-to-video and lipsync flows on fal; other providers throw when
+they're passed.
+
 ### 2. Audio Generation (Music, Sound Effects)
 
 Distinct from TTS — `generateAudio()` produces non-speech audio content.
@@ -331,6 +429,31 @@ const stream = generateVideo({
 return toServerSentEventsResponse(stream)
 ```
 
+Google Veo (`@tanstack/ai-gemini`) uses the same jobs/polling flow. Its
+`duration` option is typed per model (e.g. `4 | 6 | 8` for Veo 3.x,
+`5 | 6 | 8` for Veo 2); use `adapter.snapDuration(seconds)` to coerce raw
+seconds and `adapter.availableDurations()` to enumerate the valid set.
+Image prompt parts route by `metadata.role`: first un-roled /
+`'start_frame'` image → input image, `'end_frame'` → `lastFrame`,
+`'reference'` / `'character'` → `referenceImages`:
+
+```typescript
+import { geminiVideo } from '@tanstack/ai-gemini'
+
+const adapter = geminiVideo('veo-3.1-generate-preview')
+adapter.availableDurations() // { kind: 'discrete', values: [4, 6, 8] }
+
+const { jobId } = await generateVideo({
+  adapter,
+  prompt: 'A golden retriever playing in sunflowers',
+  size: '16:9', // Veo sizes are aspect ratios: '16:9' | '9:16'
+  duration: adapter.snapDuration(7), // 6
+  modelOptions: { resolution: '1080p', generateAudio: true },
+})
+// Note: Veo result URLs require the Google API key to download
+// (x-goog-api-key header or ?key= query parameter).
+```
+
 Client hook with job tracking:
 
 ```tsx
@@ -607,7 +730,54 @@ generateSpeech({
 
 > Source: Gemini TTS adapter validation; CodeRabbit review of PR #463.
 
-### h. LOW: Writing a logging middleware to see media chunks flow through
+### h. HIGH: Passing image prompt parts to a model that doesn't support image-conditioned generation
+
+Not every model accepts image-conditioned prompts. The `prompt` type is
+narrowed per model, so passing an image part to a text-only model
+(dall-e-3, Imagen, grok-2-image) is a **compile-time error**; adapters
+also throw a clear runtime error as a backstop, so users learn at call
+time rather than getting silently wrong output.
+
+```typescript
+// WRONG — dall-e-3 has no edit/inputs API; image parts are a type error
+generateImage({
+  adapter: openaiImage('dall-e-3'),
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } }, // ❌ type error
+  ],
+})
+
+// WRONG — Imagen is text-to-image only; same compile-time rejection
+generateImage({
+  adapter: geminiImage('imagen-4.0-generate-001'),
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } }, // ❌ type error
+  ],
+})
+
+// CORRECT — use a model that supports image-conditioned generation
+generateImage({
+  adapter: openaiImage('gpt-image-2'), // edits up to 16 images
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } },
+  ],
+})
+
+generateImage({
+  adapter: geminiImage('gemini-3.1-flash-image-preview'), // native multimodal
+  prompt: [
+    { type: 'text', content: 'Edit this' },
+    { type: 'image', source: { type: 'url', value: url } },
+  ],
+})
+```
+
+> Source: docs/media/image-generation.md, docs/media/video-generation.md.
+
+### i. LOW: Writing a logging middleware to see media chunks flow through
 
 Every media activity — `generateAudio`, `generateSpeech`,
 `generateTranscription`, `generateImage`, `generateVideo` — accepts the

package/src/activities/chat/adapter.ts

@@ -6,6 +6,7 @@ import type {
   TextOptions,
   TokenUsage,
 } from '../../types'
+import type { CapabilityHandle } from './middleware/capabilities'
 
 /**
  * Configuration for adapter instances
@@ -79,6 +80,15 @@ export interface TextAdapter<
   /** The model this adapter is configured for */
   readonly model: TModel
 
+  /**
+   * Capabilities this adapter requires at runtime. `chat()` validates that the
+   * configured middleware provides each one. Model adapters omit this; harness
+   * adapters (e.g. a future `claudeCode()`) declare e.g. `[sandboxCapability]`.
+   * Runtime access to capabilities from inside the adapter is not yet wired —
+   * this is the declaration/validation surface only.
+   */
+  readonly requires?: ReadonlyArray<CapabilityHandle>
+
   /**
    * @internal Type-only properties for inference. Not assigned at runtime.
    */
@@ -183,6 +193,7 @@ export abstract class BaseTextAdapter<
   readonly kind = 'text' as const
   abstract readonly name: string
   readonly model: TModel
+  readonly requires?: ReadonlyArray<CapabilityHandle> = undefined
 
   // Type-only property - never assigned at runtime
   declare '~types': {

package/src/activities/chat/index.ts

@@ -25,6 +25,8 @@ import {
 import { maxIterations as maxIterationsStrategy } from './agent-loop-strategies'
 import { convertMessagesToModelMessages, generateMessageId } from './messages'
 import { MiddlewareRunner } from './middleware/compose'
+import { CapabilityRegistry } from './middleware/capabilities'
+import { validateCapabilities } from './middleware/validate'
 import { MCPManager } from './mcp/manager'
 import type {
   ApprovalRequest,
@@ -54,11 +56,13 @@ import type {
   UIMessage,
 } from '../../types'
 import type {
+  AnyChatMiddleware,
   ChatMiddleware,
   ChatMiddlewareConfig,
   ChatMiddlewareContext,
   StructuredOutputMiddlewareConfig,
 } from './middleware/types'
+import type { CheckCoverage } from './middleware/builder'
 import type { SystemPrompt } from '../../system-prompts'
 import type { InternalLogger } from '../../logger/internal-logger'
 import type { DebugOption } from '../../logger/types'
@@ -141,7 +145,8 @@ type TextActivityOptionsWithContext<
   'tools' | 'middleware' | 'context'
 > & {
   tools?: TTools
-  middleware?: TMiddleware
+  middleware?: TMiddleware &
+    CheckCoverage<Extract<TMiddleware, ReadonlyArray<AnyChatMiddleware>>>
 } & RequiredContextFromInputs<TTools, TMiddleware>
 
 // ===========================
@@ -659,6 +664,15 @@ class TextEngine<
       // References
       messages: this.messages,
       createId: (prefix: string) => this.createId(prefix),
+      // Capability bookkeeping for this request (populated by middleware setup)
+      capabilities: new CapabilityRegistry(),
+      // Convenience accessors that delegate to a capability handle's own
+      // tuple getter/provider, keyed by this context. `getX(ctx)` and
+      // `ctx.get(X)` are interchangeable.
+      get: (capability) => capability[0](this.middlewareCtx),
+      getOptional: (capability) =>
+        capability[0](this.middlewareCtx, { optional: true }),
+      provide: (capability, value) => capability[1](this.middlewareCtx, value),
     }
   }
 
@@ -706,6 +720,9 @@ class TextEngine<
     })
 
     try {
+      // Provision capabilities before any consumer (onConfig onward) can read them
+      await this.middlewareRunner.runSetup(this.middlewareCtx)
+
       // Run initial onConfig (phase = init)
       this.middlewareCtx.phase = 'init'
       const initialConfig = this.buildMiddlewareConfig()
@@ -2564,6 +2581,8 @@ export function chat<
     TMiddleware
   >,
 ): TextActivityResult<TSchema, TStream> {
+  validateCapabilities(options.middleware ?? [], options.adapter)
+
   const { outputSchema, stream } = options
 
   // outputSchema + stream:true is the only branch that streams structured

package/src/activities/chat/middleware/builder.ts

@@ -0,0 +1,109 @@
+import type { CapabilityHandle } from './capabilities'
+import type { AnyChatMiddleware, ChatMiddleware } from './types'
+import type { DefinedChatMiddleware } from './define'
+
+/** Union of capability NAME literals from a tuple of handles. */
+export type NamesOf<T extends ReadonlyArray<CapabilityHandle>> =
+  T[number]['capabilityName']
+
+/** Names provided across a middleware array (imprecise middleware → `string`). */
+export type ProvidedNames<TList extends ReadonlyArray<AnyChatMiddleware>> =
+  NonNullable<TList[number]['provides']> extends infer P
+    ? P extends ReadonlyArray<CapabilityHandle>
+      ? NamesOf<P>
+      : never
+    : never
+
+/** Names required across a middleware array. */
+export type RequiredNames<TList extends ReadonlyArray<AnyChatMiddleware>> =
+  NonNullable<TList[number]['requires']> extends infer P
+    ? P extends ReadonlyArray<CapabilityHandle>
+      ? NamesOf<P>
+      : never
+    : never
+
+/**
+ * Branded marker surfaced when required capability names are missing from the
+ * provided set, so the compiler error names the gap instead of emitting an
+ * opaque "not assignable".
+ */
+export type MissingCapabilities<TMissing extends string> = {
+  // The human-readable message lives in the property KEY, so TypeScript's
+  // "Property '<key>' is missing in type ... but required in type ..." error
+  // prints the explanation instead of an opaque `__missingCapabilities`. The
+  // key distributes over a union of missing names (one required key each).
+  [K in `✖ Missing capability "${TMissing}": no configured middleware provides it. Add a middleware whose \`provides\` includes it (and, with createChatMiddleware().use(), order the provider before this consumer).`]: never
+}
+
+/**
+ * Missing capability names. When required names are imprecise (`string`, i.e.
+ * plain `ChatMiddleware` not authored via `defineChatMiddleware`), we cannot
+ * prove a gap, so we allow it (→ `never`). Otherwise the precise literals not
+ * present in the provided set.
+ */
+type MissingNames<TList extends ReadonlyArray<AnyChatMiddleware>> =
+  string extends RequiredNames<TList>
+    ? never
+    : Exclude<RequiredNames<TList>, ProvidedNames<TList>>
+
+/**
+ * Resolves to `TList` when coverage holds, otherwise to a `MissingCapabilities`
+ * marker (not assignable to a middleware array) — producing a compile error at
+ * the `middleware` option that names the missing capability.
+ */
+export type CheckCoverage<TList extends ReadonlyArray<AnyChatMiddleware>> = [
+  MissingNames<TList>,
+] extends [never]
+  ? TList
+  : MissingCapabilities<MissingNames<TList>>
+
+/**
+ * Order-aware middleware builder. Each `.use()` requires that the middleware's
+ * required capability names are already in the accumulated provided set, then
+ * adds its provided names. `.build()` returns the ordered array.
+ *
+ * `TProvided` is the running union of provided capability name literals.
+ */
+export interface ChatMiddlewareBuilder<
+  TList extends ReadonlyArray<AnyChatMiddleware>,
+  TProvided extends string,
+> {
+  use: <
+    TRequires extends ReadonlyArray<CapabilityHandle>,
+    TProvides extends ReadonlyArray<CapabilityHandle>,
+    TContext = unknown,
+  >(
+    middleware: [NamesOf<TRequires>] extends [TProvided]
+      ? DefinedChatMiddleware<TContext, TRequires, TProvides>
+      : DefinedChatMiddleware<TContext, TRequires, TProvides> &
+          MissingCapabilities<Exclude<NamesOf<TRequires>, TProvided>>,
+  ) => ChatMiddlewareBuilder<
+    readonly [...TList, DefinedChatMiddleware<TContext, TRequires, TProvides>],
+    TProvided | NamesOf<TProvides>
+  >
+
+  build: () => [...TList]
+}
+
+/** Create an order-aware middleware builder. */
+export function createChatMiddleware(): ChatMiddlewareBuilder<
+  readonly [],
+  never
+> {
+  const list: Array<ChatMiddleware<unknown>> = []
+  const builder = {
+    use(middleware: ChatMiddleware<unknown>) {
+      list.push(middleware)
+      return builder
+    },
+    build() {
+      return list
+    },
+  }
+  // The only sanctioned assertion in this PR: the runtime `builder` is a single
+  // object reused across `.use()` calls, but the type accumulates `TProvided`
+  // and `TList` per call — TypeScript cannot derive that from runtime values, so
+  // a structural `as` is impossible and the double assertion is irreducible.
+  // eslint-disable-next-line no-restricted-syntax -- irreducible: type-level accumulation cannot be expressed from a single runtime object
+  return builder as unknown as ChatMiddlewareBuilder<readonly [], never>
+}