@tanstack/ai-fal 0.8.0 → 0.9.0

package/dist/esm/image/image-inputs.js

@@ -0,0 +1,119 @@
+import { FAL_IMAGE_FIELD_OVERRIDES } from "./generated/image-field-overrides.js";
+const DEFAULT_FIELDS = {
+  single: "image_url",
+  multi: "image_urls",
+  mask: "mask_url",
+  control: "control_image_url",
+  reference: "reference_image_urls",
+  start: "start_image_url",
+  end: "end_image_url"
+};
+const LIST_FIELDS = /* @__PURE__ */ new Set([
+  "image_urls",
+  "input_image_urls",
+  "ref_image_urls",
+  "reference_image_urls"
+]);
+function fieldSpecFor(model) {
+  const overrides = FAL_IMAGE_FIELD_OVERRIDES[model];
+  return { ...DEFAULT_FIELDS, ...overrides };
+}
+function assignField(fields, field, urls, model, what) {
+  if (urls.length === 0) return;
+  const existing = fields[field];
+  if (LIST_FIELDS.has(field)) {
+    fields[field] = Array.isArray(existing) ? [...existing, ...urls] : urls;
+  } else if (existing !== void 0) {
+    throw new Error(
+      `fal: multiple inputs map to '${field}' on model ${model}. Drop one of the conflicting inputs or pass the field explicitly via modelOptions.`
+    );
+  } else if (urls.length === 1) {
+    fields[field] = urls[0];
+  } else {
+    throw new Error(
+      `fal: model ${model} accepts a single ${what} image via '${field}' (received ${urls.length}).`
+    );
+  }
+}
+function bucketByRole(imageInputs) {
+  const buckets = {
+    sources: [],
+    masks: [],
+    controls: [],
+    references: [],
+    starts: [],
+    ends: []
+  };
+  for (const part of imageInputs) {
+    const url = imagePartToUrl(part);
+    const role = part.metadata?.role;
+    if (role === "mask") buckets.masks.push(url);
+    else if (role === "control") buckets.controls.push(url);
+    else if (role === "reference" || role === "character")
+      buckets.references.push(url);
+    else if (role === "start_frame") buckets.starts.push(url);
+    else if (role === "end_frame") buckets.ends.push(url);
+    else buckets.sources.push(url);
+  }
+  return buckets;
+}
+function mapImageInputsToFalFields(model, imageInputs) {
+  if (!imageInputs || imageInputs.length === 0) return {};
+  const spec = fieldSpecFor(model);
+  const { sources, masks, controls, references, starts, ends } = bucketByRole(imageInputs);
+  const allSources = [...sources, ...starts, ...ends];
+  if (masks.length > 1) {
+    throw new Error(
+      `fal: only one input with metadata.role === 'mask' is supported per request (received ${masks.length}).`
+    );
+  }
+  if (controls.length > 1) {
+    throw new Error(
+      `fal: only one input with metadata.role === 'control' is supported per request (received ${controls.length}).`
+    );
+  }
+  const fields = {};
+  const sourceField = allSources.length > 1 ? spec.multi : spec.single;
+  assignField(fields, sourceField, allSources, model, "source");
+  assignField(fields, spec.reference, references, model, "reference");
+  assignField(fields, spec.mask, masks, model, "mask");
+  assignField(fields, spec.control, controls, model, "control");
+  return fields;
+}
+function mapImageInputsToFalVideoFields(model, imageInputs) {
+  if (!imageInputs || imageInputs.length === 0) return {};
+  const spec = fieldSpecFor(model);
+  const { sources, masks, controls, references, starts, ends } = bucketByRole(imageInputs);
+  if (masks.length > 0 || controls.length > 0) {
+    const role = masks.length > 0 ? "mask" : "control";
+    throw new Error(
+      `fal: metadata.role === '${role}' is not supported for video generation on model ${model}. Remove the role or pass the field explicitly via modelOptions.`
+    );
+  }
+  if (starts.length > 1) {
+    throw new Error(
+      `fal: only one input with metadata.role === 'start_frame' is supported (received ${starts.length}).`
+    );
+  }
+  if (ends.length > 1) {
+    throw new Error(
+      `fal: only one input with metadata.role === 'end_frame' is supported (received ${ends.length}).`
+    );
+  }
+  const fields = {};
+  const sourceField = sources.length > 1 ? spec.multi : spec.single;
+  assignField(fields, sourceField, sources, model, "source");
+  assignField(fields, spec.reference, references, model, "reference");
+  assignField(fields, spec.start, starts, model, "start frame");
+  assignField(fields, spec.end, ends, model, "end frame");
+  return fields;
+}
+function imagePartToUrl(part) {
+  if (part.source.type === "url") return part.source.value;
+  return `data:${part.source.mimeType};base64,${part.source.value}`;
+}
+export {
+  mapImageInputsToFalFields,
+  mapImageInputsToFalVideoFields
+};
+//# sourceMappingURL=image-inputs.js.map

package/dist/esm/image/image-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"image-inputs.js","sources":["../../../src/image/image-inputs.ts"],"sourcesContent":["import { FAL_IMAGE_FIELD_OVERRIDES } from './generated/image-field-overrides'\nimport type {\n  FalImageFieldName,\n  FalImageFieldOverride,\n} from './generated/image-field-overrides'\nimport type { ImagePart, MediaInputMetadata } from '@tanstack/ai'\nimport type { FalModel, FalModelInput } from '../model-meta'\n\n/**\n * The image-conditioning fields the mappers may set, narrowed to the ones\n * that actually exist on the given endpoint's input type. For endpoints\n * unknown to the installed `@fal-ai/client` this widens to all known field\n * names.\n */\nexport type FalImageInputFields<TModel extends string> = Partial<\n  Pick<\n    FalModelInput<TModel>,\n    Extract<keyof FalModelInput<TModel>, FalImageFieldName>\n  >\n>\n\n/**\n * Default field per routing role. Endpoint-specific deviations live in the\n * generated `FAL_IMAGE_FIELD_OVERRIDES` map (regenerate with\n * `pnpm generate:fal-image-fields`); these defaults must stay in sync with\n * `DEFAULTS` in scripts/generate-fal-image-field-map.ts.\n */\nconst DEFAULT_FIELDS = {\n  single: 'image_url',\n  multi: 'image_urls',\n  mask: 'mask_url',\n  control: 'control_image_url',\n  reference: 'reference_image_urls',\n  start: 'start_image_url',\n  end: 'end_image_url',\n} satisfies Required<FalImageFieldOverride>\n\n/**\n * Field names that accept an array of images. The generator asserts the\n * SDK types agree with this set, so wrap-vs-scalar decisions stay correct.\n */\nconst LIST_FIELDS = new Set<string>([\n  'image_urls',\n  'input_image_urls',\n  'ref_image_urls',\n  'reference_image_urls',\n])\n\n/** Resolve the per-role field names for a model: defaults + generated overrides. */\nfunction fieldSpecFor(model: string): Required<FalImageFieldOverride> {\n  const overrides = (\n    FAL_IMAGE_FIELD_OVERRIDES as Record<string, FalImageFieldOverride>\n  )[model]\n  return { ...DEFAULT_FIELDS, ...overrides }\n}\n\n/**\n * Assign URLs to a field, wrapping or unwrapping based on whether the field\n * takes an array. When two roles resolve to the same list field (e.g.\n * sources and references both land on `image_urls` for nano-banana edit)\n * the values are merged in assignment order; two roles resolving to the\n * same scalar field is ambiguous and throws. Throws when multiple images\n * target a scalar field.\n */\nfunction assignField(\n  fields: Record<string, unknown>,\n  field: string,\n  urls: Array<string>,\n  model: string,\n  what: string,\n): void {\n  if (urls.length === 0) return\n  const existing = fields[field]\n  if (LIST_FIELDS.has(field)) {\n    fields[field] = Array.isArray(existing) ? [...existing, ...urls] : urls\n  } else if (existing !== undefined) {\n    throw new Error(\n      `fal: multiple inputs map to '${field}' on model ${model}. Drop one of the conflicting inputs or pass the field explicitly via modelOptions.`,\n    )\n  } else if (urls.length === 1) {\n    fields[field] = urls[0]\n  } else {\n    throw new Error(\n      `fal: model ${model} accepts a single ${what} image via '${field}' (received ${urls.length}).`,\n    )\n  }\n}\n\ninterface RoleBuckets {\n  sources: Array<string>\n  masks: Array<string>\n  controls: Array<string>\n  references: Array<string>\n  starts: Array<string>\n  ends: Array<string>\n}\n\nfunction bucketByRole(\n  imageInputs: ReadonlyArray<ImagePart<MediaInputMetadata>>,\n): RoleBuckets {\n  const buckets: RoleBuckets = {\n    sources: [],\n    masks: [],\n    controls: [],\n    references: [],\n    starts: [],\n    ends: [],\n  }\n  for (const part of imageInputs) {\n    const url = imagePartToUrl(part)\n    const role = part.metadata?.role\n    if (role === 'mask') buckets.masks.push(url)\n    else if (role === 'control') buckets.controls.push(url)\n    else if (role === 'reference' || role === 'character')\n      buckets.references.push(url)\n    else if (role === 'start_frame') buckets.starts.push(url)\n    else if (role === 'end_frame') buckets.ends.push(url)\n    else buckets.sources.push(url)\n  }\n  return buckets\n}\n\n/**\n * Map the prompt's image parts onto fal.ai image-endpoint fields.\n *\n * fal endpoints use different field names for image-conditioned generation\n * (~80% use `image_url` for single; the rest use `image_urls`,\n * `reference_image_urls`, `mask_url`, `control_image_url`, etc.). Field\n * names are resolved per endpoint from the generated\n * `FAL_IMAGE_FIELD_OVERRIDES` map (derived from the fal SDK's endpoint\n * types), falling back to the defaults above for endpoints the installed\n * SDK doesn't know:\n *\n * - parts with `metadata.role === 'mask'`      → spec.mask      (single)\n * - parts with `metadata.role === 'control'`   → spec.control   (single)\n * - `role === 'reference' | 'character'`       → spec.reference\n * - `role === 'start_frame' | 'end_frame'`     → treated as sources (frame\n *   roles only apply to video generation)\n * - remaining parts                            → spec.single / spec.multi\n *\n * Users can always override the resulting field shape via `modelOptions`\n * (spread before these fields), or pass everything through `modelOptions`\n * directly when the mapping doesn't match an obscure endpoint.\n */\nexport function mapImageInputsToFalFields<TModel extends FalModel>(\n  model: TModel,\n  imageInputs?: ReadonlyArray<ImagePart<MediaInputMetadata>>,\n): FalImageInputFields<TModel> {\n  if (!imageInputs || imageInputs.length === 0) return {}\n\n  const spec = fieldSpecFor(model)\n  const { sources, masks, controls, references, starts, ends } =\n    bucketByRole(imageInputs)\n  // Frame roles aren't meaningful for image generation; treat as the\n  // primary source. The video mapper handles start/end framing.\n  const allSources = [...sources, ...starts, ...ends]\n\n  if (masks.length > 1) {\n    throw new Error(\n      `fal: only one input with metadata.role === 'mask' is supported per request (received ${masks.length}).`,\n    )\n  }\n  if (controls.length > 1) {\n    throw new Error(\n      `fal: only one input with metadata.role === 'control' is supported per request (received ${controls.length}).`,\n    )\n  }\n\n  const fields: Record<string, unknown> = {}\n  const sourceField = allSources.length > 1 ? spec.multi : spec.single\n  assignField(fields, sourceField, allSources, model, 'source')\n  assignField(fields, spec.reference, references, model, 'reference')\n  assignField(fields, spec.mask, masks, model, 'mask')\n  assignField(fields, spec.control, controls, model, 'control')\n\n  return fields as FalImageInputFields<TModel>\n}\n\n/**\n * Map the prompt's image parts onto fal.ai video-endpoint fields.\n *\n * Video endpoints often expose a start frame as `image_url` (76% of i2v\n * models) plus an optional `end_image_url`. Multi-reference video models\n * (Kling O3, Seedance reference-to-video) use `reference_image_urls` or\n * `image_urls`. Field names resolve through the same generated override\n * map as the image mapper — e.g. `role: 'start_frame'` lands on `image_url`\n * for Kling/Veo image-to-video and `first_frame_url` for Pixverse. Mapping:\n *\n * - `metadata.role === 'start_frame'`              → spec.start\n * - `metadata.role === 'end_frame'`                → spec.end\n * - `metadata.role === 'reference' | 'character'`  → spec.reference\n * - `metadata.role === 'mask' | 'control'`         → throws (no video routing)\n * - remaining parts (no role)                      → spec.single / spec.multi\n */\nexport function mapImageInputsToFalVideoFields<TModel extends FalModel>(\n  model: TModel,\n  imageInputs?: ReadonlyArray<ImagePart<MediaInputMetadata>>,\n): FalImageInputFields<TModel> {\n  if (!imageInputs || imageInputs.length === 0) return {}\n\n  const spec = fieldSpecFor(model)\n  const { sources, masks, controls, references, starts, ends } =\n    bucketByRole(imageInputs)\n  // Mask / control roles have no video-specific routing; silently repurposing\n  // them as source frames would hide the problem, so reject them instead.\n  if (masks.length > 0 || controls.length > 0) {\n    const role = masks.length > 0 ? 'mask' : 'control'\n    throw new Error(\n      `fal: metadata.role === '${role}' is not supported for video generation on model ${model}. ` +\n        `Remove the role or pass the field explicitly via modelOptions.`,\n    )\n  }\n\n  if (starts.length > 1) {\n    throw new Error(\n      `fal: only one input with metadata.role === 'start_frame' is supported (received ${starts.length}).`,\n    )\n  }\n  if (ends.length > 1) {\n    throw new Error(\n      `fal: only one input with metadata.role === 'end_frame' is supported (received ${ends.length}).`,\n    )\n  }\n\n  const fields: Record<string, unknown> = {}\n  const sourceField = sources.length > 1 ? spec.multi : spec.single\n  assignField(fields, sourceField, sources, model, 'source')\n  assignField(fields, spec.reference, references, model, 'reference')\n  // Frame roles assign last: when an endpoint routes the start frame to its\n  // generic source field (e.g. Kling image-to-video) and an unroled source\n  // was also provided, assignField rejects the ambiguous combination.\n  assignField(fields, spec.start, starts, model, 'start frame')\n  assignField(fields, spec.end, ends, model, 'end frame')\n\n  return fields as FalImageInputFields<TModel>\n}\n\n/**\n * Convert a TanStack ImagePart into a string suitable for fal's URL-based\n * input fields. URL sources pass through; data sources are emitted as a\n * `data:<mime>;base64,<value>` URI which fal endpoints accept on the wire.\n */\nfunction imagePartToUrl(part: ImagePart<MediaInputMetadata>): string {\n  if (part.source.type === 'url') return part.source.value\n  return `data:${part.source.mimeType};base64,${part.source.value}`\n}\n"],"names":[],"mappings":";AA2BA,MAAM,iBAAiB;AAAA,EACrB,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,MAAM;AAAA,EACN,SAAS;AAAA,EACT,WAAW;AAAA,EACX,OAAO;AAAA,EACP,KAAK;AACP;AAMA,MAAM,kCAAkB,IAAY;AAAA,EAClC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAGD,SAAS,aAAa,OAAgD;AACpE,QAAM,YACJ,0BACA,KAAK;AACP,SAAO,EAAE,GAAG,gBAAgB,GAAG,UAAA;AACjC;AAUA,SAAS,YACP,QACA,OACA,MACA,OACA,MACM;AACN,MAAI,KAAK,WAAW,EAAG;AACvB,QAAM,WAAW,OAAO,KAAK;AAC7B,MAAI,YAAY,IAAI,KAAK,GAAG;AAC1B,WAAO,KAAK,IAAI,MAAM,QAAQ,QAAQ,IAAI,CAAC,GAAG,UAAU,GAAG,IAAI,IAAI;AAAA,EACrE,WAAW,aAAa,QAAW;AACjC,UAAM,IAAI;AAAA,MACR,gCAAgC,KAAK,cAAc,KAAK;AAAA,IAAA;AAAA,EAE5D,WAAW,KAAK,WAAW,GAAG;AAC5B,WAAO,KAAK,IAAI,KAAK,CAAC;AAAA,EACxB,OAAO;AACL,UAAM,IAAI;AAAA,MACR,cAAc,KAAK,qBAAqB,IAAI,eAAe,KAAK,eAAe,KAAK,MAAM;AAAA,IAAA;AAAA,EAE9F;AACF;AAWA,SAAS,aACP,aACa;AACb,QAAM,UAAuB;AAAA,IAC3B,SAAS,CAAA;AAAA,IACT,OAAO,CAAA;AAAA,IACP,UAAU,CAAA;AAAA,IACV,YAAY,CAAA;AAAA,IACZ,QAAQ,CAAA;AAAA,IACR,MAAM,CAAA;AAAA,EAAC;AAET,aAAW,QAAQ,aAAa;AAC9B,UAAM,MAAM,eAAe,IAAI;AAC/B,UAAM,OAAO,KAAK,UAAU;AAC5B,QAAI,SAAS,OAAQ,SAAQ,MAAM,KAAK,GAAG;AAAA,aAClC,SAAS,UAAW,SAAQ,SAAS,KAAK,GAAG;AAAA,aAC7C,SAAS,eAAe,SAAS;AACxC,cAAQ,WAAW,KAAK,GAAG;AAAA,aACpB,SAAS,cAAe,SAAQ,OAAO,KAAK,GAAG;AAAA,aAC/C,SAAS,YAAa,SAAQ,KAAK,KAAK,GAAG;AAAA,QAC/C,SAAQ,QAAQ,KAAK,GAAG;AAAA,EAC/B;AACA,SAAO;AACT;AAwBO,SAAS,0BACd,OACA,aAC6B;AAC7B,MAAI,CAAC,eAAe,YAAY,WAAW,UAAU,CAAA;AAErD,QAAM,OAAO,aAAa,KAAK;AAC/B,QAAM,EAAE,SAAS,OAAO,UAAU,YAAY,QAAQ,KAAA,IACpD,aAAa,WAAW;AAG1B,QAAM,aAAa,CAAC,GAAG,SAAS,GAAG,QAAQ,GAAG,IAAI;AAElD,MAAI,MAAM,SAAS,GAAG;AACpB,UAAM,IAAI;AAAA,MACR,wFAAwF,MAAM,MAAM;AAAA,IAAA;AAAA,EAExG;AACA,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,IAAI;AAAA,MACR,2FAA2F,SAAS,MAAM;AAAA,IAAA;AAAA,EAE9G;AAEA,QAAM,SAAkC,CAAA;AACxC,QAAM,cAAc,WAAW,SAAS,IAAI,KAAK,QAAQ,KAAK;AAC9D,cAAY,QAAQ,aAAa,YAAY,OAAO,QAAQ;AAC5D,cAAY,QAAQ,KAAK,WAAW,YAAY,OAAO,WAAW;AAClE,cAAY,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM;AACnD,cAAY,QAAQ,KAAK,SAAS,UAAU,OAAO,SAAS;AAE5D,SAAO;AACT;AAkBO,SAAS,+BACd,OACA,aAC6B;AAC7B,MAAI,CAAC,eAAe,YAAY,WAAW,UAAU,CAAA;AAErD,QAAM,OAAO,aAAa,KAAK;AAC/B,QAAM,EAAE,SAAS,OAAO,UAAU,YAAY,QAAQ,KAAA,IACpD,aAAa,WAAW;AAG1B,MAAI,MAAM,SAAS,KAAK,SAAS,SAAS,GAAG;AAC3C,UAAM,OAAO,MAAM,SAAS,IAAI,SAAS;AACzC,UAAM,IAAI;AAAA,MACR,2BAA2B,IAAI,oDAAoD,KAAK;AAAA,IAAA;AAAA,EAG5F;AAEA,MAAI,OAAO,SAAS,GAAG;AACrB,UAAM,IAAI;AAAA,MACR,mFAAmF,OAAO,MAAM;AAAA,IAAA;AAAA,EAEpG;AACA,MAAI,KAAK,SAAS,GAAG;AACnB,UAAM,IAAI;AAAA,MACR,iFAAiF,KAAK,MAAM;AAAA,IAAA;AAAA,EAEhG;AAEA,QAAM,SAAkC,CAAA;AACxC,QAAM,cAAc,QAAQ,SAAS,IAAI,KAAK,QAAQ,KAAK;AAC3D,cAAY,QAAQ,aAAa,SAAS,OAAO,QAAQ;AACzD,cAAY,QAAQ,KAAK,WAAW,YAAY,OAAO,WAAW;AAIlE,cAAY,QAAQ,KAAK,OAAO,QAAQ,OAAO,aAAa;AAC5D,cAAY,QAAQ,KAAK,KAAK,MAAM,OAAO,WAAW;AAEtD,SAAO;AACT;AAOA,SAAS,eAAe,MAA6C;AACnE,MAAI,KAAK,OAAO,SAAS,MAAO,QAAO,KAAK,OAAO;AACnD,SAAO,QAAQ,KAAK,OAAO,QAAQ,WAAW,KAAK,OAAO,KAAK;AACjE;"}

package/dist/esm/model-meta.d.ts

@@ -1,4 +1,6 @@
 import { EndpointTypeMap } from '@fal-ai/client/endpoints';
+import { MediaPromptModality } from '@tanstack/ai';
+import { FalImageFieldName } from './image/generated/image-field-overrides.js';
 export type { EndpointTypeMap } from '@fal-ai/client/endpoints';
 /**
  * All known fal.ai model IDs with autocomplete support.
@@ -40,6 +42,21 @@ export type FalModelImageSizeInput<TModel extends string> = TModel extends keyof
 } : never : {
     image_size: string;
 };
+/**
+ * Input fields the prompt-part mappers can populate: image conditioning via
+ * the generated `FalImageFieldName` set, video conditioning via
+ * `video_url` / `video_urls` / `reference_video_urls`, audio via `audio_url`.
+ */
+type FalMediaInputFieldName = FalImageFieldName | 'video_url' | 'video_urls' | 'reference_video_urls' | 'audio_url';
+/**
+ * Demote an endpoint input's media-conditioning fields from required to
+ * optional. Image-to-video endpoints declare e.g. `image_url` as a required
+ * input, but with a multimodal `prompt` the start frame usually arrives as a
+ * prompt part — requiring it in `modelOptions` too would force redundancy.
+ * The fields stay passable via `modelOptions` as the documented escape hatch
+ * (and override-wise the mapped prompt-part fields win on conflict).
+ */
+type WithOptionalMediaInputFields<TInput> = Omit<TInput, Extract<keyof TInput, FalMediaInputFieldName>> & Partial<Pick<TInput, Extract<keyof TInput, FalMediaInputFieldName>>>;
 /**
  * Provider options for image generation, excluding fields TanStack AI handles.
  * Use this for the `modelOptions` parameter in image generation.
@@ -48,7 +65,7 @@ export type FalModelImageSizeInput<TModel extends string> = TModel extends keyof
  * type FluxOptions = FalImageProviderOptions<'fal-ai/flux/dev'>
  * // { num_inference_steps?: number; guidance_scale?: number; seed?: number; ... }
  */
-export type FalImageProviderOptions<TModel extends string> = Omit<FalModelInput<TModel>, 'prompt'>;
+export type FalImageProviderOptions<TModel extends string> = WithOptionalMediaInputFields<Omit<FalModelInput<TModel>, 'prompt'>>;
 /**
  * Extract the video size type supported by a specific fal model.
  * Video models typically use aspect_ratio and/or resolution fields.
@@ -71,11 +88,30 @@ export type FalModelVideoSizeInput<TModel extends string> = TModel extends keyof
     aspect_ratio?: string;
     resolution?: string;
 };
+/**
+ * Prompt input modalities for a fal image endpoint, derived from the SDK's
+ * endpoint input type: an endpoint accepts image prompt parts exactly when
+ * its input declares one of the known image-conditioning fields
+ * (`image_url`, `image_urls`, `mask_url`, …). Endpoints unknown to the
+ * installed SDK are unconstrained.
+ */
+export type FalImagePromptModalitiesFor<TModel extends string> = TModel extends keyof EndpointTypeMap ? ReadonlyArray<Extract<keyof FalModelInput<TModel>, FalImageFieldName> extends never ? never : 'image'> : ReadonlyArray<MediaPromptModality>;
+/**
+ * Prompt input modalities for a fal video endpoint. Image conditioning is
+ * detected via the same field set as image endpoints; video conditioning via
+ * `video_url` / `video_urls` / `reference_video_urls`; audio conditioning
+ * via `audio_url`. Endpoints unknown to the installed SDK are unconstrained.
+ */
+export type FalVideoPromptModalitiesFor<TModel extends string> = TModel extends keyof EndpointTypeMap ? ReadonlyArray<(Extract<keyof FalModelInput<TModel>, FalImageFieldName> extends never ? never : 'image') | (Extract<keyof FalModelInput<TModel>, 'video_url' | 'video_urls' | 'reference_video_urls'> extends never ? never : 'video') | (Extract<keyof FalModelInput<TModel>, 'audio_url'> extends never ? never : 'audio')> : ReadonlyArray<MediaPromptModality>;
 /**
  * Provider options for video generation, excluding fields TanStack AI handles.
  * Use this for the `modelOptions` parameter in video generation.
+ *
+ * Media-conditioning fields (start/end frame, reference images, source
+ * video/audio) are optional here even when the endpoint requires them —
+ * they're usually supplied as prompt parts instead.
  */
-export type FalVideoProviderOptions<TModel extends string> = TModel extends keyof EndpointTypeMap ? Omit<FalModelInput<TModel>, 'prompt'> : Record<string, unknown>;
+export type FalVideoProviderOptions<TModel extends string> = TModel extends keyof EndpointTypeMap ? WithOptionalMediaInputFields<Omit<FalModelInput<TModel>, 'prompt'>> : Record<string, unknown>;
 /**
  * Provider options for TTS, excluding fields TanStack AI handles.
  * Use this for the `modelOptions` parameter in speech generation.

package/package.json

@@ -1,14 +1,22 @@
 {
   "name": "@tanstack/ai-fal",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "fal.ai adapter for TanStack AI image, video, audio, speech, and transcription generation.",
-  "author": "",
+  "author": "Tanner Linsley",
   "license": "MIT",
+  "homepage": "https://tanstack.com/ai",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/TanStack/ai.git",
     "directory": "packages/ai-fal"
   },
+  "bugs": {
+    "url": "https://github.com/TanStack/ai/issues"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/tannerlinsley"
+  },
   "type": "module",
   "module": "./dist/esm/index.js",
   "types": "./dist/esm/index.d.ts",
@@ -41,15 +49,15 @@
   ],
   "dependencies": {
     "@fal-ai/client": "^1.10.1",
-    "@tanstack/ai-utils": "0.2.1"
+    "@tanstack/ai-utils": "0.2.2"
   },
   "devDependencies": {
     "@vitest/coverage-v8": "4.0.14",
     "vite": "^7.3.3",
-    "@tanstack/ai": "0.29.0"
+    "@tanstack/ai": "0.32.0"
   },
   "peerDependencies": {
-    "@tanstack/ai": "0.29.0"
+    "@tanstack/ai": "0.32.0"
   },
   "scripts": {
     "build": "vite build",

package/src/adapters/image.ts

@@ -1,4 +1,5 @@
 import { fal } from '@fal-ai/client'
+import { resolveMediaPrompt } from '@tanstack/ai'
 import { BaseImageAdapter } from '@tanstack/ai/adapters'
 import {
   buildFalUsage,
@@ -7,14 +8,17 @@ import {
   generateId as utilGenerateId,
 } from '../utils'
 import { mapSizeToFalFormat } from '../image/image-provider-options'
+import { mapImageInputsToFalFields } from '../image/image-inputs'
 import type { OutputType, Result } from '@fal-ai/client'
 import type { FalClientConfig } from '../utils'
 import type {
   GeneratedImage,
   ImageGenerationOptions,
   ImageGenerationResult,
+  ResolvedMediaPrompt,
 } from '@tanstack/ai'
 import type {
+  FalImagePromptModalitiesFor,
   FalImageProviderOptions,
   FalModel,
   FalModelImageSize,
@@ -45,7 +49,8 @@ export class FalImageAdapter<TModel extends FalModel> extends BaseImageAdapter<
   TModel,
   FalImageProviderOptions<TModel>,
   Record<TModel, FalImageProviderOptions<TModel>>,
-  Record<TModel, FalModelImageSize<TModel>>
+  Record<TModel, FalModelImageSize<TModel>>,
+  Record<TModel, FalImagePromptModalitiesFor<TModel>>
 > {
   override readonly kind = 'image' as const
   readonly name = 'fal' as const
@@ -68,8 +73,21 @@ export class FalImageAdapter<TModel extends FalModel> extends BaseImageAdapter<
       model: this.model,
     })
 
+    const resolved = resolveMediaPrompt(options.prompt)
+
+    if (resolved.videos.length > 0) {
+      throw new Error(
+        `fal.generateImages does not support video prompt parts on model ${this.model}.`,
+      )
+    }
+    if (resolved.audios.length > 0) {
+      throw new Error(
+        `fal.generateImages does not support audio prompt parts on model ${this.model}.`,
+      )
+    }
+
     try {
-      const input = this.buildInput(options)
+      const input = this.buildInput(options, resolved)
       const result = await fal.subscribe(this.model, { input })
       return this.transformResponse(result)
     } catch (error) {
@@ -86,12 +104,21 @@ export class FalImageAdapter<TModel extends FalModel> extends BaseImageAdapter<
       FalImageProviderOptions<TModel>,
       FalModelImageSize<TModel>
     >,
+    resolved: ResolvedMediaPrompt,
   ): FalModelInput<TModel> {
     const sizeParams = mapSizeToFalFormat(options.size)
+    // Order matters: size and derived image-input fields first, then
+    // modelOptions (so explicit user overrides win for mask_url /
+    // control_image_url / reference_image_urls), then the call-controlled
+    // prompt / num_images, which always take precedence.
+    const inputFields = mapImageInputsToFalFields(this.model, resolved.images)
     const input = {
-      ...options.modelOptions,
       ...sizeParams,
-      prompt: options.prompt,
+      ...inputFields,
+      ...options.modelOptions,
+      // Media-only prompts (e.g. upscalers, background removal) omit the
+      // prompt field entirely rather than sending an empty string.
+      ...(resolved.text ? { prompt: resolved.text } : {}),
       num_images: options.numberOfImages,
     } as FalModelInput<TModel>
     return input

package/src/adapters/video.ts

@@ -1,4 +1,5 @@
 import { fal } from '@fal-ai/client'
+import { resolveMediaPrompt } from '@tanstack/ai'
 import { BaseVideoAdapter } from '@tanstack/ai/adapters'
 import {
   buildFalUsage,
@@ -7,9 +8,13 @@ import {
   generateId as utilGenerateId,
 } from '../utils'
 import { mapVideoSizeToFalFormat } from '../video/video-provider-options'
+import { mapImageInputsToFalVideoFields } from '../image/image-inputs'
 import type {
+  AudioPart,
+  MediaInputMetadata,
   VideoGenerationOptions,
   VideoJobResult,
+  VideoPart,
   VideoStatusResult,
   VideoUrlResult,
 } from '@tanstack/ai'
@@ -17,10 +22,68 @@ import type {
   FalModel,
   FalModelInput,
   FalModelVideoSize,
+  FalVideoPromptModalitiesFor,
   FalVideoProviderOptions,
 } from '../model-meta'
 import type { FalClientConfig } from '../utils'
 
+/**
+ * Map video conditioning inputs onto fal field names.
+ * Video-to-video endpoints on fal almost universally use `video_url`; the
+ * occasional model takes `video_urls` (rare). Mirror the image-input logic
+ * positionally with a `reference` role escape hatch via `reference_video_urls`.
+ */
+function mapVideoInputsToFalFields(
+  videoInputs?: ReadonlyArray<VideoPart<MediaInputMetadata>>,
+): Record<string, unknown> {
+  if (!videoInputs || videoInputs.length === 0) return {}
+  const references: Array<string> = []
+  const sources: Array<string> = []
+  for (const part of videoInputs) {
+    const url = videoPartToUrl(part)
+    if (
+      part.metadata?.role === 'reference' ||
+      part.metadata?.role === 'character'
+    ) {
+      references.push(url)
+    } else {
+      sources.push(url)
+    }
+  }
+  const out: Record<string, unknown> = {}
+  if (references.length > 0) out.reference_video_urls = references
+  if (sources.length === 1) {
+    out.video_url = sources[0]
+  } else if (sources.length > 1) {
+    out.video_urls = sources
+  }
+  return out
+}
+
+function mapAudioInputsToFalFields(
+  audioInputs?: ReadonlyArray<AudioPart<MediaInputMetadata>>,
+): Record<string, unknown> {
+  if (!audioInputs || audioInputs.length === 0) return {}
+  const [part, ...rest] = audioInputs
+  if (!part || rest.length > 0) {
+    throw new Error(
+      `fal: exactly one audio prompt part is supported (received ${audioInputs.length}).`,
+    )
+  }
+  return {
+    audio_url:
+      part.source.type === 'url'
+        ? part.source.value
+        : `data:${part.source.mimeType};base64,${part.source.value}`,
+  }
+}
+
+function videoPartToUrl(part: VideoPart<MediaInputMetadata>): string {
+  return part.source.type === 'url'
+    ? part.source.value
+    : `data:${part.source.mimeType};base64,${part.source.value}`
+}
+
 type FalQueueStatus = 'IN_QUEUE' | 'IN_PROGRESS' | 'COMPLETED'
 
 interface FalStatusResponse {
@@ -69,7 +132,8 @@ export class FalVideoAdapter<TModel extends FalModel> extends BaseVideoAdapter<
   TModel,
   FalVideoProviderOptions<TModel>,
   Record<TModel, FalVideoProviderOptions<TModel>>,
-  Record<TModel, FalModelVideoSize<TModel>>
+  Record<TModel, FalModelVideoSize<TModel>>,
+  Record<TModel, FalVideoPromptModalitiesFor<TModel>>
 > {
   override readonly kind = 'video' as const
   readonly name = 'fal' as const
@@ -85,7 +149,7 @@ export class FalVideoAdapter<TModel extends FalModel> extends BaseVideoAdapter<
       FalModelVideoSize<TModel>
     >,
   ): Promise<VideoJobResult> {
-    const { prompt, size, duration, modelOptions, logger } = options
+    const { size, duration, modelOptions, logger } = options
 
     logger.request(`activity=generateVideo provider=fal model=${this.model}`, {
       provider: 'fal',
@@ -93,12 +157,26 @@ export class FalVideoAdapter<TModel extends FalModel> extends BaseVideoAdapter<
     })
 
     try {
+      const resolved = resolveMediaPrompt(options.prompt)
       const sizeParams = mapVideoSizeToFalFormat(size)
+      const inputImageFields = mapImageInputsToFalVideoFields(
+        this.model,
+        resolved.images,
+      )
+      const videoFields = mapVideoInputsToFalFields(resolved.videos)
+      const audioFields = mapAudioInputsToFalFields(resolved.audios)
 
       const input = {
-        ...modelOptions,
         ...sizeParams,
-        prompt,
+        ...inputImageFields,
+        ...videoFields,
+        ...audioFields,
+        // modelOptions applied after derived media fields so explicit user
+        // overrides (video_url, reference_video_urls, audio_url, ...) win.
+        ...modelOptions,
+        // Media-only prompts omit the prompt field rather than sending an
+        // empty string (e.g. pure image-to-video endpoints).
+        ...(resolved.text ? { prompt: resolved.text } : {}),
         ...(duration ? { duration } : {}),
       } as FalModelInput<TModel>