@broberg/ai-sdk 0.16.0 → 0.17.1

package/dist/index.d.ts

@@ -14,7 +14,7 @@ interface TierSpec {
     transport: Transport;
 }
 /** High-level capability a call exercises. Mirrors the capability layer (F5). */
-type Capability = "chat" | "vision" | "video" | "translate" | "image" | "embedding" | "transcribe" | "ocr" | "moderation" | "podcast" | "tts" | "trainStyle" | "mockup" | "design" | "extract" | "classify" | "rerank";
+type Capability = "chat" | "vision" | "video" | "translate" | "image" | "animate" | "embedding" | "transcribe" | "ocr" | "moderation" | "podcast" | "tts" | "trainStyle" | "mockup" | "design" | "extract" | "classify" | "rerank";
 type Role = "system" | "user" | "assistant" | "tool";
 /** A piece of message content. Text everywhere; image parts feed vision. */
 type ContentPart = {
@@ -188,6 +188,30 @@ interface ImageResult {
     url: string;
     usage: Usage;
 }
+/** Image-to-video generation (F024) — animate a still into a short clip. */
+interface AnimateRequest {
+    /** Input image: a URL (passed through) or raw bytes (uploaded to fal storage). */
+    image: string | Uint8Array;
+    /** Motion/scene prompt, e.g. "the subject turns and smiles". */
+    prompt?: string;
+    /** Clip length in seconds (provider-dependent; Veo ≤ 8s). */
+    durationSec?: number;
+    /** e.g. "720p" / "1080p" (provider-dependent). */
+    resolution?: string;
+    spec: TierSpec;
+}
+interface AnimateResult {
+    /** URL to the generated video. For fal this is a public hosted URL; for the
+     *  Gemini/Veo route it is Google's file URI (needs the API key to fetch — the
+     *  bytes are also returned below, already downloaded). */
+    url: string;
+    /** The downloaded video bytes — set by providers whose result URL is auth-gated
+     *  or short-lived (Gemini/Veo). Absent for providers that return a public URL (fal). */
+    bytes?: Uint8Array;
+    /** MIME type of `bytes` (e.g. "video/mp4"). */
+    mimeType?: string;
+    usage: Usage;
+}
 /** Style/brand LoRA training (F021) — fal fal-ai/flux-lora-fast-training. */
 interface TrainStyleRequest {
     /** A hosted archive URL, or an array of image URLs the SDK zips in-memory. */
@@ -304,6 +328,8 @@ interface ProviderAdapter {
     chatStream?(req: ChatRequest): AsyncIterable<ChatStreamEvent>;
     vision?(req: ChatRequest): Promise<ChatResult>;
     image?(req: ImageRequest): Promise<ImageResult>;
+    /** Image-to-video generation (F024) — animate a still into a short clip. fal. */
+    animate?(req: AnimateRequest): Promise<AnimateResult>;
     /** Train a style/brand LoRA from images (F021). fal. */
     trainStyle?(req: TrainStyleRequest): Promise<TrainStyleResult>;
     embedding?(req: EmbeddingRequest): Promise<EmbeddingResult>;
@@ -417,12 +443,12 @@ declare const toolSchema: z.ZodObject<{
     parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
 }, "strip", z.ZodTypeAny, {
     name: string;
-    description: string;
     parameters: Record<string, unknown>;
+    description: string;
 }, {
     name: string;
-    description: string;
     parameters: Record<string, unknown>;
+    description: string;
 }>;
 declare const messageSchema: z.ZodObject<{
     role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
@@ -604,12 +630,12 @@ declare const chatInputSchema: z.ZodObject<{
         parameters: z.ZodRecord<z.ZodString, z.ZodUnknown>;
     }, "strip", z.ZodTypeAny, {
         name: string;
-        description: string;
         parameters: Record<string, unknown>;
+        description: string;
     }, {
         name: string;
-        description: string;
         parameters: Record<string, unknown>;
+        description: string;
     }>, "many">>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     temperature: z.ZodOptional<z.ZodNumber>;
@@ -637,8 +663,8 @@ declare const chatInputSchema: z.ZodObject<{
     }[] | undefined;
     tools?: {
         name: string;
-        description: string;
         parameters: Record<string, unknown>;
+        description: string;
     }[] | undefined;
     temperature?: number | undefined;
     purpose?: string | undefined;
@@ -678,8 +704,8 @@ declare const chatInputSchema: z.ZodObject<{
     }[] | undefined;
     tools?: {
         name: string;
-        description: string;
         parameters: Record<string, unknown>;
+        description: string;
     }[] | undefined;
     temperature?: number | undefined;
     purpose?: string | undefined;
@@ -1032,6 +1058,81 @@ declare const imageInputSchema: z.ZodObject<{
     safetyTolerance?: number | undefined;
     retryOnBlack?: boolean | undefined;
 }>;
+declare const animateInputSchema: z.ZodObject<{
+    tier: z.ZodOptional<z.ZodEnum<["fast", "smart", "powerful", "cheap", "vision", "video", "embedding"]>>;
+    override: z.ZodOptional<z.ZodObject<{
+        provider: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+        transport: z.ZodOptional<z.ZodEnum<["http", "subprocess"]>>;
+    }, "strip", z.ZodTypeAny, {
+        provider?: string | undefined;
+        model?: string | undefined;
+        transport?: "http" | "subprocess" | undefined;
+    }, {
+        provider?: string | undefined;
+        model?: string | undefined;
+        transport?: "http" | "subprocess" | undefined;
+    }>>;
+    fallback: z.ZodOptional<z.ZodArray<z.ZodUnion<[z.ZodEnum<["fast", "smart", "powerful", "cheap", "vision", "video", "embedding"]>, z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        transport: z.ZodEnum<["http", "subprocess"]>;
+    }, "strip", z.ZodTypeAny, {
+        provider: string;
+        model: string;
+        transport: "http" | "subprocess";
+    }, {
+        provider: string;
+        model: string;
+        transport: "http" | "subprocess";
+    }>]>, "many">>;
+    purpose: z.ZodOptional<z.ZodString>;
+    labels: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    /** Input image: a URL or raw bytes (uploaded to fal storage). */
+    image: z.ZodUnion<[z.ZodString, z.ZodType<Uint8Array<ArrayBuffer>, z.ZodTypeDef, Uint8Array<ArrayBuffer>>]>;
+    /** Motion/scene prompt. */
+    prompt: z.ZodOptional<z.ZodString>;
+    /** Clip length in seconds (provider-dependent; Veo ≤ 8s). */
+    durationSec: z.ZodOptional<z.ZodNumber>;
+    /** e.g. "720p" / "1080p" (provider-dependent). */
+    resolution: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    image: string | Uint8Array<ArrayBuffer>;
+    prompt?: string | undefined;
+    resolution?: string | undefined;
+    purpose?: string | undefined;
+    tier?: "fast" | "smart" | "powerful" | "cheap" | "vision" | "video" | "embedding" | undefined;
+    override?: {
+        provider?: string | undefined;
+        model?: string | undefined;
+        transport?: "http" | "subprocess" | undefined;
+    } | undefined;
+    fallback?: ("fast" | "smart" | "powerful" | "cheap" | "vision" | "video" | "embedding" | {
+        provider: string;
+        model: string;
+        transport: "http" | "subprocess";
+    })[] | undefined;
+    labels?: Record<string, string> | undefined;
+    durationSec?: number | undefined;
+}, {
+    image: string | Uint8Array<ArrayBuffer>;
+    prompt?: string | undefined;
+    resolution?: string | undefined;
+    purpose?: string | undefined;
+    tier?: "fast" | "smart" | "powerful" | "cheap" | "vision" | "video" | "embedding" | undefined;
+    override?: {
+        provider?: string | undefined;
+        model?: string | undefined;
+        transport?: "http" | "subprocess" | undefined;
+    } | undefined;
+    fallback?: ("fast" | "smart" | "powerful" | "cheap" | "vision" | "video" | "embedding" | {
+        provider: string;
+        model: string;
+        transport: "http" | "subprocess";
+    })[] | undefined;
+    labels?: Record<string, string> | undefined;
+    durationSec?: number | undefined;
+}>;
 declare const trainStyleInputSchema: z.ZodObject<{
     tier: z.ZodOptional<z.ZodEnum<["fast", "smart", "powerful", "cheap", "vision", "video", "embedding"]>>;
     override: z.ZodOptional<z.ZodObject<{
@@ -1591,6 +1692,7 @@ type VisionInput = z.infer<typeof visionInputSchema>;
 type VideoInput = z.infer<typeof videoInputSchema>;
 type TranslateInput = z.infer<typeof translateInputSchema>;
 type ImageInput = z.infer<typeof imageInputSchema>;
+type AnimateInput = z.infer<typeof animateInputSchema>;
 type TrainStyleInput = z.infer<typeof trainStyleInputSchema>;
 type EmbeddingInput = z.infer<typeof embeddingInputSchema>;
 type TranscribeInput = z.infer<typeof transcribeInputSchema>;
@@ -1610,6 +1712,8 @@ interface AiClient {
     video(input: VideoInput): Promise<ChatResult>;
     translate(input: TranslateInput): Promise<TranslateResult>;
     image(input: ImageInput): Promise<ImageResult>;
+    /** Image-to-video (F024) — animate a still into a short clip → { url, usage }. fal. */
+    animate(input: AnimateInput): Promise<AnimateResult>;
     /** Train a style/brand LoRA from images (F021) → { loraUrl, configUrl }. fal. */
     trainStyle(input: TrainStyleInput): Promise<TrainStyleResult>;
     embedding(input: EmbeddingInput): Promise<EmbeddingResult>;
@@ -1665,6 +1769,12 @@ declare function geminiAdapter(config?: {
     baseUrl?: string;
     fetch?: typeof fetch;
     pricePerImage?: number;
+    /** Override the per-SECOND Veo video price (else estimate per model, 0 if unknown). */
+    pricePerSecond?: number;
+    /** Poll interval for Veo long-running operations (default 5s). */
+    pollIntervalMs?: number;
+    /** Deadline for a Veo job (default 300000 = 5 min). */
+    videoTimeoutMs?: number;
 }): ProviderAdapter;
 
 declare function deepinfraAdapter(config?: {
@@ -1715,11 +1825,15 @@ interface FalAdapterConfig {
     timeoutMs?: number;
     /** Deadline for training jobs — they take minutes (default 600000 = 10 min). */
     trainTimeoutMs?: number;
+    /** Deadline for video jobs — they take minutes (default 600000 = 10 min). */
+    videoTimeoutMs?: number;
     fetch?: typeof fetch;
     /** Override the per-image USD price (else a built-in estimate per model, 0 if unknown). */
     pricePerImage?: number;
     /** Override the flat per-training USD price (else ~$2 estimate). */
     pricePerTraining?: number;
+    /** Override the per-SECOND video USD price (else a built-in estimate per model, 0 if unknown). */
+    pricePerSecond?: number;
 }
 declare function falAdapter(config?: FalAdapterConfig): ProviderAdapter;
 
@@ -1781,8 +1895,8 @@ declare const falStubAdapter: ProviderAdapter;
  *  wires the live adapters. */
 declare const stubProviders: Record<string, ProviderAdapter>;
 
-declare const VERSION: "0.16.0";
-declare const SDK_TAG: "@broberg/ai-sdk@0.16.0";
+declare const VERSION: "0.17.1";
+declare const SDK_TAG: "@broberg/ai-sdk@0.17.1";
 
 /** Built-in defaults. Every entry is overridable via AiConfig.defaults or a
  *  per-call override. Model IDs are current at scaffold time; callers pin their

package/dist/index.js

@@ -870,6 +870,16 @@ function partsFrom(content) {
     return { inlineData: { mimeType: p.mimeType ?? "image/png", data } };
   });
 }
+var VEO_PRICE_PER_SEC = {
+  "veo-3.1-generate-preview": 0.4,
+  // standard; 4K = 0.60
+  "veo-3.1-fast-generate-preview": 0.1,
+  // 720p; 1080p = 0.12, 4K = 0.30
+  "veo-3.1-lite-generate-preview": 0.05,
+  // 720p; 1080p = 0.08
+  "veo-3.0-generate-001": 0.4,
+  "veo-3.0-fast-generate-001": 0.1
+};
 function geminiAdapter(config = {}) {
   const baseUrl = config.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta";
   function resolveKey() {
@@ -1025,7 +1035,82 @@ function geminiAdapter(config = {}) {
     usage.costUsd = config.pricePerImage ?? GEMINI_IMAGE_PRICE_PER_IMAGE[req.spec.model] ?? 0;
     return { url, usage };
   }
-  return { name: "gemini", chat, chatStream, image, vision: chat };
+  async function animate(req) {
+    const apiKey = resolveKey();
+    const fetchImpl = config.fetch ?? fetch;
+    const pollIntervalMs = config.pollIntervalMs ?? 5e3;
+    const deadline = Date.now() + (config.videoTimeoutMs ?? 3e5);
+    const { data, mimeType } = await toInlineImage(req.image, fetchImpl);
+    const parameters = {};
+    if (req.durationSec !== void 0) parameters.durationSeconds = req.durationSec;
+    if (req.resolution !== void 0) parameters.resolution = req.resolution;
+    const body = {
+      instances: [{ prompt: req.prompt ?? "", image: { bytesBase64Encoded: data, mimeType } }],
+      ...Object.keys(parameters).length ? { parameters } : {}
+    };
+    const submit = await fetchImpl(
+      `${baseUrl}/models/${req.spec.model}:predictLongRunning?key=${encodeURIComponent(apiKey)}`,
+      { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }
+    );
+    if (!submit.ok) {
+      throw new Error(`gemini animate ${submit.status}: ${(await submit.text().catch(() => "")).slice(0, 300)}`);
+    }
+    const op = await submit.json();
+    if (!op.name) throw new Error("gemini animate: no operation name in submit response");
+    let videoUri;
+    for (; ; ) {
+      const poll = await fetchImpl(`${baseUrl}/${op.name}?key=${encodeURIComponent(apiKey)}`, {
+        headers: { "content-type": "application/json" }
+      });
+      if (!poll.ok) throw new Error(`gemini animate poll ${poll.status}`);
+      const opData = await poll.json();
+      if (opData.error) throw new Error(`gemini animate: ${opData.error.message ?? "operation error"}`);
+      if (opData.done) {
+        videoUri = opData.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri;
+        if (!videoUri) throw new Error(`gemini animate: done but no video uri: ${JSON.stringify(opData.response).slice(0, 300)}`);
+        break;
+      }
+      if (Date.now() >= deadline) throw new Error("gemini animate: timed out");
+      await new Promise((r) => setTimeout(r, pollIntervalMs));
+    }
+    const dl = await fetchImpl(videoUri.includes("key=") ? videoUri : `${videoUri}${videoUri.includes("?") ? "&" : "?"}key=${encodeURIComponent(apiKey)}`);
+    if (!dl.ok) throw new Error(`gemini animate download ${dl.status}`);
+    const bytes = new Uint8Array(await dl.arrayBuffer());
+    const usage = freshUsage({
+      provider: "gemini",
+      model: req.spec.model,
+      transport: "http",
+      capability: "animate",
+      inputTokens: 0,
+      outputTokens: 0
+    });
+    const perSec = config.pricePerSecond ?? VEO_PRICE_PER_SEC[req.spec.model] ?? 0;
+    usage.costUsd = perSec * (req.durationSec ?? 8);
+    return { url: videoUri, bytes, mimeType: "video/mp4", usage };
+  }
+  return { name: "gemini", chat, chatStream, image, animate, vision: chat };
+}
+async function toInlineImage(image, fetchImpl) {
+  if (typeof image !== "string") {
+    return { data: Buffer.from(image).toString("base64"), mimeType: sniffMime(image) };
+  }
+  if (/^https?:\/\//i.test(image)) {
+    const res = await fetchImpl(image);
+    if (!res.ok) throw new Error(`gemini animate: failed to fetch image (${res.status})`);
+    const bytes = new Uint8Array(await res.arrayBuffer());
+    const mimeType2 = res.headers.get("content-type") ?? sniffMime(bytes);
+    return { data: Buffer.from(bytes).toString("base64"), mimeType: mimeType2 };
+  }
+  const comma = image.startsWith("data:") ? image.indexOf(",") : -1;
+  const b64 = comma >= 0 ? image.slice(comma + 1) : image;
+  const mimeType = image.startsWith("data:") ? image.slice(5, image.indexOf(";")) : "image/png";
+  return { data: b64, mimeType };
+}
+function sniffMime(b) {
+  if (b[0] === 137 && b[1] === 80) return "image/png";
+  if (b[0] === 71 && b[1] === 73) return "image/gif";
+  if (b[0] === 82 && b[1] === 73 && b[8] === 87) return "image/webp";
+  return "image/jpeg";
 }
 function mapGeminiFinish(reason) {
   switch (reason) {
@@ -1320,6 +1405,8 @@ var FAL_IMAGE_PRICE_ESTIMATE = {
   "fal-ai/flux-pro/v1.1": 0.04
 };
 var FAL_TRAIN_PRICE_ESTIMATE = 2;
+var FAL_VIDEO_PRICE_PER_SEC = {};
+var FAL_VIDEO_DEFAULT_SEC = 8;
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 function falAdapter(config = {}) {
   const doFetch = config.fetch ?? fetch;
@@ -1362,6 +1449,36 @@ function falAdapter(config = {}) {
     usage.costUsd = (config.pricePerImage ?? FAL_IMAGE_PRICE_ESTIMATE[req.spec.model] ?? 0) * calls;
     return { url, usage };
   }
+  async function animate(req) {
+    const apiKey = resolveKey();
+    if (!apiKey) throw new Error("fal adapter: FAL_KEY not set");
+    const headers = authHeaders(apiKey);
+    const imageUrl = typeof req.image === "string" && /^https?:\/\//i.test(req.image) ? req.image : await uploadToFalStorage(toBytes(req.image), sniffImageType(req.image), "input", apiKey);
+    const body = { image_url: imageUrl };
+    if (req.prompt !== void 0) body.prompt = req.prompt;
+    if (req.durationSec !== void 0) body.duration = req.durationSec;
+    if (req.resolution !== void 0) body.resolution = req.resolution;
+    const result = await queueResult(req.spec.model, headers, body, config.videoTimeoutMs ?? 6e5);
+    const url = extractVideoUrl(result);
+    if (!url) {
+      throw new Error(
+        `fal animate: no video url in result \u2014 keys [${Object.keys(
+          result ?? {}
+        ).join(", ")}]: ${JSON.stringify(result).slice(0, 500)}`
+      );
+    }
+    const usage = freshUsage({
+      provider: "fal",
+      model: req.spec.model,
+      transport: "http",
+      capability: "animate",
+      inputTokens: 0,
+      outputTokens: 0
+    });
+    const perSec = config.pricePerSecond ?? FAL_VIDEO_PRICE_PER_SEC[req.spec.model] ?? 0;
+    usage.costUsd = perSec * (req.durationSec ?? FAL_VIDEO_DEFAULT_SEC);
+    return { url, usage };
+  }
   async function trainStyle(req) {
     const apiKey = resolveKey();
     if (!apiKey) throw new Error("fal adapter: FAL_KEY not set");
@@ -1481,7 +1598,24 @@ function falAdapter(config = {}) {
     }
     return resultRes.json();
   }
-  return { name: "fal", image, trainStyle };
+  return { name: "fal", image, animate, trainStyle };
+}
+function toBytes(img) {
+  if (typeof img !== "string") return img;
+  const b64 = img.startsWith("data:") ? img.slice(img.indexOf(",") + 1) : img;
+  return new Uint8Array(Buffer.from(b64, "base64"));
+}
+function sniffImageType(img) {
+  const b = toBytes(img);
+  if (b[0] === 137 && b[1] === 80) return "image/png";
+  if (b[0] === 71 && b[1] === 73) return "image/gif";
+  if (b[0] === 82 && b[1] === 73 && b[8] === 87) return "image/webp";
+  return "image/jpeg";
+}
+function extractVideoUrl(result) {
+  const r = result;
+  const root = r?.data ?? r?.response ?? r?.output ?? r;
+  return urlOf(root?.video) ?? urlOf(root?.videos?.[0]) ?? deepFindUrl(root, (u) => /\.(mp4|webm|mov)(\?|$)/i.test(u));
 }
 function urlOf(v) {
   if (typeof v === "string") return v;
@@ -2019,6 +2153,17 @@ var imageInputSchema = z.object({
   retryOnBlack: z.boolean().optional(),
   ...callOptions
 });
+var animateInputSchema = z.object({
+  /** Input image: a URL or raw bytes (uploaded to fal storage). */
+  image: z.union([z.string(), z.instanceof(Uint8Array)]),
+  /** Motion/scene prompt. */
+  prompt: z.string().optional(),
+  /** Clip length in seconds (provider-dependent; Veo ≤ 8s). */
+  durationSec: z.number().positive().optional(),
+  /** e.g. "720p" / "1080p" (provider-dependent). */
+  resolution: z.string().optional(),
+  ...callOptions
+});
 var trainStyleInputSchema = z.object({
   /** A hosted archive URL, or an array of image URLs the SDK zips in-memory. */
   images: z.union([z.string(), z.array(z.string())]),
@@ -2097,6 +2242,12 @@ var DEFAULT_TRAINSTYLE_SPEC = {
   model: "fal-ai/flux-lora-fast-training",
   transport: "http"
 };
+var DEFAULT_ANIMATE_SPEC = {
+  provider: "gemini",
+  model: "veo-3.1-generate-preview",
+  transport: "http"
+};
+var ANIMATE_AUDIO_DIRECTIVE = "No spoken dialogue, no talking, no voiceover. Include natural ambient background sounds that match the environment.";
 var DEFAULT_BFL_FINETUNE_SPEC = {
   provider: "bfl",
   model: "flux-pro-1.1-ultra-finetuned",
@@ -2383,6 +2534,31 @@ function createAI(config = {}) {
         }
       });
     },
+    async animate(input) {
+      input = animateInputSchema.parse(input);
+      const prompt = input.prompt?.trim() ? `${input.prompt.trim()} ${ANIMATE_AUDIO_DIRECTIVE}` : ANIMATE_AUDIO_DIRECTIVE;
+      return runCapability({
+        primary: { ...DEFAULT_ANIMATE_SPEC, ...input.override },
+        fallback: input.fallback,
+        capability: "animate",
+        purpose: input.purpose,
+        labels: input.labels,
+        estIn: 0,
+        // video cost is per-second, not token-based
+        estOut: 0,
+        invoke: async (spec) => {
+          const adapter = pickProvider(spec.provider);
+          if (!adapter.animate) throw new Error(`createAI: provider "${spec.provider}" does not support animate`);
+          return adapter.animate({
+            image: input.image,
+            prompt,
+            durationSec: input.durationSec,
+            resolution: input.resolution,
+            spec
+          });
+        }
+      });
+    },
     async trainStyle(input) {
       input = trainStyleInputSchema.parse(input);
       return runCapability({
@@ -2633,8 +2809,8 @@ var stubProviders = {
 };
 
 // src/version.ts
-var VERSION = "0.16.0";
-var SDK_TAG = "@broberg/ai-sdk@0.16.0";
+var VERSION = "0.17.1";
+var SDK_TAG = "@broberg/ai-sdk@0.17.1";
 
 // src/availability/refresh.ts
 var NOT_REFRESHED = { refreshed: false, checked: 0, markedUnavailable: [] };