@ai-sdk/prodia 1.0.22 → 1.0.23

package/src/prodia-language-model-settings.ts

@@ -0,0 +1,6 @@
+/**
+ * Prodia job types for language model (multimodal img2img).
+ */
+export type ProdiaLanguageModelId =
+  | 'inference.nano-banana.img2img.v2'
+  | (string & {});

package/src/prodia-language-model.ts

@@ -0,0 +1,395 @@
+import type {
+  LanguageModelV3,
+  LanguageModelV3CallOptions,
+  LanguageModelV3Content,
+  LanguageModelV3StreamPart,
+  SharedV3Warning,
+} from '@ai-sdk/provider';
+import type { InferSchema } from '@ai-sdk/provider-utils';
+import {
+  combineHeaders,
+  convertBase64ToUint8Array,
+  generateId,
+  lazySchema,
+  parseJSON,
+  parseProviderOptions,
+  postFormDataToApi,
+  resolve,
+  zodSchema,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import type { ProdiaModelConfig } from './prodia-api';
+import {
+  buildProdiaProviderMetadata,
+  parseMultipart,
+  prodiaFailedResponseHandler,
+  prodiaJobResultSchema,
+} from './prodia-api';
+import type { ProdiaJobResult } from './prodia-api';
+import type { ProdiaLanguageModelId } from './prodia-language-model-settings';
+
+export class ProdiaLanguageModel implements LanguageModelV3 {
+  readonly specificationVersion = 'v3';
+  readonly supportedUrls = {};
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: ProdiaLanguageModelId,
+    private readonly config: ProdiaModelConfig,
+  ) {}
+
+  async doGenerate(options: LanguageModelV3CallOptions) {
+    const warnings: Array<SharedV3Warning> = [];
+
+    // Warn about unsupported LLM features
+    if (options.temperature !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'temperature' });
+    }
+    if (options.topP !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'topP' });
+    }
+    if (options.topK !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'topK' });
+    }
+    if (options.maxOutputTokens !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'maxOutputTokens' });
+    }
+    if (options.stopSequences !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'stopSequences' });
+    }
+    if (options.presencePenalty !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'presencePenalty' });
+    }
+    if (options.frequencyPenalty !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'frequencyPenalty' });
+    }
+    if (options.tools !== undefined && options.tools.length > 0) {
+      warnings.push({ type: 'unsupported', feature: 'tools' });
+    }
+    if (options.toolChoice !== undefined) {
+      warnings.push({ type: 'unsupported', feature: 'toolChoice' });
+    }
+    if (
+      options.responseFormat !== undefined &&
+      options.responseFormat.type !== 'text'
+    ) {
+      warnings.push({ type: 'unsupported', feature: 'responseFormat' });
+    }
+
+    const prodiaOptions = await parseProviderOptions({
+      provider: 'prodia',
+      providerOptions: options.providerOptions,
+      schema: prodiaLanguageModelOptionsSchema,
+    });
+
+    // Extract text prompt from messages
+    let prompt = '';
+    let systemMessage = '';
+    for (const message of options.prompt) {
+      if (message.role === 'system') {
+        systemMessage = message.content;
+      }
+    }
+    // Get text from the last user message
+    for (let i = options.prompt.length - 1; i >= 0; i--) {
+      const message = options.prompt[i];
+      if (message.role === 'user') {
+        for (const part of message.content) {
+          if (part.type === 'text') {
+            prompt += (prompt ? '\n' : '') + part.text;
+          }
+        }
+        break;
+      }
+    }
+    if (systemMessage) {
+      prompt = systemMessage + '\n' + prompt;
+    }
+
+    // Extract image from user messages
+    let imageBytes: Uint8Array | undefined;
+    let imageMediaType = 'image/png';
+    for (let i = options.prompt.length - 1; i >= 0; i--) {
+      const message = options.prompt[i];
+      if (message.role === 'user') {
+        for (const part of message.content) {
+          if (part.type === 'file' && part.mediaType.startsWith('image/')) {
+            if (part.data instanceof Uint8Array) {
+              imageBytes = part.data;
+            } else if (typeof part.data === 'string') {
+              // base64 encoded
+              imageBytes = convertBase64ToUint8Array(part.data);
+            } else if (part.data instanceof URL) {
+              const fetchFn = this.config.fetch ?? globalThis.fetch;
+              const response = await fetchFn(part.data.toString());
+              const arrayBuffer = await response.arrayBuffer();
+              imageBytes = new Uint8Array(arrayBuffer);
+            }
+            imageMediaType = part.mediaType;
+            break;
+          }
+        }
+        break;
+      }
+    }
+
+    const jobConfig: Record<string, unknown> = {
+      prompt,
+      include_messages: true,
+    };
+
+    if (prodiaOptions?.aspectRatio !== undefined) {
+      jobConfig.aspect_ratio = prodiaOptions.aspectRatio;
+    }
+
+    const body = {
+      type: this.modelId,
+      config: jobConfig,
+    };
+
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const combinedHeaders = combineHeaders(
+      await resolve(this.config.headers),
+      options.headers,
+    );
+
+    // Always use multipart form-data since img2img requires image input
+    const formData = new FormData();
+    formData.append(
+      'job',
+      new Blob([JSON.stringify(body)], { type: 'application/json' }),
+      'job.json',
+    );
+    if (imageBytes) {
+      const ext =
+        imageMediaType === 'image/png'
+          ? '.png'
+          : imageMediaType === 'image/jpeg'
+            ? '.jpg'
+            : imageMediaType === 'image/webp'
+              ? '.webp'
+              : '';
+      formData.append(
+        'input',
+        new Blob([imageBytes], { type: imageMediaType }),
+        'input' + ext,
+      );
+    }
+
+    const { value: multipartResult, responseHeaders } = await postFormDataToApi(
+      {
+        url: `${this.config.baseURL}/job?price=true`,
+        headers: {
+          ...combinedHeaders,
+          Accept: 'multipart/form-data',
+        },
+        formData,
+        failedResponseHandler: prodiaFailedResponseHandler,
+        successfulResponseHandler: createLanguageMultipartResponseHandler(),
+        abortSignal: options.abortSignal,
+        fetch: this.config.fetch,
+      },
+    );
+
+    const { jobResult, textContent, fileContent } = multipartResult;
+
+    const content: Array<LanguageModelV3Content> = [];
+    if (textContent !== undefined) {
+      content.push({ type: 'text', text: textContent });
+    }
+    for (const file of fileContent) {
+      content.push({
+        type: 'file',
+        mediaType: file.mediaType,
+        data: file.data,
+      });
+    }
+
+    return {
+      content,
+      finishReason: { unified: 'stop' as const, raw: undefined },
+      usage: {
+        inputTokens: {
+          total: undefined,
+          noCache: undefined,
+          cacheRead: undefined,
+          cacheWrite: undefined,
+        },
+        outputTokens: {
+          total: undefined,
+          text: undefined,
+          reasoning: undefined,
+        },
+      },
+      warnings,
+      providerMetadata: {
+        prodia: buildProdiaProviderMetadata(jobResult),
+      },
+      response: {
+        modelId: this.modelId,
+        timestamp: currentDate,
+        headers: responseHeaders,
+      },
+    };
+  }
+
+  async doStream(options: LanguageModelV3CallOptions) {
+    const result = await this.doGenerate(options);
+
+    const stream = new ReadableStream<LanguageModelV3StreamPart>({
+      start(controller) {
+        controller.enqueue({
+          type: 'stream-start',
+          warnings: result.warnings,
+        });
+
+        controller.enqueue({
+          type: 'response-metadata',
+          modelId: result.response?.modelId,
+          timestamp: result.response?.timestamp,
+        });
+
+        for (const part of result.content) {
+          if (part.type === 'text') {
+            const id = generateId();
+            controller.enqueue({ type: 'text-start', id });
+            controller.enqueue({
+              type: 'text-delta',
+              id,
+              delta: part.text,
+            });
+            controller.enqueue({ type: 'text-end', id });
+          } else if (part.type === 'file') {
+            controller.enqueue({
+              type: 'file',
+              mediaType: part.mediaType,
+              data: part.data,
+            });
+          }
+        }
+
+        controller.enqueue({
+          type: 'finish',
+          usage: result.usage,
+          finishReason: result.finishReason,
+          providerMetadata: result.providerMetadata,
+        });
+
+        controller.close();
+      },
+    });
+
+    return {
+      stream,
+      response: {
+        headers: result.response?.headers,
+      },
+    };
+  }
+}
+
+export const prodiaLanguageModelOptionsSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      /**
+       * Aspect ratio for the output image.
+       */
+      aspectRatio: z
+        .enum([
+          '1:1',
+          '2:3',
+          '3:2',
+          '4:5',
+          '5:4',
+          '4:7',
+          '7:4',
+          '9:16',
+          '16:9',
+          '9:21',
+          '21:9',
+        ])
+        .optional(),
+    }),
+  ),
+);
+
+export type ProdiaLanguageModelOptions = InferSchema<
+  typeof prodiaLanguageModelOptionsSchema
+>;
+
+interface LanguageMultipartResult {
+  jobResult: ProdiaJobResult;
+  textContent: string | undefined;
+  fileContent: Array<{ mediaType: string; data: Uint8Array }>;
+}
+
+function createLanguageMultipartResponseHandler() {
+  return async ({
+    response,
+  }: {
+    response: Response;
+  }): Promise<{
+    value: LanguageMultipartResult;
+    responseHeaders: Record<string, string>;
+  }> => {
+    const contentType = response.headers.get('content-type') ?? '';
+    const responseHeaders: Record<string, string> = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+
+    const boundaryMatch = contentType.match(/boundary=([^\s;]+)/);
+    if (!boundaryMatch) {
+      throw new Error(
+        `Prodia response missing multipart boundary in content-type: ${contentType}`,
+      );
+    }
+    const boundary = boundaryMatch[1];
+
+    const arrayBuffer = await response.arrayBuffer();
+    const bytes = new Uint8Array(arrayBuffer);
+
+    const parts = parseMultipart(bytes, boundary);
+
+    let jobResult: ProdiaJobResult | undefined;
+    let textContent: string | undefined;
+    const fileContent: Array<{ mediaType: string; data: Uint8Array }> = [];
+
+    for (const part of parts) {
+      const contentDisposition = part.headers['content-disposition'] ?? '';
+      const partContentType = part.headers['content-type'] ?? '';
+
+      if (contentDisposition.includes('name="job"')) {
+        const jsonStr = new TextDecoder().decode(part.body);
+        jobResult = await parseJSON({
+          text: jsonStr,
+          schema: zodSchema(prodiaJobResultSchema),
+        });
+      } else if (contentDisposition.includes('name="output"')) {
+        if (
+          partContentType.startsWith('text/') ||
+          contentDisposition.includes('.txt')
+        ) {
+          textContent = new TextDecoder().decode(part.body);
+        } else if (partContentType.startsWith('image/')) {
+          fileContent.push({
+            mediaType: partContentType,
+            data: part.body,
+          });
+        }
+      }
+    }
+
+    if (!jobResult) {
+      throw new Error('Prodia multipart response missing job part');
+    }
+
+    return {
+      value: { jobResult, textContent, fileContent },
+      responseHeaders,
+    };
+  };
+}

package/src/prodia-provider.ts

@@ -1,5 +1,7 @@
 import {
+  type Experimental_VideoModelV3,
   type ImageModelV3,
+  type LanguageModelV3,
   NoSuchModelError,
   type ProviderV3,
 } from '@ai-sdk/provider';
@@ -11,6 +13,10 @@ import {
 } from '@ai-sdk/provider-utils';
 import { ProdiaImageModel } from './prodia-image-model';
 import type { ProdiaImageModelId } from './prodia-image-settings';
+import { ProdiaLanguageModel } from './prodia-language-model';
+import type { ProdiaLanguageModelId } from './prodia-language-model-settings';
+import { ProdiaVideoModel } from './prodia-video-model';
+import type { ProdiaVideoModelId } from './prodia-video-model-settings';
 import { VERSION } from './version';
 
 export interface ProdiaProviderSettings {
@@ -37,6 +43,11 @@ export interface ProdiaProviderSettings {
 }
 
 export interface ProdiaProvider extends ProviderV3 {
+  /**
+   * Creates a language model for multimodal generation (img2img with text+image output).
+   */
+  languageModel(modelId: ProdiaLanguageModelId): LanguageModelV3;
+
   /**
    * Creates a model for image generation.
    */
@@ -47,6 +58,16 @@ export interface ProdiaProvider extends ProviderV3 {
    */
   imageModel(modelId: ProdiaImageModelId): ImageModelV3;
 
+  /**
+   * Creates a model for video generation.
+   */
+  video(modelId: ProdiaVideoModelId): Experimental_VideoModelV3;
+
+  /**
+   * Creates a model for video generation.
+   */
+  videoModel(modelId: ProdiaVideoModelId): Experimental_VideoModelV3;
+
   /**
    * @deprecated Use `embeddingModel` instead.
    */
@@ -80,25 +101,36 @@ export function createProdia(
       fetch: options.fetch,
     });
 
-  const embeddingModel = (modelId: string) => {
-    throw new NoSuchModelError({
-      modelId,
-      modelType: 'embeddingModel',
+  const createLanguageModel = (modelId: ProdiaLanguageModelId) =>
+    new ProdiaLanguageModel(modelId, {
+      provider: 'prodia.language',
+      baseURL: baseURL ?? defaultBaseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
+    });
+
+  const createVideoModel = (modelId: ProdiaVideoModelId) =>
+    new ProdiaVideoModel(modelId, {
+      provider: 'prodia.video',
+      baseURL: baseURL ?? defaultBaseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
     });
-  };
 
-  const languageModel = (modelId: string) => {
+  const embeddingModel = (modelId: string) => {
     throw new NoSuchModelError({
       modelId,
-      modelType: 'languageModel',
+      modelType: 'embeddingModel',
     });
   };
 
   return {
     specificationVersion: 'v3',
+    languageModel: createLanguageModel,
     imageModel: createImageModel,
     image: createImageModel,
-    languageModel,
+    videoModel: createVideoModel,
+    video: createVideoModel,
     embeddingModel,
     textEmbeddingModel: embeddingModel,
   };

package/src/prodia-video-model-settings.ts

@@ -0,0 +1,7 @@
+/**
+ * Prodia job types for video generation.
+ */
+export type ProdiaVideoModelId =
+  | 'inference.wan2-2.lightning.txt2vid.v0'
+  | 'inference.wan2-2.lightning.img2vid.v0'
+  | (string & {});