@aeye/models 0.1.1 → 0.3.0

package/src/transformers/index.ts

@@ -23,4 +23,4 @@ import type { ModelTransformer } from '@aeye/ai';
  */
 export const transformers: Record<string, ModelTransformer> = {
   // Transformers will be added here as they are implemented
-};
+};

package/src/transformers/replicate/abiruyt-text-extract-ocr.ts

@@ -0,0 +1,32 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    image: string;
+  };
+  Output: string;
+};
+
+export default {
+  "abiruyt/text-extract-ocr": (() => {
+    const transformer: ReplicateTransformer = {
+      imageAnalyze: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          if (!request.images || request.images.length === 0) {
+            throw new Error("Image is required");
+          }
+          return {
+            image: await toURL(request.images[0]),
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          content: response,
+          finishReason: "stop",
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adidoes-whisperx-video-transcribe.ts

@@ -0,0 +1,37 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    url: string;
+    debug?: boolean;
+    batch_size?: number;
+  };
+  Output: string;
+};
+
+export default {
+  "adidoes/whisperx-video-transcribe": (() => {
+    const transformer: ReplicateTransformer = {
+      transcribe: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          url: await toURL(request.audio),
+          batch_size: request.extra?.batch_size,
+          debug: request.extra?.debug,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => {
+          try {
+            const data = JSON.parse(response);
+            const text = data.segments 
+              ? data.segments.map((s: any) => s.text).join('').trim() 
+              : "";
+            return { text };
+          } catch (e) {
+            return { text: response };
+          }
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-e5-mistral-7b-instruct.ts

@@ -0,0 +1,40 @@
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    task?: string;
+    query?: string;
+    document: string;
+    normalize?: boolean;
+  };
+  Output: Record<string, never>;
+};
+
+export default {
+  "adirik/e5-mistral-7b-instruct": (() => {
+    const transformer: ReplicateTransformer = {
+      embed: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          // The model expects a single document string for the main input.
+          // We map the first text in the request to 'document'.
+          // 'task' and 'query' can be passed via extra if needed for asymmetric tasks.
+          return {
+            document: request.texts[0],
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => {
+          // Cast response to any to access the embeddings property not defined in the strict schema type
+          const output = response as any;
+          return {
+            embeddings: (output.embeddings || []).map((embedding: number[], index: number) => ({
+              embedding,
+              index,
+            })),
+          };
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-kosmos-g.ts

@@ -0,0 +1,48 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    seed?: number;
+    image1: string;
+    image2?: string;
+    prompt?: string;
+    num_images?: number;
+    negative_prompt?: string;
+    num_inference_steps?: number;
+    text_guidance_scale?: number;
+  };
+  Output: string[];
+};
+
+export default {
+  "adirik/kosmos-g": (() => {
+    const transformer: ReplicateTransformer = {
+      imageGenerate: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          prompt: request.prompt,
+          num_images: request.n,
+          seed: request.seed,
+          image1: request.extra?.image1, // Required by model, expected in extra for generate
+          ...request.extra,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          images: response.map((url) => ({ url })),
+        }),
+      },
+      imageEdit: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          prompt: request.prompt,
+          image1: await toURL(request.image),
+          num_images: request.n,
+          seed: request.seed,
+          ...request.extra,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          images: response.map((url) => ({ url })),
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-owlvit-base-patch32.ts

@@ -0,0 +1,35 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    image?: string;
+    query?: string;
+    threshold?: number;
+    show_visualisation?: boolean;
+  };
+  Output: Schemas["ModelOutput"];
+  ModelOutput: {
+    json_data: Record<string, never>;
+    result_image?: string;
+  };
+};
+
+export default {
+  "adirik/owlvit-base-patch32": (() => {
+    const transformer: ReplicateTransformer = {
+      imageAnalyze: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          image: await toURL(request.images[0]),
+          query: request.prompt,
+          ...request.extra,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          content: JSON.stringify(response.json_data),
+          finishReason: 'stop',
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-realvisxl-v3-0-turbo.ts

@@ -0,0 +1,85 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    mask?: string;
+    seed?: number;
+    image?: string;
+    width?: number;
+    height?: number;
+    prompt?: string;
+    refine?: Schemas["refine"];
+    scheduler?: Schemas["scheduler"];
+    num_outputs?: number;
+    refine_steps?: number;
+    guidance_scale?: number;
+    apply_watermark?: boolean;
+    high_noise_frac?: number;
+    negative_prompt?: string;
+    prompt_strength?: number;
+    num_inference_steps?: number;
+    disable_safety_checker?: boolean;
+  };
+  Output: string[];
+  refine: "no_refiner" | "expert_ensemble_refiner" | "base_image_refiner";
+  scheduler: "DDIM" | "DPMSolverMultistep" | "HeunDiscrete" | "KarrasDPM" | "K_EULER_ANCESTRAL" | "K_EULER" | "PNDM" | "DPM++_SDE_Karras";
+};
+
+export default {
+  "adirik/realvisxl-v3.0-turbo": (() => {
+    const transformer: ReplicateTransformer = {
+      imageGenerate: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          let width = 768;
+          let height = 768;
+          if (request.size) {
+            const parts = request.size.split('x');
+            if (parts.length === 2) {
+              width = parseInt(parts[0]);
+              height = parseInt(parts[1]);
+            }
+          }
+          return {
+            prompt: request.prompt,
+            width,
+            height,
+            num_outputs: request.n,
+            seed: request.seed,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          images: await Promise.all(response.map(async (url) => ({ url: await toURL(url) }))),
+        }),
+      },
+      imageEdit: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          let width = 768;
+          let height = 768;
+          if (request.size) {
+            const parts = request.size.split('x');
+            if (parts.length === 2) {
+              width = parseInt(parts[0]);
+              height = parseInt(parts[1]);
+            }
+          }
+          return {
+            prompt: request.prompt,
+            image: await toURL(request.image),
+            mask: request.mask ? await toURL(request.mask) : undefined,
+            width,
+            height,
+            num_outputs: request.n,
+            seed: request.seed,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          images: await Promise.all(response.map(async (url) => ({ url: await toURL(url) }))),
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-styletts2.ts

@@ -0,0 +1,42 @@
+import { toURL, toReadableStream } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    beta?: number;
+    seed?: number;
+    text: string;
+    alpha?: number;
+    weights?: string;
+    reference?: string;
+    diffusion_steps?: number;
+    embedding_scale?: number;
+  };
+  Output: string;
+};
+
+export default {
+  "adirik/styletts2": (() => {
+    const transformer: ReplicateTransformer = {
+      speech: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          const extra = request.extra || {};
+          return {
+            text: request.text,
+            beta: extra.beta,
+            seed: extra.seed,
+            alpha: extra.alpha,
+            weights: extra.weights,
+            reference: extra.reference ? await toURL(extra.reference) : undefined,
+            diffusion_steps: extra.diffusion_steps,
+            embedding_scale: extra.embedding_scale,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          audio: await toReadableStream(response),
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/adirik-udop-large.ts

@@ -0,0 +1,23 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+
+
+export default {
+  "adirik/udop-large": (() => {
+    const transformer: ReplicateTransformer = {
+      imageAnalyze: {
+        convertRequest: async (request, ctx) => ({
+          prompt: request.prompt,
+          image: await toURL(request.images[0]),
+          ...request.extra,
+        }),
+        parseResponse: async (response, ctx) => ({
+          content: typeof response === 'string' ? response : JSON.stringify(response),
+          finishReason: 'stop',
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/afiaka87-tortoise-tts.ts

@@ -0,0 +1,31 @@
+import { toURL, toReadableStream } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+
+
+export default {
+  "afiaka87/tortoise-tts": (() => {
+    const transformer: ReplicateTransformer = {
+      speech: {
+        convertRequest: async (request, ctx) => {
+          return {
+            text: request.text,
+            voice_a: request.voice,
+            custom_voice: request.extra?.custom_voice ? await toURL(request.extra.custom_voice) : undefined,
+            preset: request.extra?.preset,
+            seed: request.extra?.seed,
+            cvvp_amount: request.extra?.cvvp_amount,
+            voice_b: request.extra?.voice_b,
+            voice_c: request.extra?.voice_c,
+          };
+        },
+        parseResponse: async (response, ctx) => {
+          return {
+            audio: await toReadableStream(response),
+          };
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/ai-forever-kandinsky-2-2.ts

@@ -0,0 +1,54 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    seed?: number;
+    width?: Schemas["width"];
+    height?: Schemas["height"];
+    prompt?: string;
+    num_outputs?: number;
+    output_format?: Schemas["output_format"];
+    negative_prompt?: string;
+    num_inference_steps?: number;
+    num_inference_steps_prior?: number;
+  };
+  width: 384 | 512 | 576 | 640 | 704 | 768 | 960 | 1024 | 1152 | 1280 | 1536 | 1792 | 2048;
+  Output: string[];
+  height: 384 | 512 | 576 | 640 | 704 | 768 | 960 | 1024 | 1152 | 1280 | 1536 | 1792 | 2048;
+  output_format: "webp" | "jpeg" | "png";
+};
+
+export default {
+  "ai-forever/kandinsky-2.2": (() => {
+    const transformer: ReplicateTransformer = {
+      imageGenerate: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          let width: Schemas["width"] | undefined;
+          let height: Schemas["height"] | undefined;
+          if (request.size) {
+            const parts = request.size.split('x');
+            width = Number(parts[0]) as Schemas["width"];
+            height = Number(parts[1]) as Schemas["height"];
+          }
+          return {
+            prompt: request.prompt,
+            num_outputs: request.n,
+            seed: request.seed,
+            width,
+            height,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => {
+          return {
+            images: await Promise.all(response.map(async (url) => ({
+              url: await toURL(url),
+            }))),
+          };
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/ai-forever-kandinsky-2.ts

@@ -0,0 +1,62 @@
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    seed?: number;
+    width?: Schemas["width"];
+    height?: Schemas["height"];
+    prompt?: string;
+    scheduler?: Schemas["scheduler"];
+    batch_size?: Schemas["batch_size"];
+    prior_steps?: string;
+    output_format?: Schemas["output_format"];
+    guidance_scale?: number;
+    output_quality?: number;
+    prior_cf_scale?: number;
+    num_inference_steps?: number;
+  };
+  width: 256 | 288 | 432 | 512 | 576 | 768 | 1024;
+  Output: string[];
+  height: 256 | 288 | 432 | 512 | 576 | 768 | 1024;
+  scheduler: "ddim_sampler" | "p_sampler" | "plms_sampler";
+  batch_size: 1 | 2 | 3 | 4;
+  output_format: "webp" | "jpg" | "png";
+};
+
+export default {
+  "ai-forever/kandinsky-2": (() => {
+    const transformer: ReplicateTransformer = {
+      imageGenerate: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          let width: Schemas["width"] | undefined;
+          let height: Schemas["height"] | undefined;
+          
+          if (request.size) {
+            const parts = request.size.split('x').map(Number);
+            if (parts.length === 2) {
+              // Cast to specific types if they match known values, otherwise let the model validation handle it or fallback
+              const [w, h] = parts;
+              width = w as Schemas["width"];
+              height = h as Schemas["height"];
+            }
+          }
+
+          return {
+            prompt: request.prompt,
+            width,
+            height,
+            batch_size: request.n as Schemas["batch_size"],
+            seed: request.seed,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => {
+          return {
+            images: response.map((url) => ({ url })),
+          };
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/alexgenovese-upscaler.ts

@@ -0,0 +1,29 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    image: string;
+    scale?: number;
+    face_enhance?: boolean;
+  };
+  Output: string;
+};
+
+export default {
+  "alexgenovese/upscaler": (() => {
+    const transformer: ReplicateTransformer = {
+      imageEdit: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          image: await toURL(request.image),
+          scale: request.extra?.scale,
+          face_enhance: request.extra?.face_enhance,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          images: [{ url: await toURL(response) }],
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/andreasjansson-blip-2.ts

@@ -0,0 +1,38 @@
+import { toURL } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    image: string;
+    caption?: boolean;
+    context?: string;
+    question?: string;
+    temperature?: number;
+    use_nucleus_sampling?: boolean;
+  };
+  Output: string;
+};
+
+export default {
+  "andreasjansson/blip-2": (() => {
+    const transformer: ReplicateTransformer = {
+      imageAnalyze: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          return {
+            image: await toURL(request.images[0]),
+            question: request.prompt,
+            temperature: request.temperature,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => {
+          return {
+            content: response,
+            finishReason: "stop",
+          };
+        },
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/andreasjansson-clip-features.ts

@@ -0,0 +1,32 @@
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    inputs?: string;
+  };
+  Output: Schemas["NamedEmbedding"][];
+  NamedEmbedding: {
+    input: string;
+    embedding: number[];
+  };
+};
+
+export default {
+  "andreasjansson/clip-features": (() => {
+    const transformer: ReplicateTransformer = {
+      embed: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => ({
+          inputs: request.texts.join('\n'),
+          ...request.extra,
+        }),
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          embeddings: response.map((item, index) => ({
+            embedding: item.embedding,
+            index,
+          })),
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}

package/src/transformers/replicate/andreasjansson-llama-2-13b-embeddings.ts

@@ -0,0 +1,33 @@
+import { ReplicateTransformer } from '@aeye/replicate';
+
+type Schemas = {
+  Input: {
+    prompts: string;
+    prompt_separator?: string;
+  };
+  Output: number[][];
+};
+
+export default {
+  "andreasjansson/llama-2-13b-embeddings": (() => {
+    const transformer: ReplicateTransformer = {
+      embed: {
+        convertRequest: async (request, ctx): Promise<Schemas["Input"]> => {
+          const separator = (request.extra?.prompt_separator as string) || "\n\n";
+          return {
+            prompts: request.texts.join(separator),
+            prompt_separator: separator,
+            ...request.extra,
+          };
+        },
+        parseResponse: async (response: Schemas["Output"], ctx) => ({
+          embeddings: response.map((embedding, index) => ({
+            embedding,
+            index,
+          })),
+        }),
+      },
+    };
+    return transformer;
+  })(),
+}