@aeye/models 0.2.0 → 0.3.0

package/scripts/scrapers/replicate.ts

@@ -4,15 +4,18 @@
  * Fetches model information from Replicate API using the Replicate npm module
  */
 
+import type { ModelCapability, ModelInfo } from '@aeye/ai';
+import { AI, detectTier } from '@aeye/ai';
+import { OpenRouterProvider } from '@aeye/openrouter';
 import * as fs from 'fs/promises';
+import openapiTS from 'openapi-typescript';
 import * as path from 'path';
+import Replicate from 'replicate';
+import ts from 'typescript';
 import * as url from 'url';
 import { z } from 'zod';
-import Replicate from 'replicate';
-import type { ModelInfo, ModelCapability } from '@aeye/ai';
-import { AI, detectTier } from '@aeye/ai';
 import { writeModelTS } from '../codegen';
-import { OpenRouterProvider } from 'packages/openrouter/src/openrouter';
+import { exec, spawn } from 'child_process';
 
 const __filename = url.fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -187,7 +190,30 @@ const ai = AI
       hooks: {
         chat: {
           beforeRequest: (a, b, c, d) => {
-            console.log('OpenRouter Chat Request:', b);
+            const now = Date.now().toString();
+
+            const requestResolved = path.resolve(__dirname, '../../cache/replicate', now  + '-request.json');
+            fs.writeFile(requestResolved, JSON.stringify(b, null, 2));
+
+            const content = a.messages.map((m) => m.content).join('\n---\n');
+            const contentResolved = path.resolve(__dirname, '../../cache/replicate', now + '-content.txt');
+            fs.writeFile(contentResolved, content);
+          },
+          afterRequest: (a, b, c, d) => {
+            const now = Date.now().toString();
+
+            const responseResolved = path.resolve(__dirname, '../../cache/replicate', now + '-response.json');
+            fs.writeFile(responseResolved, JSON.stringify(c, null, 2));
+
+            const content = c.content;
+            const contentResolved = path.resolve(__dirname, '../../cache/replicate', now + '-response.txt');
+            fs.writeFile(contentResolved, content);
+          },
+          onError: (a, b, e, d) => {
+            const now = Date.now().toString();
+
+            const errorResolved = path.resolve(__dirname, '../../cache/replicate', now + '-error.txt');
+            fs.writeFile(errorResolved, String(e));
           },
         },
       },
@@ -196,8 +222,20 @@ const ai = AI
   .create({})
 ;
 
+const jsonReplacer = (key: string, value: any) => {
+  if (value instanceof Set) {
+    return Array.from(value);
+  }
+  return value;
+};
+
 const typeInfo = await fs.readFile(path.join(__dirname, 'extract.md'), 'utf-8');
 
+const optionalTransform = (val: any) => 
+  !val || Object.values(val).every((v) => v === null || v === undefined) 
+    ? undefined
+    : Object.fromEntries(Object.entries(val).filter(([_, v]) => v !== null && v !== undefined));
+
 const extract = ai.prompt({
   name: 'extract',
   description: 'Extract model information & a transformer definition from Replicate model data',
@@ -207,50 +245,74 @@ const extract = ai.prompt({
 {{modelData}}
 </modelData>
 
-Create a TypeScript implementation of a ModelTransformer for the model.
+{{#if modelTypes}}
+Here are the types for the model based on the OpenAPI schema:
+\`\`\`ts
+{{modelTypes}}
+\`\`\`
+
+Schemas["Input"] is what convertRequest should return.
+Schemas["Output"] is what parseResponse is given as the first argument.
+
+These types will be available and can be referenced like above to ensure type safety.
+{{/if}}
+
+Create a TypeScript implementation of a ReplicateTransformer for the model.
 Here's an example transformer implementation for reference. This is the exact format to follow:
-<exampleTransformer>
+<exampleTransformerImplementation>
 { 
   // this key comes from modelData.owner/modelData.name
-  "google/nano-banana": () => ({
-    // you must use this ModelTransformer type to avoid TypeScript errors
-    const transformer: ModelTransformer = { 
+  // // The definition is wrapped in an immediately-invoked function expression so any cached or reusable functions can be defined here.
+  "google/nano-banana": (() => {
+    // you must use this ReplicateTransformer type to avoid TypeScript errors
+    const transformer: ReplicateTransformer = { 
       imageGenerate: {
         // only use variable known to be in this request type based on the types below
-        convertRequest: async (request, ctx) => ({ 
+        convertRequest: async (request, ctx){{#if modelTypes}}: Promise<Schemas["Input"]>{{/if}} => ({ 
           prompt: request.prompt,
-          ...request.extra,
+          ...request.extra, // always spread extra to allow for additional parameters
         }),
         // only use known output schema from the model and only use properties expected on the response type based on the types below
-        parseResponse: async (response, ctx) => ({
+        parseResponse: async (response{{#if modelTypes}}: Schemas["Output"]{{/if}}, ctx) => ({
           images: [{ url: await toURL(response) }],
+          extra: { response }, // include full response in extra object
         }),
       },
       imageEdit: {
-        convertRequest: async (request, ctx) => ({
+        convertRequest: async (request, ctx){{#if modelTypes}}: Promise<Schemas["Input"]>{{/if}} => ({
           prompt: request.prompt,
           image_input: [await toURL(request.image)],
-          ...request.extra,
+          ...request.extra, // always spread extra to allow for additional parameters
         }),
-        parseResponse: async (response, ctx) => ({
+        parseResponse: async (response{{#if modelTypes}}: Schemas["Output"]{{/if}}, ctx) => ({
           images: [{ url: await toURL(response) }],
+          extra: { response }, // include full response in extra object
         }),
       },
     };
     return transformer;
-  })(),
+  })(), // immediately invoke the function to return the typed transformer
 }
-</exampleTransformer>
+</exampleTransformerImplementation>
+
+The transformer code you return should match EXACTLY the format above. It will be injected in code like so by the developer:
+<transformerResult>
+import { toURL, toBase64, toText, toStream, toReadableStream, toFile } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+const transformer: Record<string, ReplicateTransformer> = <transformerImplementation>;
+</transformerResult>
 
 <availableFunctions>
 - \`toURL(resource, mimeType?: string, fallback?: string): Promise<string>\` converts to URL string
 - \`toBase64(resource, mimeType?: string, fallback?: string): Promise<string>\`
 - \`toText(resource, fallback?: string): Promise<string>\`
 - \`toStream(resource, fallback?: Readable): Promise<Readable>\`
+- \`toReadableStream(resource, fallback?: ReadableStream): Promise<ReadableStream>\`
 - \`toFile(resource, mimeType?: string, filename?: string): Promise<File>\`
 </availableFunctions>
 
-Here's how the replicate code looks around the transformer:
+Here's how the replicate code looks around the transformer: This is an example to illustrate how the transformer you generate is used.
 <replicateCode>
 // =============================================================
 // Execute
@@ -285,9 +347,12 @@ The request parameters in the converted request should only be the properties th
 {{typeInfo}}
 </typeInfo>
   `,
-  strict: false,
+  strict: true,
+  outputRetries: 3,
   schema: z.object({
-    transformerImplementation: z.string().describe('The TypeScript implementation of the model handler (without imports or ``` marks)'),
+    transformerImplementation: z.string()
+    // .regex(/^\{\n\s+"[^"]+\/[^"]+": \(\(\) => \{(.|\r|\n)*\}\)\(\),\n\}$/)
+      .describe(`The TypeScript implementation of the model handler (without imports or \`\`\` marks) - described BETWEEN the <exampleTransformerImplementation> tags. Follow the rules outlined.`),
     modelInfo: z.object({
       capabilities: z.array(z.enum([
         'chat',
@@ -325,15 +390,17 @@ The request parameters in the converted request should only be the properties th
   - experimental: Preview/beta models`),
       pricing: z.object({
         text: z.object({
-          input: z.number(),
-          output: z.number(),
-          cached: z.number(),
-        }).optional(),
+          input: z.number().optional(),
+          output: z.number().optional(),
+          cached: z.number().optional(),
+        }).optional()
+          .transform(optionalTransform),
         audio: z.object({
           input: z.number().optional(),
           output: z.number().optional(),
           perSecond: z.number().optional(),
-        }).optional(),
+        }).optional()
+          .transform(optionalTransform),
         image: z.object({
           input: z.number().optional(),
           output: z.array(z.object({
@@ -343,18 +410,22 @@ The request parameters in the converted request should only be the properties th
               height: z.number(),
               cost: z.number().describe('Cost per image in USD'),
             })).describe('Supported image sizes and their costs'),
-          })).optional(),
-        }).optional(),
+          })).optional()
+            .transform(optionalTransform),
+        }).optional()
+          .transform(optionalTransform),
         reasoning: z.object({
           input: z.number().optional(),
           output: z.number().optional(),
           cached: z.number().optional(),
-        }).optional(),
-        embedding: z.object({
+        }).optional()
+          .transform(optionalTransform),
+        embeddings: z.object({
           cost: z.number().optional(),
-        }).optional(),
+        }).optional()
+          .transform(optionalTransform),
         perRequest: z.number().optional().describe('Flat cost per request in USD'),
-      }).describe('Pricing information. input/output/cached are per 1 million tokens unless otherwise specified.'),
+      }).describe('Pricing information. input/output/cached are per 1 million tokens unless otherwise specified. Leave null/undefined if unknown - 0 is free. Do not specify 0 unless it is free.'),
       contextWindow: z.number(),
       maxOutputTokens: z.number().optional(),
       metrics: z.object({
@@ -362,7 +433,9 @@ The request parameters in the converted request should only be the properties th
         timeToFirstToken: z.number().optional().describe('Time to first token in milliseconds'),
         averageRequestDuration: z.number().optional().describe('Average request duration in milliseconds'),
         accuracyScore: z.number().optional().describe('Accuracy score based on benchmark tests 0-1'),
-      }).optional().describe('Performance metrics if available'),
+      }).optional()
+        .transform(optionalTransform)
+        .describe('Performance metrics if available'),
       tokenizer: z.enum(([
         'Other', 'GPT', 'Mistral', 'Llama3', 'Qwen3', 'Qwen', 'Gemini', 'DeepSeek', 'Claude', 'Grok', 'Llama4', 'Llama2', 'Cohere', 'Nova', 'Router'
       ])).optional().describe('The tokenizer type used by this model if known, otherwise leave undefined'),
@@ -398,15 +471,96 @@ The request parameters in the converted request should only be the properties th
       ])).optional().describe('The supported parameters for this model'),
     }).describe('The extracted model information'),
   }),
-  input: ({ modelData }: { model: string, modelData: ReplicateModelData }) => ({
-    modelData: JSON.stringify(modelData, null, 2),
-    typeInfo,
-  }),
+  input: async ({ modelData, modelTypes }: { model: string, modelData: ReplicateModelData, modelTypes: string }) => {
+    return {
+      modelData: JSON.stringify(modelData, (k, v) => {
+        if (k === "embeddings" && Array.isArray(v) && v.length > 20) {
+          return [`...${v.length} ${typeof v[0]}s...`];
+        }
+        if (Array.isArray(v) && v.length > 20) {
+          return [`...${v.length} ${typeof v[0]}s...`];
+        }
+        return v;
+      }, 2),
+      typeInfo,
+      modelTypes,
+    };
+  },
   config: (input) => ({
     model: input?.model,
+    temperature: 0,
+    topP: 1,
   }),
 })
 
+async function extractTypes(model: ReplicateModelData): Promise<string> {
+  if (!model.latest_version?.openapi_schema) {
+    return '';
+  }
+
+  let extracted = await extractSchemasString(model.latest_version.openapi_schema as any);
+  if (!extracted) {
+    return '';
+  }
+
+  return extracted.replaceAll('components["schemas"]', 'Schemas').replaceAll('    ', '  ');
+}
+
+async function extractSchemasString(schemaObject: any): Promise<string> {
+  try {
+    const ast = await openapiTS(schemaObject, {
+      propertiesRequiredByDefault: false,
+      defaultNonNullable: false,
+    });
+        
+    const printer = ts.createPrinter({ 
+      newLine: ts.NewLineKind.LineFeed,
+      removeComments: true,
+    });
+    
+    const dummySourceFile = ts.createSourceFile("temp.ts", "", ts.ScriptTarget.ESNext, true);
+
+    const componentsNode = ast.find(node => {
+      if (ts.isInterfaceDeclaration(node)) {
+        const name = printer.printNode(ts.EmitHint.Unspecified, node.name, dummySourceFile);
+        return name === 'components';
+      }
+      return false;
+    }) as ts.InterfaceDeclaration | undefined;
+
+    if (!componentsNode) {
+      console.error("Error: Could not find the 'components' node in the generated AST.");
+      return "";
+    }
+
+    let extractedMembersString = "";
+
+    // Get the inner content of the 'schemas' property
+    for (const member of componentsNode.members) {
+      if (ts.isPropertySignature(member)) {
+        const propName = printer.printNode(ts.EmitHint.Unspecified, member.name, dummySourceFile);
+        
+        if (propName === 'schemas' && member.type && ts.isTypeLiteralNode(member.type)) {
+          member.type.members.forEach(schemaMember => {
+            extractedMembersString += printer.printNode(ts.EmitHint.Unspecified, schemaMember, dummySourceFile) + "\n";
+          });
+          break;
+        }
+      }
+    }
+
+    if (!extractedMembersString.trim()) {
+      console.error("Error: Found no user-defined schema types within the 'components.schemas' structure.");
+      return "";
+    }
+
+    return `type Schemas = {\n    ${extractedMembersString.replaceAll('\n', '\n    ').trim()}\n};`;
+  } catch (error) {
+    console.error("An unexpected error occurred during schema extraction:", error);
+    return "";
+  }
+}
+
 /**
  * Fetch all models from Replicate collections
  */
@@ -467,8 +621,6 @@ async function fetchAllModels(apiKey?: string): Promise<ReplicateModelData[]> {
     }
   }
 
-  allModels.sort((a, b) => a.name.localeCompare(b.name));
-
   console.log(`\n✓ Fetched ${allModels.length} unique Replicate models`);
 
   return allModels;
@@ -520,7 +672,6 @@ async function fetchSchemasParallel(
 
   // Find models that need schema fetching
   const modelsToFetch = models.filter((model) => {
-    const modelId = `${model.owner}/${model.name}`;
     return !model.latest_version?.openapi_schema;
   });
 
@@ -558,6 +709,18 @@ async function fetchSchemasParallel(
   return schemasCache;
 }
 
+function cleanSchema(model: ReplicateModelData) {
+  const schema = model.latest_version?.openapi_schema as any;
+  if (schema) {
+    // delete schema.components.schemas.Status;
+    delete schema.components.schemas.WebhookEvent;
+    delete schema.components.schemas.ValidationError;
+    delete schema.components.schemas.PredictionRequest;
+    delete schema.components.schemas.PredictionResponse;
+    delete schema.components.schemas.HTTPValidationError;
+  }
+}
+
 /**
  * Main scraper function
  */
@@ -568,95 +731,138 @@ export async function scrapeReplicate(
     concurrency?: number, 
     transformers?: boolean,
     transformerModel: string,
+    transformersOnly?: boolean,
+    transformConcurrency?: number,
     models?: string[],
     n?: number,
   } = {
-    transformerModel: 'google/gemini-2.5-flash',
+    transformerModel: 'google/gemini-2.5-pro',
   }
 ): Promise<void> {
   const { concurrency = 50 } = options;
 
-  console.log('\n=== Replicate Scraper ===\n');
-
   const apiKey = process.env.REPLICATE_API_KEY;
   if (!apiKey) {
     console.error('✗ REPLICATE_API_KEY environment variable is required');
     process.exit(1);
   }
 
-  // Fetch all models from collections
-  const models = await fetchAllModels(apiKey);
+  let models: ReplicateModelData[]
+
+  if (!options.transformersOnly) {
+    console.log('\n=== Replicate Scraper ===\n');
+    
+    // Fetch all models from collections
+    models = await fetchAllModels(apiKey);
+
+    // Create output directory
+    await fs.mkdir(outputDir, { recursive: true });
+
+    // Save raw models data
+    await fs.writeFile(
+      path.join(outputDir, 'replicate-models.json'),
+      JSON.stringify({ data: models }, jsonReplacer, 2)
+    );
+
+    console.log(`✓ Saved raw Replicate models to replicate-models.json`);
+  } else {
+    const modelsData = await fs.readFile(
+      path.join(outputDir, 'replicate-models.json'),
+      'utf-8'
+    );
+    const parsed = JSON.parse(modelsData);
+    models = parsed.data as ReplicateModelData[];
+
+    console.log(`\n✓ Loaded ${models.length} Replicate models from cache\n`);
+  }
+
+  models.forEach(cleanSchema);
+
+  // Sort models by owner/name
+  models.sort((a, b) => `${a.owner}/${a.name}`.localeCompare(`${b.owner}/${b.name}`));  
 
   // Convert to ModelInfo format
   const modelInfos = models.map(convertReplicateModel);
 
-  // Create output directory
-  await fs.mkdir(outputDir, { recursive: true });
-
-  // Save raw models data
-  await fs.writeFile(
-    path.join(outputDir, 'replicate-models.json'),
-    JSON.stringify({ data: models }, (key, value) => {
-      if (value instanceof Set) {
-        return Array.from(value);
+  try {
+    const modelInfoCache = await fs.readFile(
+      path.join(outputDir, 'replicate-modelinfo.json'),
+      'utf-8'
+    );
+    const parsed = JSON.parse(modelInfoCache) as ModelInfo[];
+    for (const info of parsed) {
+      const existing = modelInfos.findIndex((m) => m.id === info.id);
+      if (existing >= 0) {
+        modelInfos[existing] = info;
       }
-      return value;
-    }, 2)
-  );
-  console.log(`✓ Saved raw Replicate models to replicate-models.json`);
+    }
+  } catch (e: any) {
+    // Ignore if file doesn't exist
+    console.error(`⚠ Could not load existing model info cache: ${e.message}`);
+  }
 
   let schemasCache: Record<string, ReplicateModelData> = {};
 
   // Create cache directory if specified
   if (cacheDir) {
-    await fs.mkdir(cacheDir, { recursive: true });
+    if (!options.transformersOnly) {
+      await fs.mkdir(cacheDir, { recursive: true });
 
-    // Cache detailed model schemas (for transformer generation)
-    schemasCache = await fetchSchemasParallel(models, apiKey, concurrency);
+      // Cache detailed model schemas (for transformer generation)
+      schemasCache = await fetchSchemasParallel(models, apiKey, concurrency);
 
-    // Save schemas cache
-    await fs.writeFile(
-      path.join(cacheDir, 'replicate-schemas.json'),
-      JSON.stringify(schemasCache, (key, value) => {
-        if (value instanceof Set) {
-          return Array.from(value);
-        }
-        return value;
-      }, 2)
-    );
+      // Save schemas cache
+      await fs.writeFile(
+        path.join(cacheDir, 'replicate-schemas.json'),
+        JSON.stringify(schemasCache, jsonReplacer, 2)
+      );
 
-    console.log(`✓ Saved ${Object.keys(schemasCache).length} model schemas to ${cacheDir}/replicate-schemas.json`);
-    console.log('  (This cache file is for transformer generation and should not be committed)');
-
-    // Chunk it up to 80,000 character files for easier loading later
-    const chunkSize = 80000;
-    const schemaEntries = Object.entries(schemasCache);
-    let currentChunk: Record<string, ReplicateModelData> = {};
-    let currentSize = 0;
-    let chunkIndex = 1;
-    for (const [modelId, modelData] of schemaEntries) {
-      const entryString = JSON.stringify({ [modelId]: modelData });
-      if (currentSize + entryString.length > chunkSize && Object.keys(currentChunk).length > 0) {
-        // Save current chunk
+      console.log(`✓ Saved ${Object.keys(schemasCache).length} model schemas to ${cacheDir}/replicate-schemas.json`);
+      console.log('  (This cache file is for transformer generation and should not be committed)');
+
+      // Chunk it up to 80,000 character files for easier loading later
+      const chunkSize = 80000;
+      const schemaEntries = Object.entries(schemasCache);
+      let currentChunk: Record<string, ReplicateModelData> = {};
+      let currentSize = 0;
+      let chunkIndex = 1;
+      for (const [modelId, modelData] of schemaEntries) {
+        const entryString = JSON.stringify({ [modelId]: modelData });
+        if (currentSize + entryString.length > chunkSize && Object.keys(currentChunk).length > 0) {
+          // Save current chunk
+          await fs.writeFile(
+            path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`),
+            JSON.stringify(currentChunk, null, 2)
+          );
+          console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
+          chunkIndex++;
+          currentChunk = {};
+          currentSize = 0;
+        }
+        currentChunk[modelId] = modelData;
+        currentSize += entryString.length;
+      }
+      // Save any remaining chunk
+      if (Object.keys(currentChunk).length > 0) {
         await fs.writeFile(
           path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`),
           JSON.stringify(currentChunk, null, 2)
         );
         console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
-        chunkIndex++;
-        currentChunk = {};
-        currentSize = 0;
       }
-      currentChunk[modelId] = modelData;
-      currentSize += entryString.length;
-    }
-    // Save any remaining chunk
-    if (Object.keys(currentChunk).length > 0) {
-      await fs.writeFile(
-        path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`),
-        JSON.stringify(currentChunk, null, 2)
+    } else {
+      // Load schemas cache from file
+      const schemasData = await fs.readFile(
+        path.join(cacheDir, 'replicate-schemas.json'),
+        'utf-8'
       );
-      console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
+      schemasCache = JSON.parse(schemasData) as Record<string, ReplicateModelData>;
+
+      console.log(`\n✓ Loaded ${Object.keys(schemasCache).length} model schemas from cache\n`);
+    }
+
+    for (const modelData of Object.values(schemasCache)) {
+      cleanSchema(modelData);
     }
 
     // Calculate number of unique schemas
@@ -674,7 +880,10 @@ export async function scrapeReplicate(
       console.log('=== Generating Model Transformers ===\n');
 
       // Create replicate transformers directory
-      await fs.mkdir(path.join(cacheDir, 'replicate'), { recursive: true });
+      await fs.mkdir(
+        path.join(cacheDir, 'replicate'), 
+        { recursive: true }
+      );
 
       let transformingModels = models;
       let overwrite = false;
@@ -682,7 +891,7 @@ export async function scrapeReplicate(
       // Filter to only models with schemas if transformers are desired
       if (options.models && options.models.length > 0) {
         const modelSet = new Set(options.models);
-        transformingModels = models.filter((model) => {
+        transformingModels = transformingModels.filter((model) => {
           const modelId = `${model.owner}/${model.name}`;
           return modelSet.has(modelId);
         });
@@ -692,88 +901,168 @@ export async function scrapeReplicate(
       // If models were not specified, filter out already transformed models
       if (!overwrite) {
         const alreadyTransformed = await fs.readdir(
-          path.join(cacheDir, 'replicate'),
+          path.resolve(cacheDir, '../src/transformers/replicate'),
         );
         const alreadyTransformedSet = new Set(
-          alreadyTransformed.map((filename) => filename.replace('.ts', ''))
+          alreadyTransformed
+            .filter((filename) => filename.endsWith('.ts') && filename !== 'index.ts')
+            .map((filename) => filename.replace('.ts', ''))
         );
         transformingModels = transformingModels.filter((model) => {
           const modelId = `${model.owner}/${model.name}`;
-          const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
+          const transformedFile = modelId.replace(/[^\w]/g, '-');
+
           return !alreadyTransformedSet.has(transformedFile);
         });
       }
 
       // Only first N models if specified
       if (options.n && options.n > 0) {
-        transformingModels = models.slice(0, options.n);
+        transformingModels = transformingModels.slice(0, options.n);
       }
 
       // Call extractor for each model
       console.log(`Generating transformers for ${transformingModels.length} models...\n`);
 
-      for (const model of transformingModels) {
-        const modelId = `${model.owner}/${model.name}`;
-        const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
-        const modelData = schemasCache[modelId] || model;
+      // Use specified concurrency for transformations
+      const concurrency = options.transformConcurrency !== undefined && options.transformConcurrency <= 0 
+        ? transformingModels.length
+        : options.transformConcurrency ?? 1;
+
+      let nextModelIndex = 0;
+
+      const workers: Promise<void>[] = [];
+      for (let i = 0; i < concurrency; i++) {
+        const worker = (async () => {
+          while (nextModelIndex < transformingModels.length) {
+            const model = transformingModels[nextModelIndex];
+            nextModelIndex++;
+            const modelId = `${model.owner}/${model.name}`;
+            const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
+            const modelData = schemasCache[modelId] || model;
+
+            console.log(`\n--- Generating transformer for model: ${modelId},  ---`);
+
+            try {
+              const modelTypes = await extractTypes(modelData);
+
+              const result = await extract.get('result', { 
+                modelData, 
+                modelTypes,
+                model: options.transformerModel 
+              });
+              if (!result) {
+                console.log(`  ✗ Failed to generate transformer for model: ${modelId}`);
+                continue;
+              }
+
+              const converted = convertReplicateModel(modelData);
+
+              const translatedModelInfo: ModelInfo<'replicate'> = {
+                id: converted.id,
+                provider: 'replicate',
+                name: converted.name,
+                contextWindow: result.modelInfo.contextWindow,
+                maxOutputTokens: result.modelInfo.maxOutputTokens,
+                pricing: result.modelInfo.pricing,
+                tier: result.modelInfo.tier,
+                capabilities: new Set(result.modelInfo.capabilities),
+                metrics: result.modelInfo.metrics,
+                tokenizer: result.modelInfo.tokenizer,
+                supportedParameters: new Set(result.modelInfo.supportedParameters),
+                metadata: converted.metadata,
+              };
+
+              const replaceIndex = modelInfos.findIndex((m) => m.id === translatedModelInfo.id);
+              if (replaceIndex >= 0) {
+                modelInfos[replaceIndex] = translatedModelInfo;
+              } else {
+                console.log(`  ⚠ ModelInfo for ${modelId} not found in main list, adding new entry`);
+              }
+
+              let fixedTransformerImplementation = result.transformerImplementation;
+              if (fixedTransformerImplementation.startsWith('"')) {
+                fixedTransformerImplementation = `{\n  ${fixedTransformerImplementation}\n}`;
+              }
+
+              const coreImports = [
+                fixedTransformerImplementation.includes('toURL') ? 'toURL' : null,
+                fixedTransformerImplementation.includes('toBase64') ? 'toBase64' : null,
+                fixedTransformerImplementation.includes('toText') ? 'toText' : null,
+                fixedTransformerImplementation.includes('toStream') ? 'toStream' : null,
+                fixedTransformerImplementation.includes('toReadableStream') ? 'toReadableStream' : null,
+                fixedTransformerImplementation.includes('toFile') ? 'toFile' : null,
+              ].filter((i) => i !== null) as string[];
+
+              const validatedFile = path.resolve(cacheDir, '../src/transformers/replicate', transformedFile);
+              await fs.writeFile(
+                validatedFile,
+                `
+${coreImports.length > 0 ? `import { ${coreImports.join(', ')} } from '@aeye/core';` : ''}
+import { ReplicateTransformer } from '@aeye/replicate';
+
+${modelTypes}
+
+export default ${fixedTransformerImplementation}
+`.trim()
+              );
+
+              if (await isValidTypeScript(validatedFile)) {
+                console.log(`  ✓ Validated transformer implementation for ${modelId}`);
+              } else {
+                console.log(`  ✗ Generated transformer for model ${modelId} is not valid TypeScript, skipping save.`);
+                await fs.rename(validatedFile, validatedFile + '.invalid');
+                continue; 
+              }
+            } catch (e: any) {
+              console.log(`  ✗ Error generating transformer for model ${modelId}:`, e instanceof Error ? e.message : e);
+            }
+          }        
+        })();
+        workers.push(worker);
+      }
 
-        console.log(`\n--- Generating transformer for model: ${modelId} ---`);
+      await Promise.all(workers);
 
-        const result = await extract.get('result', { modelData, model: options.transformerModel });
-        if (!result) {
-          console.log(`  ✗ Failed to generate transformer for model: ${modelId}`);
-          continue;
-        }
+      console.log(`\n✓ Generated transformers for ${transformingModels.length} models in ${path.join(cacheDir, 'replicate')}\n`);
 
-        const converted = convertReplicateModel(modelData);
-
-        const translatedModelInfo: ModelInfo<'replicate'> = {
-          id: converted.id,
-          provider: 'replicate',
-          name: converted.name,
-          contextWindow: result.modelInfo.contextWindow,
-          maxOutputTokens: result.modelInfo.maxOutputTokens,
-          pricing: result.modelInfo.pricing,
-          tier: result.modelInfo.tier,
-          capabilities: new Set(result.modelInfo.capabilities),
-          metrics: result.modelInfo.metrics,
-          tokenizer: result.modelInfo.tokenizer,
-          supportedParameters: new Set(result.modelInfo.supportedParameters),
-          metadata: converted.metadata,
-        };
+      // Generate index file
+      const transformerTarget = path.join(__dirname, '../../src/transformers/replicate');
+      const transformerFiles = await fs.readdir(transformerTarget);
+      const transformerSet = new Set(
+        transformerFiles
+          .filter((f) => f.endsWith('.ts') && f !== 'index.ts')
+          .map((f) => f.replace('.ts', ''))
+      );
 
-        const replaceIndex = modelInfos.findIndex((m) => m.id === translatedModelInfo.id);
-        if (replaceIndex >= 0) {
-          modelInfos[replaceIndex] = translatedModelInfo;
-        } else {
-          console.log(`  ⚠ ModelInfo for ${modelId} not found in main list, adding new entry`);
-        }
+      // // For each file
+      // import name from './name.ts';
 
-        await fs.writeFile(
-          path.join(cacheDir, 'replicate', transformedFile),
-          `
-import { toURL, toBase64, toText, toStream, toFile } from '@aeye/core';
-import { ModelTransformer } from '@aeye/ai';
+      // // Final export
+      // export const replicateTransformers: Record<string, ReplicateTransformer> = {
+      //   ...name,
+      // };
 
-const transformer = ${result.transformerImplementation};
+      await fs.writeFile(
+        path.join(transformerTarget, 'index.ts'),
+        `
+import { ReplicateTransformer } from '@aeye/replicate';
 
-const modelInfo = ${JSON.stringify(translatedModelInfo, null, 2)};
+${Array.from(transformerSet).map((name) => `import ${name.replace(/-/g, '_')} from './${name}';`).join('\n')}
 
-export { transformer, modelInfo };`.trim()
-        );
-      }
+export const replicateTransformers: Record<string, ReplicateTransformer> = {
+  ${Array.from(transformerSet).map((name) => `...${name.replace(/-/g, '_')},`).join('\n  ')}
+};`.trim()
+      );
+      
+      console.log(`✓ Generated Replicate transformers index file\n`);
     }
   }
 
   // Save JSON for reference
   await fs.writeFile(
     path.join(outputDir, 'replicate-modelinfo.json'),
-    JSON.stringify(modelInfos, (key, value) => {
-      if (value instanceof Set) {
-        return Array.from(value);
-      }
-      return value;
-    }, 2)
+    JSON.stringify(modelInfos, jsonReplacer, 2)
   );
   console.log(`✓ Saved ${modelInfos.length} models to JSON`);
 
@@ -785,9 +1074,32 @@ export { transformer, modelInfo };`.trim()
   console.log('\n✓ Replicate scraping complete\n');
 }
 
+function isValidTypeScript(file: string): Promise<boolean> {
+  return new Promise((resolve, reject) => {
+    const tsc = spawn('npx', ['tsc', '--noEmit', '--skipLibCheck', file], {
+      shell: true,
+    });
+    
+    tsc.on('close', (code) => {
+      if (code === 0) {
+        resolve(true);
+      } else {
+        resolve(false);
+      }
+    });
+    
+    tsc.on('error', (error) => {
+      reject(error);
+    });
+  });
+}
+
 // CLI execution
 if (process.argv[1].endsWith('replicate.ts')) {
-  const args = process.argv.slice(2);
+  const cleanArg = (a: string) => (/^["'`](.*)["'`]$/.exec(a) || ['', a])[1];
+
+  const args = process.argv.slice(2).map(cleanArg);
+
   const outputDir = args.find((arg) => !arg.startsWith('--')) || path.join(__dirname, '../../data');
   const cacheDir = args.find((arg, i) => i > 0 && !arg.startsWith('--') && !args[i - 1].startsWith('--')) || path.join(__dirname, '../../cache');
 
@@ -798,21 +1110,34 @@ if (process.argv[1].endsWith('replicate.ts')) {
 
   const transformers = args.includes('--transformers');
 
+  const transformersOnly = args.includes('--transformers-only');
+
   const transformerModelArg = args.find((arg) => arg.startsWith('--transformer-model='));
   const transformerModel = transformerModelArg
-    ? transformerModelArg.split('=', 2)[1]
+    ? cleanArg(transformerModelArg.split('=', 2)[1])
     : 'google/gemini-2.5-pro';
 
   const modelsArg = args.find((arg) => arg.startsWith('--models='));
   let models: string[] | undefined = undefined;
   if (modelsArg) {
     models = modelsArg.substring(9).split(',').map((m) => m.trim());
+    models = models.map(cleanArg)
+    models = models.filter((m) => m.length > 0);
   }
 
   const nArg = args.find((arg) => arg.startsWith('--n='));;
-  const n = nArg ? parseInt(nArg.split('=')[1], 10) : undefined;
+  const n = nArg ? parseInt(cleanArg(nArg.split('=')[1]), 10) : undefined;
+
+  const transformConcurrencyArg = args.find((arg) => arg.startsWith('--transform-concurrency='));
+  const transformConcurrency = transformConcurrencyArg
+    ? parseInt(transformConcurrencyArg.split('=')[1], 10)
+    : concurrency;
+
+  const params = { concurrency, transformers, transformersOnly, transformerModel, transformConcurrency, models, n };
+
+  console.debug('Scrape parameters:', { outputDir, cacheDir, ...params });
 
-  scrapeReplicate(outputDir, cacheDir, { concurrency, transformers, transformerModel, models, n }).catch((error) => {
+  scrapeReplicate(outputDir, cacheDir, params).catch((error) => {
     console.error('✗ Replicate scraping failed:', error);
     process.exit(1);
   });