@aeye/models 0.2.0 → 0.3.0

package/dist/scripts/scrapers/replicate.js

@@ -3,14 +3,17 @@
  *
  * Fetches model information from Replicate API using the Replicate npm module
  */
+import { AI, detectTier } from '@aeye/ai';
+import { OpenRouterProvider } from '@aeye/openrouter';
 import * as fs from 'fs/promises';
+import openapiTS from 'openapi-typescript';
 import * as path from 'path';
+import Replicate from 'replicate';
+import ts from 'typescript';
 import * as url from 'url';
 import { z } from 'zod';
-import Replicate from 'replicate';
-import { AI, detectTier } from '@aeye/ai';
 import { writeModelTS } from '../codegen';
-import { OpenRouterProvider } from 'packages/openrouter/src/openrouter';
+import { spawn } from 'child_process';
 const __filename = url.fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 /**
@@ -116,14 +119,41 @@ const ai = AI
         hooks: {
             chat: {
                 beforeRequest: (a, b, c, d) => {
-                    console.log('OpenRouter Chat Request:', b);
+                    const now = Date.now().toString();
+                    const requestResolved = path.resolve(__dirname, '../../cache/replicate', now + '-request.json');
+                    fs.writeFile(requestResolved, JSON.stringify(b, null, 2));
+                    const content = a.messages.map((m) => m.content).join('\n---\n');
+                    const contentResolved = path.resolve(__dirname, '../../cache/replicate', now + '-content.txt');
+                    fs.writeFile(contentResolved, content);
+                },
+                afterRequest: (a, b, c, d) => {
+                    const now = Date.now().toString();
+                    const responseResolved = path.resolve(__dirname, '../../cache/replicate', now + '-response.json');
+                    fs.writeFile(responseResolved, JSON.stringify(c, null, 2));
+                    const content = c.content;
+                    const contentResolved = path.resolve(__dirname, '../../cache/replicate', now + '-response.txt');
+                    fs.writeFile(contentResolved, content);
+                },
+                onError: (a, b, e, d) => {
+                    const now = Date.now().toString();
+                    const errorResolved = path.resolve(__dirname, '../../cache/replicate', now + '-error.txt');
+                    fs.writeFile(errorResolved, String(e));
                 },
             },
         },
     }),
 })
     .create({});
+const jsonReplacer = (key, value) => {
+    if (value instanceof Set) {
+        return Array.from(value);
+    }
+    return value;
+};
 const typeInfo = await fs.readFile(path.join(__dirname, 'extract.md'), 'utf-8');
+const optionalTransform = (val) => !val || Object.values(val).every((v) => v === null || v === undefined)
+    ? undefined
+    : Object.fromEntries(Object.entries(val).filter(([_, v]) => v !== null && v !== undefined));
 const extract = ai.prompt({
     name: 'extract',
     description: 'Extract model information & a transformer definition from Replicate model data',
@@ -133,50 +163,74 @@ const extract = ai.prompt({
 {{modelData}}
 </modelData>
 
-Create a TypeScript implementation of a ModelTransformer for the model.
+{{#if modelTypes}}
+Here are the types for the model based on the OpenAPI schema:
+\`\`\`ts
+{{modelTypes}}
+\`\`\`
+
+Schemas["Input"] is what convertRequest should return.
+Schemas["Output"] is what parseResponse is given as the first argument.
+
+These types will be available and can be referenced like above to ensure type safety.
+{{/if}}
+
+Create a TypeScript implementation of a ReplicateTransformer for the model.
 Here's an example transformer implementation for reference. This is the exact format to follow:
-<exampleTransformer>
+<exampleTransformerImplementation>
 { 
   // this key comes from modelData.owner/modelData.name
-  "google/nano-banana": () => ({
-    // you must use this ModelTransformer type to avoid TypeScript errors
-    const transformer: ModelTransformer = { 
+  // // The definition is wrapped in an immediately-invoked function expression so any cached or reusable functions can be defined here.
+  "google/nano-banana": (() => {
+    // you must use this ReplicateTransformer type to avoid TypeScript errors
+    const transformer: ReplicateTransformer = { 
       imageGenerate: {
         // only use variable known to be in this request type based on the types below
-        convertRequest: async (request, ctx) => ({ 
+        convertRequest: async (request, ctx){{#if modelTypes}}: Promise<Schemas["Input"]>{{/if}} => ({ 
           prompt: request.prompt,
-          ...request.extra,
+          ...request.extra, // always spread extra to allow for additional parameters
         }),
         // only use known output schema from the model and only use properties expected on the response type based on the types below
-        parseResponse: async (response, ctx) => ({
+        parseResponse: async (response{{#if modelTypes}}: Schemas["Output"]{{/if}}, ctx) => ({
           images: [{ url: await toURL(response) }],
+          extra: { response }, // include full response in extra object
         }),
       },
       imageEdit: {
-        convertRequest: async (request, ctx) => ({
+        convertRequest: async (request, ctx){{#if modelTypes}}: Promise<Schemas["Input"]>{{/if}} => ({
           prompt: request.prompt,
           image_input: [await toURL(request.image)],
-          ...request.extra,
+          ...request.extra, // always spread extra to allow for additional parameters
         }),
-        parseResponse: async (response, ctx) => ({
+        parseResponse: async (response{{#if modelTypes}}: Schemas["Output"]{{/if}}, ctx) => ({
           images: [{ url: await toURL(response) }],
+          extra: { response }, // include full response in extra object
         }),
       },
     };
     return transformer;
-  })(),
+  })(), // immediately invoke the function to return the typed transformer
 }
-</exampleTransformer>
+</exampleTransformerImplementation>
+
+The transformer code you return should match EXACTLY the format above. It will be injected in code like so by the developer:
+<transformerResult>
+import { toURL, toBase64, toText, toStream, toReadableStream, toFile } from '@aeye/core';
+import { ReplicateTransformer } from '@aeye/replicate';
+
+const transformer: Record<string, ReplicateTransformer> = <transformerImplementation>;
+</transformerResult>
 
 <availableFunctions>
 - \`toURL(resource, mimeType?: string, fallback?: string): Promise<string>\` converts to URL string
 - \`toBase64(resource, mimeType?: string, fallback?: string): Promise<string>\`
 - \`toText(resource, fallback?: string): Promise<string>\`
 - \`toStream(resource, fallback?: Readable): Promise<Readable>\`
+- \`toReadableStream(resource, fallback?: ReadableStream): Promise<ReadableStream>\`
 - \`toFile(resource, mimeType?: string, filename?: string): Promise<File>\`
 </availableFunctions>
 
-Here's how the replicate code looks around the transformer:
+Here's how the replicate code looks around the transformer: This is an example to illustrate how the transformer you generate is used.
 <replicateCode>
 // =============================================================
 // Execute
@@ -211,9 +265,12 @@ The request parameters in the converted request should only be the properties th
 {{typeInfo}}
 </typeInfo>
   `,
-    strict: false,
+    strict: true,
+    outputRetries: 3,
     schema: z.object({
-        transformerImplementation: z.string().describe('The TypeScript implementation of the model handler (without imports or ``` marks)'),
+        transformerImplementation: z.string()
+            // .regex(/^\{\n\s+"[^"]+\/[^"]+": \(\(\) => \{(.|\r|\n)*\}\)\(\),\n\}$/)
+            .describe(`The TypeScript implementation of the model handler (without imports or \`\`\` marks) - described BETWEEN the <exampleTransformerImplementation> tags. Follow the rules outlined.`),
         modelInfo: z.object({
             capabilities: z.array(z.enum([
                 'chat',
@@ -251,15 +308,17 @@ The request parameters in the converted request should only be the properties th
   - experimental: Preview/beta models`),
             pricing: z.object({
                 text: z.object({
-                    input: z.number(),
-                    output: z.number(),
-                    cached: z.number(),
-                }).optional(),
+                    input: z.number().optional(),
+                    output: z.number().optional(),
+                    cached: z.number().optional(),
+                }).optional()
+                    .transform(optionalTransform),
                 audio: z.object({
                     input: z.number().optional(),
                     output: z.number().optional(),
                     perSecond: z.number().optional(),
-                }).optional(),
+                }).optional()
+                    .transform(optionalTransform),
                 image: z.object({
                     input: z.number().optional(),
                     output: z.array(z.object({
@@ -269,18 +328,22 @@ The request parameters in the converted request should only be the properties th
                             height: z.number(),
                             cost: z.number().describe('Cost per image in USD'),
                         })).describe('Supported image sizes and their costs'),
-                    })).optional(),
-                }).optional(),
+                    })).optional()
+                        .transform(optionalTransform),
+                }).optional()
+                    .transform(optionalTransform),
                 reasoning: z.object({
                     input: z.number().optional(),
                     output: z.number().optional(),
                     cached: z.number().optional(),
-                }).optional(),
-                embedding: z.object({
+                }).optional()
+                    .transform(optionalTransform),
+                embeddings: z.object({
                     cost: z.number().optional(),
-                }).optional(),
+                }).optional()
+                    .transform(optionalTransform),
                 perRequest: z.number().optional().describe('Flat cost per request in USD'),
-            }).describe('Pricing information. input/output/cached are per 1 million tokens unless otherwise specified.'),
+            }).describe('Pricing information. input/output/cached are per 1 million tokens unless otherwise specified. Leave null/undefined if unknown - 0 is free. Do not specify 0 unless it is free.'),
             contextWindow: z.number(),
             maxOutputTokens: z.number().optional(),
             metrics: z.object({
@@ -288,7 +351,9 @@ The request parameters in the converted request should only be the properties th
                 timeToFirstToken: z.number().optional().describe('Time to first token in milliseconds'),
                 averageRequestDuration: z.number().optional().describe('Average request duration in milliseconds'),
                 accuracyScore: z.number().optional().describe('Accuracy score based on benchmark tests 0-1'),
-            }).optional().describe('Performance metrics if available'),
+            }).optional()
+                .transform(optionalTransform)
+                .describe('Performance metrics if available'),
             tokenizer: z.enum(([
                 'Other', 'GPT', 'Mistral', 'Llama3', 'Qwen3', 'Qwen', 'Gemini', 'DeepSeek', 'Claude', 'Grok', 'Llama4', 'Llama2', 'Cohere', 'Nova', 'Router'
             ])).optional().describe('The tokenizer type used by this model if known, otherwise leave undefined'),
@@ -324,14 +389,83 @@ The request parameters in the converted request should only be the properties th
             ])).optional().describe('The supported parameters for this model'),
         }).describe('The extracted model information'),
     }),
-    input: ({ modelData }) => ({
-        modelData: JSON.stringify(modelData, null, 2),
-        typeInfo,
-    }),
+    input: async ({ modelData, modelTypes }) => {
+        return {
+            modelData: JSON.stringify(modelData, (k, v) => {
+                if (k === "embeddings" && Array.isArray(v) && v.length > 20) {
+                    return [`...${v.length} ${typeof v[0]}s...`];
+                }
+                if (Array.isArray(v) && v.length > 20) {
+                    return [`...${v.length} ${typeof v[0]}s...`];
+                }
+                return v;
+            }, 2),
+            typeInfo,
+            modelTypes,
+        };
+    },
     config: (input) => ({
         model: input?.model,
+        temperature: 0,
+        topP: 1,
     }),
 });
+async function extractTypes(model) {
+    if (!model.latest_version?.openapi_schema) {
+        return '';
+    }
+    let extracted = await extractSchemasString(model.latest_version.openapi_schema);
+    if (!extracted) {
+        return '';
+    }
+    return extracted.replaceAll('components["schemas"]', 'Schemas').replaceAll('    ', '  ');
+}
+async function extractSchemasString(schemaObject) {
+    try {
+        const ast = await openapiTS(schemaObject, {
+            propertiesRequiredByDefault: false,
+            defaultNonNullable: false,
+        });
+        const printer = ts.createPrinter({
+            newLine: ts.NewLineKind.LineFeed,
+            removeComments: true,
+        });
+        const dummySourceFile = ts.createSourceFile("temp.ts", "", ts.ScriptTarget.ESNext, true);
+        const componentsNode = ast.find(node => {
+            if (ts.isInterfaceDeclaration(node)) {
+                const name = printer.printNode(ts.EmitHint.Unspecified, node.name, dummySourceFile);
+                return name === 'components';
+            }
+            return false;
+        });
+        if (!componentsNode) {
+            console.error("Error: Could not find the 'components' node in the generated AST.");
+            return "";
+        }
+        let extractedMembersString = "";
+        // Get the inner content of the 'schemas' property
+        for (const member of componentsNode.members) {
+            if (ts.isPropertySignature(member)) {
+                const propName = printer.printNode(ts.EmitHint.Unspecified, member.name, dummySourceFile);
+                if (propName === 'schemas' && member.type && ts.isTypeLiteralNode(member.type)) {
+                    member.type.members.forEach(schemaMember => {
+                        extractedMembersString += printer.printNode(ts.EmitHint.Unspecified, schemaMember, dummySourceFile) + "\n";
+                    });
+                    break;
+                }
+            }
+        }
+        if (!extractedMembersString.trim()) {
+            console.error("Error: Found no user-defined schema types within the 'components.schemas' structure.");
+            return "";
+        }
+        return `type Schemas = {\n    ${extractedMembersString.replaceAll('\n', '\n    ').trim()}\n};`;
+    }
+    catch (error) {
+        console.error("An unexpected error occurred during schema extraction:", error);
+        return "";
+    }
+}
 /**
  * Fetch all models from Replicate collections
  */
@@ -382,7 +516,6 @@ async function fetchAllModels(apiKey) {
             console.log(`    ⚠ Failed to fetch collection ${collectionSlug}:`, error instanceof Error ? error.message : error);
         }
     }
-    allModels.sort((a, b) => a.name.localeCompare(b.name));
     console.log(`\n✓ Fetched ${allModels.length} unique Replicate models`);
     return allModels;
 }
@@ -418,7 +551,6 @@ async function fetchSchemasParallel(models, apiKey, concurrency = 50) {
     }
     // Find models that need schema fetching
     const modelsToFetch = models.filter((model) => {
-        const modelId = `${model.owner}/${model.name}`;
         return !model.latest_version?.openapi_schema;
     });
     if (modelsToFetch.length === 0) {
@@ -444,71 +576,109 @@ async function fetchSchemasParallel(models, apiKey, concurrency = 50) {
     console.log(`\n✓ Cached ${Object.keys(schemasCache).length} model schemas\n`);
     return schemasCache;
 }
+function cleanSchema(model) {
+    const schema = model.latest_version?.openapi_schema;
+    if (schema) {
+        // delete schema.components.schemas.Status;
+        delete schema.components.schemas.WebhookEvent;
+        delete schema.components.schemas.ValidationError;
+        delete schema.components.schemas.PredictionRequest;
+        delete schema.components.schemas.PredictionResponse;
+        delete schema.components.schemas.HTTPValidationError;
+    }
+}
 /**
  * Main scraper function
  */
 export async function scrapeReplicate(outputDir, cacheDir, options = {
-    transformerModel: 'google/gemini-2.5-flash',
+    transformerModel: 'google/gemini-2.5-pro',
 }) {
     const { concurrency = 50 } = options;
-    console.log('\n=== Replicate Scraper ===\n');
     const apiKey = process.env.REPLICATE_API_KEY;
     if (!apiKey) {
         console.error('✗ REPLICATE_API_KEY environment variable is required');
         process.exit(1);
     }
-    // Fetch all models from collections
-    const models = await fetchAllModels(apiKey);
+    let models;
+    if (!options.transformersOnly) {
+        console.log('\n=== Replicate Scraper ===\n');
+        // Fetch all models from collections
+        models = await fetchAllModels(apiKey);
+        // Create output directory
+        await fs.mkdir(outputDir, { recursive: true });
+        // Save raw models data
+        await fs.writeFile(path.join(outputDir, 'replicate-models.json'), JSON.stringify({ data: models }, jsonReplacer, 2));
+        console.log(`✓ Saved raw Replicate models to replicate-models.json`);
+    }
+    else {
+        const modelsData = await fs.readFile(path.join(outputDir, 'replicate-models.json'), 'utf-8');
+        const parsed = JSON.parse(modelsData);
+        models = parsed.data;
+        console.log(`\n✓ Loaded ${models.length} Replicate models from cache\n`);
+    }
+    models.forEach(cleanSchema);
+    // Sort models by owner/name
+    models.sort((a, b) => `${a.owner}/${a.name}`.localeCompare(`${b.owner}/${b.name}`));
     // Convert to ModelInfo format
     const modelInfos = models.map(convertReplicateModel);
-    // Create output directory
-    await fs.mkdir(outputDir, { recursive: true });
-    // Save raw models data
-    await fs.writeFile(path.join(outputDir, 'replicate-models.json'), JSON.stringify({ data: models }, (key, value) => {
-        if (value instanceof Set) {
-            return Array.from(value);
+    try {
+        const modelInfoCache = await fs.readFile(path.join(outputDir, 'replicate-modelinfo.json'), 'utf-8');
+        const parsed = JSON.parse(modelInfoCache);
+        for (const info of parsed) {
+            const existing = modelInfos.findIndex((m) => m.id === info.id);
+            if (existing >= 0) {
+                modelInfos[existing] = info;
+            }
         }
-        return value;
-    }, 2));
-    console.log(`✓ Saved raw Replicate models to replicate-models.json`);
+    }
+    catch (e) {
+        // Ignore if file doesn't exist
+        console.error(`⚠ Could not load existing model info cache: ${e.message}`);
+    }
     let schemasCache = {};
     // Create cache directory if specified
     if (cacheDir) {
-        await fs.mkdir(cacheDir, { recursive: true });
-        // Cache detailed model schemas (for transformer generation)
-        schemasCache = await fetchSchemasParallel(models, apiKey, concurrency);
-        // Save schemas cache
-        await fs.writeFile(path.join(cacheDir, 'replicate-schemas.json'), JSON.stringify(schemasCache, (key, value) => {
-            if (value instanceof Set) {
-                return Array.from(value);
+        if (!options.transformersOnly) {
+            await fs.mkdir(cacheDir, { recursive: true });
+            // Cache detailed model schemas (for transformer generation)
+            schemasCache = await fetchSchemasParallel(models, apiKey, concurrency);
+            // Save schemas cache
+            await fs.writeFile(path.join(cacheDir, 'replicate-schemas.json'), JSON.stringify(schemasCache, jsonReplacer, 2));
+            console.log(`✓ Saved ${Object.keys(schemasCache).length} model schemas to ${cacheDir}/replicate-schemas.json`);
+            console.log('  (This cache file is for transformer generation and should not be committed)');
+            // Chunk it up to 80,000 character files for easier loading later
+            const chunkSize = 80000;
+            const schemaEntries = Object.entries(schemasCache);
+            let currentChunk = {};
+            let currentSize = 0;
+            let chunkIndex = 1;
+            for (const [modelId, modelData] of schemaEntries) {
+                const entryString = JSON.stringify({ [modelId]: modelData });
+                if (currentSize + entryString.length > chunkSize && Object.keys(currentChunk).length > 0) {
+                    // Save current chunk
+                    await fs.writeFile(path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`), JSON.stringify(currentChunk, null, 2));
+                    console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
+                    chunkIndex++;
+                    currentChunk = {};
+                    currentSize = 0;
+                }
+                currentChunk[modelId] = modelData;
+                currentSize += entryString.length;
             }
-            return value;
-        }, 2));
-        console.log(`✓ Saved ${Object.keys(schemasCache).length} model schemas to ${cacheDir}/replicate-schemas.json`);
-        console.log('  (This cache file is for transformer generation and should not be committed)');
-        // Chunk it up to 80,000 character files for easier loading later
-        const chunkSize = 80000;
-        const schemaEntries = Object.entries(schemasCache);
-        let currentChunk = {};
-        let currentSize = 0;
-        let chunkIndex = 1;
-        for (const [modelId, modelData] of schemaEntries) {
-            const entryString = JSON.stringify({ [modelId]: modelData });
-            if (currentSize + entryString.length > chunkSize && Object.keys(currentChunk).length > 0) {
-                // Save current chunk
+            // Save any remaining chunk
+            if (Object.keys(currentChunk).length > 0) {
                 await fs.writeFile(path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`), JSON.stringify(currentChunk, null, 2));
                 console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
-                chunkIndex++;
-                currentChunk = {};
-                currentSize = 0;
             }
-            currentChunk[modelId] = modelData;
-            currentSize += entryString.length;
         }
-        // Save any remaining chunk
-        if (Object.keys(currentChunk).length > 0) {
-            await fs.writeFile(path.join(cacheDir, `replicate-schemas-chunk-${chunkIndex}.json`), JSON.stringify(currentChunk, null, 2));
-            console.log(`✓ Saved schema chunk ${chunkIndex} with ${Object.keys(currentChunk).length} models`);
+        else {
+            // Load schemas cache from file
+            const schemasData = await fs.readFile(path.join(cacheDir, 'replicate-schemas.json'), 'utf-8');
+            schemasCache = JSON.parse(schemasData);
+            console.log(`\n✓ Loaded ${Object.keys(schemasCache).length} model schemas from cache\n`);
+        }
+        for (const modelData of Object.values(schemasCache)) {
+            cleanSchema(modelData);
         }
         // Calculate number of unique schemas
         const uniqueSchemas = new Set();
@@ -528,7 +698,7 @@ export async function scrapeReplicate(outputDir, cacheDir, options = {
             // Filter to only models with schemas if transformers are desired
             if (options.models && options.models.length > 0) {
                 const modelSet = new Set(options.models);
-                transformingModels = models.filter((model) => {
+                transformingModels = transformingModels.filter((model) => {
                     const modelId = `${model.owner}/${model.name}`;
                     return modelSet.has(modelId);
                 });
@@ -536,71 +706,134 @@ export async function scrapeReplicate(outputDir, cacheDir, options = {
             }
             // If models were not specified, filter out already transformed models
             if (!overwrite) {
-                const alreadyTransformed = await fs.readdir(path.join(cacheDir, 'replicate'));
-                const alreadyTransformedSet = new Set(alreadyTransformed.map((filename) => filename.replace('.ts', '')));
+                const alreadyTransformed = await fs.readdir(path.resolve(cacheDir, '../src/transformers/replicate'));
+                const alreadyTransformedSet = new Set(alreadyTransformed
+                    .filter((filename) => filename.endsWith('.ts') && filename !== 'index.ts')
+                    .map((filename) => filename.replace('.ts', '')));
                 transformingModels = transformingModels.filter((model) => {
                     const modelId = `${model.owner}/${model.name}`;
-                    const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
+                    const transformedFile = modelId.replace(/[^\w]/g, '-');
                     return !alreadyTransformedSet.has(transformedFile);
                 });
             }
             // Only first N models if specified
             if (options.n && options.n > 0) {
-                transformingModels = models.slice(0, options.n);
+                transformingModels = transformingModels.slice(0, options.n);
             }
             // Call extractor for each model
             console.log(`Generating transformers for ${transformingModels.length} models...\n`);
-            for (const model of transformingModels) {
-                const modelId = `${model.owner}/${model.name}`;
-                const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
-                const modelData = schemasCache[modelId] || model;
-                console.log(`\n--- Generating transformer for model: ${modelId} ---`);
-                const result = await extract.get('result', { modelData, model: options.transformerModel });
-                if (!result) {
-                    console.log(`  ✗ Failed to generate transformer for model: ${modelId}`);
-                    continue;
-                }
-                const converted = convertReplicateModel(modelData);
-                const translatedModelInfo = {
-                    id: converted.id,
-                    provider: 'replicate',
-                    name: converted.name,
-                    contextWindow: result.modelInfo.contextWindow,
-                    maxOutputTokens: result.modelInfo.maxOutputTokens,
-                    pricing: result.modelInfo.pricing,
-                    tier: result.modelInfo.tier,
-                    capabilities: new Set(result.modelInfo.capabilities),
-                    metrics: result.modelInfo.metrics,
-                    tokenizer: result.modelInfo.tokenizer,
-                    supportedParameters: new Set(result.modelInfo.supportedParameters),
-                    metadata: converted.metadata,
-                };
-                const replaceIndex = modelInfos.findIndex((m) => m.id === translatedModelInfo.id);
-                if (replaceIndex >= 0) {
-                    modelInfos[replaceIndex] = translatedModelInfo;
-                }
-                else {
-                    console.log(`  ⚠ ModelInfo for ${modelId} not found in main list, adding new entry`);
-                }
-                await fs.writeFile(path.join(cacheDir, 'replicate', transformedFile), `
-import { toURL, toBase64, toText, toStream, toFile } from '@aeye/core';
-import { ModelTransformer } from '@aeye/ai';
-
-const transformer = ${result.transformerImplementation};
+            // Use specified concurrency for transformations
+            const concurrency = options.transformConcurrency !== undefined && options.transformConcurrency <= 0
+                ? transformingModels.length
+                : options.transformConcurrency ?? 1;
+            let nextModelIndex = 0;
+            const workers = [];
+            for (let i = 0; i < concurrency; i++) {
+                const worker = (async () => {
+                    while (nextModelIndex < transformingModels.length) {
+                        const model = transformingModels[nextModelIndex];
+                        nextModelIndex++;
+                        const modelId = `${model.owner}/${model.name}`;
+                        const transformedFile = modelId.replace(/[^\w]/g, '-') + '.ts';
+                        const modelData = schemasCache[modelId] || model;
+                        console.log(`\n--- Generating transformer for model: ${modelId},  ---`);
+                        try {
+                            const modelTypes = await extractTypes(modelData);
+                            const result = await extract.get('result', {
+                                modelData,
+                                modelTypes,
+                                model: options.transformerModel
+                            });
+                            if (!result) {
+                                console.log(`  ✗ Failed to generate transformer for model: ${modelId}`);
+                                continue;
+                            }
+                            const converted = convertReplicateModel(modelData);
+                            const translatedModelInfo = {
+                                id: converted.id,
+                                provider: 'replicate',
+                                name: converted.name,
+                                contextWindow: result.modelInfo.contextWindow,
+                                maxOutputTokens: result.modelInfo.maxOutputTokens,
+                                pricing: result.modelInfo.pricing,
+                                tier: result.modelInfo.tier,
+                                capabilities: new Set(result.modelInfo.capabilities),
+                                metrics: result.modelInfo.metrics,
+                                tokenizer: result.modelInfo.tokenizer,
+                                supportedParameters: new Set(result.modelInfo.supportedParameters),
+                                metadata: converted.metadata,
+                            };
+                            const replaceIndex = modelInfos.findIndex((m) => m.id === translatedModelInfo.id);
+                            if (replaceIndex >= 0) {
+                                modelInfos[replaceIndex] = translatedModelInfo;
+                            }
+                            else {
+                                console.log(`  ⚠ ModelInfo for ${modelId} not found in main list, adding new entry`);
+                            }
+                            let fixedTransformerImplementation = result.transformerImplementation;
+                            if (fixedTransformerImplementation.startsWith('"')) {
+                                fixedTransformerImplementation = `{\n  ${fixedTransformerImplementation}\n}`;
+                            }
+                            const coreImports = [
+                                fixedTransformerImplementation.includes('toURL') ? 'toURL' : null,
+                                fixedTransformerImplementation.includes('toBase64') ? 'toBase64' : null,
+                                fixedTransformerImplementation.includes('toText') ? 'toText' : null,
+                                fixedTransformerImplementation.includes('toStream') ? 'toStream' : null,
+                                fixedTransformerImplementation.includes('toReadableStream') ? 'toReadableStream' : null,
+                                fixedTransformerImplementation.includes('toFile') ? 'toFile' : null,
+                            ].filter((i) => i !== null);
+                            const validatedFile = path.resolve(cacheDir, '../src/transformers/replicate', transformedFile);
+                            await fs.writeFile(validatedFile, `
+${coreImports.length > 0 ? `import { ${coreImports.join(', ')} } from '@aeye/core';` : ''}
+import { ReplicateTransformer } from '@aeye/replicate';
 
-const modelInfo = ${JSON.stringify(translatedModelInfo, null, 2)};
+${modelTypes}
 
-export { transformer, modelInfo };`.trim());
+export default ${fixedTransformerImplementation}
+`.trim());
+                            if (await isValidTypeScript(validatedFile)) {
+                                console.log(`  ✓ Validated transformer implementation for ${modelId}`);
+                            }
+                            else {
+                                console.log(`  ✗ Generated transformer for model ${modelId} is not valid TypeScript, skipping save.`);
+                                await fs.rename(validatedFile, validatedFile + '.invalid');
+                                continue;
+                            }
+                        }
+                        catch (e) {
+                            console.log(`  ✗ Error generating transformer for model ${modelId}:`, e instanceof Error ? e.message : e);
+                        }
+                    }
+                })();
+                workers.push(worker);
             }
+            await Promise.all(workers);
+            console.log(`\n✓ Generated transformers for ${transformingModels.length} models in ${path.join(cacheDir, 'replicate')}\n`);
+            // Generate index file
+            const transformerTarget = path.join(__dirname, '../../src/transformers/replicate');
+            const transformerFiles = await fs.readdir(transformerTarget);
+            const transformerSet = new Set(transformerFiles
+                .filter((f) => f.endsWith('.ts') && f !== 'index.ts')
+                .map((f) => f.replace('.ts', '')));
+            // // For each file
+            // import name from './name.ts';
+            // // Final export
+            // export const replicateTransformers: Record<string, ReplicateTransformer> = {
+            //   ...name,
+            // };
+            await fs.writeFile(path.join(transformerTarget, 'index.ts'), `
+import { ReplicateTransformer } from '@aeye/replicate';
+
+${Array.from(transformerSet).map((name) => `import ${name.replace(/-/g, '_')} from './${name}';`).join('\n')}
+
+export const replicateTransformers: Record<string, ReplicateTransformer> = {
+  ${Array.from(transformerSet).map((name) => `...${name.replace(/-/g, '_')},`).join('\n  ')}
+};`.trim());
+            console.log(`✓ Generated Replicate transformers index file\n`);
         }
     }
     // Save JSON for reference
-    await fs.writeFile(path.join(outputDir, 'replicate-modelinfo.json'), JSON.stringify(modelInfos, (key, value) => {
-        if (value instanceof Set) {
-            return Array.from(value);
-        }
-        return value;
-    }, 2));
+    await fs.writeFile(path.join(outputDir, 'replicate-modelinfo.json'), JSON.stringify(modelInfos, jsonReplacer, 2));
     console.log(`✓ Saved ${modelInfos.length} models to JSON`);
     // Generate TypeScript file
     const srcDir = path.join(__dirname, '../../src/models');
@@ -608,9 +841,28 @@ export { transformer, modelInfo };`.trim());
     console.log(`✓ Generated TypeScript file: src/models/replicate.ts`);
     console.log('\n✓ Replicate scraping complete\n');
 }
+function isValidTypeScript(file) {
+    return new Promise((resolve, reject) => {
+        const tsc = spawn('npx', ['tsc', '--noEmit', '--skipLibCheck', file], {
+            shell: true,
+        });
+        tsc.on('close', (code) => {
+            if (code === 0) {
+                resolve(true);
+            }
+            else {
+                resolve(false);
+            }
+        });
+        tsc.on('error', (error) => {
+            reject(error);
+        });
+    });
+}
 // CLI execution
 if (process.argv[1].endsWith('replicate.ts')) {
-    const args = process.argv.slice(2);
+    const cleanArg = (a) => (/^["'`](.*)["'`]$/.exec(a) || ['', a])[1];
+    const args = process.argv.slice(2).map(cleanArg);
     const outputDir = args.find((arg) => !arg.startsWith('--')) || path.join(__dirname, '../../data');
     const cacheDir = args.find((arg, i) => i > 0 && !arg.startsWith('--') && !args[i - 1].startsWith('--')) || path.join(__dirname, '../../cache');
     const concurrencyArg = args.find((arg) => arg.startsWith('--concurrency='));
@@ -618,19 +870,28 @@ if (process.argv[1].endsWith('replicate.ts')) {
         ? parseInt(concurrencyArg.split('=')[1], 10)
         : 50;
     const transformers = args.includes('--transformers');
+    const transformersOnly = args.includes('--transformers-only');
     const transformerModelArg = args.find((arg) => arg.startsWith('--transformer-model='));
     const transformerModel = transformerModelArg
-        ? transformerModelArg.split('=', 2)[1]
+        ? cleanArg(transformerModelArg.split('=', 2)[1])
         : 'google/gemini-2.5-pro';
     const modelsArg = args.find((arg) => arg.startsWith('--models='));
     let models = undefined;
     if (modelsArg) {
         models = modelsArg.substring(9).split(',').map((m) => m.trim());
+        models = models.map(cleanArg);
+        models = models.filter((m) => m.length > 0);
     }
     const nArg = args.find((arg) => arg.startsWith('--n='));
     ;
-    const n = nArg ? parseInt(nArg.split('=')[1], 10) : undefined;
-    scrapeReplicate(outputDir, cacheDir, { concurrency, transformers, transformerModel, models, n }).catch((error) => {
+    const n = nArg ? parseInt(cleanArg(nArg.split('=')[1]), 10) : undefined;
+    const transformConcurrencyArg = args.find((arg) => arg.startsWith('--transform-concurrency='));
+    const transformConcurrency = transformConcurrencyArg
+        ? parseInt(transformConcurrencyArg.split('=')[1], 10)
+        : concurrency;
+    const params = { concurrency, transformers, transformersOnly, transformerModel, transformConcurrency, models, n };
+    console.debug('Scrape parameters:', { outputDir, cacheDir, ...params });
+    scrapeReplicate(outputDir, cacheDir, params).catch((error) => {
         console.error('✗ Replicate scraping failed:', error);
         process.exit(1);
     });