@aeye/models 0.1.0

package/dist/scripts/scrapers/aws.js

@@ -0,0 +1,314 @@
+/**
+ * AWS Bedrock Model Scraper
+ *
+ * Fetches model information from AWS Bedrock using the AWS SDK
+ */
+import { BedrockClient, ListFoundationModelsCommand, } from '@aws-sdk/client-bedrock';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import * as url from 'url';
+import { writeModelTS } from '../codegen';
+const __filename = url.fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+/**
+ * Model pricing information (per million tokens)
+ * Source: https://aws.amazon.com/bedrock/pricing/
+ */
+const MODEL_PRICING = {
+    // Anthropic Claude 3.5 Sonnet
+    'anthropic.claude-3-5-sonnet-20240620-v1:0': { input: 3, output: 15 },
+    'anthropic.claude-3-5-sonnet-20241022-v2:0': { input: 3, output: 15 },
+    // Anthropic Claude 3 Opus
+    'anthropic.claude-3-opus-20240229-v1:0': { input: 15, output: 75 },
+    // Anthropic Claude 3 Sonnet
+    'anthropic.claude-3-sonnet-20240229-v1:0': { input: 3, output: 15 },
+    // Anthropic Claude 3 Haiku
+    'anthropic.claude-3-haiku-20240307-v1:0': { input: 0.25, output: 1.25 },
+    // Anthropic Claude 2.x
+    'anthropic.claude-v2:1': { input: 8, output: 24 },
+    'anthropic.claude-v2': { input: 8, output: 24 },
+    'anthropic.claude-instant-v1': { input: 0.8, output: 2.4 },
+    // Meta Llama 3.2
+    'meta.llama3-2-1b-instruct-v1:0': { input: 0.1, output: 0.1 },
+    'meta.llama3-2-3b-instruct-v1:0': { input: 0.15, output: 0.15 },
+    'meta.llama3-2-11b-instruct-v1:0': { input: 0.35, output: 0.35 },
+    'meta.llama3-2-90b-instruct-v1:0': { input: 2.65, output: 2.65 },
+    // Meta Llama 3.1
+    'meta.llama3-1-8b-instruct-v1:0': { input: 0.3, output: 0.6 },
+    'meta.llama3-1-70b-instruct-v1:0': { input: 2.65, output: 3.5 },
+    'meta.llama3-1-405b-instruct-v1:0': { input: 5.32, output: 16 },
+    // Meta Llama 3
+    'meta.llama3-8b-instruct-v1:0': { input: 0.3, output: 0.6 },
+    'meta.llama3-70b-instruct-v1:0': { input: 2.65, output: 3.5 },
+    // Meta Llama 2
+    'meta.llama2-13b-chat-v1': { input: 0.75, output: 1 },
+    'meta.llama2-70b-chat-v1': { input: 1.95, output: 2.56 },
+    // Mistral AI
+    'mistral.mistral-7b-instruct-v0:2': { input: 0.15, output: 0.2 },
+    'mistral.mixtral-8x7b-instruct-v0:1': { input: 0.45, output: 0.7 },
+    'mistral.mistral-large-2402-v1:0': { input: 4, output: 12 },
+    'mistral.mistral-large-2407-v1:0': { input: 3, output: 9 },
+    // Cohere
+    'cohere.command-text-v14': { input: 1.5, output: 2 },
+    'cohere.command-light-text-v14': { input: 0.3, output: 0.6 },
+    'cohere.command-r-v1:0': { input: 0.5, output: 1.5 },
+    'cohere.command-r-plus-v1:0': { input: 3, output: 15 },
+    // Amazon Titan Text
+    'amazon.titan-text-lite-v1': { input: 0.15, output: 0.2 },
+    'amazon.titan-text-express-v1': { input: 0.2, output: 0.6 },
+    'amazon.titan-text-premier-v1:0': { input: 0.5, output: 1.5 },
+};
+/**
+ * Model context window information (in tokens)
+ * Source: AWS Bedrock documentation
+ */
+const MODEL_CONTEXT_WINDOWS = {
+    // Anthropic Claude 3.5 Sonnet
+    'anthropic.claude-3-5-sonnet-20240620-v1:0': { context: 200000, maxOutput: 8192 },
+    'anthropic.claude-3-5-sonnet-20241022-v2:0': { context: 200000, maxOutput: 8192 },
+    // Anthropic Claude 3 Opus
+    'anthropic.claude-3-opus-20240229-v1:0': { context: 200000, maxOutput: 4096 },
+    // Anthropic Claude 3 Sonnet
+    'anthropic.claude-3-sonnet-20240229-v1:0': { context: 200000, maxOutput: 4096 },
+    // Anthropic Claude 3 Haiku
+    'anthropic.claude-3-haiku-20240307-v1:0': { context: 200000, maxOutput: 4096 },
+    // Anthropic Claude 2.x
+    'anthropic.claude-v2:1': { context: 100000, maxOutput: 4096 },
+    'anthropic.claude-v2': { context: 100000, maxOutput: 4096 },
+    'anthropic.claude-instant-v1': { context: 100000, maxOutput: 4096 },
+    // Meta Llama 3.2
+    'meta.llama3-2-1b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    'meta.llama3-2-3b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    'meta.llama3-2-11b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    'meta.llama3-2-90b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    // Meta Llama 3.1
+    'meta.llama3-1-8b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    'meta.llama3-1-70b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
+    'meta.llama3-1-405b-instruct-v1:0': { context: 128000, maxOutput: 4096 },
+    // Meta Llama 3
+    'meta.llama3-8b-instruct-v1:0': { context: 8192, maxOutput: 2048 },
+    'meta.llama3-70b-instruct-v1:0': { context: 8192, maxOutput: 2048 },
+    // Meta Llama 2
+    'meta.llama2-13b-chat-v1': { context: 4096, maxOutput: 2048 },
+    'meta.llama2-70b-chat-v1': { context: 4096, maxOutput: 2048 },
+    // Mistral AI
+    'mistral.mistral-7b-instruct-v0:2': { context: 32000, maxOutput: 8192 },
+    'mistral.mixtral-8x7b-instruct-v0:1': { context: 32000, maxOutput: 8192 },
+    'mistral.mistral-large-2402-v1:0': { context: 32000, maxOutput: 8192 },
+    'mistral.mistral-large-2407-v1:0': { context: 128000, maxOutput: 8192 },
+    // Cohere
+    'cohere.command-text-v14': { context: 4096, maxOutput: 4096 },
+    'cohere.command-light-text-v14': { context: 4096, maxOutput: 4096 },
+    'cohere.command-r-v1:0': { context: 128000, maxOutput: 4096 },
+    'cohere.command-r-plus-v1:0': { context: 128000, maxOutput: 4096 },
+    // Amazon Titan Text
+    'amazon.titan-text-lite-v1': { context: 4096, maxOutput: 4096 },
+    'amazon.titan-text-express-v1': { context: 8192, maxOutput: 8192 },
+    'amazon.titan-text-premier-v1:0': { context: 32000, maxOutput: 3072 },
+};
+/**
+ * Detect model family from model ID
+ */
+function detectModelFamily(modelId) {
+    if (modelId.startsWith('anthropic.'))
+        return 'anthropic';
+    if (modelId.startsWith('meta.'))
+        return 'meta';
+    if (modelId.startsWith('mistral.'))
+        return 'mistral';
+    if (modelId.startsWith('cohere.'))
+        return 'cohere';
+    if (modelId.startsWith('ai21.'))
+        return 'ai21';
+    if (modelId.startsWith('amazon.'))
+        return 'amazon';
+    if (modelId.startsWith('stability.'))
+        return 'stability';
+    return 'unknown';
+}
+/**
+ * Detect capabilities from model information
+ */
+function detectCapabilities(model) {
+    const capabilities = new Set();
+    const modelId = model.modelId || '';
+    const family = detectModelFamily(modelId);
+    // Check for chat capability (text-to-text models)
+    if (model.inputModalities?.includes('TEXT') &&
+        model.outputModalities?.includes('TEXT')) {
+        // Chat models
+        if (family === 'anthropic' ||
+            family === 'meta' ||
+            family === 'mistral' ||
+            family === 'cohere' ||
+            family === 'ai21' ||
+            (family === 'amazon' && modelId.includes('text'))) {
+            capabilities.add('chat');
+            // Streaming support
+            if (model.responseStreamingSupported) {
+                capabilities.add('streaming');
+            }
+        }
+    }
+    // Image generation
+    if (model.inputModalities?.includes('TEXT') &&
+        model.outputModalities?.includes('IMAGE')) {
+        capabilities.add('image');
+    }
+    // Vision (image understanding)
+    if (model.inputModalities?.includes('IMAGE') &&
+        model.outputModalities?.includes('TEXT')) {
+        capabilities.add('vision');
+    }
+    // Embeddings
+    if (family === 'amazon' && modelId.includes('embed')) {
+        capabilities.add('embedding');
+    }
+    if (family === 'cohere' && modelId.includes('embed')) {
+        capabilities.add('embedding');
+    }
+    // Tool calling for supported models
+    if (family === 'anthropic' && modelId.includes('claude-3')) {
+        capabilities.add('tools');
+    }
+    return capabilities;
+}
+/**
+ * Convert AWS Bedrock model to ModelInfo format
+ */
+function convertAWSModel(model) {
+    const modelId = model.modelId;
+    if (!modelId)
+        return null;
+    const family = detectModelFamily(modelId);
+    const capabilities = detectCapabilities(model);
+    // Determine tier based on model family and name
+    let tier = 'efficient';
+    if (family === 'anthropic' && modelId.includes('opus')) {
+        tier = 'flagship';
+    }
+    else if (family === 'anthropic' && modelId.includes('sonnet')) {
+        tier = 'efficient';
+    }
+    else if (family === 'anthropic' && modelId.includes('haiku')) {
+        tier = 'efficient';
+    }
+    else if (family === 'meta' && (modelId.includes('405b') || modelId.includes('90b'))) {
+        tier = 'flagship';
+    }
+    else if (family === 'meta' && (modelId.includes('70b') || modelId.includes('13b'))) {
+        tier = 'efficient';
+    }
+    else if (family === 'meta' && (modelId.includes('8b') || modelId.includes('7b') || modelId.includes('3b') || modelId.includes('1b'))) {
+        tier = 'efficient';
+    }
+    else if (family === 'mistral' && modelId.includes('large')) {
+        tier = 'flagship';
+    }
+    else if (family === 'mistral' && modelId.includes('mixtral')) {
+        tier = 'efficient';
+    }
+    else if (family === 'mistral' && modelId.includes('7b')) {
+        tier = 'efficient';
+    }
+    else if (family === 'cohere' && modelId.includes('plus')) {
+        tier = 'flagship';
+    }
+    else {
+        tier = 'efficient';
+    }
+    // Get pricing information
+    const pricing = MODEL_PRICING[modelId];
+    const contextInfo = MODEL_CONTEXT_WINDOWS[modelId];
+    return {
+        provider: 'aws',
+        id: modelId,
+        name: model.modelName || modelId,
+        capabilities,
+        tier,
+        pricing: pricing ? {
+            text: {
+                input: pricing.input,
+                output: pricing.output,
+            },
+        } : {},
+        contextWindow: contextInfo?.context || 0,
+        maxOutputTokens: contextInfo?.maxOutput,
+        metadata: {
+            modelArn: model.modelArn,
+            providerName: model.providerName,
+            responseStreamingSupported: model.responseStreamingSupported,
+            customizationsSupported: model.customizationsSupported,
+            inferenceTypesSupported: model.inferenceTypesSupported,
+            inputModalities: model.inputModalities,
+            outputModalities: model.outputModalities,
+        },
+    };
+}
+/**
+ * Main scraper function
+ */
+export async function scrapeAWS(outputDir, options = {}) {
+    const { region = process.env.AWS_REGION || 'us-east-1' } = options;
+    console.log('\n=== AWS Bedrock Scraper ===\n');
+    console.log(`Using region: ${region}`);
+    // Create Bedrock client
+    const client = new BedrockClient({
+        region,
+        // Credentials are automatically picked up from environment
+    });
+    try {
+        // List foundation models
+        console.log('Fetching models from AWS Bedrock...');
+        const command = new ListFoundationModelsCommand({});
+        const response = await client.send(command);
+        if (!response.modelSummaries) {
+            console.log('✗ No models returned from AWS Bedrock');
+            return;
+        }
+        console.log(`✓ Fetched ${response.modelSummaries.length} AWS Bedrock models`);
+        // Create output directory
+        await fs.mkdir(outputDir, { recursive: true });
+        // Save raw data
+        await fs.writeFile(path.join(outputDir, 'aws-models.json'), JSON.stringify({ data: response.modelSummaries }, (key, value) => {
+            if (value instanceof Set) {
+                return Array.from(value);
+            }
+            return value;
+        }, 2));
+        console.log(`✓ Saved raw AWS models to aws-models.json`);
+        // Convert to ModelInfo format
+        const modelInfos = response.modelSummaries
+            .map(convertAWSModel)
+            .filter((m) => m !== null);
+        // Save JSON for reference
+        await fs.writeFile(path.join(outputDir, 'aws-modelinfo.json'), JSON.stringify(modelInfos, (key, value) => {
+            if (value instanceof Set) {
+                return Array.from(value);
+            }
+            return value;
+        }, 2));
+        console.log(`✓ Saved ${modelInfos.length} models to JSON`);
+        // Generate TypeScript file
+        const srcDir = path.join(__dirname, '../../src/models');
+        await writeModelTS(modelInfos, 'awsModels', path.join(srcDir, 'aws.ts'));
+        console.log(`✓ Generated TypeScript file: src/models/aws.ts`);
+        console.log('\n✓ AWS Bedrock scraping complete\n');
+    }
+    catch (error) {
+        console.error('✗ AWS Bedrock scraping failed:', error);
+        throw error;
+    }
+}
+// CLI execution
+if (process.argv[1].endsWith('aws.ts')) {
+    const args = process.argv.slice(2);
+    const outputDir = args.find((arg) => !arg.startsWith('--')) || path.join(__dirname, '../../data');
+    const regionArg = args.find((arg) => arg.startsWith('--region='));
+    const region = regionArg ? regionArg.split('=')[1] : undefined;
+    scrapeAWS(outputDir, { region }).catch((error) => {
+        console.error('✗ AWS Bedrock scraping failed:', error);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=aws.js.map

package/dist/scripts/scrapers/aws.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"aws.js","sourceRoot":"","sources":["../../../scripts/scrapers/aws.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,aAAa,EACb,2BAA2B,GAE5B,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C;;;GAGG;AACH,MAAM,aAAa,GAAsD;IACvE,8BAA8B;IAC9B,2CAA2C,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IACrE,2CAA2C,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IAErE,0BAA0B;IAC1B,uCAAuC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;IAElE,4BAA4B;IAC5B,yCAAyC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IAEnE,2BAA2B;IAC3B,wCAAwC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAEvE,uBAAuB;IACvB,uBAAuB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IACjD,qBAAqB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IAC/C,6BAA6B,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAE1D,iBAAiB;IACjB,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7D,gCAAgC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/D,iCAAiC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAChE,iCAAiC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAEhE,iBAAiB;IACjB,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7D,iCAAiC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC/D,kCAAkC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE;IAE/D,eAAe;IACf,8BAA8B,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAC3D,+BAA+B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAE7D,eAAe;IACf,yBAAyB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;IACrD,yBAAyB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAExD,aAAa;IACb,kCAAkC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAChE,oCAAoC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAClE,iCAAiC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IAC3D,iCAAiC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;IAE1D,SAAS;IACT,yBAAyB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE;IACpD,+BAA+B,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAC5D,uBAAuB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IACpD,4BAA4B,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;IAEtD,oBAAoB;IACpB,2BAA2B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IACzD,8BAA8B,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;IAC3D,gCAAgC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;CAC9D,CAAC;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAA4D;IACrF,8BAA8B;IAC9B,2CAA2C,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACjF,2CAA2C,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAEjF,0BAA0B;IAC1B,uCAAuC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAE7E,4BAA4B;IAC5B,yCAAyC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAE/E,2BAA2B;IAC3B,wCAAwC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAE9E,uBAAuB;IACvB,uBAAuB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAC7D,qBAAqB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAC3D,6BAA6B,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAEnE,iBAAiB;IACjB,gCAAgC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACtE,gCAAgC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACtE,iCAAiC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACvE,iCAAiC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAEvE,iBAAiB;IACjB,gCAAgC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACtE,iCAAiC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IACvE,kCAAkC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAExE,eAAe;IACf,8BAA8B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAClE,+BAA+B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAEnE,eAAe;IACf,yBAAyB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAC7D,yBAAyB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAE7D,aAAa;IACb,kCAAkC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE;IACvE,oCAAoC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE;IACzE,iCAAiC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE;IACtE,iCAAiC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAEvE,SAAS;IACT,yBAAyB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAC7D,+BAA+B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IACnE,uBAAuB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAC7D,4BAA4B,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE;IAElE,oBAAoB;IACpB,2BAA2B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAC/D,8BAA8B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;IAClE,gCAAgC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE;CACtE,CAAC;AAEF;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAe;IACxC,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,WAAW,CAAC;IACzD,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC/C,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IACrD,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,QAAQ,CAAC;IACnD,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC/C,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,QAAQ,CAAC;IACnD,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,WAAW,CAAC;IACzD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAA6B;IACvD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAmB,CAAC;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE1C,kDAAkD;IAClD,IACE,KAAK,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC;QACvC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,EACxC,CAAC;QACD,cAAc;QACd,IACE,MAAM,KAAK,WAAW;YACtB,MAAM,KAAK,MAAM;YACjB,MAAM,KAAK,SAAS;YACpB,MAAM,KAAK,QAAQ;YACnB,MAAM,KAAK,MAAM;YACjB,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EACjD,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEzB,oBAAoB;YACpB,IAAI,KAAK,CAAC,0BAA0B,EAAE,CAAC;gBACrC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IACE,KAAK,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC;QACvC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,OAAO,CAAC,EACzC,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,+BAA+B;IAC/B,IACE,KAAK,CAAC,eAAe,EAAE,QAAQ,CAAC,OAAO,CAAC;QACxC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,EACxC,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED,aAAa;IACb,IAAI,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAChC,CAAC;IAED,oCAAoC;IACpC,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3D,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAA6B;IACpD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,gDAAgD;IAChD,IAAI,IAAI,GAAc,WAAW,CAAC;IAClC,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;SAAM,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChE,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/D,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACtF,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;SAAM,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACrF,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvI,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;SAAM,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/D,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;SAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3D,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,WAAW,CAAC;IACrB,CAAC;IAED,0BAA0B;IAC1B,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAEnD,OAAO;QACL,QAAQ,EAAE,KAAK;QACf,EAAE,EAAE,OAAO;QACX,IAAI,EAAE,KAAK,CAAC,SAAS,IAAI,OAAO;QAChC,YAAY;QACZ,IAAI;QACJ,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACjB,IAAI,EAAE;gBACJ,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB;SACF,CAAC,CAAC,CAAC,EAAE;QACN,aAAa,EAAE,WAAW,EAAE,OAAO,IAAI,CAAC;QACxC,eAAe,EAAE,WAAW,EAAE,SAAS;QACvC,QAAQ,EAAE;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;YAC5D,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;YACtD,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;YACtD,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAiB,EACjB,UAA+B,EAAE;IAEjC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,WAAW,EAAE,GAAG,OAAO,CAAC;IAEnE,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,EAAE,CAAC,CAAC;IAEvC,wBAAwB;IACxB,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;QAC/B,MAAM;QACN,2DAA2D;KAC5D,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,yBAAyB;QACzB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,2BAA2B,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE5C,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAE9E,0BAA0B;QAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,gBAAgB;QAChB,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EACvC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC/D,IAAI,KAAK,YAAY,GAAG,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,EAAE,CAAC,CAAC,CACN,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QAEzD,8BAA8B;QAC9B,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc;aACvC,GAAG,CAAC,eAAe,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAE7C,0BAA0B;QAC1B,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC,EAC1C,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACxC,IAAI,KAAK,YAAY,GAAG,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC,EAAE,CAAC,CAAC,CACN,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAC;QAE3D,2BAA2B;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QACxD,MAAM,YAAY,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAE9D,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,gBAAgB;AAChB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAClG,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE/D,SAAS,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/scripts/scrapers/openai.d.ts

@@ -0,0 +1,12 @@
+/**
+ * OpenAI Model Scraper
+ *
+ * Scrapes model information from OpenAI documentation using Puppeteer
+ */
+/**
+ * Main scraper function
+ */
+export declare function scrapeOpenAI(outputDir: string, options?: {
+    concurrency?: number;
+}): Promise<void>;
+//# sourceMappingURL=openai.d.ts.map

package/dist/scripts/scrapers/openai.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../scripts/scrapers/openai.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAgiBH;;GAEG;AACH,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACrC,OAAO,CAAC,IAAI,CAAC,CAgDf"}