@aeye/models 0.1.0 → 0.2.0

package/scripts/scrapers/aws.ts

@@ -4,11 +4,11 @@
  * Fetches model information from AWS Bedrock using the AWS SDK
  */
 
-import type { ModelCapability, ModelInfo, ModelTier } from '@aeye/ai';
+import type { ModelInfo } from '@aeye/ai';
+import { convertAWSModel } from '@aeye/aws';
 import {
   BedrockClient,
   ListFoundationModelsCommand,
-  type FoundationModelSummary,
 } from '@aws-sdk/client-bedrock';
 import * as fs from 'fs/promises';
 import * as path from 'path';
@@ -18,269 +18,6 @@ import { writeModelTS } from '../codegen';
 const __filename = url.fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-/**
- * Model pricing information (per million tokens)
- * Source: https://aws.amazon.com/bedrock/pricing/
- */
-const MODEL_PRICING: Record<string, { input: number; output: number }> = {
-  // Anthropic Claude 3.5 Sonnet
-  'anthropic.claude-3-5-sonnet-20240620-v1:0': { input: 3, output: 15 },
-  'anthropic.claude-3-5-sonnet-20241022-v2:0': { input: 3, output: 15 },
-  
-  // Anthropic Claude 3 Opus
-  'anthropic.claude-3-opus-20240229-v1:0': { input: 15, output: 75 },
-  
-  // Anthropic Claude 3 Sonnet
-  'anthropic.claude-3-sonnet-20240229-v1:0': { input: 3, output: 15 },
-  
-  // Anthropic Claude 3 Haiku
-  'anthropic.claude-3-haiku-20240307-v1:0': { input: 0.25, output: 1.25 },
-  
-  // Anthropic Claude 2.x
-  'anthropic.claude-v2:1': { input: 8, output: 24 },
-  'anthropic.claude-v2': { input: 8, output: 24 },
-  'anthropic.claude-instant-v1': { input: 0.8, output: 2.4 },
-  
-  // Meta Llama 3.2
-  'meta.llama3-2-1b-instruct-v1:0': { input: 0.1, output: 0.1 },
-  'meta.llama3-2-3b-instruct-v1:0': { input: 0.15, output: 0.15 },
-  'meta.llama3-2-11b-instruct-v1:0': { input: 0.35, output: 0.35 },
-  'meta.llama3-2-90b-instruct-v1:0': { input: 2.65, output: 2.65 },
-  
-  // Meta Llama 3.1
-  'meta.llama3-1-8b-instruct-v1:0': { input: 0.3, output: 0.6 },
-  'meta.llama3-1-70b-instruct-v1:0': { input: 2.65, output: 3.5 },
-  'meta.llama3-1-405b-instruct-v1:0': { input: 5.32, output: 16 },
-  
-  // Meta Llama 3
-  'meta.llama3-8b-instruct-v1:0': { input: 0.3, output: 0.6 },
-  'meta.llama3-70b-instruct-v1:0': { input: 2.65, output: 3.5 },
-  
-  // Meta Llama 2
-  'meta.llama2-13b-chat-v1': { input: 0.75, output: 1 },
-  'meta.llama2-70b-chat-v1': { input: 1.95, output: 2.56 },
-  
-  // Mistral AI
-  'mistral.mistral-7b-instruct-v0:2': { input: 0.15, output: 0.2 },
-  'mistral.mixtral-8x7b-instruct-v0:1': { input: 0.45, output: 0.7 },
-  'mistral.mistral-large-2402-v1:0': { input: 4, output: 12 },
-  'mistral.mistral-large-2407-v1:0': { input: 3, output: 9 },
-  
-  // Cohere
-  'cohere.command-text-v14': { input: 1.5, output: 2 },
-  'cohere.command-light-text-v14': { input: 0.3, output: 0.6 },
-  'cohere.command-r-v1:0': { input: 0.5, output: 1.5 },
-  'cohere.command-r-plus-v1:0': { input: 3, output: 15 },
-  
-  // Amazon Titan Text
-  'amazon.titan-text-lite-v1': { input: 0.15, output: 0.2 },
-  'amazon.titan-text-express-v1': { input: 0.2, output: 0.6 },
-  'amazon.titan-text-premier-v1:0': { input: 0.5, output: 1.5 },
-};
-
-/**
- * Model context window information (in tokens)
- * Source: AWS Bedrock documentation
- */
-const MODEL_CONTEXT_WINDOWS: Record<string, { context: number; maxOutput?: number }> = {
-  // Anthropic Claude 3.5 Sonnet
-  'anthropic.claude-3-5-sonnet-20240620-v1:0': { context: 200000, maxOutput: 8192 },
-  'anthropic.claude-3-5-sonnet-20241022-v2:0': { context: 200000, maxOutput: 8192 },
-  
-  // Anthropic Claude 3 Opus
-  'anthropic.claude-3-opus-20240229-v1:0': { context: 200000, maxOutput: 4096 },
-  
-  // Anthropic Claude 3 Sonnet
-  'anthropic.claude-3-sonnet-20240229-v1:0': { context: 200000, maxOutput: 4096 },
-  
-  // Anthropic Claude 3 Haiku
-  'anthropic.claude-3-haiku-20240307-v1:0': { context: 200000, maxOutput: 4096 },
-  
-  // Anthropic Claude 2.x
-  'anthropic.claude-v2:1': { context: 100000, maxOutput: 4096 },
-  'anthropic.claude-v2': { context: 100000, maxOutput: 4096 },
-  'anthropic.claude-instant-v1': { context: 100000, maxOutput: 4096 },
-  
-  // Meta Llama 3.2
-  'meta.llama3-2-1b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  'meta.llama3-2-3b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  'meta.llama3-2-11b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  'meta.llama3-2-90b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  
-  // Meta Llama 3.1
-  'meta.llama3-1-8b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  'meta.llama3-1-70b-instruct-v1:0': { context: 128000, maxOutput: 2048 },
-  'meta.llama3-1-405b-instruct-v1:0': { context: 128000, maxOutput: 4096 },
-  
-  // Meta Llama 3
-  'meta.llama3-8b-instruct-v1:0': { context: 8192, maxOutput: 2048 },
-  'meta.llama3-70b-instruct-v1:0': { context: 8192, maxOutput: 2048 },
-  
-  // Meta Llama 2
-  'meta.llama2-13b-chat-v1': { context: 4096, maxOutput: 2048 },
-  'meta.llama2-70b-chat-v1': { context: 4096, maxOutput: 2048 },
-  
-  // Mistral AI
-  'mistral.mistral-7b-instruct-v0:2': { context: 32000, maxOutput: 8192 },
-  'mistral.mixtral-8x7b-instruct-v0:1': { context: 32000, maxOutput: 8192 },
-  'mistral.mistral-large-2402-v1:0': { context: 32000, maxOutput: 8192 },
-  'mistral.mistral-large-2407-v1:0': { context: 128000, maxOutput: 8192 },
-  
-  // Cohere
-  'cohere.command-text-v14': { context: 4096, maxOutput: 4096 },
-  'cohere.command-light-text-v14': { context: 4096, maxOutput: 4096 },
-  'cohere.command-r-v1:0': { context: 128000, maxOutput: 4096 },
-  'cohere.command-r-plus-v1:0': { context: 128000, maxOutput: 4096 },
-  
-  // Amazon Titan Text
-  'amazon.titan-text-lite-v1': { context: 4096, maxOutput: 4096 },
-  'amazon.titan-text-express-v1': { context: 8192, maxOutput: 8192 },
-  'amazon.titan-text-premier-v1:0': { context: 32000, maxOutput: 3072 },
-};
-
-/**
- * Detect model family from model ID
- */
-function detectModelFamily(modelId: string): string {
-  if (modelId.startsWith('anthropic.')) return 'anthropic';
-  if (modelId.startsWith('meta.')) return 'meta';
-  if (modelId.startsWith('mistral.')) return 'mistral';
-  if (modelId.startsWith('cohere.')) return 'cohere';
-  if (modelId.startsWith('ai21.')) return 'ai21';
-  if (modelId.startsWith('amazon.')) return 'amazon';
-  if (modelId.startsWith('stability.')) return 'stability';
-  return 'unknown';
-}
-
-/**
- * Detect capabilities from model information
- */
-function detectCapabilities(model: FoundationModelSummary): Set<ModelCapability> {
-  const capabilities = new Set<ModelCapability>();
-  const modelId = model.modelId || '';
-  const family = detectModelFamily(modelId);
-
-  // Check for chat capability (text-to-text models)
-  if (
-    model.inputModalities?.includes('TEXT') &&
-    model.outputModalities?.includes('TEXT')
-  ) {
-    // Chat models
-    if (
-      family === 'anthropic' ||
-      family === 'meta' ||
-      family === 'mistral' ||
-      family === 'cohere' ||
-      family === 'ai21' ||
-      (family === 'amazon' && modelId.includes('text'))
-    ) {
-      capabilities.add('chat');
-      
-      // Streaming support
-      if (model.responseStreamingSupported) {
-        capabilities.add('streaming');
-      }
-    }
-  }
-
-  // Image generation
-  if (
-    model.inputModalities?.includes('TEXT') &&
-    model.outputModalities?.includes('IMAGE')
-  ) {
-    capabilities.add('image');
-  }
-
-  // Vision (image understanding)
-  if (
-    model.inputModalities?.includes('IMAGE') &&
-    model.outputModalities?.includes('TEXT')
-  ) {
-    capabilities.add('vision');
-  }
-
-  // Embeddings
-  if (family === 'amazon' && modelId.includes('embed')) {
-    capabilities.add('embedding');
-  }
-  if (family === 'cohere' && modelId.includes('embed')) {
-    capabilities.add('embedding');
-  }
-
-  // Tool calling for supported models
-  if (family === 'anthropic' && modelId.includes('claude-3')) {
-    capabilities.add('tools');
-  }
-
-  return capabilities;
-}
-
-/**
- * Convert AWS Bedrock model to ModelInfo format
- */
-function convertAWSModel(model: FoundationModelSummary): ModelInfo | null {
-  const modelId = model.modelId;
-  if (!modelId) return null;
-
-  const family = detectModelFamily(modelId);
-  const capabilities = detectCapabilities(model);
-  
-  // Determine tier based on model family and name
-  let tier: ModelTier = 'efficient';
-  if (family === 'anthropic' && modelId.includes('opus')) {
-    tier = 'flagship';
-  } else if (family === 'anthropic' && modelId.includes('sonnet')) {
-    tier = 'efficient';
-  } else if (family === 'anthropic' && modelId.includes('haiku')) {
-    tier = 'efficient';
-  } else if (family === 'meta' && (modelId.includes('405b') || modelId.includes('90b'))) {
-    tier = 'flagship';
-  } else if (family === 'meta' && (modelId.includes('70b') || modelId.includes('13b'))) {
-    tier = 'efficient';
-  } else if (family === 'meta' && (modelId.includes('8b') || modelId.includes('7b') || modelId.includes('3b') || modelId.includes('1b'))) {
-    tier = 'efficient';
-  } else if (family === 'mistral' && modelId.includes('large')) {
-    tier = 'flagship';
-  } else if (family === 'mistral' && modelId.includes('mixtral')) {
-    tier = 'efficient';
-  } else if (family === 'mistral' && modelId.includes('7b')) {
-    tier = 'efficient';
-  } else if (family === 'cohere' && modelId.includes('plus')) {
-    tier = 'flagship';
-  } else {
-    tier = 'efficient';
-  }
-
-  // Get pricing information
-  const pricing = MODEL_PRICING[modelId];
-  const contextInfo = MODEL_CONTEXT_WINDOWS[modelId];
-
-  return {
-    provider: 'aws',
-    id: modelId,
-    name: model.modelName || modelId,
-    capabilities,
-    tier,
-    pricing: pricing ? {
-      text: {
-        input: pricing.input,
-        output: pricing.output,
-      },
-    } : {},
-    contextWindow: contextInfo?.context || 0,
-    maxOutputTokens: contextInfo?.maxOutput,
-    metadata: {
-      modelArn: model.modelArn,
-      providerName: model.providerName,
-      responseStreamingSupported: model.responseStreamingSupported,
-      customizationsSupported: model.customizationsSupported,
-      inferenceTypesSupported: model.inferenceTypesSupported,
-      inputModalities: model.inputModalities,
-      outputModalities: model.outputModalities,
-    },
-  };
-}
-
 /**
  * Main scraper function
  */
@@ -288,7 +25,7 @@ export async function scrapeAWS(
   outputDir: string,
   options: { region?: string } = {}
 ): Promise<void> {
-  const { region = process.env.AWS_REGION || 'us-east-1' } = options;
+  const { region = process.env.AWS_REGION || 'us-east-2' } = options;
 
   console.log('\n=== AWS Bedrock Scraper ===\n');
   console.log(`Using region: ${region}`);

package/scripts/scrapers/extract.md

@@ -0,0 +1,274 @@
+Run `npm run scrape:replicate`
+
+I want you to consume one model chunk file at a time. They are in cache/replicate-schemas-chunk-#.json. Each model has latest_version and in the components of the schema has Input & Output. You need to do one at a time because each file is ~ 20k tokens.
+
+What you need to do is replace/create a src/replicate.ts file with all model handlers that make sense for each model. A model handler allows you to link a particular model ID with a set of supported functions. Like chat prediction, image generation, etc.
+
+Here's the shape of the ModelHandler from the `@aeye/ai` package:
+
+```ts
+export interface ModelTransformer {
+  chat?: {
+    convertRequest?: (request: Request, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<Response>;
+    parseChunk?: (chunk: object, ctx: AIContextAny) => Promise<Chunk>;
+  };
+
+  imageGenerate?: {
+    convertRequest?: (request: ImageGenerationRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<ImageGenerationResponse>;
+    parseChunk?: (chunk: object, ctx: AIContextAny) => Promise<ImageGenerationChunk>;
+  };
+
+  imageEdit?: {
+    convertRequest?: (request: ImageEditRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<ImageGenerationResponse>;
+    parseChunk?: (chunk: object, ctx: AIContextAny) => Promise<ImageGenerationChunk>;
+  };
+
+  imageAnalyze?: {
+    convertRequest?: (request: ImageAnalyzeRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<Response>;
+    parseChunk?: (chunk: object, ctx: AIContextAny) => Promise<Chunk>;
+  };
+
+  transcribe?: {
+    convertRequest?: (request: TranscriptionRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<TranscriptionResponse>;
+    parseChunk?: (chunk: object, ctx: AIContextAny) => Promise<TranscriptionChunk>;
+  };
+
+  speech?: {
+    convertRequest?: (request: SpeechRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<SpeechResponse>;
+  };
+
+  embed?: {
+    convertRequest?: (request: EmbeddingRequest, ctx: AIContextAny) => Promise<object>;
+    parseResponse?: (response: object, ctx: AIContextAny) => Promise<EmbeddingResponse>;
+  };
+}
+```
+
+And here are the referenced types:
+```ts
+export type MessageRole = 'system' | 'user' | 'assistant' | 'tool';
+export interface Message { 
+  role: MessageRole;
+  content: string | MessageContent[];
+  tokens?: number;
+  name?: string;
+  toolCallId?: string;
+  toolCalls?: ToolCall[];
+  refusal?: string;
+  cache?: Record<string, any>;
+}
+export type MessageContentType = 'text' | 'image' | 'file' | 'audio';
+export interface MessageContent {
+  type: MessageContentType;
+  content: Resource;
+  format?: string;
+}
+export interface ToolDefinition {
+  name: string;
+  description?: string;
+  parameters: z.ZodType<object>;
+  strict?: boolean;
+}
+export interface ToolCall {
+  id: string;
+  name: string;
+  arguments: string;
+}
+export type ToolChoice =
+  | 'auto'
+  | 'none'
+  | 'required'
+  | { tool: string };
+export type ResponseFormat =
+  | 'text'
+  | 'json'
+  | { type: z.ZodType<object, object>, strict: boolean };
+export interface Usage {
+  text?: {
+    input?: number;
+    output?: number;
+    cached?: number;
+  };
+  audio?: {
+    input?: number;
+    output?: number;
+    seconds?: number;
+  };
+  image?: {
+    input?: number;
+    output?: {
+      quality: string;
+      size: { width: number; height: number; };
+      count: number;
+    }[];
+  };
+  reasoning?: {
+    input?: number;
+    output?: number;
+    cached?: number;
+  };
+  embeddings?: {
+    count?: number;
+    tokens?: number;
+  };
+  cost?: number;
+}
+export type ReasoningEffort = 'low' | 'medium' | 'high';
+export interface Request extends BaseRequest {
+  name?: string;
+  messages: Message[];
+  temperature?: number;
+  maxTokens?: number;
+  topP?: number;
+  frequencyPenalty?: number;
+  presencePenalty?: number;
+  stop?: string | string[];
+  logProbabilities?: boolean;
+  logitBias?: Record<string, number>;
+  tools?: ToolDefinition[];
+  toolsOneAtATime?: boolean;
+  toolChoice?: ToolChoice;
+  responseFormat?: ResponseFormat;
+  reason?: { effort?: ReasoningEffort, maxTokens?: number };
+  cacheKey?: string;
+  userKey?: string;
+}
+export type FinishReason = 'stop' | 'length' | 'tool_calls' | 'content_filter' | 'refusal';
+export interface Model {
+  id: string;
+  contextWindow?: number;
+  maxOutputTokens?: number;
+}
+export type ModelInput = string | Model;
+export interface BaseRequest {
+  model?: ModelInput;
+  extra?: Record<string, any>;
+}
+export interface BaseResponse {
+  usage?: Usage;
+  model: ModelInput;
+}
+export interface BaseChunk {
+  usage?: Usage;
+  model?: ModelInput;
+}
+export interface Response extends BaseResponse {
+  content: string;
+  toolCalls?: ToolCall[];
+  finishReason: FinishReason;
+  refusal?: string;
+  reasoning?: string;
+}
+export interface Chunk extends BaseChunk {
+  content?: string;
+  toolCallNamed?: ToolCall;
+  toolCallArguments?: ToolCall;
+  toolCall?: ToolCall;
+  finishReason?: FinishReason;
+  refusal?: string;
+  reasoning?: string;
+}
+export interface ImageGenerationRequest extends BaseRequest {
+  prompt: string;
+  n?: number;
+  size?: string;
+  quality?: 'low' | 'medium' | 'high';
+  style?: 'vivid' | 'natural'; 
+  responseFormat?: 'url' | 'b64_json';
+  background?: 'transparent' | 'opaque' | 'auto';
+  streamCount?: number;
+  seed?: number;
+  userIdentifier?: string;
+}
+export interface ImageEditRequest extends BaseRequest {
+  prompt: string;
+  image: Resource;
+  mask?: Resource;
+  n?: number;
+  size?: string;
+  responseFormat?: 'url' | 'b64_json';
+  seed?: number;
+  streamCount?: number;
+  userIdentifier?: string;
+}
+export interface ImageGenerationResponse extends BaseResponse {
+  images: Array<{
+    url?: string;
+    b64_json?: string;
+    revisedPrompt?: string;
+  }>;
+}
+export interface ImageGenerationChunk extends BaseChunk {
+  imageData?: string;
+  progress?: number;
+  done?: boolean;
+  image?: {
+    url?: string;
+    b64_json?: string;
+  };
+}
+export interface TranscriptionRequest extends BaseRequest {
+  audio: Resource;
+  language?: string;
+  prompt?: string;
+  temperature?: number;
+  responseFormat?: 'json' | 'text' | 'srt' | 'vtt' | 'verbose_json';
+  timestampGranularities?: Array<'word' | 'segment'>;
+}
+export interface TranscriptionResponse extends BaseResponse {
+  text: string;
+}
+export interface TranscriptionChunk extends BaseChunk {
+  delta?: string;
+  text?: string;
+  segment?: { start: number; end: number; speaker: string, text: string, id: string };
+  status?: string;
+}
+export interface SpeechRequest extends BaseRequest{
+  text: string;
+  instructions?: string;
+  voice?: string;
+  speed?: number;
+  responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
+}
+export interface SpeechResponse extends BaseResponse {
+  audio: ReadableStream<any>;
+  responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
+}
+export interface EmbeddingRequest extends BaseRequest {
+  texts: string[];
+  dimensions?: number;
+  encodingFormat?: 'float' | 'base64';
+  userIdentifier?: string;
+}
+export interface EmbeddingResponse extends BaseResponse {
+  embeddings: Array<{
+    embedding: number[];
+    index: number;
+  }>;
+}
+export interface ImageAnalyzeRequest extends BaseRequest{
+  prompt: string;
+  images: string[];
+  maxTokens?: number;
+  temperature?: number;
+}
+export type Resource = 
+ | string // plain text, or data URL, or http(s) URL, or file:// URL
+ | AsyncIterable<Uint8Array> // fs.ReadStream, ReadableStream
+ | Blob // File
+ | Uint8Array 
+ | URL
+ | ArrayBuffer
+ | DataView
+ | Buffer
+ | { blob(): Promise<Blob> | Blob }
+ | { url(): string }
+ | { read(): Promise<ReadableStream> | ReadableStream }
+```