ai-functions 0.3.0 → 0.4.0

package/src/eval/models.ts

@@ -0,0 +1,158 @@
+/**
+ * Model Registry for AI Functions Eval Suite
+ *
+ * Simple model list for running evals across providers.
+ * Uses ai-providers/language-models for resolution and pricing.
+ *
+ * @packageDocumentation
+ */
+
+import { resolve, get, list, type ModelInfo } from 'language-models'
+
+export type ModelTier = 'best' | 'fast' | 'cheap'
+
+export interface EvalModel {
+  /** Alias or full model ID */
+  id: string
+  /** Human-readable name */
+  name: string
+  /** Provider slug */
+  provider: string
+  /** Capability tier */
+  tier: ModelTier
+  /** Optional notes */
+  notes?: string
+}
+
+// ============================================================================
+// Models to evaluate - using aliases from language-models
+// ============================================================================
+
+/**
+ * Core models to test - one per tier per major provider
+ * These resolve via ai-providers to OpenRouter or direct SDKs
+ *
+ * Updated: December 2025
+ *
+ * Note: Some models use OpenRouter format (provider/model) to avoid
+ * resolution issues with provider_model_id mismatches.
+ */
+export const EVAL_MODELS: EvalModel[] = [
+  // Anthropic Claude 4.5 - via AWS Bedrock (uses AWS credits with bearer token auth)
+  // All Claude models should be 4.5 - older versions are deprecated
+  { id: 'bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0', name: 'Claude Opus 4.5', provider: 'anthropic', tier: 'best', notes: 'Bedrock' },
+  { id: 'bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0', name: 'Claude Sonnet 4.5', provider: 'anthropic', tier: 'fast', notes: 'Bedrock' },
+  { id: 'bedrock:us.anthropic.claude-haiku-4-5-20251001-v1:0', name: 'Claude Haiku 4.5', provider: 'anthropic', tier: 'cheap', notes: 'Bedrock' },
+
+  // OpenAI - GPT-5.1 variants + GPT-oss (open source)
+  { id: 'openai/o3', name: 'o3', provider: 'openai', tier: 'best' },
+  { id: 'openai/gpt-5.1', name: 'GPT-5.1', provider: 'openai', tier: 'best' },
+  { id: 'openai/gpt-5-mini', name: 'GPT-5 Mini', provider: 'openai', tier: 'fast' },
+  { id: 'openai/gpt-5-nano', name: 'GPT-5 Nano', provider: 'openai', tier: 'cheap' },
+  // GPT-oss 120B removed - times out frequently
+  { id: 'openai/gpt-oss-20b', name: 'GPT-oss 20B', provider: 'openai', tier: 'fast', notes: 'Open source' },
+
+  // Google - Gemini 3 (November 2025)
+  { id: 'google/gemini-3-pro-preview', name: 'Gemini 3 Pro', provider: 'google', tier: 'best', notes: '1M context, #1 LMArena' },
+  // Gemini 2.5 Pro removed - times out frequently
+  { id: 'flash', name: 'Gemini 2.5 Flash', provider: 'google', tier: 'fast' },
+
+  // Meta (via OpenRouter)
+  { id: 'meta-llama/llama-4-maverick', name: 'Llama 4 Maverick', provider: 'meta-llama', tier: 'best' },
+  { id: 'meta-llama/llama-3.3-70b-instruct', name: 'Llama 3.3 70B', provider: 'meta-llama', tier: 'fast' },
+
+  // DeepSeek - V3.2 (December 2025)
+  { id: 'deepseek/deepseek-v3.2', name: 'DeepSeek V3.2', provider: 'deepseek', tier: 'best', notes: 'GPT-5 class reasoning' },
+  // DeepSeek V3.2 Speciale removed - no tool use support on OpenRouter
+  { id: 'deepseek/deepseek-chat', name: 'DeepSeek Chat', provider: 'deepseek', tier: 'fast' },
+
+  // Mistral - Mistral 3 family (December 2025)
+  { id: 'mistralai/mistral-large-2512', name: 'Mistral Large 3', provider: 'mistralai', tier: 'best', notes: '675B MoE, 41B active' },
+  { id: 'mistralai/mistral-medium-3.1', name: 'Mistral Medium 3.1', provider: 'mistralai', tier: 'fast' },
+  // Ministral 3 14B removed - often fails structured output
+
+  // Qwen - Qwen3 family (2025)
+  { id: 'qwen/qwen3-coder', name: 'Qwen3 Coder 480B', provider: 'qwen', tier: 'best', notes: 'Agentic coding' },
+  { id: 'qwen/qwen3-30b-a3b', name: 'Qwen3 30B', provider: 'qwen', tier: 'fast', notes: 'MoE 30B/3B active' },
+  { id: 'qwen/qwen3-next-80b-a3b-instruct', name: 'Qwen3 Next 80B', provider: 'qwen', tier: 'best', notes: 'Ultra-long context' },
+
+  // xAI - Grok 4 family (December 2025)
+  { id: 'x-ai/grok-4', name: 'Grok 4', provider: 'x-ai', tier: 'best', notes: '256K context, reasoning' },
+  { id: 'x-ai/grok-4.1-fast', name: 'Grok 4.1 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context, agentic' },
+  { id: 'x-ai/grok-4-fast', name: 'Grok 4 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context' },
+]
+
+/**
+ * Get models by tier
+ */
+export function getModelsByTier(tier: ModelTier): EvalModel[] {
+  return EVAL_MODELS.filter(m => m.tier === tier)
+}
+
+/**
+ * Get models by provider
+ */
+export function getModelsByProvider(provider: string): EvalModel[] {
+  return EVAL_MODELS.filter(m => m.provider === provider)
+}
+
+/**
+ * Get model info from language-models package (includes pricing)
+ */
+export function getModelInfo(id: string): ModelInfo | undefined {
+  const resolved = resolve(id)
+  return get(resolved)
+}
+
+/**
+ * Get pricing for a model (from OpenRouter data)
+ */
+export function getModelPricing(id: string): { prompt: number; completion: number } | undefined {
+  const info = getModelInfo(id)
+  if (!info?.pricing) return undefined
+
+  return {
+    prompt: parseFloat(info.pricing.prompt) * 1_000_000, // Convert to per-million
+    completion: parseFloat(info.pricing.completion) * 1_000_000,
+  }
+}
+
+/**
+ * Create evalite variants for model testing
+ */
+export function createModelVariants(opts?: {
+  tiers?: ModelTier[]
+  providers?: string[]
+}): Array<{ name: string; input: EvalModel }> {
+  let models = EVAL_MODELS
+
+  if (opts?.tiers) {
+    models = models.filter(m => opts.tiers!.includes(m.tier))
+  }
+
+  if (opts?.providers) {
+    models = models.filter(m => opts.providers!.includes(m.provider))
+  }
+
+  return models.map(model => ({
+    name: `${model.provider}/${model.name}`,
+    input: model,
+  }))
+}
+
+/**
+ * Get a representative model from each provider for a given tier
+ */
+export function getRepresentativeModels(tier: ModelTier): EvalModel[] {
+  const seen = new Set<string>()
+  const result: EvalModel[] = []
+
+  for (const model of EVAL_MODELS) {
+    if (model.tier === tier && !seen.has(model.provider)) {
+      seen.add(model.provider)
+      result.push(model)
+    }
+  }
+
+  return result
+}

package/src/eval/runner.ts

@@ -0,0 +1,217 @@
+/**
+ * Simple eval runner for AI Functions
+ *
+ * Runs evals across multiple models and collects results.
+ * Does not depend on evalite - uses our own infrastructure.
+ */
+
+import { generateObject, generateText } from '../generate.js'
+import { schema } from '../schema.js'
+import { createModelVariants, getModelPricing, type EvalModel, type ModelTier } from './models.js'
+
+export interface EvalCase<TInput = unknown, TExpected = unknown> {
+  name: string
+  input: TInput
+  expected?: TExpected
+}
+
+export interface EvalScore {
+  name: string
+  score: number
+  description?: string
+  metadata?: unknown
+}
+
+export interface EvalResult<TOutput = unknown> {
+  model: EvalModel
+  case: EvalCase
+  output: TOutput
+  scores: EvalScore[]
+  latencyMs: number
+  cost: number
+  error?: string
+}
+
+export interface EvalSummary {
+  name: string
+  results: EvalResult[]
+  avgScore: number
+  byModel: Record<string, { avgScore: number; count: number }>
+  totalCost: number
+  totalTime: number
+}
+
+export interface RunEvalOptions<TInput, TOutput, TExpected> {
+  name: string
+  cases: EvalCase<TInput, TExpected>[]
+  task: (input: TInput, model: EvalModel) => Promise<TOutput>
+  scorers: Array<{
+    name: string
+    description?: string
+    scorer: (args: { input: TInput; output: TOutput; expected?: TExpected }) => number | Promise<number>
+  }>
+  models?: EvalModel[]
+  tiers?: ModelTier[]
+  providers?: string[]
+  concurrency?: number
+}
+
+/**
+ * Run an eval suite across models
+ */
+export async function runEval<TInput, TOutput, TExpected>(
+  options: RunEvalOptions<TInput, TOutput, TExpected>
+): Promise<EvalSummary> {
+  const { name, cases, task, scorers, concurrency = 3 } = options
+
+  // Get models to test
+  const models = options.models ?? createModelVariants({
+    tiers: options.tiers,
+    providers: options.providers,
+  }).map(v => v.input)
+
+  const results: EvalResult<TOutput>[] = []
+  const startTime = Date.now()
+
+  console.log(`\n🧪 Running eval: ${name}`)
+  console.log(`   Models: ${models.map(m => m.name).join(', ')}`)
+  console.log(`   Cases: ${cases.length}`)
+  console.log('')
+
+  // Run all model/case combinations
+  const jobs: Array<{ model: EvalModel; case: EvalCase<TInput, TExpected> }> = []
+  for (const model of models) {
+    for (const evalCase of cases) {
+      jobs.push({ model, case: evalCase })
+    }
+  }
+
+  // Process in batches with concurrency limit
+  for (let i = 0; i < jobs.length; i += concurrency) {
+    const batch = jobs.slice(i, i + concurrency)
+
+    const batchResults = await Promise.all(
+      batch.map(async (job) => {
+        const caseStart = Date.now()
+
+        try {
+          // Run the task
+          const output = await task(job.case.input, job.model)
+          const latencyMs = Date.now() - caseStart
+
+          // Run scorers
+          const scores: EvalScore[] = []
+          for (const s of scorers) {
+            try {
+              const score = await s.scorer({
+                input: job.case.input,
+                output,
+                expected: job.case.expected,
+              })
+              scores.push({
+                name: s.name,
+                score: Math.max(0, Math.min(1, score)),
+                description: s.description,
+              })
+            } catch (err) {
+              scores.push({
+                name: s.name,
+                score: 0,
+                description: s.description,
+                metadata: { error: String(err) },
+              })
+            }
+          }
+
+          // Calculate cost
+          const pricing = getModelPricing(job.model.id)
+          // Estimate tokens - rough approximation
+          const estimatedPromptTokens = 100
+          const estimatedCompletionTokens = 200
+          const cost = pricing
+            ? (estimatedPromptTokens * pricing.prompt + estimatedCompletionTokens * pricing.completion) / 1_000_000
+            : 0
+
+          const avgScore = scores.length > 0
+            ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length
+            : 0
+
+          const symbol = avgScore >= 0.8 ? '✓' : avgScore >= 0.5 ? '~' : '✗'
+          console.log(`   ${symbol} ${job.model.name} | ${job.case.name} | ${(avgScore * 100).toFixed(0)}% | ${latencyMs}ms`)
+
+          return {
+            model: job.model,
+            case: job.case,
+            output,
+            scores,
+            latencyMs,
+            cost,
+          }
+        } catch (err) {
+          console.log(`   ✗ ${job.model.name} | ${job.case.name} | ERROR: ${err}`)
+
+          return {
+            model: job.model,
+            case: job.case,
+            output: null as unknown as TOutput,
+            scores: scorers.map(s => ({ name: s.name, score: 0 })),
+            latencyMs: Date.now() - caseStart,
+            cost: 0,
+            error: String(err),
+          }
+        }
+      })
+    )
+
+    results.push(...batchResults)
+  }
+
+  // Calculate summary
+  const totalTime = Date.now() - startTime
+  const totalCost = results.reduce((sum, r) => sum + r.cost, 0)
+  const allScores = results.flatMap(r => r.scores.map(s => s.score))
+  const avgScore = allScores.length > 0
+    ? allScores.reduce((a, b) => a + b, 0) / allScores.length
+    : 0
+
+  // Group by model
+  const byModel: Record<string, { avgScore: number; count: number }> = {}
+  for (const result of results) {
+    const modelKey = result.model.id
+    if (!byModel[modelKey]) {
+      byModel[modelKey] = { avgScore: 0, count: 0 }
+    }
+    const resultAvg = result.scores.reduce((sum, s) => sum + s.score, 0) / result.scores.length
+    byModel[modelKey].avgScore += resultAvg
+    byModel[modelKey].count++
+  }
+  for (const key of Object.keys(byModel)) {
+    const entry = byModel[key]
+    if (entry) {
+      entry.avgScore /= entry.count
+    }
+  }
+
+  console.log('')
+  console.log(`📊 Results:`)
+  console.log(`   Overall: ${(avgScore * 100).toFixed(1)}%`)
+  console.log(`   Time: ${(totalTime / 1000).toFixed(1)}s`)
+  console.log(`   Cost: $${totalCost.toFixed(4)}`)
+  console.log('')
+  console.log('   By Model:')
+  for (const [modelId, stats] of Object.entries(byModel)) {
+    console.log(`   - ${modelId}: ${(stats.avgScore * 100).toFixed(1)}%`)
+  }
+
+  return {
+    name,
+    results,
+    avgScore,
+    byModel,
+    totalCost,
+    totalTime,
+  }
+}
+
+// Re-export helpers
+export { generateObject, generateText, schema }

package/src/generate.ts

@@ -0,0 +1,245 @@
+/**
+ * AI Generation functions with automatic model resolution and routing
+ *
+ * Wraps AI SDK generateObject and generateText with smart model routing:
+ * - Simple aliases: 'opus', 'sonnet', 'gpt-4o'
+ * - Full IDs: 'anthropic/claude-sonnet-4.5'
+ * - Auto-routes to native SDKs for openai/anthropic/google
+ *
+ * @packageDocumentation
+ */
+
+import {
+  generateObject as sdkGenerateObject,
+  generateText as sdkGenerateText,
+  streamObject as sdkStreamObject,
+  streamText as sdkStreamText,
+  type GenerateObjectResult,
+  type GenerateTextResult,
+  type StreamObjectResult,
+  type StreamTextResult,
+  type LanguageModel
+} from 'ai'
+import { schema as convertSchema, type SimpleSchema } from './schema.js'
+import type { ZodTypeAny } from 'zod'
+
+type ModelArg = string | LanguageModel
+type SchemaArg = ZodTypeAny | SimpleSchema
+
+interface GenerateObjectOptions<T> {
+  model: ModelArg
+  schema: T
+  prompt?: string
+  messages?: Array<{ role: 'user' | 'assistant' | 'system'; content: string }>
+  system?: string
+  mode?: 'auto' | 'json' | 'tool'
+  maxTokens?: number
+  temperature?: number
+  topP?: number
+  topK?: number
+  presencePenalty?: number
+  frequencyPenalty?: number
+  seed?: number
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+  experimental_telemetry?: { isEnabled?: boolean; functionId?: string; metadata?: Record<string, string> }
+}
+
+interface GenerateTextOptions {
+  model: ModelArg
+  prompt?: string
+  messages?: Array<{ role: 'user' | 'assistant' | 'system'; content: string }>
+  system?: string
+  maxTokens?: number
+  temperature?: number
+  topP?: number
+  topK?: number
+  presencePenalty?: number
+  frequencyPenalty?: number
+  seed?: number
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+  tools?: Record<string, unknown>
+  toolChoice?: 'auto' | 'none' | 'required' | { type: 'tool'; toolName: string }
+  maxSteps?: number
+  experimental_telemetry?: { isEnabled?: boolean; functionId?: string; metadata?: Record<string, string> }
+}
+
+/**
+ * Resolve model string to LanguageModel instance
+ * Uses ai-providers for model routing with Cloudflare AI Gateway support
+ */
+async function resolveModel(modelArg: ModelArg): Promise<LanguageModel> {
+  // Already a LanguageModel instance
+  if (typeof modelArg !== 'string') {
+    return modelArg
+  }
+
+  // Use ai-providers for model resolution
+  const { model } = await import('ai-providers')
+  return model(modelArg)
+}
+
+/**
+ * Check if value is a Zod schema
+ */
+function isZodSchema(value: unknown): value is ZodTypeAny {
+  return value !== null &&
+    typeof value === 'object' &&
+    '_def' in value &&
+    'parse' in value
+}
+
+/**
+ * Convert schema to Zod if needed
+ */
+function resolveSchema(schemaArg: SchemaArg): ZodTypeAny {
+  if (isZodSchema(schemaArg)) {
+    return schemaArg
+  }
+  return convertSchema(schemaArg as SimpleSchema)
+}
+
+/**
+ * Generate a typed object from a prompt using AI
+ *
+ * Automatically resolves model aliases and routes to the best provider.
+ * Supports both Zod schemas and simplified schema syntax.
+ *
+ * @example
+ * ```ts
+ * import { generateObject } from 'ai-functions'
+ *
+ * // Simplified schema syntax
+ * const { object } = await generateObject({
+ *   model: 'sonnet',
+ *   schema: {
+ *     recipe: {
+ *       name: 'What is the recipe name?',
+ *       type: 'food | drink | dessert',
+ *       ingredients: ['List all ingredients'],
+ *       steps: ['List all cooking steps'],
+ *     },
+ *   },
+ *   prompt: 'Generate a lasagna recipe.',
+ * })
+ *
+ * // Zod schema also works
+ * import { z } from 'zod'
+ * const { object } = await generateObject({
+ *   model: 'sonnet',
+ *   schema: z.object({
+ *     name: z.string(),
+ *     ingredients: z.array(z.string()),
+ *   }),
+ *   prompt: 'Generate a lasagna recipe.',
+ * })
+ * ```
+ */
+export async function generateObject<T>(
+  options: GenerateObjectOptions<T>
+): Promise<GenerateObjectResult<T>> {
+  const model = await resolveModel(options.model)
+  const schema = resolveSchema(options.schema as SchemaArg)
+  // Use 'as any' to handle AI SDK v4 API variance
+  return sdkGenerateObject({
+    ...options,
+    model,
+    schema,
+    output: 'object'
+  } as any) as Promise<GenerateObjectResult<T>>
+}
+
+/**
+ * Generate text from a prompt using AI
+ *
+ * Automatically resolves model aliases and routes to the best provider.
+ *
+ * @example
+ * ```ts
+ * import { generateText } from 'ai-functions'
+ *
+ * const { text } = await generateText({
+ *   model: 'opus',  // → anthropic/claude-opus-4.5
+ *   prompt: 'Write a haiku about programming.',
+ * })
+ *
+ * // With tools
+ * const { text, toolResults } = await generateText({
+ *   model: 'gpt-4o',  // → openai/gpt-4o
+ *   prompt: 'What is the weather in San Francisco?',
+ *   tools: { ... },
+ *   maxSteps: 5,
+ * })
+ * ```
+ */
+export async function generateText(
+  options: GenerateTextOptions
+): Promise<Awaited<ReturnType<typeof sdkGenerateText>>> {
+  const model = await resolveModel(options.model)
+  return sdkGenerateText({
+    ...options,
+    model
+  } as Parameters<typeof sdkGenerateText>[0])
+}
+
+/**
+ * Stream a typed object from a prompt using AI
+ *
+ * @example
+ * ```ts
+ * import { streamObject } from 'ai-functions'
+ *
+ * const { partialObjectStream } = streamObject({
+ *   model: 'sonnet',
+ *   schema: { story: 'Write a creative story' },
+ *   prompt: 'Write a short story.',
+ * })
+ *
+ * for await (const partial of partialObjectStream) {
+ *   console.log(partial.story)
+ * }
+ * ```
+ */
+export async function streamObject<T>(
+  options: GenerateObjectOptions<T>
+): Promise<StreamObjectResult<T, T, never>> {
+  const model = await resolveModel(options.model)
+  const schema = resolveSchema(options.schema as SchemaArg)
+  // Use 'as any' to handle AI SDK API variance
+  return sdkStreamObject({
+    ...options,
+    model,
+    schema,
+    output: 'object'
+  } as any) as unknown as StreamObjectResult<T, T, never>
+}
+
+/**
+ * Stream text from a prompt using AI
+ *
+ * @example
+ * ```ts
+ * import { streamText } from 'ai-functions'
+ *
+ * const { textStream } = streamText({
+ *   model: 'gemini',  // → google/gemini-2.5-flash
+ *   prompt: 'Explain quantum computing.',
+ * })
+ *
+ * for await (const chunk of textStream) {
+ *   process.stdout.write(chunk)
+ * }
+ * ```
+ */
+export async function streamText(
+  options: GenerateTextOptions
+): Promise<ReturnType<typeof sdkStreamText>> {
+  const model = await resolveModel(options.model)
+  return sdkStreamText({
+    ...options,
+    model
+  } as Parameters<typeof sdkStreamText>[0])
+}