ai-functions 0.3.0 → 0.4.0

package/evals/marketing.eval.ts

@@ -0,0 +1,370 @@
+#!/usr/bin/env npx tsx
+/**
+ * Marketing Copy Eval with LLM-as-Judge ELO Ranking
+ *
+ * Generates marketing copy (title, description, hero headline/subhead, CTAs)
+ * and uses pairwise comparison with an LLM judge to create ELO rankings.
+ *
+ * Usage:
+ *   npx tsx evals/marketing.eval.ts
+ *   npx tsx evals/marketing.eval.ts --judge=opus        # Use specific judge model
+ *   npx tsx evals/marketing.eval.ts --judge=haiku       # Test cheap judge
+ *   npx tsx evals/marketing.eval.ts --judge=flash       # Test fast judge
+ *   npx tsx evals/marketing.eval.ts --all               # Run all tiers
+ *   npx tsx evals/marketing.eval.ts --all --judge=haiku # All tiers + cheap judge
+ */
+
+// Load .env from project root
+import { config } from 'dotenv'
+import { resolve } from 'path'
+config({ path: resolve(import.meta.dirname, '../../../.env') })
+
+import { generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { EVAL_MODELS, type EvalModel, type ModelTier } from '../src/eval/models.js'
+
+// Parse CLI args
+const args = process.argv.slice(2)
+const judgeArg = args.find(a => a.startsWith('--judge='))
+const JUDGE_MODEL = judgeArg ? judgeArg.split('=')[1] : 'sonnet'
+const runAll = args.includes('--all')
+
+const tiers: ModelTier[] = runAll ? ['best', 'fast', 'cheap'] : ['fast']
+
+// Marketing copy schema
+const marketingCopySchema = schema({
+  title: 'Product/page title (5-10 words)',
+  description: 'Meta description for SEO (150-160 characters)',
+  hero: {
+    headline: 'Hero headline (5-8 words, compelling)',
+    subhead: 'Supporting subheadline (10-20 words)',
+    primaryCTA: 'Primary call-to-action button text (2-4 words)',
+    secondaryCTA: 'Secondary call-to-action link text (3-6 words)',
+  },
+})
+
+// Test cases - different product/service scenarios
+const TEST_CASES = [
+  {
+    name: 'SaaS Analytics Platform',
+    prompt: `Create marketing copy for a B2B SaaS analytics platform called "InsightFlow" that helps companies understand their customer behavior with AI-powered insights. Target audience: Product managers and growth teams at mid-size tech companies.`,
+  },
+  {
+    name: 'E-commerce Fashion Brand',
+    prompt: `Create marketing copy for a sustainable fashion e-commerce brand called "EcoThread" that sells organic, ethically-made clothing. Target audience: Environmentally conscious millennials aged 25-35.`,
+  },
+  {
+    name: 'Developer Tool',
+    prompt: `Create marketing copy for a CLI tool called "DeployFast" that simplifies Kubernetes deployments with one-command deploys. Target audience: DevOps engineers and backend developers.`,
+  },
+  {
+    name: 'Mobile Fitness App',
+    prompt: `Create marketing copy for a fitness app called "FitPulse" that uses AI to create personalized workout plans and tracks progress with smart watch integration. Target audience: Busy professionals aged 30-45.`,
+  },
+]
+
+interface MarketingCopy {
+  title: string
+  description: string
+  hero: {
+    headline: string
+    subhead: string
+    primaryCTA: string
+    secondaryCTA: string
+  }
+}
+
+interface GeneratedCopy {
+  model: EvalModel
+  testCase: typeof TEST_CASES[0]
+  copy: MarketingCopy
+  latencyMs: number
+}
+
+interface ELORating {
+  modelId: string
+  modelName: string
+  rating: number
+  wins: number
+  losses: number
+  draws: number
+}
+
+// ELO calculation
+const K_FACTOR = 32
+const INITIAL_ELO = 1500
+
+function calculateEloChange(ratingA: number, ratingB: number, scoreA: number): { deltaA: number; deltaB: number } {
+  const expectedA = 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400))
+  const expectedB = 1 - expectedA
+
+  const deltaA = K_FACTOR * (scoreA - expectedA)
+  const deltaB = K_FACTOR * ((1 - scoreA) - expectedB)
+
+  return { deltaA, deltaB }
+}
+
+// LLM Judge for pairwise comparison
+async function judgePair(
+  copyA: MarketingCopy,
+  copyB: MarketingCopy,
+  testCase: typeof TEST_CASES[0],
+  judgeModel: string
+): Promise<'A' | 'B' | 'TIE'> {
+  const prompt = `You are an expert marketing copywriter and brand strategist. Compare these two marketing copy options for: ${testCase.name}
+
+Context: ${testCase.prompt}
+
+=== OPTION A ===
+Title: ${copyA.title}
+Description: ${copyA.description}
+Hero Headline: ${copyA.hero.headline}
+Hero Subhead: ${copyA.hero.subhead}
+Primary CTA: ${copyA.hero.primaryCTA}
+Secondary CTA: ${copyA.hero.secondaryCTA}
+
+=== OPTION B ===
+Title: ${copyB.title}
+Description: ${copyB.description}
+Hero Headline: ${copyB.hero.headline}
+Hero Subhead: ${copyB.hero.subhead}
+Primary CTA: ${copyB.hero.primaryCTA}
+Secondary CTA: ${copyB.hero.secondaryCTA}
+
+Evaluate based on:
+1. Clarity and impact of messaging
+2. Target audience alignment
+3. Emotional appeal and persuasiveness
+4. CTA effectiveness
+5. Overall brand voice consistency
+
+Which option is better? Answer A, B, or TIE if they're roughly equal.`
+
+  try {
+    const { object } = await generateObject({
+      model: judgeModel,
+      schema: schema({
+        reasoning: 'Brief explanation of your judgment (2-3 sentences)',
+        winner: 'A | B | TIE',
+      }),
+      prompt,
+      temperature: 0.3,
+    })
+
+    const result = object as { reasoning: string; winner: string }
+    const winner = result.winner.toUpperCase().trim()
+
+    if (winner === 'A' || winner === 'B' || winner === 'TIE') {
+      return winner
+    }
+    return 'TIE'
+  } catch (err) {
+    console.error(`   Judge error: ${err}`)
+    return 'TIE'
+  }
+}
+
+// Generate marketing copy for a model
+async function generateCopy(model: EvalModel, testCase: typeof TEST_CASES[0]): Promise<GeneratedCopy> {
+  const start = Date.now()
+
+  const { object } = await generateObject({
+    model: model.id,
+    schema: marketingCopySchema,
+    prompt: testCase.prompt,
+    temperature: 0.7,
+  })
+
+  return {
+    model,
+    testCase,
+    copy: object as MarketingCopy,
+    latencyMs: Date.now() - start,
+  }
+}
+
+// Run pairwise comparisons and calculate ELO
+async function runEloTournament(
+  copies: GeneratedCopy[],
+  judgeModel: string
+): Promise<ELORating[]> {
+  // Initialize ELO ratings
+  const ratings: Map<string, ELORating> = new Map()
+
+  for (const copy of copies) {
+    if (!ratings.has(copy.model.id)) {
+      ratings.set(copy.model.id, {
+        modelId: copy.model.id,
+        modelName: copy.model.name,
+        rating: INITIAL_ELO,
+        wins: 0,
+        losses: 0,
+        draws: 0,
+      })
+    }
+  }
+
+  // Group copies by test case
+  const byTestCase = new Map<string, GeneratedCopy[]>()
+  for (const copy of copies) {
+    const key = copy.testCase.name
+    if (!byTestCase.has(key)) {
+      byTestCase.set(key, [])
+    }
+    byTestCase.get(key)!.push(copy)
+  }
+
+  console.log(`\n⚖️  Running pairwise comparisons with ${JUDGE_MODEL} as judge...\n`)
+
+  let comparisonCount = 0
+  const totalComparisons = Array.from(byTestCase.values()).reduce(
+    (sum, copies) => sum + (copies.length * (copies.length - 1)) / 2,
+    0
+  )
+
+  // Run pairwise comparisons within each test case
+  for (const [testCaseName, testCaseCopies] of byTestCase) {
+    console.log(`   📝 ${testCaseName}:`)
+
+    for (let i = 0; i < testCaseCopies.length; i++) {
+      for (let j = i + 1; j < testCaseCopies.length; j++) {
+        const copyA = testCaseCopies[i]
+        const copyB = testCaseCopies[j]
+
+        comparisonCount++
+        process.stdout.write(`      ${comparisonCount}/${totalComparisons} ${copyA.model.name} vs ${copyB.model.name}... `)
+
+        const winner = await judgePair(copyA.copy, copyB.copy, copyA.testCase, judgeModel)
+
+        const ratingA = ratings.get(copyA.model.id)!
+        const ratingB = ratings.get(copyB.model.id)!
+
+        let scoreA: number
+        if (winner === 'A') {
+          scoreA = 1
+          ratingA.wins++
+          ratingB.losses++
+          console.log(`${copyA.model.name} wins`)
+        } else if (winner === 'B') {
+          scoreA = 0
+          ratingA.losses++
+          ratingB.wins++
+          console.log(`${copyB.model.name} wins`)
+        } else {
+          scoreA = 0.5
+          ratingA.draws++
+          ratingB.draws++
+          console.log(`TIE`)
+        }
+
+        const { deltaA, deltaB } = calculateEloChange(ratingA.rating, ratingB.rating, scoreA)
+        ratingA.rating += deltaA
+        ratingB.rating += deltaB
+      }
+    }
+  }
+
+  // Sort by ELO rating
+  return Array.from(ratings.values()).sort((a, b) => b.rating - a.rating)
+}
+
+// Main
+async function main() {
+  console.log('╔════════════════════════════════════════════════════════════════╗')
+  console.log('║              Marketing Copy Eval (LLM-as-Judge)                ║')
+  console.log('╚════════════════════════════════════════════════════════════════╝')
+  console.log('')
+  console.log(`Judge Model: ${JUDGE_MODEL}`)
+  console.log(`Tiers: ${tiers.join(', ')}`)
+
+  // Get models to test
+  const models = EVAL_MODELS.filter(m => tiers.includes(m.tier))
+  console.log(`Models: ${models.map(m => m.name).join(', ')}`)
+  console.log(`Test Cases: ${TEST_CASES.length}`)
+  console.log('')
+
+  // Generate copy from each model for each test case
+  console.log('🎨 Generating marketing copy...\n')
+
+  const allCopies: GeneratedCopy[] = []
+  const startTime = Date.now()
+
+  for (const testCase of TEST_CASES) {
+    console.log(`   📦 ${testCase.name}:`)
+
+    const jobs = models.map(async model => {
+      try {
+        const copy = await generateCopy(model, testCase)
+        console.log(`      ✓ ${model.name} (${copy.latencyMs}ms)`)
+        return copy
+      } catch (err) {
+        console.log(`      ✗ ${model.name}: ${err}`)
+        return null
+      }
+    })
+
+    const results = await Promise.all(jobs)
+    allCopies.push(...results.filter((r): r is GeneratedCopy => r !== null))
+  }
+
+  const generateTime = Date.now() - startTime
+  console.log(`\n   Generated ${allCopies.length} copies in ${(generateTime / 1000).toFixed(1)}s`)
+
+  // Run ELO tournament
+  const tournamentStart = Date.now()
+  const eloRatings = await runEloTournament(allCopies, JUDGE_MODEL)
+  const tournamentTime = Date.now() - tournamentStart
+
+  // Display results
+  console.log('')
+  console.log('╔════════════════════════════════════════════════════════════════╗')
+  console.log('║                     ELO Rankings                               ║')
+  console.log('╚════════════════════════════════════════════════════════════════╝')
+  console.log('')
+  console.log('   Rank | Model                  | ELO    | W   | L   | D   |')
+  console.log('   -----|------------------------|--------|-----|-----|-----|')
+
+  eloRatings.forEach((rating, idx) => {
+    const rank = `${idx + 1}`.padStart(4)
+    const name = rating.modelName.padEnd(22)
+    const elo = Math.round(rating.rating).toString().padStart(6)
+    const wins = rating.wins.toString().padStart(3)
+    const losses = rating.losses.toString().padStart(3)
+    const draws = rating.draws.toString().padStart(3)
+    console.log(`   ${rank} | ${name} | ${elo} | ${wins} | ${losses} | ${draws} |`)
+  })
+
+  console.log('')
+  console.log(`   Judge: ${JUDGE_MODEL}`)
+  console.log(`   Generation Time: ${(generateTime / 1000).toFixed(1)}s`)
+  console.log(`   Tournament Time: ${(tournamentTime / 1000).toFixed(1)}s`)
+  console.log(`   Total Time: ${((generateTime + tournamentTime) / 1000).toFixed(1)}s`)
+
+  // Show sample outputs from top 3
+  console.log('')
+  console.log('╔════════════════════════════════════════════════════════════════╗')
+  console.log('║                  Sample Outputs (Top 3)                        ║')
+  console.log('╚════════════════════════════════════════════════════════════════╝')
+
+  const top3Models = eloRatings.slice(0, 3).map(r => r.modelId)
+  const sampleTestCase = TEST_CASES[0]
+
+  for (const modelId of top3Models) {
+    const copy = allCopies.find(c => c.model.id === modelId && c.testCase.name === sampleTestCase.name)
+    if (copy) {
+      const rank = eloRatings.findIndex(r => r.modelId === modelId) + 1
+      console.log(`\n   #${rank} ${copy.model.name} (${sampleTestCase.name}):`)
+      console.log(`   ─────────────────────────────────────────`)
+      console.log(`   Title: ${copy.copy.title}`)
+      console.log(`   Description: ${copy.copy.description}`)
+      console.log(`   Headline: ${copy.copy.hero.headline}`)
+      console.log(`   Subhead: ${copy.copy.hero.subhead}`)
+      console.log(`   Primary CTA: [${copy.copy.hero.primaryCTA}]`)
+      console.log(`   Secondary CTA: ${copy.copy.hero.secondaryCTA}`)
+    }
+  }
+
+  console.log('')
+}
+
+main().catch(console.error)

package/evals/math.eval.ts

@@ -0,0 +1,94 @@
+/**
+ * Math Eval
+ *
+ * Tests model mathematical reasoning from simple arithmetic
+ * to word problems.
+ */
+
+import { evalite } from 'evalite'
+import { generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, type EvalModel } from '../src/eval/models.js'
+
+// Math test cases
+const TEST_CASES = [
+  // Arithmetic
+  { problem: 'What is 15 + 27?', expected: 42, difficulty: 'easy' },
+  { problem: 'What is 144 / 12?', expected: 12, difficulty: 'easy' },
+  { problem: 'What is 7 * 8?', expected: 56, difficulty: 'easy' },
+
+  // Word problems
+  { problem: 'A store sells 45 apples at $2 each. What is the total revenue?', expected: 90, difficulty: 'medium' },
+  { problem: 'A train travels 240 miles in 4 hours. What is the average speed in mph?', expected: 60, difficulty: 'medium' },
+
+  // Multi-step
+  { problem: 'A company has 120 employees. 40% work in engineering, and 25% of engineers are senior. How many senior engineers?', expected: 12, difficulty: 'hard' },
+]
+
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Math', {
+  data: TEST_CASES.map(tc => ({ input: tc, expected: tc.expected })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    const { object, usage } = await generateObject({
+      model: model.id,
+      schema: schema({
+        answer: 'The numeric answer (number)',
+        reasoning: 'Step by step reasoning',
+      }),
+      prompt: `Solve this math problem:\n\n${input.problem}`,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    return {
+      answer: object.answer,
+      reasoning: object.reasoning,
+      expected: input.expected,
+      problem: input.problem,
+      difficulty: input.difficulty,
+      modelId: model.id,
+      modelName: model.name,
+      latencyMs,
+      usage,
+    }
+  },
+
+  scorers: [
+    // Exact answer
+    {
+      name: 'Correct Answer',
+      description: 'Whether the numeric answer is correct',
+      scorer: ({ output, expected }) => {
+        const answer = output.answer as number
+        const exp = expected as number
+        // Allow small floating point tolerance
+        return { score: Math.abs(answer - exp) < 0.01 ? 1 : 0 }
+      },
+    },
+
+    // Shows reasoning
+    {
+      name: 'Shows Work',
+      description: 'Whether model explains reasoning',
+      scorer: ({ output }) => {
+        const reasoning = output.reasoning as string
+        if (!reasoning || reasoning.length < 20) return { score: 0.2 }
+        if (reasoning.length > 50) return { score: 1 }
+        return { score: 0.6 }
+      },
+    },
+  ],
+
+  columns: ({ output, expected }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Difficulty', value: output.difficulty },
+    { label: 'Expected', value: expected },
+    { label: 'Got', value: output.answer },
+    { label: 'Correct', value: Math.abs((output.answer as number) - (expected as number)) < 0.01 ? 'Yes' : 'No' },
+  ],
+})

package/evals/run-evals.ts

@@ -0,0 +1,166 @@
+#!/usr/bin/env npx tsx
+/**
+ * Run AI Functions Eval Suite
+ *
+ * Usage:
+ *   npx tsx evals/run-evals.ts [--fast] [--all]
+ *
+ * Options:
+ *   --fast    Only run fast-tier models (default)
+ *   --all     Run all models
+ *   --math    Run only math eval
+ *   --class   Run only classification eval
+ */
+
+import { runEval, generateObject, generateText, schema } from '../src/eval/runner.js'
+import type { EvalModel, ModelTier } from '../src/eval/models.js'
+
+// Parse CLI args
+const args = process.argv.slice(2)
+const runAll = args.includes('--all')
+const runMath = args.includes('--math')
+const runClass = args.includes('--class')
+const runSingle = runMath || runClass
+
+const tiers: ModelTier[] = runAll ? ['best', 'fast', 'cheap'] : ['fast']
+
+console.log('╔════════════════════════════════════════════════════════════════╗')
+console.log('║                   AI Functions Eval Suite                      ║')
+console.log('╚════════════════════════════════════════════════════════════════╝')
+console.log('')
+console.log(`Tiers: ${tiers.join(', ')}`)
+
+// Math eval
+async function runMathEval() {
+  const cases = [
+    { name: 'Simple addition', input: { problem: 'What is 15 + 27?' }, expected: 42 },
+    { name: 'Division', input: { problem: 'What is 144 / 12?' }, expected: 12 },
+    { name: 'Multiplication', input: { problem: 'What is 7 * 8?' }, expected: 56 },
+    { name: 'Word problem', input: { problem: 'A store sells 45 apples at $2 each. What is the total revenue?' }, expected: 90 },
+    { name: 'Multi-step', input: { problem: 'A company has 120 employees. 40% work in engineering, and 25% of engineers are senior. How many senior engineers?' }, expected: 12 },
+  ]
+
+  return runEval({
+    name: 'Math',
+    cases,
+    tiers,
+    task: async (input, model) => {
+      const { object } = await generateObject({
+        model: model.id,
+        schema: schema({
+          answer: 'The numeric answer (number)',
+          reasoning: 'Step by step reasoning',
+        }),
+        prompt: `Solve this math problem:\n\n${input.problem}`,
+      })
+      return object
+    },
+    scorers: [
+      {
+        name: 'Correct Answer',
+        description: 'Whether the numeric answer is correct',
+        scorer: ({ output, expected }) => {
+          const answer = (output as { answer: number }).answer
+          const exp = expected as number
+          return Math.abs(answer - exp) < 0.01 ? 1 : 0
+        },
+      },
+      {
+        name: 'Shows Work',
+        description: 'Whether model explains reasoning',
+        scorer: ({ output }) => {
+          const reasoning = (output as { reasoning: string }).reasoning
+          if (!reasoning || reasoning.length < 20) return 0.2
+          if (reasoning.length > 50) return 1
+          return 0.6
+        },
+      },
+    ],
+  })
+}
+
+// Classification eval
+async function runClassificationEval() {
+  const cases = [
+    { name: 'Positive sentiment', input: { text: 'This product exceeded my expectations!', options: ['positive', 'negative', 'neutral'] }, expected: 'positive' },
+    { name: 'Negative sentiment', input: { text: 'The delivery was late and packaging damaged.', options: ['positive', 'negative', 'neutral'] }, expected: 'negative' },
+    { name: 'Neutral sentiment', input: { text: 'The product arrived as described.', options: ['positive', 'negative', 'neutral'] }, expected: 'neutral' },
+    { name: 'Account ticket', input: { text: 'I need to reset my password', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'account' },
+    { name: 'Billing ticket', input: { text: 'When will my refund be processed?', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'billing' },
+    { name: 'Technical ticket', input: { text: 'The app crashes when uploading images', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'technical' },
+  ]
+
+  return runEval({
+    name: 'Classification',
+    cases,
+    tiers,
+    task: async (input, model) => {
+      const enumStr = input.options.join(' | ')
+      const { object } = await generateObject({
+        model: model.id,
+        schema: schema({
+          category: enumStr,
+          confidence: 'Confidence 0-1 (number)',
+        }),
+        prompt: `Classify this text into one of: ${input.options.join(', ')}\n\nText: "${input.text}"`,
+      })
+      return object
+    },
+    scorers: [
+      {
+        name: 'Accuracy',
+        description: 'Whether classification is correct',
+        scorer: ({ output, expected }) => {
+          const predicted = (output as { category: string }).category
+          return predicted === expected ? 1 : 0
+        },
+      },
+      {
+        name: 'Valid Category',
+        description: 'Whether output is a valid option',
+        scorer: ({ input, output }) => {
+          const predicted = (output as { category: string }).category
+          const options = (input as { options: string[] }).options
+          return options.includes(predicted) ? 1 : 0
+        },
+      },
+    ],
+  })
+}
+
+// Run evals
+async function main() {
+  const results = []
+
+  if (!runSingle || runMath) {
+    results.push(await runMathEval())
+  }
+
+  if (!runSingle || runClass) {
+    results.push(await runClassificationEval())
+  }
+
+  // Overall summary
+  console.log('')
+  console.log('╔════════════════════════════════════════════════════════════════╗')
+  console.log('║                          Summary                               ║')
+  console.log('╚════════════════════════════════════════════════════════════════╝')
+
+  let totalScore = 0
+  let totalCost = 0
+  let totalTime = 0
+
+  for (const result of results) {
+    console.log(`\n${result.name}: ${(result.avgScore * 100).toFixed(1)}%`)
+    totalScore += result.avgScore
+    totalCost += result.totalCost
+    totalTime += result.totalTime
+  }
+
+  console.log('')
+  console.log(`Overall: ${((totalScore / results.length) * 100).toFixed(1)}%`)
+  console.log(`Total Cost: $${totalCost.toFixed(4)}`)
+  console.log(`Total Time: ${(totalTime / 1000).toFixed(1)}s`)
+}
+
+main().catch(console.error)