ai-functions 0.2.19 → 0.4.0

package/evals/math.eval.ts

@@ -0,0 +1,94 @@
+/**
+ * Math Eval
+ *
+ * Tests model mathematical reasoning from simple arithmetic
+ * to word problems.
+ */
+
+import { evalite } from 'evalite'
+import { generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, type EvalModel } from '../src/eval/models.js'
+
+// Math test cases
+const TEST_CASES = [
+  // Arithmetic
+  { problem: 'What is 15 + 27?', expected: 42, difficulty: 'easy' },
+  { problem: 'What is 144 / 12?', expected: 12, difficulty: 'easy' },
+  { problem: 'What is 7 * 8?', expected: 56, difficulty: 'easy' },
+
+  // Word problems
+  { problem: 'A store sells 45 apples at $2 each. What is the total revenue?', expected: 90, difficulty: 'medium' },
+  { problem: 'A train travels 240 miles in 4 hours. What is the average speed in mph?', expected: 60, difficulty: 'medium' },
+
+  // Multi-step
+  { problem: 'A company has 120 employees. 40% work in engineering, and 25% of engineers are senior. How many senior engineers?', expected: 12, difficulty: 'hard' },
+]
+
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Math', {
+  data: TEST_CASES.map(tc => ({ input: tc, expected: tc.expected })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    const { object, usage } = await generateObject({
+      model: model.id,
+      schema: schema({
+        answer: 'The numeric answer (number)',
+        reasoning: 'Step by step reasoning',
+      }),
+      prompt: `Solve this math problem:\n\n${input.problem}`,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    return {
+      answer: object.answer,
+      reasoning: object.reasoning,
+      expected: input.expected,
+      problem: input.problem,
+      difficulty: input.difficulty,
+      modelId: model.id,
+      modelName: model.name,
+      latencyMs,
+      usage,
+    }
+  },
+
+  scorers: [
+    // Exact answer
+    {
+      name: 'Correct Answer',
+      description: 'Whether the numeric answer is correct',
+      scorer: ({ output, expected }) => {
+        const answer = output.answer as number
+        const exp = expected as number
+        // Allow small floating point tolerance
+        return { score: Math.abs(answer - exp) < 0.01 ? 1 : 0 }
+      },
+    },
+
+    // Shows reasoning
+    {
+      name: 'Shows Work',
+      description: 'Whether model explains reasoning',
+      scorer: ({ output }) => {
+        const reasoning = output.reasoning as string
+        if (!reasoning || reasoning.length < 20) return { score: 0.2 }
+        if (reasoning.length > 50) return { score: 1 }
+        return { score: 0.6 }
+      },
+    },
+  ],
+
+  columns: ({ output, expected }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Difficulty', value: output.difficulty },
+    { label: 'Expected', value: expected },
+    { label: 'Got', value: output.answer },
+    { label: 'Correct', value: Math.abs((output.answer as number) - (expected as number)) < 0.01 ? 'Yes' : 'No' },
+  ],
+})

package/evals/run-evals.ts

@@ -0,0 +1,166 @@
+#!/usr/bin/env npx tsx
+/**
+ * Run AI Functions Eval Suite
+ *
+ * Usage:
+ *   npx tsx evals/run-evals.ts [--fast] [--all]
+ *
+ * Options:
+ *   --fast    Only run fast-tier models (default)
+ *   --all     Run all models
+ *   --math    Run only math eval
+ *   --class   Run only classification eval
+ */
+
+import { runEval, generateObject, generateText, schema } from '../src/eval/runner.js'
+import type { EvalModel, ModelTier } from '../src/eval/models.js'
+
+// Parse CLI args
+const args = process.argv.slice(2)
+const runAll = args.includes('--all')
+const runMath = args.includes('--math')
+const runClass = args.includes('--class')
+const runSingle = runMath || runClass
+
+const tiers: ModelTier[] = runAll ? ['best', 'fast', 'cheap'] : ['fast']
+
+console.log('╔════════════════════════════════════════════════════════════════╗')
+console.log('║                   AI Functions Eval Suite                      ║')
+console.log('╚════════════════════════════════════════════════════════════════╝')
+console.log('')
+console.log(`Tiers: ${tiers.join(', ')}`)
+
+// Math eval
+async function runMathEval() {
+  const cases = [
+    { name: 'Simple addition', input: { problem: 'What is 15 + 27?' }, expected: 42 },
+    { name: 'Division', input: { problem: 'What is 144 / 12?' }, expected: 12 },
+    { name: 'Multiplication', input: { problem: 'What is 7 * 8?' }, expected: 56 },
+    { name: 'Word problem', input: { problem: 'A store sells 45 apples at $2 each. What is the total revenue?' }, expected: 90 },
+    { name: 'Multi-step', input: { problem: 'A company has 120 employees. 40% work in engineering, and 25% of engineers are senior. How many senior engineers?' }, expected: 12 },
+  ]
+
+  return runEval({
+    name: 'Math',
+    cases,
+    tiers,
+    task: async (input, model) => {
+      const { object } = await generateObject({
+        model: model.id,
+        schema: schema({
+          answer: 'The numeric answer (number)',
+          reasoning: 'Step by step reasoning',
+        }),
+        prompt: `Solve this math problem:\n\n${input.problem}`,
+      })
+      return object
+    },
+    scorers: [
+      {
+        name: 'Correct Answer',
+        description: 'Whether the numeric answer is correct',
+        scorer: ({ output, expected }) => {
+          const answer = (output as { answer: number }).answer
+          const exp = expected as number
+          return Math.abs(answer - exp) < 0.01 ? 1 : 0
+        },
+      },
+      {
+        name: 'Shows Work',
+        description: 'Whether model explains reasoning',
+        scorer: ({ output }) => {
+          const reasoning = (output as { reasoning: string }).reasoning
+          if (!reasoning || reasoning.length < 20) return 0.2
+          if (reasoning.length > 50) return 1
+          return 0.6
+        },
+      },
+    ],
+  })
+}
+
+// Classification eval
+async function runClassificationEval() {
+  const cases = [
+    { name: 'Positive sentiment', input: { text: 'This product exceeded my expectations!', options: ['positive', 'negative', 'neutral'] }, expected: 'positive' },
+    { name: 'Negative sentiment', input: { text: 'The delivery was late and packaging damaged.', options: ['positive', 'negative', 'neutral'] }, expected: 'negative' },
+    { name: 'Neutral sentiment', input: { text: 'The product arrived as described.', options: ['positive', 'negative', 'neutral'] }, expected: 'neutral' },
+    { name: 'Account ticket', input: { text: 'I need to reset my password', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'account' },
+    { name: 'Billing ticket', input: { text: 'When will my refund be processed?', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'billing' },
+    { name: 'Technical ticket', input: { text: 'The app crashes when uploading images', options: ['account', 'billing', 'technical', 'shipping'] }, expected: 'technical' },
+  ]
+
+  return runEval({
+    name: 'Classification',
+    cases,
+    tiers,
+    task: async (input, model) => {
+      const enumStr = input.options.join(' | ')
+      const { object } = await generateObject({
+        model: model.id,
+        schema: schema({
+          category: enumStr,
+          confidence: 'Confidence 0-1 (number)',
+        }),
+        prompt: `Classify this text into one of: ${input.options.join(', ')}\n\nText: "${input.text}"`,
+      })
+      return object
+    },
+    scorers: [
+      {
+        name: 'Accuracy',
+        description: 'Whether classification is correct',
+        scorer: ({ output, expected }) => {
+          const predicted = (output as { category: string }).category
+          return predicted === expected ? 1 : 0
+        },
+      },
+      {
+        name: 'Valid Category',
+        description: 'Whether output is a valid option',
+        scorer: ({ input, output }) => {
+          const predicted = (output as { category: string }).category
+          const options = (input as { options: string[] }).options
+          return options.includes(predicted) ? 1 : 0
+        },
+      },
+    ],
+  })
+}
+
+// Run evals
+async function main() {
+  const results = []
+
+  if (!runSingle || runMath) {
+    results.push(await runMathEval())
+  }
+
+  if (!runSingle || runClass) {
+    results.push(await runClassificationEval())
+  }
+
+  // Overall summary
+  console.log('')
+  console.log('╔════════════════════════════════════════════════════════════════╗')
+  console.log('║                          Summary                               ║')
+  console.log('╚════════════════════════════════════════════════════════════════╝')
+
+  let totalScore = 0
+  let totalCost = 0
+  let totalTime = 0
+
+  for (const result of results) {
+    console.log(`\n${result.name}: ${(result.avgScore * 100).toFixed(1)}%`)
+    totalScore += result.avgScore
+    totalCost += result.totalCost
+    totalTime += result.totalTime
+  }
+
+  console.log('')
+  console.log(`Overall: ${((totalScore / results.length) * 100).toFixed(1)}%`)
+  console.log(`Total Cost: $${totalCost.toFixed(4)}`)
+  console.log(`Total Time: ${(totalTime / 1000).toFixed(1)}s`)
+}
+
+main().catch(console.error)

package/evals/structured-output.eval.ts

@@ -0,0 +1,143 @@
+/**
+ * Structured Output Eval
+ *
+ * Tests model ability to generate valid structured JSON output
+ * matching specified schemas across all providers.
+ */
+
+import { evalite } from 'evalite'
+import { generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, getModelPricing, type EvalModel } from '../src/eval/models.js'
+
+// Test cases for structured output
+const TEST_CASES = [
+  {
+    name: 'Simple object',
+    prompt: 'Generate a greeting in French',
+    schema: {
+      greeting: 'A friendly greeting',
+      language: 'The language of the greeting',
+    },
+    expectedTypes: { greeting: 'string', language: 'string' },
+  },
+  {
+    name: 'With numbers',
+    prompt: 'Generate info about Tokyo',
+    schema: {
+      name: 'City name',
+      population: 'Population in millions (number)',
+      area: 'Area in square kilometers (number)',
+    },
+    expectedTypes: { name: 'string', population: 'number', area: 'number' },
+  },
+  {
+    name: 'With arrays',
+    prompt: 'Generate a simple pasta recipe',
+    schema: {
+      title: 'Recipe title',
+      ingredients: ['List of ingredients'],
+      steps: ['Cooking steps'],
+    },
+    expectedTypes: { title: 'string', ingredients: 'array', steps: 'array' },
+  },
+  {
+    name: 'With enum',
+    prompt: 'Analyze sentiment: "I love this product!"',
+    schema: {
+      sentiment: 'positive | negative | neutral',
+      confidence: 'Confidence score 0-1 (number)',
+    },
+    expectedTypes: { sentiment: 'string', confidence: 'number' },
+  },
+  {
+    name: 'Nested object',
+    prompt: 'Generate a fictional person living in Japan',
+    schema: {
+      person: { name: 'Full name', age: 'Age (number)' },
+      address: { city: 'City name', country: 'Country name' },
+    },
+    expectedTypes: { person: 'object', address: 'object' },
+  },
+]
+
+// Test across models - start with fast tier for quick iteration
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Structured Output', {
+  data: TEST_CASES.map(tc => ({ input: tc })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    const { object, usage } = await generateObject({
+      model: model.id,
+      schema: schema(input.schema),
+      prompt: input.prompt,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    // Calculate cost from language-models pricing
+    const pricing = getModelPricing(model.id)
+    const cost = pricing
+      ? ((usage?.promptTokens ?? 0) * pricing.prompt + (usage?.completionTokens ?? 0) * pricing.completion) / 1_000_000
+      : 0
+
+    return {
+      object,
+      expectedTypes: input.expectedTypes,
+      testName: input.name,
+      modelId: model.id,
+      modelName: model.name,
+      provider: model.provider,
+      latencyMs,
+      cost,
+      usage,
+    }
+  },
+
+  scorers: [
+    // Type accuracy
+    {
+      name: 'Type Accuracy',
+      description: 'Whether output fields have correct types',
+      scorer: ({ output }) => {
+        const obj = output.object as Record<string, unknown>
+        const expected = output.expectedTypes as Record<string, string>
+        const fields = Object.keys(expected)
+
+        let correct = 0
+        for (const field of fields) {
+          const val = obj[field]
+          const expectedType = expected[field]
+          const actualType = Array.isArray(val) ? 'array' : typeof val
+
+          if (actualType === expectedType) correct++
+        }
+
+        return { score: correct / fields.length }
+      },
+    },
+
+    // Latency
+    {
+      name: 'Latency',
+      description: 'Response time (target < 3s)',
+      scorer: ({ output }) => {
+        const ms = output.latencyMs as number
+        if (ms < 2000) return { score: 1 }
+        if (ms > 10000) return { score: 0 }
+        return { score: 1 - (ms - 2000) / 8000 }
+      },
+    },
+  ],
+
+  columns: ({ output, scores }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Test', value: output.testName },
+    { label: 'Latency', value: `${output.latencyMs}ms` },
+    { label: 'Cost', value: `$${(output.cost as number).toFixed(6)}` },
+  ],
+})

package/evals/writing.eval.ts

@@ -0,0 +1,117 @@
+/**
+ * Writing Quality Eval (LLM-as-Judge)
+ *
+ * Tests model writing capabilities using LLM-as-judge scoring.
+ * Uses a strong model (sonnet) to judge output quality.
+ */
+
+import { evalite } from 'evalite'
+import { generateText, generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, type EvalModel } from '../src/eval/models.js'
+
+// Use sonnet as the judge model
+const JUDGE_MODEL = 'sonnet'
+
+// Writing test cases
+const TEST_CASES = [
+  {
+    name: 'Professional email',
+    prompt: 'Write a professional email declining a meeting invitation politely.',
+    criteria: ['Polite tone', 'Clear explanation', 'Proper email format'],
+  },
+  {
+    name: 'Product description',
+    prompt: 'Write a product description for wireless earbuds targeting tech-savvy consumers.',
+    criteria: ['Highlights features', 'Compelling language', 'Clear value proposition'],
+  },
+  {
+    name: 'Explanation',
+    prompt: 'Explain how photosynthesis works in simple terms for a high school student.',
+    criteria: ['Accurate content', 'Clear language', 'Logical flow'],
+  },
+]
+
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Writing Quality', {
+  data: TEST_CASES.map(tc => ({ input: tc })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    // Generate the writing
+    const { text, usage } = await generateText({
+      model: model.id,
+      prompt: input.prompt,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    return {
+      text,
+      testName: input.name,
+      criteria: input.criteria,
+      modelId: model.id,
+      modelName: model.name,
+      provider: model.provider,
+      latencyMs,
+      usage,
+    }
+  },
+
+  scorers: [
+    // LLM-as-judge for quality
+    {
+      name: 'Writing Quality',
+      description: 'LLM judge evaluation of writing quality',
+      scorer: async ({ input, output }) => {
+        const { object } = await generateObject({
+          model: JUDGE_MODEL,
+          schema: schema({
+            clarity: 'How clear is the writing? (number 0-1)',
+            engagement: 'How engaging is the content? (number 0-1)',
+            accuracy: 'How well does it meet the criteria? (number 0-1)',
+            reasoning: 'Brief explanation',
+          }),
+          prompt: `Evaluate this writing on a scale of 0-1.
+
+Criteria: ${(input.criteria as string[]).join(', ')}
+
+Writing:
+"""
+${output.text}
+"""`,
+        })
+
+        const avg = ((object.clarity as number) + (object.engagement as number) + (object.accuracy as number)) / 3
+        return {
+          score: avg,
+          metadata: object,
+        }
+      },
+    },
+
+    // Word count check
+    {
+      name: 'Appropriate Length',
+      description: 'Whether output has reasonable length',
+      scorer: ({ output }) => {
+        const words = (output.text as string).split(/\s+/).length
+        if (words < 20) return { score: 0.3, metadata: { words } }
+        if (words > 500) return { score: 0.7, metadata: { words } }
+        return { score: 1, metadata: { words } }
+      },
+    },
+  ],
+
+  columns: ({ output }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Test', value: output.testName },
+    { label: 'Words', value: (output.text as string).split(/\s+/).length },
+    { label: 'Latency', value: `${output.latencyMs}ms` },
+  ],
+
+  trialCount: 2, // Run twice for variance
+})

package/examples/batch-blog-posts.ts

@@ -0,0 +1,160 @@
+/**
+ * Batch Blog Post Generation Example
+ *
+ * This example demonstrates the new IMPLICIT batch processing:
+ *
+ * ```ts
+ * // Configure once (or use environment variables)
+ * configure({ provider: 'openai', model: 'gpt-4o', batchMode: 'auto' })
+ *
+ * // Use naturally - batching is automatic!
+ * const titles = await list`10 blog post titles about startups`
+ * const posts = titles.map(title => write`blog post: # ${title}`)
+ * console.log(await posts) // Batched automatically!
+ * ```
+ *
+ * Environment variables:
+ * - AI_PROVIDER: openai | anthropic | cloudflare | bedrock
+ * - AI_MODEL: model name (e.g., gpt-4o, claude-sonnet-4-20250514)
+ * - AI_BATCH_MODE: auto | immediate | deferred
+ * - AI_BATCH_THRESHOLD: minimum items for auto batch (default: 5)
+ *
+ * @example
+ * ```bash
+ * # Using environment variables
+ * AI_PROVIDER=openai AI_MODEL=gpt-4o npx tsx examples/batch-blog-posts.ts
+ *
+ * # Or with API keys
+ * OPENAI_API_KEY=sk-... npx tsx examples/batch-blog-posts.ts
+ * ```
+ */
+
+import {
+  list,
+  write,
+  configure,
+  withContext,
+  type BatchProvider,
+} from '../src/index.js'
+
+// Import the batch adapter for your provider
+// import '../src/batch/openai.js'
+// import '../src/batch/anthropic.js'
+// import '../src/batch/cloudflare.js'
+// import '../src/batch/bedrock.js'
+
+// For testing, use the memory adapter
+import '../src/batch/memory.js'
+
+async function main() {
+  console.log('\n🚀 Implicit Batch Blog Post Generation\n')
+
+  // ============================================================================
+  // Option 1: Global Configuration (recommended)
+  // ============================================================================
+
+  configure({
+    provider: 'openai',
+    model: 'gpt-4o',
+    batchMode: 'auto',      // 'auto' | 'immediate' | 'deferred'
+    batchThreshold: 5,       // Use batch API when >= 5 items
+  })
+
+  console.log('📝 Step 1: Generate titles (executes immediately)...')
+  const titles = await list`10 blog post titles about building startups in 2026`
+
+  console.log(`\nGenerated ${(titles as any).length || 10} titles`)
+
+  // ============================================================================
+  // Option 2: The Clean API (what you asked for!)
+  // ============================================================================
+
+  console.log('\n⚡ Step 2: Map titles to blog posts (automatic batching)...')
+  console.log('   Code: titles.map(title => write`blog post: # ${title}`)')
+
+  // This is the API you wanted!
+  // - No explicit batch creation
+  // - No provider/model in the code
+  // - Automatic batch detection based on context
+  const posts = (titles as string[]).map(title =>
+    write`Write a comprehensive blog post for startup founders:
+
+# ${title}
+
+Include:
+- Attention-grabbing introduction
+- 3-5 key sections with actionable insights
+- Real-world examples
+- Compelling conclusion with call-to-action`
+  )
+
+  console.log(`   Created ${posts.length} deferred operations`)
+
+  // When you await, it resolves via batch API if beneficial
+  console.log('\n⏳ Step 3: Await results (batched automatically)...')
+  // Note: Each item is an AIPromise, we'd await them all
+  // const results = await Promise.all(posts)
+
+  console.log('\n✅ Done!')
+
+  // ============================================================================
+  // Option 3: Scoped Context (for different providers in same code)
+  // ============================================================================
+
+  console.log('\n🔄 Bonus: Using withContext for scoped configuration...')
+
+  await withContext(
+    { provider: 'anthropic', model: 'claude-sonnet-4-20250514', batchMode: 'deferred' },
+    async () => {
+      console.log('   Inside context: Using Anthropic with deferred batching')
+      // All operations here use Anthropic
+      // const summaries = titles.map(title => write`summarize: ${title}`)
+    }
+  )
+
+  console.log('   Outside context: Back to OpenAI')
+}
+
+// ============================================================================
+// Summary of the API
+// ============================================================================
+
+/*
+The new API is clean and implicit:
+
+1. Configure once (globally or via environment):
+   ```ts
+   configure({ provider: 'openai', model: 'gpt-4o', batchMode: 'auto' })
+   ```
+
+2. Use naturally:
+   ```ts
+   const titles = await list`10 blog post titles`
+   const posts = titles.map(title => write`blog post: # ${title}`)
+   ```
+
+3. Batching happens automatically when:
+   - batchMode is 'auto' and items >= batchThreshold
+   - batchMode is 'deferred' (always batch)
+
+4. No batching when:
+   - batchMode is 'immediate'
+   - batchMode is 'auto' and items < batchThreshold
+
+5. Provider batch APIs supported:
+   - OpenAI: 50% discount, 24hr turnaround
+   - Anthropic: 50% discount, 24hr turnaround
+   - Cloudflare: Via AI Gateway
+   - AWS Bedrock: Native batch inference
+*/
+
+// Run the example
+main()
+  .then(() => {
+    console.log('\n✨ Example complete!\n')
+    process.exit(0)
+  })
+  .catch((error) => {
+    console.error('\n❌ Error:', error.message)
+    process.exit(1)
+  })