ai-functions 0.3.0 → 0.4.0

package/evals/structured-output.eval.ts

@@ -0,0 +1,143 @@
+/**
+ * Structured Output Eval
+ *
+ * Tests model ability to generate valid structured JSON output
+ * matching specified schemas across all providers.
+ */
+
+import { evalite } from 'evalite'
+import { generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, getModelPricing, type EvalModel } from '../src/eval/models.js'
+
+// Test cases for structured output
+const TEST_CASES = [
+  {
+    name: 'Simple object',
+    prompt: 'Generate a greeting in French',
+    schema: {
+      greeting: 'A friendly greeting',
+      language: 'The language of the greeting',
+    },
+    expectedTypes: { greeting: 'string', language: 'string' },
+  },
+  {
+    name: 'With numbers',
+    prompt: 'Generate info about Tokyo',
+    schema: {
+      name: 'City name',
+      population: 'Population in millions (number)',
+      area: 'Area in square kilometers (number)',
+    },
+    expectedTypes: { name: 'string', population: 'number', area: 'number' },
+  },
+  {
+    name: 'With arrays',
+    prompt: 'Generate a simple pasta recipe',
+    schema: {
+      title: 'Recipe title',
+      ingredients: ['List of ingredients'],
+      steps: ['Cooking steps'],
+    },
+    expectedTypes: { title: 'string', ingredients: 'array', steps: 'array' },
+  },
+  {
+    name: 'With enum',
+    prompt: 'Analyze sentiment: "I love this product!"',
+    schema: {
+      sentiment: 'positive | negative | neutral',
+      confidence: 'Confidence score 0-1 (number)',
+    },
+    expectedTypes: { sentiment: 'string', confidence: 'number' },
+  },
+  {
+    name: 'Nested object',
+    prompt: 'Generate a fictional person living in Japan',
+    schema: {
+      person: { name: 'Full name', age: 'Age (number)' },
+      address: { city: 'City name', country: 'Country name' },
+    },
+    expectedTypes: { person: 'object', address: 'object' },
+  },
+]
+
+// Test across models - start with fast tier for quick iteration
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Structured Output', {
+  data: TEST_CASES.map(tc => ({ input: tc })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    const { object, usage } = await generateObject({
+      model: model.id,
+      schema: schema(input.schema),
+      prompt: input.prompt,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    // Calculate cost from language-models pricing
+    const pricing = getModelPricing(model.id)
+    const cost = pricing
+      ? ((usage?.promptTokens ?? 0) * pricing.prompt + (usage?.completionTokens ?? 0) * pricing.completion) / 1_000_000
+      : 0
+
+    return {
+      object,
+      expectedTypes: input.expectedTypes,
+      testName: input.name,
+      modelId: model.id,
+      modelName: model.name,
+      provider: model.provider,
+      latencyMs,
+      cost,
+      usage,
+    }
+  },
+
+  scorers: [
+    // Type accuracy
+    {
+      name: 'Type Accuracy',
+      description: 'Whether output fields have correct types',
+      scorer: ({ output }) => {
+        const obj = output.object as Record<string, unknown>
+        const expected = output.expectedTypes as Record<string, string>
+        const fields = Object.keys(expected)
+
+        let correct = 0
+        for (const field of fields) {
+          const val = obj[field]
+          const expectedType = expected[field]
+          const actualType = Array.isArray(val) ? 'array' : typeof val
+
+          if (actualType === expectedType) correct++
+        }
+
+        return { score: correct / fields.length }
+      },
+    },
+
+    // Latency
+    {
+      name: 'Latency',
+      description: 'Response time (target < 3s)',
+      scorer: ({ output }) => {
+        const ms = output.latencyMs as number
+        if (ms < 2000) return { score: 1 }
+        if (ms > 10000) return { score: 0 }
+        return { score: 1 - (ms - 2000) / 8000 }
+      },
+    },
+  ],
+
+  columns: ({ output, scores }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Test', value: output.testName },
+    { label: 'Latency', value: `${output.latencyMs}ms` },
+    { label: 'Cost', value: `$${(output.cost as number).toFixed(6)}` },
+  ],
+})

package/evals/writing.eval.ts

@@ -0,0 +1,117 @@
+/**
+ * Writing Quality Eval (LLM-as-Judge)
+ *
+ * Tests model writing capabilities using LLM-as-judge scoring.
+ * Uses a strong model (sonnet) to judge output quality.
+ */
+
+import { evalite } from 'evalite'
+import { generateText, generateObject } from '../src/generate.js'
+import { schema } from '../src/schema.js'
+import { createModelVariants, type EvalModel } from '../src/eval/models.js'
+
+// Use sonnet as the judge model
+const JUDGE_MODEL = 'sonnet'
+
+// Writing test cases
+const TEST_CASES = [
+  {
+    name: 'Professional email',
+    prompt: 'Write a professional email declining a meeting invitation politely.',
+    criteria: ['Polite tone', 'Clear explanation', 'Proper email format'],
+  },
+  {
+    name: 'Product description',
+    prompt: 'Write a product description for wireless earbuds targeting tech-savvy consumers.',
+    criteria: ['Highlights features', 'Compelling language', 'Clear value proposition'],
+  },
+  {
+    name: 'Explanation',
+    prompt: 'Explain how photosynthesis works in simple terms for a high school student.',
+    criteria: ['Accurate content', 'Clear language', 'Logical flow'],
+  },
+]
+
+const modelVariants = createModelVariants({ tiers: ['fast'] })
+
+evalite.each(modelVariants)('Writing Quality', {
+  data: TEST_CASES.map(tc => ({ input: tc })),
+
+  task: async (input, variant) => {
+    const model = variant as EvalModel
+    const startTime = Date.now()
+
+    // Generate the writing
+    const { text, usage } = await generateText({
+      model: model.id,
+      prompt: input.prompt,
+    })
+
+    const latencyMs = Date.now() - startTime
+
+    return {
+      text,
+      testName: input.name,
+      criteria: input.criteria,
+      modelId: model.id,
+      modelName: model.name,
+      provider: model.provider,
+      latencyMs,
+      usage,
+    }
+  },
+
+  scorers: [
+    // LLM-as-judge for quality
+    {
+      name: 'Writing Quality',
+      description: 'LLM judge evaluation of writing quality',
+      scorer: async ({ input, output }) => {
+        const { object } = await generateObject({
+          model: JUDGE_MODEL,
+          schema: schema({
+            clarity: 'How clear is the writing? (number 0-1)',
+            engagement: 'How engaging is the content? (number 0-1)',
+            accuracy: 'How well does it meet the criteria? (number 0-1)',
+            reasoning: 'Brief explanation',
+          }),
+          prompt: `Evaluate this writing on a scale of 0-1.
+
+Criteria: ${(input.criteria as string[]).join(', ')}
+
+Writing:
+"""
+${output.text}
+"""`,
+        })
+
+        const avg = ((object.clarity as number) + (object.engagement as number) + (object.accuracy as number)) / 3
+        return {
+          score: avg,
+          metadata: object,
+        }
+      },
+    },
+
+    // Word count check
+    {
+      name: 'Appropriate Length',
+      description: 'Whether output has reasonable length',
+      scorer: ({ output }) => {
+        const words = (output.text as string).split(/\s+/).length
+        if (words < 20) return { score: 0.3, metadata: { words } }
+        if (words > 500) return { score: 0.7, metadata: { words } }
+        return { score: 1, metadata: { words } }
+      },
+    },
+  ],
+
+  columns: ({ output }) => [
+    { label: 'Model', value: output.modelName },
+    { label: 'Test', value: output.testName },
+    { label: 'Words', value: (output.text as string).split(/\s+/).length },
+    { label: 'Latency', value: `${output.latencyMs}ms` },
+  ],
+
+  trialCount: 2, // Run twice for variance
+})

package/examples/batch-blog-posts.ts

@@ -0,0 +1,160 @@
+/**
+ * Batch Blog Post Generation Example
+ *
+ * This example demonstrates the new IMPLICIT batch processing:
+ *
+ * ```ts
+ * // Configure once (or use environment variables)
+ * configure({ provider: 'openai', model: 'gpt-4o', batchMode: 'auto' })
+ *
+ * // Use naturally - batching is automatic!
+ * const titles = await list`10 blog post titles about startups`
+ * const posts = titles.map(title => write`blog post: # ${title}`)
+ * console.log(await posts) // Batched automatically!
+ * ```
+ *
+ * Environment variables:
+ * - AI_PROVIDER: openai | anthropic | cloudflare | bedrock
+ * - AI_MODEL: model name (e.g., gpt-4o, claude-sonnet-4-20250514)
+ * - AI_BATCH_MODE: auto | immediate | deferred
+ * - AI_BATCH_THRESHOLD: minimum items for auto batch (default: 5)
+ *
+ * @example
+ * ```bash
+ * # Using environment variables
+ * AI_PROVIDER=openai AI_MODEL=gpt-4o npx tsx examples/batch-blog-posts.ts
+ *
+ * # Or with API keys
+ * OPENAI_API_KEY=sk-... npx tsx examples/batch-blog-posts.ts
+ * ```
+ */
+
+import {
+  list,
+  write,
+  configure,
+  withContext,
+  type BatchProvider,
+} from '../src/index.js'
+
+// Import the batch adapter for your provider
+// import '../src/batch/openai.js'
+// import '../src/batch/anthropic.js'
+// import '../src/batch/cloudflare.js'
+// import '../src/batch/bedrock.js'
+
+// For testing, use the memory adapter
+import '../src/batch/memory.js'
+
+async function main() {
+  console.log('\n🚀 Implicit Batch Blog Post Generation\n')
+
+  // ============================================================================
+  // Option 1: Global Configuration (recommended)
+  // ============================================================================
+
+  configure({
+    provider: 'openai',
+    model: 'gpt-4o',
+    batchMode: 'auto',      // 'auto' | 'immediate' | 'deferred'
+    batchThreshold: 5,       // Use batch API when >= 5 items
+  })
+
+  console.log('📝 Step 1: Generate titles (executes immediately)...')
+  const titles = await list`10 blog post titles about building startups in 2026`
+
+  console.log(`\nGenerated ${(titles as any).length || 10} titles`)
+
+  // ============================================================================
+  // Option 2: The Clean API (what you asked for!)
+  // ============================================================================
+
+  console.log('\n⚡ Step 2: Map titles to blog posts (automatic batching)...')
+  console.log('   Code: titles.map(title => write`blog post: # ${title}`)')
+
+  // This is the API you wanted!
+  // - No explicit batch creation
+  // - No provider/model in the code
+  // - Automatic batch detection based on context
+  const posts = (titles as string[]).map(title =>
+    write`Write a comprehensive blog post for startup founders:
+
+# ${title}
+
+Include:
+- Attention-grabbing introduction
+- 3-5 key sections with actionable insights
+- Real-world examples
+- Compelling conclusion with call-to-action`
+  )
+
+  console.log(`   Created ${posts.length} deferred operations`)
+
+  // When you await, it resolves via batch API if beneficial
+  console.log('\n⏳ Step 3: Await results (batched automatically)...')
+  // Note: Each item is an AIPromise, we'd await them all
+  // const results = await Promise.all(posts)
+
+  console.log('\n✅ Done!')
+
+  // ============================================================================
+  // Option 3: Scoped Context (for different providers in same code)
+  // ============================================================================
+
+  console.log('\n🔄 Bonus: Using withContext for scoped configuration...')
+
+  await withContext(
+    { provider: 'anthropic', model: 'claude-sonnet-4-20250514', batchMode: 'deferred' },
+    async () => {
+      console.log('   Inside context: Using Anthropic with deferred batching')
+      // All operations here use Anthropic
+      // const summaries = titles.map(title => write`summarize: ${title}`)
+    }
+  )
+
+  console.log('   Outside context: Back to OpenAI')
+}
+
+// ============================================================================
+// Summary of the API
+// ============================================================================
+
+/*
+The new API is clean and implicit:
+
+1. Configure once (globally or via environment):
+   ```ts
+   configure({ provider: 'openai', model: 'gpt-4o', batchMode: 'auto' })
+   ```
+
+2. Use naturally:
+   ```ts
+   const titles = await list`10 blog post titles`
+   const posts = titles.map(title => write`blog post: # ${title}`)
+   ```
+
+3. Batching happens automatically when:
+   - batchMode is 'auto' and items >= batchThreshold
+   - batchMode is 'deferred' (always batch)
+
+4. No batching when:
+   - batchMode is 'immediate'
+   - batchMode is 'auto' and items < batchThreshold
+
+5. Provider batch APIs supported:
+   - OpenAI: 50% discount, 24hr turnaround
+   - Anthropic: 50% discount, 24hr turnaround
+   - Cloudflare: Via AI Gateway
+   - AWS Bedrock: Native batch inference
+*/
+
+// Run the example
+main()
+  .then(() => {
+    console.log('\n✨ Example complete!\n')
+    process.exit(0)
+  })
+  .catch((error) => {
+    console.error('\n❌ Error:', error.message)
+    process.exit(1)
+  })

package/package.json

@@ -1,72 +1,72 @@
 {
   "name": "ai-functions",
-  "version": "0.3.0",
-  "description": "A powerful TypeScript library for building AI-powered applications with template literals and structured outputs",
+  "version": "0.4.0",
+  "description": "Core AI primitives for building intelligent applications",
   "type": "module",
   "main": "dist/index.js",
-  "module": "dist/index.js",
   "types": "dist/index.d.ts",
-  "bin": {
-    "ai-functions": "bin/cli.js"
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./embeddings": {
+      "import": "./dist/embeddings.js",
+      "types": "./dist/embeddings.d.ts"
+    },
+    "./providers": {
+      "import": "./dist/providers/index.js",
+      "types": "./dist/providers/index.d.ts"
+    },
+    "./providers/cloudflare": {
+      "import": "./dist/providers/cloudflare.js",
+      "types": "./dist/providers/cloudflare.d.ts"
+    }
   },
-  "files": [
-    "dist",
-    "bin"
-  ],
   "scripts": {
-    "build": "tsc",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "lint": "eslint src",
-    "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx}\" \"*.{md,mdx}\"",
-    "prepublishOnly": "pnpm run build",
-    "semantic-release": "semantic-release"
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsc -p tsconfig.json --watch",
+    "test": "vitest",
+    "test:unit": "vitest run --exclude 'test/evals/**' --exclude 'test/e2e-*'",
+    "test:evals": "vitest run test/evals/",
+    "test:evals:primitives": "vitest run test/evals/primitives.eval.test.ts",
+    "test:evals:define": "vitest run test/evals/define-function.eval.test.ts",
+    "eval": "tsx evals/run-evals.ts",
+    "eval:fast": "tsx evals/run-evals.ts",
+    "eval:all": "tsx evals/run-evals.ts --all",
+    "eval:math": "tsx evals/run-evals.ts --math",
+    "eval:class": "tsx evals/run-evals.ts --class",
+    "eval:marketing": "tsx evals/marketing.eval.ts",
+    "eval:marketing:all": "tsx evals/marketing.eval.ts --all",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@ai-sdk/amazon-bedrock": "^3.0.0",
+    "ai": "^5.0.0",
+    "ai-providers": "workspace:^",
+    "language-models": "workspace:*",
+    "rpc.do": "^0.1.0",
+    "yaml": "^2.8.0",
+    "zod": "^3.23.0"
+  },
+  "optionalDependencies": {
+    "oauth.do": "^0.0.1"
   },
   "keywords": [
     "ai",
-    "openai",
-    "template-literals",
-    "typescript",
-    "structured-output",
-    "cli"
+    "functions",
+    "embeddings",
+    "cloudflare",
+    "oauth",
+    "primitives"
   ],
-  "author": "AI Primitives",
   "license": "MIT",
-  "homepage": "https://github.com/ai-primitives/ai-functions#readme",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/ai-primitives/ai-functions.git"
-  },
-  "bugs": {
-    "url": "https://github.com/ai-primitives/ai-functions/issues"
-  },
   "devDependencies": {
-    "@ai-sdk/provider": "^1.0.2",
-    "@ai-sdk/provider-utils": "^2.0.4",
-    "@ai-sdk/ui-utils": "^1.0.5",
-    "@eslint/js": "^9.17.0",
-    "@opentelemetry/api": "^1.9.0",
-    "@semantic-release/commit-analyzer": "^13.0.0",
-    "@semantic-release/github": "^11.0.1",
-    "@semantic-release/npm": "^12.0.1",
-    "@semantic-release/release-notes-generator": "^14.0.1",
-    "@types/node": "^22.10.2",
-    "@typescript-eslint/eslint-plugin": "^8.18.0",
-    "@typescript-eslint/parser": "^8.18.0",
-    "dotenv": "^16.4.7",
-    "eslint": "^9.17.0",
-    "prettier": "^3.4.2",
-    "semantic-release": "^24.2.0",
-    "typescript": "^5.7.2",
-    "undici": "^7.2.0",
-    "vitest": "^2.1.8",
-    "web-streams-polyfill": "^4.0.0"
-  },
-  "dependencies": {
-    "@ai-sdk/openai": "^1.0.10",
-    "@ai-sdk/openai-compatible": "^0.0.9",
-    "ai": "^4.0.20",
-    "p-queue": "^8.0.1",
-    "zod": "^3.24.1"
+    "dotenv": "^17.2.3",
+    "evalite": "^0.19.0",
+    "tsx": "^4.0.0",
+    "vitest": "^2.1.0"
   }
 }