ai-functions 2.1.3 → 2.4.0

package/src/eval/models.js

@@ -1,119 +0,0 @@
-/**
- * Model Registry for AI Functions Eval Suite
- *
- * Simple model list for running evals across providers.
- * Uses ai-providers/language-models for resolution and pricing.
- *
- * @packageDocumentation
- */
-import { resolve, get } from 'language-models';
-// ============================================================================
-// Models to evaluate - using aliases from language-models
-// ============================================================================
-/**
- * Core models to test - one per tier per major provider
- * These resolve via ai-providers to OpenRouter or direct SDKs
- *
- * Updated: December 2025
- *
- * Note: Some models use OpenRouter format (provider/model) to avoid
- * resolution issues with provider_model_id mismatches.
- */
-export const EVAL_MODELS = [
-    // Anthropic Claude 4.5 - via AWS Bedrock (uses AWS credits with bearer token auth)
-    // All Claude models should be 4.5 - older versions are deprecated
-    { id: 'bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0', name: 'Claude Opus 4.5', provider: 'anthropic', tier: 'best', notes: 'Bedrock' },
-    { id: 'bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0', name: 'Claude Sonnet 4.5', provider: 'anthropic', tier: 'fast', notes: 'Bedrock' },
-    { id: 'bedrock:us.anthropic.claude-haiku-4-5-20251001-v1:0', name: 'Claude Haiku 4.5', provider: 'anthropic', tier: 'cheap', notes: 'Bedrock' },
-    // OpenAI - GPT-5.1 variants + GPT-oss (open source)
-    { id: 'openai/o3', name: 'o3', provider: 'openai', tier: 'best' },
-    { id: 'openai/gpt-5.1', name: 'GPT-5.1', provider: 'openai', tier: 'best' },
-    { id: 'openai/gpt-5-mini', name: 'GPT-5 Mini', provider: 'openai', tier: 'fast' },
-    { id: 'openai/gpt-5-nano', name: 'GPT-5 Nano', provider: 'openai', tier: 'cheap' },
-    // GPT-oss 120B removed - times out frequently
-    { id: 'openai/gpt-oss-20b', name: 'GPT-oss 20B', provider: 'openai', tier: 'fast', notes: 'Open source' },
-    // Google - Gemini 3 (November 2025)
-    { id: 'google/gemini-3-pro-preview', name: 'Gemini 3 Pro', provider: 'google', tier: 'best', notes: '1M context, #1 LMArena' },
-    // Gemini 2.5 Pro removed - times out frequently
-    { id: 'flash', name: 'Gemini 2.5 Flash', provider: 'google', tier: 'fast' },
-    // Meta (via OpenRouter)
-    { id: 'meta-llama/llama-4-maverick', name: 'Llama 4 Maverick', provider: 'meta-llama', tier: 'best' },
-    { id: 'meta-llama/llama-3.3-70b-instruct', name: 'Llama 3.3 70B', provider: 'meta-llama', tier: 'fast' },
-    // DeepSeek - V3.2 (December 2025)
-    { id: 'deepseek/deepseek-v3.2', name: 'DeepSeek V3.2', provider: 'deepseek', tier: 'best', notes: 'GPT-5 class reasoning' },
-    // DeepSeek V3.2 Speciale removed - no tool use support on OpenRouter
-    { id: 'deepseek/deepseek-chat', name: 'DeepSeek Chat', provider: 'deepseek', tier: 'fast' },
-    // Mistral - Mistral 3 family (December 2025)
-    { id: 'mistralai/mistral-large-2512', name: 'Mistral Large 3', provider: 'mistralai', tier: 'best', notes: '675B MoE, 41B active' },
-    { id: 'mistralai/mistral-medium-3.1', name: 'Mistral Medium 3.1', provider: 'mistralai', tier: 'fast' },
-    // Ministral 3 14B removed - often fails structured output
-    // Qwen - Qwen3 family (2025)
-    { id: 'qwen/qwen3-coder', name: 'Qwen3 Coder 480B', provider: 'qwen', tier: 'best', notes: 'Agentic coding' },
-    { id: 'qwen/qwen3-30b-a3b', name: 'Qwen3 30B', provider: 'qwen', tier: 'fast', notes: 'MoE 30B/3B active' },
-    { id: 'qwen/qwen3-next-80b-a3b-instruct', name: 'Qwen3 Next 80B', provider: 'qwen', tier: 'best', notes: 'Ultra-long context' },
-    // xAI - Grok 4 family (December 2025)
-    { id: 'x-ai/grok-4', name: 'Grok 4', provider: 'x-ai', tier: 'best', notes: '256K context, reasoning' },
-    { id: 'x-ai/grok-4.1-fast', name: 'Grok 4.1 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context, agentic' },
-    { id: 'x-ai/grok-4-fast', name: 'Grok 4 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context' },
-];
-/**
- * Get models by tier
- */
-export function getModelsByTier(tier) {
-    return EVAL_MODELS.filter(m => m.tier === tier);
-}
-/**
- * Get models by provider
- */
-export function getModelsByProvider(provider) {
-    return EVAL_MODELS.filter(m => m.provider === provider);
-}
-/**
- * Get model info from language-models package (includes pricing)
- */
-export function getModelInfo(id) {
-    const resolved = resolve(id);
-    return get(resolved);
-}
-/**
- * Get pricing for a model (from OpenRouter data)
- */
-export function getModelPricing(id) {
-    const info = getModelInfo(id);
-    if (!info?.pricing)
-        return undefined;
-    return {
-        prompt: parseFloat(info.pricing.prompt) * 1_000_000, // Convert to per-million
-        completion: parseFloat(info.pricing.completion) * 1_000_000,
-    };
-}
-/**
- * Create evalite variants for model testing
- */
-export function createModelVariants(opts) {
-    let models = EVAL_MODELS;
-    if (opts?.tiers) {
-        models = models.filter(m => opts.tiers.includes(m.tier));
-    }
-    if (opts?.providers) {
-        models = models.filter(m => opts.providers.includes(m.provider));
-    }
-    return models.map(model => ({
-        name: `${model.provider}/${model.name}`,
-        input: model,
-    }));
-}
-/**
- * Get a representative model from each provider for a given tier
- */
-export function getRepresentativeModels(tier) {
-    const seen = new Set();
-    const result = [];
-    for (const model of EVAL_MODELS) {
-        if (model.tier === tier && !seen.has(model.provider)) {
-            seen.add(model.provider);
-            result.push(model);
-        }
-    }
-    return result;
-}

package/src/eval/runner.js

@@ -1,147 +0,0 @@
-/**
- * Simple eval runner for AI Functions
- *
- * Runs evals across multiple models and collects results.
- * Does not depend on evalite - uses our own infrastructure.
- */
-import { generateObject, generateText } from '../generate.js';
-import { schema } from '../schema.js';
-import { createModelVariants, getModelPricing } from './models.js';
-/**
- * Run an eval suite across models
- */
-export async function runEval(options) {
-    const { name, cases, task, scorers, concurrency = 3 } = options;
-    // Get models to test
-    const models = options.models ?? createModelVariants({
-        tiers: options.tiers,
-        providers: options.providers,
-    }).map(v => v.input);
-    const results = [];
-    const startTime = Date.now();
-    console.log(`\n🧪 Running eval: ${name}`);
-    console.log(`   Models: ${models.map(m => m.name).join(', ')}`);
-    console.log(`   Cases: ${cases.length}`);
-    console.log('');
-    // Run all model/case combinations
-    const jobs = [];
-    for (const model of models) {
-        for (const evalCase of cases) {
-            jobs.push({ model, case: evalCase });
-        }
-    }
-    // Process in batches with concurrency limit
-    for (let i = 0; i < jobs.length; i += concurrency) {
-        const batch = jobs.slice(i, i + concurrency);
-        const batchResults = await Promise.all(batch.map(async (job) => {
-            const caseStart = Date.now();
-            try {
-                // Run the task
-                const output = await task(job.case.input, job.model);
-                const latencyMs = Date.now() - caseStart;
-                // Run scorers
-                const scores = [];
-                for (const s of scorers) {
-                    try {
-                        const score = await s.scorer({
-                            input: job.case.input,
-                            output,
-                            expected: job.case.expected,
-                        });
-                        scores.push({
-                            name: s.name,
-                            score: Math.max(0, Math.min(1, score)),
-                            description: s.description,
-                        });
-                    }
-                    catch (err) {
-                        scores.push({
-                            name: s.name,
-                            score: 0,
-                            description: s.description,
-                            metadata: { error: String(err) },
-                        });
-                    }
-                }
-                // Calculate cost
-                const pricing = getModelPricing(job.model.id);
-                // Estimate tokens - rough approximation
-                const estimatedPromptTokens = 100;
-                const estimatedCompletionTokens = 200;
-                const cost = pricing
-                    ? (estimatedPromptTokens * pricing.prompt + estimatedCompletionTokens * pricing.completion) / 1_000_000
-                    : 0;
-                const avgScore = scores.length > 0
-                    ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length
-                    : 0;
-                const symbol = avgScore >= 0.8 ? '✓' : avgScore >= 0.5 ? '~' : '✗';
-                console.log(`   ${symbol} ${job.model.name} | ${job.case.name} | ${(avgScore * 100).toFixed(0)}% | ${latencyMs}ms`);
-                return {
-                    model: job.model,
-                    case: job.case,
-                    output,
-                    scores,
-                    latencyMs,
-                    cost,
-                };
-            }
-            catch (err) {
-                console.log(`   ✗ ${job.model.name} | ${job.case.name} | ERROR: ${err}`);
-                return {
-                    model: job.model,
-                    case: job.case,
-                    output: null,
-                    scores: scorers.map(s => ({ name: s.name, score: 0 })),
-                    latencyMs: Date.now() - caseStart,
-                    cost: 0,
-                    error: String(err),
-                };
-            }
-        }));
-        results.push(...batchResults);
-    }
-    // Calculate summary
-    const totalTime = Date.now() - startTime;
-    const totalCost = results.reduce((sum, r) => sum + r.cost, 0);
-    const allScores = results.flatMap(r => r.scores.map(s => s.score));
-    const avgScore = allScores.length > 0
-        ? allScores.reduce((a, b) => a + b, 0) / allScores.length
-        : 0;
-    // Group by model
-    const byModel = {};
-    for (const result of results) {
-        const modelKey = result.model.id;
-        if (!byModel[modelKey]) {
-            byModel[modelKey] = { avgScore: 0, count: 0 };
-        }
-        const resultAvg = result.scores.reduce((sum, s) => sum + s.score, 0) / result.scores.length;
-        byModel[modelKey].avgScore += resultAvg;
-        byModel[modelKey].count++;
-    }
-    for (const key of Object.keys(byModel)) {
-        const entry = byModel[key];
-        if (entry) {
-            entry.avgScore /= entry.count;
-        }
-    }
-    console.log('');
-    console.log(`📊 Results:`);
-    console.log(`   Overall: ${(avgScore * 100).toFixed(1)}%`);
-    console.log(`   Time: ${(totalTime / 1000).toFixed(1)}s`);
-    console.log(`   Cost: $${totalCost.toFixed(4)}`);
-    console.log('');
-    console.log('   By Model:');
-    for (const [modelId, stats] of Object.entries(byModel)) {
-        console.log(`   - ${modelId}: ${(stats.avgScore * 100).toFixed(1)}%`);
-    }
-    return {
-        name,
-        results,
-        avgScore,
-        byModel,
-        totalCost,
-        totalTime,
-    };
-}
-// Re-export helpers
-export { generateObject, generateText, schema };

package/test/schema.test.js

@@ -1,96 +0,0 @@
-/**
- * Tests for schema conversion
- *
- * These are pure unit tests - no AI calls needed.
- */
-import { describe, it, expect } from 'vitest';
-import { schema } from '../src/index.js';
-import { z } from 'zod';
-describe('schema', () => {
-    describe('string types', () => {
-        it('converts simple string description to z.string()', () => {
-            const result = schema('User name');
-            expect(result._def.typeName).toBe('ZodString');
-            expect(result._def.description).toBe('User name');
-        });
-        it('converts (number) hint to z.number()', () => {
-            const result = schema('User age (number)');
-            expect(result._def.typeName).toBe('ZodNumber');
-            expect(result._def.description).toBe('User age');
-        });
-        it('converts (boolean) hint to z.boolean()', () => {
-            const result = schema('Is active (boolean)');
-            expect(result._def.typeName).toBe('ZodBoolean');
-            expect(result._def.description).toBe('Is active');
-        });
-        it('converts (integer) hint to z.number().int()', () => {
-            const result = schema('Item count (integer)');
-            expect(result._def.typeName).toBe('ZodNumber');
-            expect(result._def.checks?.some((c) => c.kind === 'int')).toBe(true);
-        });
-        it('converts (date) hint to z.string().datetime()', () => {
-            const result = schema('Created at (date)');
-            expect(result._def.typeName).toBe('ZodString');
-            expect(result._def.checks?.some((c) => c.kind === 'datetime')).toBe(true);
-        });
-    });
-    describe('enum types', () => {
-        it('converts pipe-separated values to z.enum()', () => {
-            const result = schema('pending | done | cancelled');
-            expect(result._def.typeName).toBe('ZodEnum');
-            expect(result._def.values).toEqual(['pending', 'done', 'cancelled']);
-        });
-        it('handles spaces around pipe', () => {
-            const result = schema('yes | no | maybe');
-            expect(result._def.values).toEqual(['yes', 'no', 'maybe']);
-        });
-    });
-    describe('array types', () => {
-        it('converts [string] to z.array(z.string())', () => {
-            const result = schema(['List of items']);
-            expect(result._def.typeName).toBe('ZodArray');
-            expect(result._def.type._def.typeName).toBe('ZodString');
-            expect(result._def.description).toBe('List of items');
-        });
-    });
-    describe('object types', () => {
-        it('converts object to z.object()', () => {
-            const result = schema({
-                name: 'User name',
-                age: 'Age (number)',
-            });
-            expect(result._def.typeName).toBe('ZodObject');
-        });
-        it('handles nested objects', () => {
-            const result = schema({
-                user: {
-                    name: 'Name',
-                    profile: {
-                        bio: 'Bio',
-                    },
-                },
-            });
-            expect(result._def.typeName).toBe('ZodObject');
-        });
-        it('handles mixed types in object', () => {
-            const result = schema({
-                name: 'Name',
-                count: 'Count (number)',
-                active: 'Active (boolean)',
-                status: 'pending | done',
-                tags: ['Tags'],
-            });
-            expect(result._def.typeName).toBe('ZodObject');
-        });
-    });
-    describe('zod passthrough', () => {
-        it('passes through existing zod schemas', () => {
-            const zodSchema = z.object({
-                name: z.string(),
-                age: z.number(),
-            });
-            const result = schema(zodSchema);
-            expect(result).toBe(zodSchema);
-        });
-    });
-});