ai-functions 2.0.2 → 2.1.3

package/src/eval/models.js

@@ -0,0 +1,119 @@
+/**
+ * Model Registry for AI Functions Eval Suite
+ *
+ * Simple model list for running evals across providers.
+ * Uses ai-providers/language-models for resolution and pricing.
+ *
+ * @packageDocumentation
+ */
+import { resolve, get } from 'language-models';
+// ============================================================================
+// Models to evaluate - using aliases from language-models
+// ============================================================================
+/**
+ * Core models to test - one per tier per major provider
+ * These resolve via ai-providers to OpenRouter or direct SDKs
+ *
+ * Updated: December 2025
+ *
+ * Note: Some models use OpenRouter format (provider/model) to avoid
+ * resolution issues with provider_model_id mismatches.
+ */
+export const EVAL_MODELS = [
+    // Anthropic Claude 4.5 - via AWS Bedrock (uses AWS credits with bearer token auth)
+    // All Claude models should be 4.5 - older versions are deprecated
+    { id: 'bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0', name: 'Claude Opus 4.5', provider: 'anthropic', tier: 'best', notes: 'Bedrock' },
+    { id: 'bedrock:us.anthropic.claude-sonnet-4-5-20250929-v1:0', name: 'Claude Sonnet 4.5', provider: 'anthropic', tier: 'fast', notes: 'Bedrock' },
+    { id: 'bedrock:us.anthropic.claude-haiku-4-5-20251001-v1:0', name: 'Claude Haiku 4.5', provider: 'anthropic', tier: 'cheap', notes: 'Bedrock' },
+    // OpenAI - GPT-5.1 variants + GPT-oss (open source)
+    { id: 'openai/o3', name: 'o3', provider: 'openai', tier: 'best' },
+    { id: 'openai/gpt-5.1', name: 'GPT-5.1', provider: 'openai', tier: 'best' },
+    { id: 'openai/gpt-5-mini', name: 'GPT-5 Mini', provider: 'openai', tier: 'fast' },
+    { id: 'openai/gpt-5-nano', name: 'GPT-5 Nano', provider: 'openai', tier: 'cheap' },
+    // GPT-oss 120B removed - times out frequently
+    { id: 'openai/gpt-oss-20b', name: 'GPT-oss 20B', provider: 'openai', tier: 'fast', notes: 'Open source' },
+    // Google - Gemini 3 (November 2025)
+    { id: 'google/gemini-3-pro-preview', name: 'Gemini 3 Pro', provider: 'google', tier: 'best', notes: '1M context, #1 LMArena' },
+    // Gemini 2.5 Pro removed - times out frequently
+    { id: 'flash', name: 'Gemini 2.5 Flash', provider: 'google', tier: 'fast' },
+    // Meta (via OpenRouter)
+    { id: 'meta-llama/llama-4-maverick', name: 'Llama 4 Maverick', provider: 'meta-llama', tier: 'best' },
+    { id: 'meta-llama/llama-3.3-70b-instruct', name: 'Llama 3.3 70B', provider: 'meta-llama', tier: 'fast' },
+    // DeepSeek - V3.2 (December 2025)
+    { id: 'deepseek/deepseek-v3.2', name: 'DeepSeek V3.2', provider: 'deepseek', tier: 'best', notes: 'GPT-5 class reasoning' },
+    // DeepSeek V3.2 Speciale removed - no tool use support on OpenRouter
+    { id: 'deepseek/deepseek-chat', name: 'DeepSeek Chat', provider: 'deepseek', tier: 'fast' },
+    // Mistral - Mistral 3 family (December 2025)
+    { id: 'mistralai/mistral-large-2512', name: 'Mistral Large 3', provider: 'mistralai', tier: 'best', notes: '675B MoE, 41B active' },
+    { id: 'mistralai/mistral-medium-3.1', name: 'Mistral Medium 3.1', provider: 'mistralai', tier: 'fast' },
+    // Ministral 3 14B removed - often fails structured output
+    // Qwen - Qwen3 family (2025)
+    { id: 'qwen/qwen3-coder', name: 'Qwen3 Coder 480B', provider: 'qwen', tier: 'best', notes: 'Agentic coding' },
+    { id: 'qwen/qwen3-30b-a3b', name: 'Qwen3 30B', provider: 'qwen', tier: 'fast', notes: 'MoE 30B/3B active' },
+    { id: 'qwen/qwen3-next-80b-a3b-instruct', name: 'Qwen3 Next 80B', provider: 'qwen', tier: 'best', notes: 'Ultra-long context' },
+    // xAI - Grok 4 family (December 2025)
+    { id: 'x-ai/grok-4', name: 'Grok 4', provider: 'x-ai', tier: 'best', notes: '256K context, reasoning' },
+    { id: 'x-ai/grok-4.1-fast', name: 'Grok 4.1 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context, agentic' },
+    { id: 'x-ai/grok-4-fast', name: 'Grok 4 Fast', provider: 'x-ai', tier: 'fast', notes: '2M context' },
+];
+/**
+ * Get models by tier
+ */
+export function getModelsByTier(tier) {
+    return EVAL_MODELS.filter(m => m.tier === tier);
+}
+/**
+ * Get models by provider
+ */
+export function getModelsByProvider(provider) {
+    return EVAL_MODELS.filter(m => m.provider === provider);
+}
+/**
+ * Get model info from language-models package (includes pricing)
+ */
+export function getModelInfo(id) {
+    const resolved = resolve(id);
+    return get(resolved);
+}
+/**
+ * Get pricing for a model (from OpenRouter data)
+ */
+export function getModelPricing(id) {
+    const info = getModelInfo(id);
+    if (!info?.pricing)
+        return undefined;
+    return {
+        prompt: parseFloat(info.pricing.prompt) * 1_000_000, // Convert to per-million
+        completion: parseFloat(info.pricing.completion) * 1_000_000,
+    };
+}
+/**
+ * Create evalite variants for model testing
+ */
+export function createModelVariants(opts) {
+    let models = EVAL_MODELS;
+    if (opts?.tiers) {
+        models = models.filter(m => opts.tiers.includes(m.tier));
+    }
+    if (opts?.providers) {
+        models = models.filter(m => opts.providers.includes(m.provider));
+    }
+    return models.map(model => ({
+        name: `${model.provider}/${model.name}`,
+        input: model,
+    }));
+}
+/**
+ * Get a representative model from each provider for a given tier
+ */
+export function getRepresentativeModels(tier) {
+    const seen = new Set();
+    const result = [];
+    for (const model of EVAL_MODELS) {
+        if (model.tier === tier && !seen.has(model.provider)) {
+            seen.add(model.provider);
+            result.push(model);
+        }
+    }
+    return result;
+}

package/src/eval/runner.js

@@ -0,0 +1,147 @@
+/**
+ * Simple eval runner for AI Functions
+ *
+ * Runs evals across multiple models and collects results.
+ * Does not depend on evalite - uses our own infrastructure.
+ */
+import { generateObject, generateText } from '../generate.js';
+import { schema } from '../schema.js';
+import { createModelVariants, getModelPricing } from './models.js';
+/**
+ * Run an eval suite across models
+ */
+export async function runEval(options) {
+    const { name, cases, task, scorers, concurrency = 3 } = options;
+    // Get models to test
+    const models = options.models ?? createModelVariants({
+        tiers: options.tiers,
+        providers: options.providers,
+    }).map(v => v.input);
+    const results = [];
+    const startTime = Date.now();
+    console.log(`\n🧪 Running eval: ${name}`);
+    console.log(`   Models: ${models.map(m => m.name).join(', ')}`);
+    console.log(`   Cases: ${cases.length}`);
+    console.log('');
+    // Run all model/case combinations
+    const jobs = [];
+    for (const model of models) {
+        for (const evalCase of cases) {
+            jobs.push({ model, case: evalCase });
+        }
+    }
+    // Process in batches with concurrency limit
+    for (let i = 0; i < jobs.length; i += concurrency) {
+        const batch = jobs.slice(i, i + concurrency);
+        const batchResults = await Promise.all(batch.map(async (job) => {
+            const caseStart = Date.now();
+            try {
+                // Run the task
+                const output = await task(job.case.input, job.model);
+                const latencyMs = Date.now() - caseStart;
+                // Run scorers
+                const scores = [];
+                for (const s of scorers) {
+                    try {
+                        const score = await s.scorer({
+                            input: job.case.input,
+                            output,
+                            expected: job.case.expected,
+                        });
+                        scores.push({
+                            name: s.name,
+                            score: Math.max(0, Math.min(1, score)),
+                            description: s.description,
+                        });
+                    }
+                    catch (err) {
+                        scores.push({
+                            name: s.name,
+                            score: 0,
+                            description: s.description,
+                            metadata: { error: String(err) },
+                        });
+                    }
+                }
+                // Calculate cost
+                const pricing = getModelPricing(job.model.id);
+                // Estimate tokens - rough approximation
+                const estimatedPromptTokens = 100;
+                const estimatedCompletionTokens = 200;
+                const cost = pricing
+                    ? (estimatedPromptTokens * pricing.prompt + estimatedCompletionTokens * pricing.completion) / 1_000_000
+                    : 0;
+                const avgScore = scores.length > 0
+                    ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length
+                    : 0;
+                const symbol = avgScore >= 0.8 ? '✓' : avgScore >= 0.5 ? '~' : '✗';
+                console.log(`   ${symbol} ${job.model.name} | ${job.case.name} | ${(avgScore * 100).toFixed(0)}% | ${latencyMs}ms`);
+                return {
+                    model: job.model,
+                    case: job.case,
+                    output,
+                    scores,
+                    latencyMs,
+                    cost,
+                };
+            }
+            catch (err) {
+                console.log(`   ✗ ${job.model.name} | ${job.case.name} | ERROR: ${err}`);
+                return {
+                    model: job.model,
+                    case: job.case,
+                    output: null,
+                    scores: scorers.map(s => ({ name: s.name, score: 0 })),
+                    latencyMs: Date.now() - caseStart,
+                    cost: 0,
+                    error: String(err),
+                };
+            }
+        }));
+        results.push(...batchResults);
+    }
+    // Calculate summary
+    const totalTime = Date.now() - startTime;
+    const totalCost = results.reduce((sum, r) => sum + r.cost, 0);
+    const allScores = results.flatMap(r => r.scores.map(s => s.score));
+    const avgScore = allScores.length > 0
+        ? allScores.reduce((a, b) => a + b, 0) / allScores.length
+        : 0;
+    // Group by model
+    const byModel = {};
+    for (const result of results) {
+        const modelKey = result.model.id;
+        if (!byModel[modelKey]) {
+            byModel[modelKey] = { avgScore: 0, count: 0 };
+        }
+        const resultAvg = result.scores.reduce((sum, s) => sum + s.score, 0) / result.scores.length;
+        byModel[modelKey].avgScore += resultAvg;
+        byModel[modelKey].count++;
+    }
+    for (const key of Object.keys(byModel)) {
+        const entry = byModel[key];
+        if (entry) {
+            entry.avgScore /= entry.count;
+        }
+    }
+    console.log('');
+    console.log(`📊 Results:`);
+    console.log(`   Overall: ${(avgScore * 100).toFixed(1)}%`);
+    console.log(`   Time: ${(totalTime / 1000).toFixed(1)}s`);
+    console.log(`   Cost: $${totalCost.toFixed(4)}`);
+    console.log('');
+    console.log('   By Model:');
+    for (const [modelId, stats] of Object.entries(byModel)) {
+        console.log(`   - ${modelId}: ${(stats.avgScore * 100).toFixed(1)}%`);
+    }
+    return {
+        name,
+        results,
+        avgScore,
+        byModel,
+        totalCost,
+        totalTime,
+    };
+}
+// Re-export helpers
+export { generateObject, generateText, schema };

package/src/eval/runner.ts

@@ -25,7 +25,8 @@ export interface EvalScore {
 export interface EvalResult<TOutput = unknown> {
   model: EvalModel
   case: EvalCase
-  output: TOutput
+  /** The output from the task. Will be null if an error occurred. */
+  output: TOutput | null
   scores: EvalScore[]
   latencyMs: number
   cost: number
@@ -153,7 +154,7 @@ export async function runEval<TInput, TOutput, TExpected>(
           return {
             model: job.model,
             case: job.case,
-            output: null as unknown as TOutput,
+            output: null,
             scores: scorers.map(s => ({ name: s.name, score: 0 })),
             latencyMs: Date.now() - caseStart,
             cost: 0,

package/src/generate.ts

@@ -21,6 +21,7 @@ import {
   type LanguageModel
 } from 'ai'
 import { schema as convertSchema, type SimpleSchema } from './schema.js'
+import { isZodSchema } from '@org.ai/core'
 import type { ZodTypeAny } from 'zod'
 
 type ModelArg = string | LanguageModel
@@ -82,16 +83,6 @@ async function resolveModel(modelArg: ModelArg): Promise<LanguageModel> {
   return model(modelArg)
 }
 
-/**
- * Check if value is a Zod schema
- */
-function isZodSchema(value: unknown): value is ZodTypeAny {
-  return value !== null &&
-    typeof value === 'object' &&
-    '_def' in value &&
-    'parse' in value
-}
-
 /**
  * Convert schema to Zod if needed
  */
@@ -208,13 +199,17 @@ export async function streamObject<T>(
 ): Promise<StreamObjectResult<T, T, never>> {
   const model = await resolveModel(options.model)
   const schema = resolveSchema(options.schema as SchemaArg)
-  // Use 'as any' to handle AI SDK API variance
+  // NOTE: Type assertion required due to AI SDK's complex conditional return types.
+  // The SDK uses different return types based on output mode ('object' | 'array' | 'enum' | 'no-schema')
+  // and schema definition presence. Our wrapper simplifies this to always use 'object' output mode.
+  // This cast is safe because we control the input parameters.
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   return sdkStreamObject({
     ...options,
     model,
     schema,
     output: 'object'
-  } as any) as unknown as StreamObjectResult<T, T, never>
+  } as any) as StreamObjectResult<T, T, never>
 }
 
 /**

package/src/index.ts

@@ -1,39 +1,67 @@
 /**
- * ai-functions - Core AI primitives for building intelligent applications
+ * ai-functions - Full-featured AI primitives for building intelligent applications
  *
- * This is the foundational package that all other primitives depend on.
- * It provides:
- * - RPC primitives via rpc.do
- * - AI function types and interfaces
- * - Core AI() and ai() constructors
- * - Embedding utilities from AI SDK
- * - Cloudflare Workers AI provider (default for embeddings)
- * - Tagged template literal support for all functions
- * - Async iterator support for list/extract
+ * This package provides the complete feature set including:
+ * - Core primitives from ai-core (generate, AIPromise, templates, context)
+ * - Batch processing (BatchQueue, BatchMapPromise)
+ * - Retry/resilience patterns (RetryPolicy, CircuitBreaker, FallbackChain)
+ * - Budget tracking (BudgetTracker, TokenCounter)
+ * - Caching (MemoryCache, EmbeddingCache, GenerationCache)
+ * - Tool orchestration (AgenticLoop, ToolRouter)
+ * - Embeddings
+ * - Provider integrations
+ *
+ * For lightweight usage with just core primitives, use ai-core directly.
  *
  * @packageDocumentation
  */
 
-// Re-export RPC primitives from rpc.do
-export * from 'rpc.do'
-
-// Export AI function types and interfaces
-export * from './types.js'
-export * from './ai.js'
-
-// Export embedding utilities
-export * from './embeddings.js'
-
-// Export generation functions with smart model routing
-export { generateObject, generateText, streamObject, streamText } from './generate.js'
+// ============================================================================
+// Re-export core primitives from ai-core for backward compatibility
+// ============================================================================
 
-// Export simplified schema helper
-export { schema, type SimpleSchema } from './schema.js'
+// Types from ai-core
+export type {
+  AIFunctionDefinition,
+  JSONSchema,
+  AIGenerateOptions,
+  AIGenerateResult,
+  AIFunctionCall,
+  AIClient,
+  ImageOptions,
+  ImageResult,
+  VideoOptions,
+  VideoResult,
+  WriteOptions,
+  TemplateFunction as CoreTemplateFunction,
+  ListItem,
+  ListResult,
+  NamedList,
+  ListsResult,
+  CodeLanguage,
+  GenerativeOutputType,
+  HumanChannel,
+  LegacyHumanChannel,
+  SchemaLimitations,
+  BaseFunctionDefinition,
+  CodeFunctionDefinition,
+  CodeFunctionResult,
+  GenerativeFunctionDefinition,
+  GenerativeFunctionResult,
+  AgenticFunctionDefinition,
+  AgenticExecutionState,
+  HumanFunctionDefinition,
+  HumanFunctionResult,
+  FunctionDefinition,
+  DefinedFunction,
+  FunctionRegistry,
+  AutoDefineResult,
+} from '@org.ai/core'
 
-// Export providers
-export * from './providers/index.js'
+// Schema exports from ai-core
+export { schema, type SimpleSchema } from '@org.ai/core'
 
-// Export template utilities
+// Template exports from ai-core
 export {
   parseTemplate,
   createTemplateFunction,
@@ -45,9 +73,9 @@ export {
   type BatchableFunction,
   type StreamableList,
   type ChainablePromise,
-} from './template.js'
+} from '@org.ai/core'
 
-// Export AIPromise utilities for promise pipelining
+// AIPromise exports from ai-core
 export {
   AIPromise,
   isAIPromise,
@@ -63,17 +91,19 @@ export {
   AI_PROMISE_SYMBOL,
   RAW_PROMISE_SYMBOL,
   type AIPromiseOptions,
-} from './ai-promise.js'
+  type StreamingAIPromise,
+  type StreamOptions,
+} from '@org.ai/core'
 
-// Export AI primitives
+// Generation exports from ai-core
+export { generateObject, generateText, streamObject, streamText } from '@org.ai/core'
+
+// Primitives from ai-core
 export {
-  // Core generate primitive
   generate,
   type GenerateType,
   type GenerateOptions,
-
-  // Generative functions
-  ai as aiPrompt, // Renamed to avoid conflict with AIProxy from ai.ts
+  ai,
   write,
   code,
   list,
@@ -85,25 +115,52 @@ export {
   slides,
   image,
   video,
-
-  // Agentic functions
   do,
   research,
-
-  // Web functions
   read,
   browse,
-
-  // Decision functions
   decide,
-
-  // Human-in-the-loop functions
   ask,
   approve,
   review,
   type HumanOptions,
   type HumanResult,
-} from './primitives.js'
+} from '@org.ai/core'
+
+// Context exports from ai-core (basic version)
+export {
+  configure,
+  getContext,
+  withContext,
+  getGlobalContext,
+  resetContext,
+  getModel,
+  getProvider,
+  type ExecutionContext,
+} from '@org.ai/core'
+
+// Type guards from ai-core
+export { isZodSchema } from '@org.ai/core'
+
+// ============================================================================
+// Export AI Proxy (the AI class/proxy from ai.ts)
+// This is separate from the core 'ai' template function
+// ============================================================================
+
+export { AI, define, defineFunction, type AIProxy } from './ai.js'
+
+// Also export 'ai' primitive as 'aiPrompt' to avoid conflict with AIProxy
+export { ai as aiPrompt } from '@org.ai/core'
+
+// Export embedding utilities (not in ai-core)
+export * from './embeddings.js'
+
+// Export providers (not in ai-core)
+export * from './providers/index.js'
+
+// ============================================================================
+// Extended exports (NOT in ai-core)
+// ============================================================================
 
 // Export batch processing
 export {
@@ -137,18 +194,139 @@ export {
   type BatchMapOptions,
 } from './batch-map.js'
 
-// Export execution context
+// Export additional execution context features (full version with batch/budget)
 export {
-  configure,
-  getContext,
-  withContext,
-  getGlobalContext,
-  resetContext,
-  getModel,
-  getProvider,
   getBatchMode,
   getBatchThreshold,
   shouldUseBatchAPI,
-  type ExecutionContext,
+  getFlexThreshold,
+  getExecutionTier,
+  isFlexAvailable,
   type BatchMode,
+  type ContextBudgetConfig,
+  type ExecutionTier,
 } from './context.js'
+
+// Export budget tracking and request tracing
+export {
+  BudgetTracker,
+  TokenCounter,
+  RequestContext,
+  BudgetExceededError,
+  createRequestContext,
+  withBudget,
+  type BudgetConfig,
+  type BudgetAlert,
+  type BudgetSnapshot,
+  type TokenUsage,
+  type RequestInfo,
+  type RequestContextOptions,
+  type ModelPricing,
+  type WithBudgetOptions,
+  type RemainingBudget,
+  type CheckBudgetOptions,
+} from './budget.js'
+
+// Export agentic tool orchestration
+export {
+  // Core classes
+  AgenticLoop,
+  ToolRouter,
+  ToolValidator,
+
+  // Tool composition utilities
+  createTool,
+  createToolset,
+  wrapTool,
+  cachedTool,
+  rateLimitedTool,
+  timeoutTool,
+  createAgenticLoop,
+
+  // Types
+  type Tool,
+  type ToolCall,
+  type ToolResult,
+  type FormattedToolResult,
+  type ValidationResult,
+  type ModelResponse,
+  type Message,
+  type StepInfo,
+  type LoopOptions,
+  type RunOptions,
+  type ToolCallResult,
+  type SDKToolResult,
+  type LoopResult,
+  type LoopStreamEvent,
+} from './tool-orchestration.js'
+
+// Export caching layer for embeddings and generations
+export {
+  // Core cache storage
+  MemoryCache,
+
+  // Specialized caches
+  EmbeddingCache,
+  GenerationCache,
+
+  // Cache wrapper
+  withCache,
+
+  // Utilities
+  hashKey,
+  createCacheKey,
+
+  // Types
+  type CacheStorage,
+  type CacheEntry,
+  type CacheOptions,
+  type CacheStats,
+  type MemoryCacheOptions,
+  type CacheKeyType,
+  type EmbeddingCacheOptions,
+  type BatchEmbeddingResult,
+  type GenerationParams,
+  type GenerationCacheGetOptions,
+  type WithCacheOptions,
+  type CachedFunction,
+} from './cache.js'
+
+// Export retry/fallback patterns with exponential backoff
+export {
+  // Error types and classification
+  RetryableError,
+  NonRetryableError,
+  NetworkError,
+  RateLimitError,
+  CircuitOpenError,
+  ErrorCategory,
+  classifyError,
+
+  // Backoff calculation
+  calculateBackoff,
+
+  // Retry policy
+  RetryPolicy,
+
+  // Circuit breaker
+  CircuitBreaker,
+
+  // Fallback chain
+  FallbackChain,
+
+  // Convenience helper
+  withRetry,
+
+  // Types
+  type JitterStrategy,
+  type BackoffOptions,
+  type RetryOptions,
+  type RetryInfo,
+  type BatchItemResult,
+  type CircuitState,
+  type CircuitBreakerOptions,
+  type CircuitBreakerMetrics,
+  type FallbackModel,
+  type FallbackOptions,
+  type FallbackMetrics,
+} from './retry.js'