ai-functions 2.1.1 → 2.3.0

package/src/errors.ts

@@ -0,0 +1,37 @@
+/**
+ * Error classes for AI primitives
+ */
+
+/**
+ * Error thrown when a function is not yet implemented.
+ *
+ * This is used to clearly indicate at runtime that a function exists
+ * in the API but does not have a working implementation yet.
+ *
+ * @example
+ * ```ts
+ * throw new NotImplementedError('human', 'Human-in-the-loop functions require channel integrations')
+ * ```
+ */
+export class NotImplementedError extends Error {
+  /** The name of the function that is not implemented */
+  readonly functionName: string
+
+  /** Additional details about why it's not implemented or what's needed */
+  readonly details?: string
+
+  constructor(functionName: string, details?: string) {
+    const message = details
+      ? `Function '${functionName}' is not implemented: ${details}`
+      : `Function '${functionName}' is not implemented`
+    super(message)
+    this.name = 'NotImplementedError'
+    this.functionName = functionName
+    if (details !== undefined) this.details = details
+
+    // Maintain proper stack trace for where the error was thrown (V8 engines)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError)
+    }
+  }
+}

package/src/eval/runner.ts

@@ -8,6 +8,17 @@
 import { generateObject, generateText } from '../generate.js'
 import { schema } from '../schema.js'
 import { createModelVariants, getModelPricing, type EvalModel, type ModelTier } from './models.js'
+import { getLogger } from '../logger.js'
+
+/**
+ * Output function type for eval progress reporting
+ */
+export type EvalOutputFn = (message: string) => void
+
+/**
+ * Default output function uses logger.info
+ */
+const defaultOutput: EvalOutputFn = (message: string) => getLogger().info(message)
 
 export interface EvalCase<TInput = unknown, TExpected = unknown> {
   name: string
@@ -25,7 +36,8 @@ export interface EvalScore {
 export interface EvalResult<TOutput = unknown> {
   model: EvalModel
   case: EvalCase
-  output: TOutput
+  /** The output from the task. Will be null if an error occurred. */
+  output: TOutput | null
   scores: EvalScore[]
   latencyMs: number
   cost: number
@@ -48,12 +60,20 @@ export interface RunEvalOptions<TInput, TOutput, TExpected> {
   scorers: Array<{
     name: string
     description?: string
-    scorer: (args: { input: TInput; output: TOutput; expected?: TExpected }) => number | Promise<number>
+    scorer: (args: {
+      input: TInput
+      output: TOutput
+      expected?: TExpected
+    }) => number | Promise<number>
   }>
   models?: EvalModel[]
   tiers?: ModelTier[]
   providers?: string[]
   concurrency?: number
+  /** Custom output function for progress reporting (defaults to logger.info) */
+  output?: EvalOutputFn
+  /** Whether to suppress progress output (defaults to false) */
+  quiet?: boolean
 }
 
 /**
@@ -62,21 +82,22 @@ export interface RunEvalOptions<TInput, TOutput, TExpected> {
 export async function runEval<TInput, TOutput, TExpected>(
   options: RunEvalOptions<TInput, TOutput, TExpected>
 ): Promise<EvalSummary> {
-  const { name, cases, task, scorers, concurrency = 3 } = options
+  const { name, cases, task, scorers, concurrency = 3, quiet = false } = options
+  const log = quiet ? () => {} : options.output ?? defaultOutput
 
   // Get models to test
-  const models = options.models ?? createModelVariants({
-    tiers: options.tiers,
-    providers: options.providers,
-  }).map(v => v.input)
+  const variantOptions: { tiers?: ModelTier[]; providers?: string[] } = {}
+  if (options.tiers !== undefined) variantOptions.tiers = options.tiers
+  if (options.providers !== undefined) variantOptions.providers = options.providers
+  const models = options.models ?? createModelVariants(variantOptions).map((v) => v.input)
 
   const results: EvalResult<TOutput>[] = []
   const startTime = Date.now()
 
-  console.log(`\n🧪 Running eval: ${name}`)
-  console.log(`   Models: ${models.map(m => m.name).join(', ')}`)
-  console.log(`   Cases: ${cases.length}`)
-  console.log('')
+  log(`\nRunning eval: ${name}`)
+  log(`   Models: ${models.map((m) => m.name).join(', ')}`)
+  log(`   Cases: ${cases.length}`)
+  log('')
 
   // Run all model/case combinations
   const jobs: Array<{ model: EvalModel; case: EvalCase<TInput, TExpected> }> = []
@@ -96,7 +117,7 @@ export async function runEval<TInput, TOutput, TExpected>(
 
         try {
           // Run the task
-          const output = await task(job.case.input, job.model)
+          const taskOutput = await task(job.case.input, job.model)
           const latencyMs = Date.now() - caseStart
 
           // Run scorers
@@ -105,19 +126,19 @@ export async function runEval<TInput, TOutput, TExpected>(
             try {
               const score = await s.scorer({
                 input: job.case.input,
-                output,
-                expected: job.case.expected,
+                output: taskOutput,
+                ...(job.case.expected !== undefined && { expected: job.case.expected }),
               })
               scores.push({
                 name: s.name,
                 score: Math.max(0, Math.min(1, score)),
-                description: s.description,
+                ...(s.description && { description: s.description }),
               })
             } catch (err) {
               scores.push({
                 name: s.name,
                 score: 0,
-                description: s.description,
+                ...(s.description && { description: s.description }),
                 metadata: { error: String(err) },
               })
             }
@@ -129,32 +150,37 @@ export async function runEval<TInput, TOutput, TExpected>(
           const estimatedPromptTokens = 100
           const estimatedCompletionTokens = 200
           const cost = pricing
-            ? (estimatedPromptTokens * pricing.prompt + estimatedCompletionTokens * pricing.completion) / 1_000_000
+            ? (estimatedPromptTokens * pricing.prompt +
+                estimatedCompletionTokens * pricing.completion) /
+              1_000_000
             : 0
 
-          const avgScore = scores.length > 0
-            ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length
-            : 0
+          const avgScore =
+            scores.length > 0 ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length : 0
 
-          const symbol = avgScore >= 0.8 ? '✓' : avgScore >= 0.5 ? '~' : '✗'
-          console.log(`   ${symbol} ${job.model.name} | ${job.case.name} | ${(avgScore * 100).toFixed(0)}% | ${latencyMs}ms`)
+          const symbol = avgScore >= 0.8 ? 'PASS' : avgScore >= 0.5 ? 'WARN' : 'FAIL'
+          log(
+            `   ${symbol} ${job.model.name} | ${job.case.name} | ${(avgScore * 100).toFixed(
+              0
+            )}% | ${latencyMs}ms`
+          )
 
           return {
             model: job.model,
             case: job.case,
-            output,
+            output: taskOutput,
             scores,
             latencyMs,
             cost,
           }
         } catch (err) {
-          console.log(`   ✗ ${job.model.name} | ${job.case.name} | ERROR: ${err}`)
+          log(`   FAIL ${job.model.name} | ${job.case.name} | ERROR: ${err}`)
 
           return {
             model: job.model,
             case: job.case,
-            output: null as unknown as TOutput,
-            scores: scorers.map(s => ({ name: s.name, score: 0 })),
+            output: null,
+            scores: scorers.map((s) => ({ name: s.name, score: 0 })),
             latencyMs: Date.now() - caseStart,
             cost: 0,
             error: String(err),
@@ -169,10 +195,9 @@ export async function runEval<TInput, TOutput, TExpected>(
   // Calculate summary
   const totalTime = Date.now() - startTime
   const totalCost = results.reduce((sum, r) => sum + r.cost, 0)
-  const allScores = results.flatMap(r => r.scores.map(s => s.score))
-  const avgScore = allScores.length > 0
-    ? allScores.reduce((a, b) => a + b, 0) / allScores.length
-    : 0
+  const allScores = results.flatMap((r) => r.scores.map((s) => s.score))
+  const avgScore =
+    allScores.length > 0 ? allScores.reduce((a, b) => a + b, 0) / allScores.length : 0
 
   // Group by model
   const byModel: Record<string, { avgScore: number; count: number }> = {}
@@ -192,15 +217,15 @@ export async function runEval<TInput, TOutput, TExpected>(
     }
   }
 
-  console.log('')
-  console.log(`📊 Results:`)
-  console.log(`   Overall: ${(avgScore * 100).toFixed(1)}%`)
-  console.log(`   Time: ${(totalTime / 1000).toFixed(1)}s`)
-  console.log(`   Cost: $${totalCost.toFixed(4)}`)
-  console.log('')
-  console.log('   By Model:')
+  log('')
+  log(`Results:`)
+  log(`   Overall: ${(avgScore * 100).toFixed(1)}%`)
+  log(`   Time: ${(totalTime / 1000).toFixed(1)}s`)
+  log(`   Cost: $${totalCost.toFixed(4)}`)
+  log('')
+  log('   By Model:')
   for (const [modelId, stats] of Object.entries(byModel)) {
-    console.log(`   - ${modelId}: ${(stats.avgScore * 100).toFixed(1)}%`)
+    log(`   - ${modelId}: ${(stats.avgScore * 100).toFixed(1)}%`)
   }
 
   return {

package/src/eval-log/in-memory.ts

@@ -0,0 +1,90 @@
+/**
+ * InMemoryEvalLogStore — Map-backed default implementation of
+ * {@link EvalLogStore}.
+ *
+ * Matches Evalite v1's default backend: process-local Map keyed on `$id`,
+ * insertion-ordered for "most recent first" listing without sorting. Suitable
+ * for single-process tests, evals, and the cascade walker's in-flight log;
+ * not suitable for cross-process or multi-worker setups (use a disk/SQLite
+ * backend for those — same contract).
+ *
+ * @packageDocumentation
+ */
+
+import { randomUUID } from 'crypto'
+import type { EvalLogEntry, EvalLogListOptions, EvalLogStore } from './types.js'
+
+/**
+ * In-memory implementation of {@link EvalLogStore}.
+ */
+export class InMemoryEvalLogStore implements EvalLogStore {
+  /**
+   * Map keyed on `$id`. Insertion order on a JS Map is preserved, so we
+   * walk it in reverse for "most recent first" listing.
+   */
+  private readonly entries: Map<string, EvalLogEntry> = new Map()
+
+  async record(
+    entry: Omit<EvalLogEntry, '$id' | 'createdAt'> &
+      Partial<Pick<EvalLogEntry, '$id' | 'createdAt'>>
+  ): Promise<EvalLogEntry> {
+    const $id = entry.$id ?? randomUUID()
+    const createdAt = entry.createdAt ?? Date.now()
+    const stored: EvalLogEntry = {
+      $id,
+      createdAt,
+      model: entry.model,
+      prompt: entry.prompt,
+      response: entry.response,
+      usage: entry.usage,
+      costUsd: entry.costUsd,
+      durationMs: entry.durationMs,
+      ...(entry.traceId !== undefined ? { traceId: entry.traceId } : {}),
+      ...(entry.tags !== undefined ? { tags: entry.tags } : {}),
+    }
+    this.entries.set($id, stored)
+    return stored
+  }
+
+  async get(id: string): Promise<EvalLogEntry | undefined> {
+    return this.entries.get(id)
+  }
+
+  async list(options: EvalLogListOptions = {}): Promise<EvalLogEntry[]> {
+    const { traceId, model, tags, limit } = options
+    const out: EvalLogEntry[] = []
+    // Iterate in reverse insertion order — Map preserves order; we walk
+    // values into an array, then reverse for most-recent-first.
+    const all = Array.from(this.entries.values()).reverse()
+    for (const entry of all) {
+      if (traceId !== undefined && entry.traceId !== traceId) continue
+      if (model !== undefined && entry.model !== model) continue
+      if (tags !== undefined) {
+        let matchesAll = true
+        for (const k of Object.keys(tags)) {
+          if (entry.tags?.[k] !== tags[k]) {
+            matchesAll = false
+            break
+          }
+        }
+        if (!matchesAll) continue
+      }
+      out.push(entry)
+      if (limit !== undefined && out.length >= limit) break
+    }
+    return out
+  }
+
+  async delete(id: string): Promise<boolean> {
+    return this.entries.delete(id)
+  }
+
+  /**
+   * Convenience for tests: drop every entry. Not on the public
+   * {@link EvalLogStore} interface because the disk/SQLite backends may not
+   * want to expose a one-shot wipe.
+   */
+  clear(): void {
+    this.entries.clear()
+  }
+}

package/src/eval-log/index.ts

@@ -0,0 +1,46 @@
+/**
+ * EvalLogStore — pluggable persistence primitive for trace/eval entries.
+ *
+ * Exports the {@link EvalLogStore} contract, the
+ * {@link InMemoryEvalLogStore} default implementation, and a global
+ * accessor pair (`getEvalLogStore` / `configureEvalLogStore`) mirroring the
+ * marketplace persistence pattern from round 9.
+ *
+ * @packageDocumentation
+ */
+
+import { InMemoryEvalLogStore } from './in-memory.js'
+import type { EvalLogStore } from './types.js'
+
+export type { EvalLogEntry, EvalLogListOptions, EvalLogStore } from './types.js'
+export { InMemoryEvalLogStore } from './in-memory.js'
+
+// ============================================================================
+// Global accessor (lazy default + override)
+// ============================================================================
+
+let _store: EvalLogStore | null = null
+
+/**
+ * Get the global {@link EvalLogStore}. Lazily constructs an
+ * {@link InMemoryEvalLogStore} on first call when no store has been
+ * configured.
+ *
+ * Match the round-9 marketplace persistence accessor: callers that don't
+ * care about isolation read the global; callers that do (tests, multi-tenant
+ * apps) install their own via {@link configureEvalLogStore}.
+ */
+export function getEvalLogStore(): EvalLogStore {
+  if (_store === null) {
+    _store = new InMemoryEvalLogStore()
+  }
+  return _store
+}
+
+/**
+ * Install a global {@link EvalLogStore}. Pass `null` to reset to the lazy
+ * in-memory default (useful in test teardown).
+ */
+export function configureEvalLogStore(store: EvalLogStore | null): void {
+  _store = store
+}

package/src/eval-log/types.ts

@@ -0,0 +1,110 @@
+/**
+ * EvalLogStore — pluggable persistence primitive for trace/eval entries.
+ *
+ * Forward-looking primitive matching Evalite v1's EvalLogStore pattern:
+ * the in-memory default ships today; the disk/SQLite/durable backends can
+ * land later without breaking the trace middleware contract.
+ *
+ * Used downstream by `traceMiddleware` (in `../middleware/trace.ts`) as the
+ * sink for per-call prompt+response+usage records. The cascade-walker in
+ * services-as-software will consume `list()` / `get()` to populate the
+ * InvocationEvent stream once round 16+ adds the `'persona-trace'` variant.
+ *
+ * @packageDocumentation
+ */
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * A single entry in the eval log — one LLM call with its full payload.
+ *
+ * Shape mirrors what `traceMiddleware` emits, with optional `tags` for
+ * caller-supplied dimensions (persona name, evaluator role, cascade depth).
+ */
+export interface EvalLogEntry {
+  /** MDXLD identity — typically a UUID generated at insert time. */
+  $id: string
+  /**
+   * Optional caller-supplied trace correlation ID. When the cascade walker
+   * spans multiple LLM calls under one user request, all entries share the
+   * same `traceId` so `list({ traceId })` rolls them up.
+   */
+  traceId?: string
+  /** Model identifier (e.g. `'anthropic/claude-sonnet-4.5'` or `'sonnet'`). */
+  model: string
+  /**
+   * Stringified prompt as submitted to the model. We don't store the
+   * structured `LanguageModelV3Prompt` shape because (a) it's bulky and (b)
+   * downstream consumers (replay, fixture diff) only need the text payload.
+   */
+  prompt: string
+  /** The model's text response. Tool calls/files are not stored here. */
+  response: string
+  /** Token usage as reported by the AI SDK. */
+  usage: {
+    inputTokens: number
+    outputTokens: number
+  }
+  /** Computed USD cost (caller-supplied via the `pricing` overlay). */
+  costUsd: number
+  /** Wall-clock duration of the underlying `doGenerate` / `doStream` call. */
+  durationMs: number
+  /** Caller-supplied dimensions (persona, evaluator role, cascade step). */
+  tags?: Record<string, string>
+  /** Insert timestamp (epoch ms). */
+  createdAt: number
+}
+
+/**
+ * Options accepted by `EvalLogStore.list`. All fields are AND-combined.
+ */
+export interface EvalLogListOptions {
+  /** Filter to entries with this trace correlation ID. */
+  traceId?: string
+  /** Filter to entries for a specific model. */
+  model?: string
+  /**
+   * Filter to entries whose `tags` are a *superset* of the supplied object.
+   * (E.g. `{ persona: 'cfo' }` matches entries tagged
+   * `{ persona: 'cfo', step: '3' }` but not entries tagged
+   * `{ persona: 'cto' }`.)
+   */
+  tags?: Record<string, string>
+  /** Maximum number of entries to return (most recent first). */
+  limit?: number
+}
+
+/**
+ * Pluggable persistence interface for eval log entries.
+ *
+ * Modeled after the Evalite v1 EvalLogStore contract: in-memory default,
+ * disk JSON / SQLite / durable backends supplied via
+ * `configureEvalLogStore`.
+ *
+ * All methods are async to keep the contract uniform across backends — the
+ * in-memory implementation resolves synchronously under the hood.
+ */
+export interface EvalLogStore {
+  /**
+   * Persist a new entry. Returns the stored entry (with `$id` and
+   * `createdAt` filled in if the caller omitted them).
+   */
+  record(
+    entry: Omit<EvalLogEntry, '$id' | 'createdAt'> &
+      Partial<Pick<EvalLogEntry, '$id' | 'createdAt'>>
+  ): Promise<EvalLogEntry>
+  /**
+   * Read an entry by `$id`. Returns `undefined` when not found.
+   */
+  get(id: string): Promise<EvalLogEntry | undefined>
+  /**
+   * List entries matching the supplied filter. Returns most recent first.
+   */
+  list(options?: EvalLogListOptions): Promise<EvalLogEntry[]>
+  /**
+   * Delete an entry. Returns `true` if an entry was actually removed.
+   */
+  delete(id: string): Promise<boolean>
+}