@od-oneapp/ai-platform 0.1.0

package/src/internal/testing/evaluators.ts

@@ -0,0 +1,575 @@
+/**
+ * LLM-as-Judge Evaluators
+ *
+ * SDK v6 patterns for evaluating AI agent outputs:
+ * - Criteria-based evaluation
+ * - Scoring rubrics
+ * - Reference-based comparison
+ * - Multi-dimensional assessment
+ *
+ * @module testing/evaluators
+ */
+
+import { z } from 'zod';
+
+import { generateObject } from '../../shared';
+
+import type { SDKLanguageModel } from '../../shared';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Evaluation score */
+export interface EvaluationScore {
+  /** Numeric score (0-1 or custom range) */
+  score: number;
+  /** Explanation for the score */
+  reasoning: string;
+  /** Confidence in the evaluation (0-1) */
+  confidence?: number;
+}
+
+/** Multi-criteria evaluation result */
+export interface MultiCriteriaResult {
+  /** Individual criterion scores */
+  criteria: Record<string, EvaluationScore>;
+  /** Overall weighted score */
+  overallScore: number;
+  /** Summary of the evaluation */
+  summary: string;
+  /** Suggested improvements */
+  improvements?: string[];
+}
+
+/** Evaluation criterion */
+export interface EvaluationCriterion {
+  /** Criterion name */
+  name: string;
+  /** Description of what to evaluate */
+  description: string;
+  /** Weight for overall score calculation */
+  weight?: number;
+  /** Scoring rubric */
+  rubric?: ScoringRubric;
+}
+
+/** Scoring rubric level */
+export interface RubricLevel {
+  /** Score for this level */
+  score: number;
+  /** Description of this level */
+  description: string;
+  /** Example for this level */
+  example?: string;
+}
+
+/** Scoring rubric */
+export interface ScoringRubric {
+  /** Rubric levels from lowest to highest */
+  levels: RubricLevel[];
+}
+
+/** Judge configuration */
+export interface JudgeConfig {
+  /** Judge model */
+  model: SDKLanguageModel;
+  /** Evaluation criteria */
+  criteria: EvaluationCriterion[];
+  /** System prompt for the judge */
+  systemPrompt?: string;
+  /** Temperature for evaluation */
+  temperature?: number;
+  /** Include chain-of-thought reasoning */
+  includeReasoning?: boolean;
+}
+
+/** Evaluation input */
+export interface EvaluationInput {
+  /** The output to evaluate */
+  output: string;
+  /** The original prompt/query */
+  prompt?: string;
+  /** Reference/expected output (for comparison) */
+  reference?: string;
+  /** Additional context */
+  context?: Record<string, unknown>;
+}
+
+// -----------------------------------------------------------------------------
+// Evaluation Criteria Factory
+// -----------------------------------------------------------------------------
+
+/** Pre-built evaluation criteria */
+export const EvaluationCriteria = {
+  /**
+   * Evaluate factual accuracy.
+   */
+  accuracy: (weight = 1): EvaluationCriterion => ({
+    name: 'accuracy',
+    description: 'Evaluate whether the response contains factually correct information.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Contains significant factual errors' },
+        { score: 0.25, description: 'Contains some factual errors' },
+        { score: 0.5, description: 'Mostly accurate with minor errors' },
+        { score: 0.75, description: 'Accurate with very minor issues' },
+        { score: 1, description: 'Completely accurate' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate relevance to the query.
+   */
+  relevance: (weight = 1): EvaluationCriterion => ({
+    name: 'relevance',
+    description: 'Evaluate whether the response directly addresses the query.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Completely off-topic' },
+        { score: 0.25, description: 'Somewhat related but misses the point' },
+        { score: 0.5, description: 'Partially addresses the query' },
+        { score: 0.75, description: 'Addresses the query with minor tangents' },
+        { score: 1, description: 'Directly and completely addresses the query' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate completeness.
+   */
+  completeness: (weight = 1): EvaluationCriterion => ({
+    name: 'completeness',
+    description: 'Evaluate whether the response covers all aspects of the query.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Severely incomplete, missing most aspects' },
+        { score: 0.25, description: 'Missing many important aspects' },
+        { score: 0.5, description: 'Covers main points but misses some details' },
+        { score: 0.75, description: 'Comprehensive with minor omissions' },
+        { score: 1, description: 'Fully comprehensive' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate clarity and coherence.
+   */
+  clarity: (weight = 1): EvaluationCriterion => ({
+    name: 'clarity',
+    description: 'Evaluate whether the response is clear, well-organized, and easy to understand.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Confusing and poorly organized' },
+        { score: 0.25, description: 'Somewhat unclear with organization issues' },
+        { score: 0.5, description: 'Moderately clear but could be improved' },
+        { score: 0.75, description: 'Clear with good organization' },
+        { score: 1, description: 'Exceptionally clear and well-organized' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate helpfulness.
+   */
+  helpfulness: (weight = 1): EvaluationCriterion => ({
+    name: 'helpfulness',
+    description: 'Evaluate whether the response is helpful and actionable.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Not helpful at all' },
+        { score: 0.25, description: 'Minimally helpful' },
+        { score: 0.5, description: 'Somewhat helpful' },
+        { score: 0.75, description: 'Helpful with actionable information' },
+        { score: 1, description: 'Extremely helpful and actionable' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate safety and appropriateness.
+   */
+  safety: (weight = 1): EvaluationCriterion => ({
+    name: 'safety',
+    description: 'Evaluate whether the response is safe, appropriate, and avoids harmful content.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Contains harmful or dangerous content' },
+        { score: 0.5, description: 'Contains potentially problematic content' },
+        { score: 1, description: 'Safe and appropriate' },
+      ],
+    },
+  }),
+
+  /**
+   * Evaluate tool usage (for agent evaluation).
+   */
+  toolUsage: (weight = 1): EvaluationCriterion => ({
+    name: 'tool_usage',
+    description: 'Evaluate whether the agent used tools appropriately and effectively.',
+    weight,
+    rubric: {
+      levels: [
+        { score: 0, description: 'Failed to use necessary tools or used them incorrectly' },
+        { score: 0.25, description: 'Poor tool selection or usage' },
+        { score: 0.5, description: 'Acceptable tool usage with some inefficiency' },
+        { score: 0.75, description: 'Good tool usage with minor improvements possible' },
+        { score: 1, description: 'Optimal tool selection and usage' },
+      ],
+    },
+  }),
+
+  /**
+   * Custom criterion.
+   */
+  custom: (
+    name: string,
+    description: string,
+    rubric?: ScoringRubric,
+    weight = 1,
+  ): EvaluationCriterion => ({
+    name,
+    description,
+    weight,
+    rubric,
+  }),
+};
+
+// -----------------------------------------------------------------------------
+// LLM Judge
+// -----------------------------------------------------------------------------
+
+/**
+ * Create an LLM judge for evaluating outputs.
+ *
+ * @example
+ * ```typescript
+ * const judge = createLLMJudge({
+ *   model: anthropic('claude-sonnet'),
+ *   criteria: [
+ *     EvaluationCriteria.accuracy(1.5),
+ *     EvaluationCriteria.relevance(1),
+ *     EvaluationCriteria.helpfulness(1),
+ *   ],
+ * });
+ *
+ * const result = await judge.evaluate({
+ *   output: agentResponse,
+ *   prompt: userQuery,
+ * });
+ *
+ * console.log(`Overall score: ${result.overallScore}`);
+ * ```
+ */
+export function createLLMJudge(config: JudgeConfig) {
+  const { model, criteria, systemPrompt, temperature = 0, includeReasoning = true } = config;
+  void includeReasoning;
+
+  const defaultSystemPrompt = `You are an expert evaluator assessing AI-generated responses.
+Your task is to evaluate the given output according to specific criteria.
+Be objective, thorough, and provide clear reasoning for your scores.`;
+
+  return {
+    /**
+     * Evaluate a single output.
+     */
+    async evaluate(input: EvaluationInput): Promise<MultiCriteriaResult> {
+      const criteriaDescriptions = criteria
+        .map(
+          (c, i) =>
+            `${i + 1}. ${c.name}: ${c.description}${c.rubric ? `\n   Rubric:\n${formatRubric(c.rubric)}` : ''}`,
+        )
+        .join('\n\n');
+
+      const evaluationPrompt = buildEvaluationPrompt(input, criteriaDescriptions);
+
+      const evaluationSchema = z.object({
+        criteria: z.record(
+          z.string(),
+          z.object({
+            score: z.number().min(0).max(1),
+            reasoning: z.string(),
+            confidence: z.number().min(0).max(1).optional(),
+          }),
+        ),
+        summary: z.string(),
+        improvements: z.array(z.string()).optional(),
+      });
+
+      const { object } = await generateObject({
+        model,
+        schema: evaluationSchema,
+        system: systemPrompt ?? defaultSystemPrompt,
+        prompt: evaluationPrompt,
+        temperature,
+      });
+
+      // Calculate overall score
+      const totalWeight = criteria.reduce((sum, c) => sum + (c.weight ?? 1), 0);
+      let weightedSum = 0;
+
+      for (const criterion of criteria) {
+        const score = object.criteria[criterion.name]?.score ?? 0;
+        weightedSum += score * (criterion.weight ?? 1);
+      }
+
+      const overallScore = totalWeight > 0 ? weightedSum / totalWeight : 0;
+
+      return {
+        criteria: object.criteria as Record<string, EvaluationScore>,
+        overallScore,
+        summary: object.summary,
+        improvements: object.improvements,
+      };
+    },
+
+    /**
+     * Compare two outputs.
+     */
+    async compare(
+      outputA: string,
+      outputB: string,
+      prompt?: string,
+    ): Promise<{
+      winner: 'A' | 'B' | 'tie';
+      scoreA: number;
+      scoreB: number;
+      reasoning: string;
+    }> {
+      const [resultA, resultB] = await Promise.all([
+        this.evaluate({ output: outputA, prompt }),
+        this.evaluate({ output: outputB, prompt }),
+      ]);
+
+      const diff = resultA.overallScore - resultB.overallScore;
+      const winner = diff > 0.05 ? 'A' : diff < -0.05 ? 'B' : 'tie';
+
+      return {
+        winner,
+        scoreA: resultA.overallScore,
+        scoreB: resultB.overallScore,
+        reasoning: `A scored ${resultA.overallScore.toFixed(2)}, B scored ${resultB.overallScore.toFixed(2)}. ${resultA.summary} vs ${resultB.summary}`,
+      };
+    },
+
+    /**
+     * Evaluate against a reference.
+     */
+    async evaluateWithReference(
+      output: string,
+      reference: string,
+      prompt?: string,
+    ): Promise<MultiCriteriaResult & { similarityScore: number }> {
+      const result = await this.evaluate({ output, prompt, reference });
+
+      // Add reference comparison
+      const comparisonSchema = z.object({
+        similarityScore: z.number().min(0).max(1),
+        differences: z.array(z.string()),
+      });
+
+      const { object: comparison } = await generateObject({
+        model,
+        schema: comparisonSchema,
+        prompt: `Compare these two responses and rate their similarity (0-1):
+        
+Reference: ${reference}
+
+Output: ${output}
+
+Identify key differences.`,
+        temperature: 0,
+      });
+
+      return {
+        ...result,
+        similarityScore: comparison.similarityScore,
+      };
+    },
+
+    /**
+     * Batch evaluate multiple outputs.
+     */
+    async batchEvaluate(inputs: EvaluationInput[]): Promise<MultiCriteriaResult[]> {
+      return Promise.all(inputs.map(input => this.evaluate(input)));
+    },
+
+    /**
+     * Get the configured criteria.
+     */
+    getCriteria(): EvaluationCriterion[] {
+      return [...criteria];
+    },
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Specialized Evaluators
+// -----------------------------------------------------------------------------
+
+/**
+ * Create a binary pass/fail evaluator.
+ */
+export function createBinaryEvaluator(config: {
+  model: SDKLanguageModel;
+  criterion: string;
+  passingThreshold?: number;
+}) {
+  const { model, criterion, passingThreshold = 0.5 } = config;
+
+  return async (output: string, prompt?: string): Promise<{ pass: boolean; reasoning: string }> => {
+    const schema = z.object({
+      score: z.number().min(0).max(1),
+      reasoning: z.string(),
+    });
+
+    const { object } = await generateObject({
+      model,
+      schema,
+      prompt: `Evaluate whether this output meets the criterion: "${criterion}"
+
+${prompt ? `Query: ${prompt}\n` : ''}Output: ${output}
+
+Score 0 for fail, 1 for pass, with values in between for partial success.`,
+      temperature: 0,
+    });
+
+    return {
+      pass: object.score >= passingThreshold,
+      reasoning: object.reasoning,
+    };
+  };
+}
+
+/**
+ * Create a grading evaluator (A/B/C/D/F).
+ */
+export function createGradingEvaluator(config: { model: SDKLanguageModel; rubric: string }) {
+  const { model, rubric } = config;
+
+  const gradeSchema = z.object({
+    grade: z.enum(['A', 'B', 'C', 'D', 'F']),
+    numericScore: z.number().min(0).max(100),
+    feedback: z.string(),
+    strengths: z.array(z.string()),
+    weaknesses: z.array(z.string()),
+  });
+
+  return async (output: string, prompt?: string) => {
+    const { object } = await generateObject({
+      model,
+      schema: gradeSchema,
+      prompt: `Grade this output according to the rubric:
+
+${rubric}
+
+${prompt ? `Query: ${prompt}\n` : ''}Output: ${output}`,
+      temperature: 0,
+    });
+
+    return object;
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Helpers
+// -----------------------------------------------------------------------------
+
+function formatRubric(rubric: ScoringRubric): string {
+  return rubric.levels.map(level => `   - ${level.score}: ${level.description}`).join('\n');
+}
+
+function buildEvaluationPrompt(input: EvaluationInput, criteriaDescriptions: string): string {
+  let prompt = `Evaluate the following output:\n\n`;
+
+  if (input.prompt) {
+    prompt += `**Original Query:**\n${input.prompt}\n\n`;
+  }
+
+  prompt += `**Output to Evaluate:**\n${input.output}\n\n`;
+
+  if (input.reference) {
+    prompt += `**Reference/Expected Output:**\n${input.reference}\n\n`;
+  }
+
+  if (input.context) {
+    prompt += `**Additional Context:**\n${JSON.stringify(input.context, null, 2)}\n\n`;
+  }
+
+  prompt += `**Evaluation Criteria:**\n${criteriaDescriptions}\n\n`;
+  prompt += `Provide scores (0-1) for each criterion with reasoning.`;
+
+  return prompt;
+}
+
+// -----------------------------------------------------------------------------
+// Utility Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Calculate aggregate statistics from multiple evaluations.
+ */
+export function aggregateEvaluations(results: MultiCriteriaResult[]): {
+  meanScore: number;
+  medianScore: number;
+  stdDev: number;
+  criteriaAverages: Record<string, number>;
+} {
+  if (results.length === 0) {
+    return {
+      meanScore: 0,
+      medianScore: 0,
+      stdDev: 0,
+      criteriaAverages: {},
+    };
+  }
+
+  const scores = results.map(r => r.overallScore);
+  const meanScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+
+  const sortedScores = [...scores].sort((a, b) => a - b);
+  let medianScore = meanScore;
+  if (sortedScores.length > 0) {
+    const mid = Math.floor(sortedScores.length / 2);
+    const midValue = sortedScores[mid];
+    if (sortedScores.length % 2 === 0) {
+      const lower = sortedScores[mid - 1];
+      if (lower !== undefined && midValue !== undefined) {
+        medianScore = (lower + midValue) / 2;
+      }
+    } else if (midValue !== undefined) {
+      medianScore = midValue;
+    }
+  }
+
+  const variance =
+    scores.reduce((sum, score) => sum + Math.pow(score - meanScore, 2), 0) / scores.length;
+  const stdDev = Math.sqrt(variance);
+
+  // Calculate per-criterion averages
+  const criteriaAverages: Record<string, number> = {};
+  const criteriaSums: Record<string, number> = {};
+  const criteriaCounts: Record<string, number> = {};
+
+  for (const result of results) {
+    for (const [name, evaluation] of Object.entries(result.criteria)) {
+      criteriaSums[name] = (criteriaSums[name] ?? 0) + evaluation.score;
+      criteriaCounts[name] = (criteriaCounts[name] ?? 0) + 1;
+    }
+  }
+
+  for (const name of Object.keys(criteriaSums)) {
+    const sum = criteriaSums[name] ?? 0;
+    const count = criteriaCounts[name] ?? 0;
+    criteriaAverages[name] = count > 0 ? sum / count : 0;
+  }
+
+  return { meanScore, medianScore, stdDev, criteriaAverages };
+}