outcome-cli 1.0.0

package/src/eval/ai-judge.ts

@@ -0,0 +1,368 @@
+/**
+ * AI-Powered Evaluation System
+ *
+ * Implements AI judges for subjective bounty criteria evaluation.
+ * Supports GPT-4o and Claude Opus models for fair evaluation of creative tasks.
+ *
+ * @module eval/ai-judge
+ * @see Requirements 10.1, 10.2, 10.3, 10.4, 10.5
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import OpenAI from 'openai';
+import { getJudgeCache, type JudgeCache, type JudgeResult, type JudgeModel } from './judge-cache.js';
+import { createHash } from 'crypto';
+
+// Re-export types from judge-cache for convenience
+export type { JudgeModel, JudgeResult } from './judge-cache.js';
+
+/**
+ * Configuration for an AI judge evaluation.
+ *
+ * @see Requirements 10.1, 10.5
+ */
+export interface JudgeConfig {
+  /** The AI model to use for evaluation */
+  model: JudgeModel;
+  /** The rubric describing evaluation criteria */
+  rubric: string;
+  /** Maximum score that can be awarded */
+  maxScore: number;
+  /** Optional temperature for model responses (default: 0.3 for consistency) */
+  temperature?: number;
+  /** Optional maximum tokens for response (default: 1024) */
+  maxTokens?: number;
+}
+
+/**
+ * Internal structure for parsed judge response.
+ */
+interface ParsedJudgeResponse {
+  score: number;
+  reasoning: string;
+  highlights: string[];
+}
+
+/**
+ * Error thrown when AI judge evaluation fails.
+ */
+export class AIJudgeError extends Error {
+  constructor(
+    message: string,
+    public readonly model: JudgeModel,
+    public readonly cause?: Error
+  ) {
+    super(message);
+    this.name = 'AIJudgeError';
+  }
+}
+
+/**
+ * Creates a hash key for caching judge results.
+ * Combines artifact content and rubric to ensure cache consistency.
+ *
+ * @param artifact - The artifact being evaluated
+ * @param rubric - The evaluation rubric
+ * @returns SHA-256 hash string for cache key
+ *
+ * @see Requirements 10.4
+ */
+export function hashArtifact(artifact: unknown, rubric: string): string {
+  const content = JSON.stringify({ artifact, rubric });
+  return createHash('sha256').update(content).digest('hex');
+}
+
+/**
+ * Builds the evaluation prompt for the AI judge.
+ *
+ * @param artifact - The artifact to evaluate
+ * @param rubric - The evaluation rubric
+ * @param maxScore - Maximum score possible
+ * @returns Formatted prompt string
+ *
+ * @see Requirements 10.2
+ */
+function buildJudgePrompt(artifact: unknown, rubric: string, maxScore: number): string {
+  const artifactStr = typeof artifact === 'string' 
+    ? artifact 
+    : JSON.stringify(artifact, null, 2);
+
+  return `You are an expert AI judge evaluating an agent's output against a specific rubric.
+
+## Evaluation Rubric
+${rubric}
+
+## Agent Output to Evaluate
+${artifactStr}
+
+## Instructions
+1. Carefully evaluate the agent's output against each criterion in the rubric
+2. Provide a score from 0 to ${maxScore} based on how well the output meets the criteria
+3. Explain your reasoning in detail
+4. Highlight any notable strengths or weaknesses
+
+## Response Format
+Respond with a JSON object in this exact format:
+{
+  "score": <number from 0 to ${maxScore}>,
+  "reasoning": "<detailed explanation of your evaluation>",
+  "highlights": ["<notable aspect 1>", "<notable aspect 2>", ...]
+}
+
+Respond ONLY with the JSON object, no additional text.`;
+}
+
+/**
+ * Parses the AI judge response into a structured format.
+ *
+ * @param response - Raw response from the AI model
+ * @param maxScore - Maximum score for validation
+ * @returns Parsed judge response
+ * @throws AIJudgeError if parsing fails
+ */
+function parseJudgeResponse(response: string, maxScore: number, model: JudgeModel): ParsedJudgeResponse {
+  try {
+    // Try to extract JSON from the response
+    const jsonMatch = response.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      throw new Error('No JSON object found in response');
+    }
+
+    const parsed = JSON.parse(jsonMatch[0]) as Record<string, unknown>;
+
+    // Validate required fields
+    if (typeof parsed.score !== 'number') {
+      throw new Error('Missing or invalid "score" field');
+    }
+    if (typeof parsed.reasoning !== 'string' || parsed.reasoning.trim() === '') {
+      throw new Error('Missing or invalid "reasoning" field');
+    }
+
+    // Clamp score to valid range
+    const score = Math.max(0, Math.min(maxScore, parsed.score));
+
+    // Parse highlights (optional, default to empty array)
+    const highlights = Array.isArray(parsed.highlights)
+      ? parsed.highlights.filter((h): h is string => typeof h === 'string')
+      : [];
+
+    return {
+      score,
+      reasoning: parsed.reasoning,
+      highlights,
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown parsing error';
+    throw new AIJudgeError(`Failed to parse judge response: ${errorMessage}`, model);
+  }
+}
+
+/**
+ * Evaluates an artifact using GPT-4o as the judge.
+ *
+ * @param artifact - The artifact to evaluate
+ * @param config - Judge configuration
+ * @returns Parsed judge response
+ */
+async function evaluateWithGPT4o(
+  artifact: unknown,
+  config: JudgeConfig
+): Promise<ParsedJudgeResponse> {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new AIJudgeError('OPENAI_API_KEY not configured', 'gpt-4o');
+  }
+
+  const client = new OpenAI({ apiKey });
+  const prompt = buildJudgePrompt(artifact, config.rubric, config.maxScore);
+
+  try {
+    const response = await client.chat.completions.create({
+      model: 'gpt-4o',
+      messages: [
+        {
+          role: 'system',
+          content: 'You are an expert AI judge. Evaluate outputs fairly and provide detailed reasoning.',
+        },
+        {
+          role: 'user',
+          content: prompt,
+        },
+      ],
+      temperature: config.temperature ?? 0.3,
+      max_tokens: config.maxTokens ?? 1024,
+    });
+
+    const content = response.choices[0]?.message?.content;
+    if (!content) {
+      throw new AIJudgeError('Empty response from GPT-4o', 'gpt-4o');
+    }
+
+    return parseJudgeResponse(content, config.maxScore, 'gpt-4o');
+  } catch (error) {
+    if (error instanceof AIJudgeError) {
+      throw error;
+    }
+    const cause = error instanceof Error ? error : new Error(String(error));
+    throw new AIJudgeError(`GPT-4o evaluation failed: ${cause.message}`, 'gpt-4o', cause);
+  }
+}
+
+/**
+ * Evaluates an artifact using Claude Opus as the judge.
+ *
+ * @param artifact - The artifact to evaluate
+ * @param config - Judge configuration
+ * @returns Parsed judge response
+ */
+async function evaluateWithClaudeOpus(
+  artifact: unknown,
+  config: JudgeConfig
+): Promise<ParsedJudgeResponse> {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    throw new AIJudgeError('ANTHROPIC_API_KEY not configured', 'claude-opus');
+  }
+
+  const client = new Anthropic({ apiKey });
+  const prompt = buildJudgePrompt(artifact, config.rubric, config.maxScore);
+
+  try {
+    const response = await client.messages.create({
+      model: 'claude-3-opus-20240229',
+      max_tokens: config.maxTokens ?? 1024,
+      system: 'You are an expert AI judge. Evaluate outputs fairly and provide detailed reasoning.',
+      messages: [
+        {
+          role: 'user',
+          content: prompt,
+        },
+      ],
+    });
+
+    const content = response.content[0];
+    if (content.type !== 'text' || !content.text) {
+      throw new AIJudgeError('Empty response from Claude Opus', 'claude-opus');
+    }
+
+    return parseJudgeResponse(content.text, config.maxScore, 'claude-opus');
+  } catch (error) {
+    if (error instanceof AIJudgeError) {
+      throw error;
+    }
+    const cause = error instanceof Error ? error : new Error(String(error));
+    throw new AIJudgeError(`Claude Opus evaluation failed: ${cause.message}`, 'claude-opus', cause);
+  }
+}
+
+/**
+ * Evaluates an artifact using an AI judge model.
+ *
+ * This function:
+ * 1. Checks the cache for existing results (idempotence)
+ * 2. If not cached, invokes the specified AI model
+ * 3. Parses and validates the response
+ * 4. Caches the result for future requests
+ *
+ * @param artifact - The artifact to evaluate
+ * @param config - Judge configuration including model, rubric, and maxScore
+ * @param cache - Optional cache instance (defaults to global cache)
+ * @returns JudgeResult with score, reasoning, and highlights
+ *
+ * @example
+ * const result = await evaluateWithAIJudge(
+ *   { message: "Hello world", quality: "high" },
+ *   {
+ *     model: 'gpt-4o',
+ *     rubric: 'Evaluate the message for clarity and professionalism...',
+ *     maxScore: 10
+ *   }
+ * );
+ *
+ * @see Requirements 10.1, 10.2, 10.3, 10.4, 10.5
+ */
+export async function evaluateWithAIJudge(
+  artifact: unknown,
+  config: JudgeConfig,
+  cache?: JudgeCache
+): Promise<JudgeResult> {
+  // Use provided cache or get global cache
+  const judgeCache = cache ?? getJudgeCache();
+
+  // Generate cache key from artifact + rubric
+  const cacheKey = hashArtifact(artifact, config.rubric);
+
+  // Check cache first (idempotence)
+  const cached = await judgeCache.get(cacheKey);
+  if (cached) {
+    return {
+      ...cached,
+      cached: true,
+    };
+  }
+
+  // Evaluate with the specified model
+  let parsed: ParsedJudgeResponse;
+
+  switch (config.model) {
+    case 'gpt-4o':
+      parsed = await evaluateWithGPT4o(artifact, config);
+      break;
+    case 'claude-opus':
+      parsed = await evaluateWithClaudeOpus(artifact, config);
+      break;
+    default: {
+      const exhaustiveCheck: never = config.model;
+      throw new AIJudgeError(`Unsupported judge model: ${exhaustiveCheck}`, config.model);
+    }
+  }
+
+  // Build result
+  const result: JudgeResult = {
+    score: parsed.score,
+    normalizedScore: config.maxScore > 0 ? parsed.score / config.maxScore : 0,
+    reasoning: parsed.reasoning,
+    highlights: parsed.highlights,
+    model: config.model,
+    cached: false,
+    evaluatedAt: new Date().toISOString(),
+  };
+
+  // Cache the result (without the 'cached' flag)
+  const cacheableResult: Omit<JudgeResult, 'cached'> & { cached?: boolean } = { ...result };
+  delete cacheableResult.cached;
+  await judgeCache.set(cacheKey, cacheableResult as JudgeResult);
+
+  return result;
+}
+
+/**
+ * Validates a JudgeConfig object.
+ *
+ * @param config - Configuration to validate
+ * @returns true if valid
+ * @throws Error if invalid
+ */
+export function validateJudgeConfig(config: JudgeConfig): boolean {
+  if (!['gpt-4o', 'claude-opus'].includes(config.model)) {
+    throw new Error(`Invalid judge model: ${config.model}`);
+  }
+
+  if (typeof config.rubric !== 'string' || config.rubric.trim() === '') {
+    throw new Error('Rubric must be a non-empty string');
+  }
+
+  if (typeof config.maxScore !== 'number' || config.maxScore <= 0) {
+    throw new Error('maxScore must be a positive number');
+  }
+
+  if (config.temperature !== undefined && (config.temperature < 0 || config.temperature > 2)) {
+    throw new Error('temperature must be between 0 and 2');
+  }
+
+  if (config.maxTokens !== undefined && (config.maxTokens < 1 || config.maxTokens > 4096)) {
+    throw new Error('maxTokens must be between 1 and 4096');
+  }
+
+  return true;
+}