@od-oneapp/ai-platform 0.1.0

package/src/agents/patterns/evaluator-optimizer-pattern.ts

@@ -0,0 +1,653 @@
+/**
+ * Evaluator-Optimizer Feedback Loop Pattern
+ * Based on VERCEL_AI_SDK_V6_PATTERNS.md section 4.2
+ *
+ * A powerful feedback loop architecture for iterative quality improvement.
+ * Agents can evaluate output against criteria and optimize based on feedback
+ * until quality thresholds are met.
+ *
+ * WORKFLOW:
+ * 1. Generate initial content based on user request
+ * 2. Evaluate quality against criteria
+ * 3. If quality < threshold, optimize the content
+ * 4. Re-evaluate optimized content
+ * 5. Repeat steps 3-4 until threshold met or max iterations reached
+ */
+
+import { logInfo } from '@repo/shared/logs';
+
+/**
+ * Evaluation result with multi-dimensional assessment
+ * Provides detailed feedback on content quality
+ *
+ * Note: Named PatternEvaluationResult to avoid collision with
+ * EvaluationResult from evaluator-optimizer/schema.ts which uses
+ * a different schema structure (9-dimensional vs multi-dimensional).
+ */
+export interface PatternEvaluationResult {
+  /**
+   * Overall quality score (0-1 or 1-10)
+   */
+  score: number;
+
+  /**
+   * Multi-dimensional quality assessment
+   */
+  dimensions: {
+    name: string;
+    score: number;
+    feedback: string;
+  }[];
+
+  /**
+   * Actionable improvement suggestions
+   */
+  suggestions: string[];
+
+  /**
+   * Whether content should be retried/optimized
+   */
+  shouldRetry: boolean;
+
+  /**
+   * Detailed assessment text
+   */
+  assessment?: string;
+
+  /**
+   * Strengths identified in the content
+   */
+  strengths?: string[];
+
+  /**
+   * Issues found that prevent threshold achievement
+   */
+  issues?: string[];
+}
+
+/**
+ * Context for evaluations
+ */
+export interface EvalContext {
+  request: string;
+  threshold?: number;
+  maxScore?: number;
+  focusAreas?: string[];
+  iterationNumber?: number;
+}
+
+/**
+ * Configuration for evaluator-optimizer
+ */
+export interface EvaluatorOptimizerOptions {
+  /**
+   * Evaluator function that assesses output quality
+   * Returns detailed feedback on multiple dimensions
+   */
+  evaluator: (output: string, context: EvalContext) => Promise<PatternEvaluationResult>;
+
+  /**
+   * Optimizer function that improves content based on evaluation
+   */
+  optimizer: (output: string, evaluation: PatternEvaluationResult) => Promise<string>;
+
+  /**
+   * Maximum optimization iterations (default: 5)
+   */
+  maxIterations?: number;
+
+  /**
+   * Target quality score to achieve (default: 0.8 or 8/10)
+   */
+  targetScore?: number;
+
+  /**
+   * Enable detailed logging of each cycle
+   */
+  enableLogging?: boolean;
+
+  /**
+   * Stop immediately when threshold is met
+   */
+  stopOnThreshold?: boolean;
+
+  /**
+   * Detect diminishing returns after N non-improving cycles
+   */
+  diminishingReturnThreshold?: number;
+}
+
+/**
+ * Complete optimization result
+ */
+export interface PatternOptimizedResult {
+  finalContent: string;
+  finalEvaluation: PatternEvaluationResult;
+  cyclesCompleted: number;
+  meetsThreshold: boolean;
+  scoreHistory: number[];
+  contentHistory: string[];
+  terminationReason: 'threshold_met' | 'max_iterations' | 'diminishing_returns' | 'error';
+  error?: string;
+  metadata?: {
+    startTime: string;
+    endTime: string;
+    processingTimeMs: number;
+    averageScoreImprovement: number;
+  };
+}
+
+/**
+ * Main EvaluatorOptimizer class
+ * Orchestrates the iterative evaluation-optimization feedback loop
+ */
+export class PatternEvaluatorOptimizer {
+  private options: Required<EvaluatorOptimizerOptions>;
+  private logger: (msg: string) => void;
+
+  constructor(options: EvaluatorOptimizerOptions) {
+    this.options = {
+      evaluator: options.evaluator,
+      optimizer: options.optimizer,
+      maxIterations: options.maxIterations ?? 5,
+      targetScore: options.targetScore ?? 0.8,
+      enableLogging: options.enableLogging ?? false,
+      stopOnThreshold: options.stopOnThreshold ?? true,
+      diminishingReturnThreshold: options.diminishingReturnThreshold ?? 2,
+    };
+
+    this.logger = this.options.enableLogging ? logInfo : () => {};
+  }
+
+  /**
+   * Run the feedback loop
+   *
+   * @param initialContent - Initial content to optimize
+   * @param context - Evaluation context with request and criteria
+   * @returns Optimization result with full history
+   */
+  async run(initialContent: string, context: EvalContext): Promise<PatternOptimizedResult> {
+    const startTime = new Date().toISOString();
+    const scoreHistory: number[] = [];
+    const contentHistory: string[] = [initialContent];
+    let currentContent = initialContent;
+    let currentEvaluation: PatternEvaluationResult | null = null;
+    let cyclesCompleted = 0;
+    let consecutiveNonImprovingCycles = 0;
+
+    this.logger(`\n${'='.repeat(70)}`);
+    this.logger('Evaluator-Optimizer Feedback Loop Started');
+    this.logger(`Target Score: ${this.options.targetScore}`);
+    this.logger(`Max Iterations: ${this.options.maxIterations}`);
+    this.logger(`${'='.repeat(70)}\n`);
+
+    try {
+      // STEP 2: Initial evaluation
+      this.logger('STEP 2: Initial Evaluation');
+      currentEvaluation = await this.evaluateContent(currentContent, context, 1);
+      scoreHistory.push(currentEvaluation.score);
+
+      this.logger(`Score: ${currentEvaluation.score}`);
+      this.logger(`Meets Threshold: ${!currentEvaluation.shouldRetry}\n`);
+
+      // STEP 3-5: Iterative optimization loop
+      while (cyclesCompleted < this.options.maxIterations && currentEvaluation.shouldRetry) {
+        cyclesCompleted += 1;
+
+        this.logger(`${'─'.repeat(70)}`);
+        this.logger(`STEP 3: Optimization (Cycle ${cyclesCompleted})`);
+
+        // Check for diminishing returns
+        if (cyclesCompleted > 1) {
+          const previousScore = scoreHistory[scoreHistory.length - 1];
+          if (currentEvaluation.score === previousScore) {
+            consecutiveNonImprovingCycles += 1;
+            this.logger(`⚠ No improvement detected (${consecutiveNonImprovingCycles} consecutive)`);
+
+            if (consecutiveNonImprovingCycles >= this.options.diminishingReturnThreshold) {
+              this.logger('Diminishing returns detected. Stopping optimization.');
+              return this.buildResult(
+                currentContent,
+                currentEvaluation,
+                cyclesCompleted,
+                scoreHistory,
+                contentHistory,
+                'diminishing_returns',
+                startTime,
+              );
+            }
+          } else {
+            consecutiveNonImprovingCycles = 0;
+          }
+        }
+
+        // Optimize content
+        const optimizedContent = await this.optimizeContent(currentContent, currentEvaluation);
+        currentContent = optimizedContent;
+        contentHistory.push(optimizedContent);
+
+        this.logger('Content optimized');
+
+        // STEP 4: Re-evaluate
+        this.logger('STEP 4: Re-evaluation');
+        currentEvaluation = await this.evaluateContent(
+          currentContent,
+          context,
+          cyclesCompleted + 1,
+        );
+        scoreHistory.push(currentEvaluation.score);
+
+        this.logger(`Score: ${currentEvaluation.score}`);
+        this.logger(`Meets Threshold: ${!currentEvaluation.shouldRetry}`);
+        this.logger(`Issues: ${currentEvaluation.issues?.length ?? 0}\n`);
+
+        // Check if threshold met
+        if (!currentEvaluation.shouldRetry && this.options.stopOnThreshold) {
+          this.logger(`✓ Threshold reached! Score: ${currentEvaluation.score}`);
+          return this.buildResult(
+            currentContent,
+            currentEvaluation,
+            cyclesCompleted,
+            scoreHistory,
+            contentHistory,
+            'threshold_met',
+            startTime,
+          );
+        }
+      }
+
+      // Loop ended - either threshold met or max iterations reached
+      if (!currentEvaluation.shouldRetry) {
+        this.logger(`✓ Threshold reached after ${cyclesCompleted} cycles`);
+        return this.buildResult(
+          currentContent,
+          currentEvaluation,
+          cyclesCompleted,
+          scoreHistory,
+          contentHistory,
+          'threshold_met',
+          startTime,
+        );
+      } else {
+        this.logger(
+          `⚠ Max iterations (${this.options.maxIterations}) reached without meeting threshold`,
+        );
+        return this.buildResult(
+          currentContent,
+          currentEvaluation,
+          cyclesCompleted,
+          scoreHistory,
+          contentHistory,
+          'max_iterations',
+          startTime,
+        );
+      }
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.logger(`✗ Error in feedback loop: ${errorMessage}`);
+
+      return this.buildResult(
+        currentContent,
+        currentEvaluation || {
+          score: 0,
+          dimensions: [],
+          suggestions: [],
+          shouldRetry: true,
+        },
+        cyclesCompleted,
+        scoreHistory,
+        contentHistory,
+        'error',
+        startTime,
+        errorMessage,
+      );
+    }
+  }
+
+  /**
+   * Evaluate content quality
+   */
+  private async evaluateContent(
+    content: string,
+    context: EvalContext,
+    iterationNumber: number,
+  ): Promise<PatternEvaluationResult> {
+    const evalContext: EvalContext = {
+      ...context,
+      threshold: context.threshold ?? this.options.targetScore,
+      iterationNumber,
+    };
+
+    const evaluation = await this.options.evaluator(content, evalContext);
+
+    // Determine if should retry based on score vs threshold
+    const shouldRetry = evaluation.score < (context.threshold ?? this.options.targetScore);
+
+    return {
+      ...evaluation,
+      shouldRetry,
+    };
+  }
+
+  /**
+   * Optimize content based on evaluation feedback
+   */
+  private async optimizeContent(
+    content: string,
+    evaluation: PatternEvaluationResult,
+  ): Promise<string> {
+    return this.options.optimizer(content, evaluation);
+  }
+
+  /**
+   * Build final result object
+   */
+  private buildResult(
+    finalContent: string,
+    finalEvaluation: PatternEvaluationResult,
+    cyclesCompleted: number,
+    scoreHistory: number[],
+    contentHistory: string[],
+    terminationReason: 'threshold_met' | 'max_iterations' | 'diminishing_returns' | 'error',
+    startTime: string,
+    error?: string,
+  ): PatternOptimizedResult {
+    const endTime = new Date().toISOString();
+    const startMs = new Date(startTime).getTime();
+    const endMs = new Date(endTime).getTime();
+
+    const firstScore = scoreHistory[0] ?? 0;
+    const lastScore = scoreHistory[scoreHistory.length - 1] ?? firstScore;
+    const averageScoreImprovement =
+      scoreHistory.length > 1 ? (lastScore - firstScore) / (scoreHistory.length - 1) : 0;
+
+    this.logger(`\n${'='.repeat(70)}`);
+    this.logger('Feedback Loop Complete');
+    this.logger(`Cycles: ${cyclesCompleted}`);
+    this.logger(`Final Score: ${finalEvaluation.score}`);
+    this.logger(`Average Improvement: ${averageScoreImprovement.toFixed(3)}/cycle`);
+    this.logger(`Time: ${endMs - startMs}ms`);
+    this.logger(`Reason: ${terminationReason}`);
+    this.logger(`${'='.repeat(70)}\n`);
+
+    return {
+      finalContent,
+      finalEvaluation,
+      cyclesCompleted,
+      meetsThreshold: !finalEvaluation.shouldRetry,
+      scoreHistory,
+      contentHistory,
+      terminationReason,
+      error,
+      metadata: {
+        startTime,
+        endTime,
+        processingTimeMs: endMs - startMs,
+        averageScoreImprovement,
+      },
+    };
+  }
+}
+
+/**
+ * Factory function to create evaluator-optimizer
+ *
+ * @param options - Configuration object
+ * @returns EvaluatorOptimizer instance
+ *
+ * @example
+ * ```typescript
+ * const loop = createEvaluatorOptimizer({
+ *   evaluator: async (output, context) => ({
+ *     score: 0.85,
+ *     dimensions: [
+ *       { name: 'clarity', score: 0.9, feedback: 'Clear and concise' },
+ *       { name: 'completeness', score: 0.8, feedback: 'Missing examples' }
+ *     ],
+ *     suggestions: ['Add practical examples'],
+ *     shouldRetry: false
+ *   }),
+ *   optimizer: async (output, evaluation) => {
+ *     // Improve based on feedback
+ *     return improvedOutput;
+ *   },
+ *   targetScore: 0.85,
+ *   maxIterations: 5
+ * });
+ * ```
+ */
+export function createPatternEvaluatorOptimizer(
+  options: EvaluatorOptimizerOptions,
+): PatternEvaluatorOptimizer {
+  return new PatternEvaluatorOptimizer(options);
+}
+
+/**
+ * @deprecated Use createPatternEvaluatorOptimizer instead
+ */
+export const createEvaluatorOptimizer = createPatternEvaluatorOptimizer;
+
+/**
+ * Pre-built evaluators for common quality dimensions
+ *
+ * ⚠️ WARNING: These are MOCK implementations returning hardcoded values.
+ * They are intended for testing and demonstration purposes only.
+ * Replace with actual LLM-based or rule-based evaluators for production use.
+ */
+export const evaluators = {
+  /**
+   * Factuality evaluator checks for factual accuracy
+   * @remarks Mock implementation - always returns score 0.85
+   */
+  factuality: (threshold: number = 0.8) => {
+    return async (output: string, context: EvalContext): Promise<PatternEvaluationResult> => {
+      void output;
+      void context;
+      // In real implementation, this would call an LLM or fact-checking service
+      return {
+        score: 0.85,
+        dimensions: [
+          {
+            name: 'factuality',
+            score: 0.85,
+            feedback: 'Most facts are accurate with minor uncertainties',
+          },
+        ],
+        suggestions: ['Verify statistics and dates', 'Add citations for claims'],
+        shouldRetry: 0.85 < threshold,
+        assessment: 'Content is largely factually accurate',
+        strengths: ['Well-sourced citations'],
+        issues: ['One unverified statistic'],
+      };
+    };
+  },
+
+  /**
+   * Relevance evaluator checks if content addresses the request
+   * @remarks Mock implementation - always returns score 0.9
+   */
+  relevance: (threshold: number = 0.8) => {
+    return async (output: string, context: EvalContext): Promise<PatternEvaluationResult> => {
+      void output;
+      void context;
+      return {
+        score: 0.9,
+        dimensions: [
+          {
+            name: 'relevance',
+            score: 0.9,
+            feedback: 'Content directly addresses the request',
+          },
+          {
+            name: 'completeness',
+            score: 0.85,
+            feedback: 'Covers main points but misses some details',
+          },
+        ],
+        suggestions: ['Add more context about edge cases', 'Include real-world examples'],
+        shouldRetry: 0.9 < threshold,
+        assessment: 'Content is highly relevant to the request',
+        strengths: ['Directly addresses user intent', 'Well-structured'],
+        issues: [],
+      };
+    };
+  },
+
+  /**
+   * Safety evaluator checks for harmful content
+   * @remarks Mock implementation - always returns score 0.98
+   */
+  safety: (threshold: number = 0.95) => {
+    return async (output: string, context: EvalContext): Promise<PatternEvaluationResult> => {
+      void output;
+      void context;
+      return {
+        score: 0.98,
+        dimensions: [
+          {
+            name: 'safety',
+            score: 0.98,
+            feedback: 'Content contains no harmful or dangerous information',
+          },
+          {
+            name: 'bias',
+            score: 0.95,
+            feedback: 'Minimal bias detected; generally balanced perspective',
+          },
+        ],
+        suggestions: ['Review pronouns for inclusive language'],
+        shouldRetry: 0.98 < threshold,
+        assessment: 'Content is safe and appropriate',
+        strengths: ['No harmful content', 'Balanced perspective'],
+        issues: [],
+      };
+    };
+  },
+
+  /**
+   * Quality evaluator provides comprehensive assessment
+   * @remarks Mock implementation - always returns score 0.82
+   */
+  quality: (threshold: number = 0.8) => {
+    return async (output: string, context: EvalContext): Promise<PatternEvaluationResult> => {
+      void output;
+      void context;
+      return {
+        score: 0.82,
+        dimensions: [
+          {
+            name: 'clarity',
+            score: 0.85,
+            feedback: 'Clear writing with good structure',
+          },
+          {
+            name: 'completeness',
+            score: 0.8,
+            feedback: 'Covers main topics but lacks depth in places',
+          },
+          {
+            name: 'accuracy',
+            score: 0.82,
+            feedback: 'Factually sound with minor uncertainties',
+          },
+          {
+            name: 'engagement',
+            score: 0.8,
+            feedback: 'Moderately engaging, could be more compelling',
+          },
+        ],
+        suggestions: [
+          'Add more specific examples',
+          'Improve opening sentence for engagement',
+          'Expand technical sections',
+        ],
+        shouldRetry: 0.82 < threshold,
+        assessment: 'Content is good quality with room for improvement',
+        strengths: ['Well-organized', 'Clear explanations'],
+        issues: ['Lacks specific examples', 'Some sections too brief'],
+      };
+    };
+  },
+};
+
+/**
+ * Helper function to run feedback loop with sensible defaults
+ *
+ * @param initialContent - Content to optimize
+ * @param context - Evaluation context
+ * @param options - Optional configuration overrides
+ * @returns Optimized result
+ *
+ * @example
+ * ```typescript
+ * const result = await runFeedbackLoop(
+ *   "Generated content",
+ *   { request: "Write an article about AI" },
+ *   {
+ *     evaluator: evaluators.quality(0.85),
+ *     optimizer: async (content, evaluation) => {
+ *       // Improve content based on feedback
+ *       return improvedContent;
+ *     },
+ *     targetScore: 0.85,
+ *     maxIterations: 5
+ *   }
+ * );
+ * ```
+ */
+export async function runPatternFeedbackLoop(
+  initialContent: string,
+  context: EvalContext,
+  options: EvaluatorOptimizerOptions,
+): Promise<PatternOptimizedResult> {
+  const optimizer = createPatternEvaluatorOptimizer(options);
+  return optimizer.run(initialContent, context);
+}
+
+/**
+ * Utility function to normalize evaluation scores
+ * Converts between different scoring scales (0-1, 1-10, etc.)
+ */
+export function normalizeScore(score: number, fromMax: number = 10, toMax: number = 1): number {
+  return (score / fromMax) * toMax;
+}
+
+/**
+ * Utility function to calculate aggregate score from dimensions
+ * Normalizes weights so they sum to 1, handling mixed explicit/default weights.
+ */
+export function calculateAggregateScore(
+  dimensions: PatternEvaluationResult['dimensions'],
+  weights?: Record<string, number>,
+): number {
+  if (dimensions.length === 0) return 0;
+
+  // Calculate effective weights for each dimension
+  const defaultWeight = 1 / dimensions.length;
+  const effectiveWeights = dimensions.map(dim => weights?.[dim.name] ?? defaultWeight);
+  const totalWeight = effectiveWeights.reduce((a, b) => a + b, 0);
+
+  // Normalize and calculate weighted sum
+  const sum = dimensions.reduce((acc, dim, idx) => {
+    const weight = effectiveWeights[idx] ?? defaultWeight;
+    const normalizedWeight = weight / totalWeight;
+    return acc + dim.score * normalizedWeight;
+  }, 0);
+
+  return Math.min(sum, 1);
+}
+
+/**
+ * @deprecated Use PatternEvaluationResult instead to avoid collision with EvaluationResult from evaluator-optimizer/schema.ts
+ */
+export type EvaluationResult = PatternEvaluationResult;
+
+/**
+ * @deprecated Use PatternOptimizedResult instead to avoid collision with OptimizedResult from evaluator-optimizer/schema.ts
+ */
+export type OptimizedResult = PatternOptimizedResult;
+
+/**
+ * @deprecated Use runPatternFeedbackLoop instead to avoid collision with runFeedbackLoop from evaluator-optimizer/index.ts
+ */
+export const runFeedbackLoop = runPatternFeedbackLoop;

package/src/agents/patterns/index.ts

@@ -0,0 +1,26 @@
+/**
+ * @fileoverview Reusable agent architecture patterns (evaluator-optimizer, etc.).
+ * @module @od-oneapp/ai-platform/agents/patterns
+ */
+
+export {
+  PatternEvaluatorOptimizer,
+  createPatternEvaluatorOptimizer,
+  // Deprecated aliases
+  createEvaluatorOptimizer,
+  runPatternFeedbackLoop,
+  runFeedbackLoop,
+  evaluators,
+  normalizeScore,
+  calculateAggregateScore,
+} from './evaluator-optimizer-pattern';
+
+export type {
+  PatternEvaluationResult,
+  EvalContext,
+  EvaluatorOptimizerOptions,
+  PatternOptimizedResult,
+  // Deprecated aliases
+  EvaluationResult,
+  OptimizedResult,
+} from './evaluator-optimizer-pattern';