@od-oneapp/ai-platform 0.1.0

package/src/agents/evaluator-optimizer/evaluator.ts

@@ -0,0 +1,282 @@
+/**
+ * Evaluator Module
+ * Implements LLM-as-judge pattern for content quality assessment
+ * Based on VERCEL_AI_SDK_V6_PATTERNS.md section 4.2
+ */
+
+import { generateObject } from '../../shared';
+
+import {
+  type EvaluationResult,
+  EvaluationResultSchema,
+  type EvaluatorInput,
+  type Evaluator,
+} from './schema';
+
+/**
+ * LLM-as-Judge Evaluator
+ * Uses language model to assess content quality across 9 dimensions
+ */
+export class LLMEvaluator implements Evaluator {
+  /**
+   * Model identifier for evaluation (typically a fast model)
+   * @default 'openai/gpt-4o-mini'
+   */
+  private modelId: string;
+
+  constructor(modelId: string = 'openai/gpt-4o-mini') {
+    this.modelId = modelId;
+  }
+
+  /**
+   * Evaluate content using LLM-as-judge pattern
+   * Provides comprehensive assessment across multiple quality dimensions
+   *
+   * @param input - Evaluation input containing content and context
+   * @returns Promise resolving to detailed evaluation result
+   *
+   * @example
+   * ```typescript
+   * const evaluator = new LLMEvaluator();
+   * const result = await evaluator.evaluate({
+   *   content: "Generated content here",
+   *   originalRequest: "Write a technical article about TypeScript",
+   *   criteria: { threshold: 8 }
+   * });
+   * ```
+   */
+  async evaluate(input: EvaluatorInput): Promise<EvaluationResult> {
+    const { content, originalRequest, criteria, iterationNumber } = input;
+    const threshold = criteria?.threshold ?? 8;
+
+    const systemPrompt = `You are an expert content evaluator with deep knowledge in quality assessment.
+Evaluate content across these 9 dimensions:
+
+1. **Quality Score** (1-10): Overall quality from poor (1-3) to excellent (9-10)
+2. **Tone Preservation**: Whether original intended tone is maintained
+3. **Nuance Preservation**: Whether subtle meanings and context are preserved
+4. **Cultural Accuracy**: Sensitivity and accuracy regarding cultural references
+5. **Technical Correctness**: Accuracy of any technical, factual, or domain-specific claims
+6. **Specific Issues**: Concrete problems, gaps, or errors found
+7. **Improvement Suggestions**: Actionable feedback for enhancement
+8. **Strengths**: Aspects to preserve during optimization
+9. **Overall Assessment**: Summary of quality and readiness
+
+Be rigorous and specific. Identify real issues, not minor nitpicks.
+Provide constructive, actionable feedback.`;
+
+    const userPrompt = `${iterationNumber ? `[Iteration ${iterationNumber}]\n` : ''}Evaluate this content:
+
+ORIGINAL REQUEST: ${originalRequest}
+
+CONTENT TO EVALUATE:
+${content}
+
+QUALITY THRESHOLD: ${threshold}/10
+
+Provide a comprehensive evaluation following the schema. Content must score at least ${threshold}/10 to pass.
+For meetsThreshold, set to true only if qualityScore >= ${threshold}.`;
+
+    try {
+      const { object: evaluation } = await generateObject({
+        model: this.modelId,
+        schema: EvaluationResultSchema,
+        system: systemPrompt,
+        prompt: userPrompt,
+      });
+
+      return evaluation;
+    } catch (error) {
+      throw new Error(
+        `LLM evaluation failed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+/**
+ * Rubric-Based Evaluator
+ * Provides structured evaluation using predefined criteria and scoring rubric
+ * Useful for consistent, deterministic evaluation across multiple runs
+ */
+export class RubricEvaluator implements Evaluator {
+  /**
+   * Scoring rubric mapping criteria to score ranges
+   */
+  private rubric: Map<string, (content: string, request: string) => number>;
+
+  constructor(customRubric?: Map<string, (content: string, request: string) => number>) {
+    this.rubric = customRubric || this.createDefaultRubric();
+  }
+
+  /**
+   * Create default rubric for evaluation
+   */
+  private createDefaultRubric(): Map<string, (content: string, request: string) => number> {
+    return new Map([
+      ['length', content => (content.length > 100 ? 2 : 1)],
+      ['clarity', content => (content.split(' ').length > 10 ? 2 : 1)],
+      ['structure', content => (content.includes('\n') ? 2 : 1)],
+      ['completeness', (content, request) => (content.length > request.length * 0.5 ? 2 : 1)],
+    ]);
+  }
+
+  /**
+   * Evaluate content using predefined rubric
+   * Faster than LLM evaluation but less nuanced
+   */
+  async evaluate(input: EvaluatorInput): Promise<EvaluationResult> {
+    const { content, originalRequest, criteria } = input;
+    const threshold = criteria?.threshold ?? 8;
+
+    // Calculate score from rubric criteria
+    let totalScore = 0;
+    let maxScore = 0;
+    const issues: string[] = [];
+
+    for (const scorer of this.rubric.values()) {
+      const score = scorer(content, originalRequest);
+      totalScore += score;
+      maxScore += 2; // Each criterion in default rubric scores 1-2
+    }
+
+    // Normalize to 1-10 scale
+    const qualityScore = Math.max(1, Math.min(10, Math.round((totalScore / maxScore) * 10)));
+
+    // Determine issues based on score
+    if (qualityScore < 4) {
+      issues.push('Content is too brief or lacks detail');
+      issues.push('Structure could be clearer');
+    } else if (qualityScore < 7) {
+      issues.push('Could provide more specific examples');
+      issues.push('Organization could be improved');
+    }
+
+    return {
+      qualityScore,
+      preservesTone: true,
+      preservesNuance: qualityScore >= 7,
+      culturallyAccurate: true,
+      technicallyCorrect: qualityScore >= 6,
+      specificIssues: issues,
+      improvementSuggestions: [
+        'Expand with concrete examples',
+        'Improve paragraph transitions',
+        'Add supporting evidence',
+      ],
+      strengths: ['Clear writing', 'Relevant content'],
+      overallAssessment: `Content quality score: ${qualityScore}/10. ${
+        qualityScore >= threshold ? 'Meets threshold.' : 'Needs optimization.'
+      }`,
+      meetsThreshold: qualityScore >= threshold,
+    };
+  }
+}
+
+/**
+ * Composite Evaluator
+ * Combines multiple evaluators (LLM, rubric, custom) for multi-dimensional assessment
+ * Weights results from different evaluators to produce robust evaluation
+ */
+export class CompositeEvaluator implements Evaluator {
+  /**
+   * Array of evaluators with their weights (0-1)
+   */
+  private evaluators: Array<{
+    evaluator: Evaluator;
+    weight: number;
+  }>;
+
+  constructor(evaluators: Array<{ evaluator: Evaluator; weight: number }> = []) {
+    this.evaluators = evaluators;
+
+    // Default composite if none provided
+    if (this.evaluators.length === 0) {
+      this.evaluators = [
+        { evaluator: new LLMEvaluator(), weight: 0.7 },
+        { evaluator: new RubricEvaluator(), weight: 0.3 },
+      ];
+    }
+
+    // Normalize weights
+    const totalWeight = this.evaluators.reduce((sum, e) => sum + e.weight, 0);
+    if (totalWeight === 0) {
+      throw new Error('CompositeEvaluator requires weights > 0');
+    }
+    this.evaluators = this.evaluators.map(e => ({
+      ...e,
+      weight: e.weight / totalWeight,
+    }));
+  }
+
+  /**
+   * Run composite evaluation combining multiple perspectives
+   */
+  async evaluate(input: EvaluatorInput): Promise<EvaluationResult> {
+    const results = await Promise.all(this.evaluators.map(e => e.evaluator.evaluate(input)));
+
+    // Weighted average score
+    const weightedScore = results.reduce(
+      (sum, result, idx) => sum + result.qualityScore * (this.evaluators[idx]?.weight ?? 0),
+      0,
+    );
+
+    // Combine boolean assessments (true if majority agree)
+    const agreeingEvaluators = (
+      key: 'preservesTone' | 'preservesNuance' | 'culturallyAccurate' | 'technicallyCorrect',
+    ) => {
+      const trueCount = results.filter(r => r[key]).length;
+      return trueCount > results.length / 2;
+    };
+
+    // Aggregate issues and suggestions (unique)
+    const allIssues = results.flatMap(r => r.specificIssues);
+    const uniqueIssues = [...new Set(allIssues)];
+
+    const allSuggestions = results.flatMap(r => r.improvementSuggestions);
+    const uniqueSuggestions = [...new Set(allSuggestions)];
+
+    const allStrengths = results.flatMap(r => r.strengths);
+    const uniqueStrengths = [...new Set(allStrengths)];
+
+    const meetsThreshold = weightedScore >= (input.criteria?.threshold ?? 8);
+
+    return {
+      qualityScore: Math.round(weightedScore),
+      preservesTone: agreeingEvaluators('preservesTone'),
+      preservesNuance: agreeingEvaluators('preservesNuance'),
+      culturallyAccurate: agreeingEvaluators('culturallyAccurate'),
+      technicallyCorrect: agreeingEvaluators('technicallyCorrect'),
+      specificIssues: uniqueIssues.slice(0, 5), // Top 5 issues
+      improvementSuggestions: uniqueSuggestions.slice(0, 5), // Top 5 suggestions
+      strengths: uniqueStrengths.slice(0, 5), // Top 5 strengths
+      overallAssessment: `Composite evaluation: ${weightedScore.toFixed(1)}/10. Based on ${this.evaluators.length} evaluator(s).`,
+      meetsThreshold,
+    };
+  }
+}
+
+/**
+ * Factory function to create LLM evaluator
+ */
+export function createLLMEvaluator(modelId?: string): Evaluator {
+  return new LLMEvaluator(modelId);
+}
+
+/**
+ * Factory function to create rubric evaluator
+ */
+export function createRubricEvaluator(
+  rubric?: Map<string, (content: string, request: string) => number>,
+): Evaluator {
+  return new RubricEvaluator(rubric);
+}
+
+/**
+ * Factory function to create composite evaluator
+ */
+export function createCompositeEvaluator(
+  evaluators: Array<{ evaluator: Evaluator; weight: number }> = [],
+): Evaluator {
+  return new CompositeEvaluator(evaluators);
+}

package/src/agents/evaluator-optimizer/index.test.ts

@@ -0,0 +1,416 @@
+/**
+ * Evaluator-Optimizer Tests
+ * Unit tests for the feedback loop pattern implementation
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+import { createLLMEvaluator, createRubricEvaluator, createCompositeEvaluator } from './evaluator';
+import { createPromptOptimizer, createOutputOptimizer } from './optimizer';
+import {
+  type Evaluator,
+  type Optimizer,
+  type EvaluationResult,
+  type OptimizerInput,
+} from './schema';
+
+import { createEvaluatorOptimizer, type EvaluatorOptimizer } from ".";
+
+/**
+ * Mock Evaluator for testing
+ * Returns predetermined scores for consistent testing
+ */
+class MockEvaluator implements Evaluator {
+  private scores: number[];
+  private currentCall: number = 0;
+
+  constructor(scores: number[] = [7, 8, 9]) {
+    this.scores = scores;
+  }
+
+  async evaluate(): Promise<EvaluationResult> {
+    const score = this.scores[Math.min(this.currentCall, this.scores.length - 1)];
+    this.currentCall += 1;
+
+    return {
+      qualityScore: score,
+      preservesTone: score >= 7,
+      preservesNuance: score >= 8,
+      culturallyAccurate: true,
+      technicallyCorrect: score >= 6,
+      specificIssues: score < 8 ? ['Quality needs improvement'] : [],
+      improvementSuggestions: score < 8 ? ['Add more detail'] : [],
+      strengths: ['Clear writing'],
+      overallAssessment: `Score: ${score}/10`,
+      meetsThreshold: score >= 8,
+    };
+  }
+}
+
+/**
+ * Mock Optimizer for testing
+ */
+class MockOptimizer implements Optimizer {
+  private callCount: number = 0;
+
+  async optimize(input: OptimizerInput): Promise<string> {
+    this.callCount += 1;
+    // Simulate improvement by appending iteration marker
+    return `${input.content} [optimized iteration ${this.callCount}]`;
+  }
+
+  getCallCount(): number {
+    return this.callCount;
+  }
+}
+
+describe('Evaluator-Optimizer Feedback Loop', () => {
+  let mockEvaluator: MockEvaluator;
+  let mockOptimizer: MockOptimizer;
+  let optimizer: EvaluatorOptimizer;
+
+  beforeEach(() => {
+    mockEvaluator = new MockEvaluator([6, 7, 8, 9]);
+    mockOptimizer = new MockOptimizer();
+
+    optimizer = createEvaluatorOptimizer({
+      maxIterations: 5,
+      targetScore: 8,
+      evaluator: mockEvaluator,
+      optimizer: mockOptimizer,
+      enableLogging: false,
+    });
+  });
+
+  describe('initialization', () => {
+    it('should create optimizer with config', () => {
+      expect(optimizer).toBeDefined();
+      expect(optimizer.getEvaluator()).toBeDefined();
+      expect(optimizer.getOptimizer()).toBeDefined();
+    });
+
+    it('should preserve evaluator and optimizer instances', () => {
+      expect(optimizer.getEvaluator()).toBe(mockEvaluator);
+      expect(optimizer.getOptimizer()).toBe(mockOptimizer);
+    });
+  });
+
+  describe('feedback Loop Execution', () => {
+    it('should complete feedback loop successfully', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request');
+
+      expect(result).toBeDefined();
+      expect(result.finalContent).toBeDefined();
+      expect(result.finalEvaluation).toBeDefined();
+      expect(result.cyclesCompleted).toBeGreaterThan(0);
+    });
+
+    it('should meet threshold when score reaches target', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request');
+
+      expect(result.meetsThreshold).toBe(true);
+      expect(result.finalEvaluation.qualityScore).toBeGreaterThanOrEqual(8);
+      expect(result.terminationReason).toBe('threshold_met');
+    });
+
+    it('should record score history', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request');
+
+      expect(result.scoreHistory).toBeDefined();
+      expect(result.scoreHistory.length).toBeGreaterThan(0);
+      expect(result.scoreHistory[0]).toBe(6); // Initial score
+    });
+
+    it('should record content history', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request');
+
+      expect(result.contentHistory).toBeDefined();
+      expect(result.contentHistory.length).toBeGreaterThan(0);
+      expect(result.contentHistory[0]).toBe('Initial content');
+    });
+
+    it('should terminate when max iterations reached', async () => {
+      // Create evaluator that never reaches threshold
+      const lowScoreEvaluator = new MockEvaluator([5, 5.5, 6, 6.5, 7]);
+
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 3,
+        targetScore: 9,
+        evaluator: lowScoreEvaluator,
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      const result = await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(result.cyclesCompleted).toBeLessThanOrEqual(3);
+      expect(result.meetsThreshold).toBe(false);
+      expect(result.terminationReason).toBe('max_iterations');
+    });
+
+    it('should stop on threshold when stopOnThreshold is true', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request', {
+        stopOnThreshold: true,
+      });
+
+      // Should stop as soon as threshold is met
+      expect(result.meetsThreshold).toBe(true);
+      expect(result.finalEvaluation.qualityScore).toBeGreaterThanOrEqual(8);
+    });
+
+    it('should calculate average score improvement', async () => {
+      const result = await optimizer.runFeedbackLoop('Initial content', 'Test request');
+
+      const avgImprovement = result.metadata?.averageScoreImprovement ?? 0;
+      expect(avgImprovement).toBeGreaterThanOrEqual(0);
+    });
+  });
+
+  describe('optimization Cycles', () => {
+    it('should call optimizer only when score below threshold', async () => {
+      mockOptimizer = new MockOptimizer();
+      optimizer = createEvaluatorOptimizer({
+        maxIterations: 5,
+        targetScore: 8,
+        evaluator: mockEvaluator,
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      await optimizer.runFeedbackLoop('Content', 'Request');
+
+      // With mock scores [6, 7, 8, 9], optimizer should be called
+      // for first 2 scores (below 8)
+      expect(mockOptimizer.getCallCount()).toBeGreaterThan(0);
+    });
+
+    it('should improve content through iterations', async () => {
+      const result = await optimizer.runFeedbackLoop('Content', 'Request');
+
+      // Final content should be different from initial
+      // (modified by optimizer)
+      if (result.cyclesCompleted > 0) {
+        expect(result.finalContent).not.toBe(result.contentHistory[0]);
+      }
+    });
+  });
+
+  describe('evaluators', () => {
+    it('should work with LLMEvaluator type', () => {
+      // Just test that factory creates right type
+      const evaluator = createLLMEvaluator('openai/gpt-4o-mini');
+      expect(evaluator).toBeDefined();
+    });
+
+    it('should work with RubricEvaluator type', () => {
+      const evaluator = createRubricEvaluator();
+      expect(evaluator).toBeDefined();
+    });
+
+    it('should work with CompositeEvaluator type', () => {
+      const composite = createCompositeEvaluator([
+        { evaluator: createRubricEvaluator(), weight: 1 },
+      ]);
+      expect(composite).toBeDefined();
+    });
+  });
+
+  describe('optimizers', () => {
+    it('should work with PromptOptimizer', () => {
+      const optimizer = createPromptOptimizer('openai/gpt-4o-mini');
+      expect(optimizer).toBeDefined();
+    });
+
+    it('should work with OutputOptimizer', () => {
+      const optimizer = createOutputOptimizer('openai/gpt-4o-mini', 'refine');
+      expect(optimizer).toBeDefined();
+    });
+  });
+
+  describe('error Handling', () => {
+    it('should handle evaluation errors gracefully', async () => {
+      // Create evaluator that throws
+      class ErrorEvaluator implements Evaluator {
+        async evaluate(): Promise<EvaluationResult> {
+          throw new Error('Evaluation failed');
+        }
+      }
+
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 2,
+        targetScore: 8,
+        evaluator: new ErrorEvaluator(),
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      const result = await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(result.terminationReason).toBe('error');
+      expect(result.error).toBeDefined();
+    });
+
+    it('should handle optimizer errors gracefully', async () => {
+      // Create optimizer that throws
+      class ErrorOptimizer implements Optimizer {
+        async optimize(): Promise<string> {
+          throw new Error('Optimization failed');
+        }
+      }
+
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 2,
+        targetScore: 8,
+        evaluator: mockEvaluator,
+        optimizer: new ErrorOptimizer(),
+        enableLogging: false,
+      });
+
+      const result = await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(result.terminationReason).toBe('error');
+      expect(result.error).toBeDefined();
+    });
+  });
+
+  describe('result Structure', () => {
+    it('should include all required fields in result', async () => {
+      const result = await optimizer.runFeedbackLoop('Content', 'Request');
+
+      expect(result.finalContent).toBeDefined();
+      expect(result.finalEvaluation).toBeDefined();
+      expect(result.cyclesCompleted).toBeDefined();
+      expect(result.meetsThreshold).toBeDefined();
+      expect(result.scoreHistory).toBeDefined();
+      expect(result.contentHistory).toBeDefined();
+      expect(result.terminationReason).toBeDefined();
+      expect(result.metadata).toBeDefined();
+    });
+
+    it('should include metadata with timing information', async () => {
+      const result = await optimizer.runFeedbackLoop('Content', 'Request');
+
+      expect(result.metadata?.startTime).toBeDefined();
+      expect(result.metadata?.endTime).toBeDefined();
+      expect(result.metadata?.processingTimeMs).toBeGreaterThan(0);
+      expect(result.metadata?.averageScoreImprovement).toBeDefined();
+    });
+
+    it('should include error only on error termination', async () => {
+      const result = await optimizer.runFeedbackLoop('Content', 'Request');
+
+      if (result.terminationReason === 'error') {
+        expect(result.error).toBeDefined();
+      } else {
+        // May or may not have error field when not erroring
+        expect(result.error).toBeUndefined();
+      }
+    });
+  });
+
+  describe('configuration', () => {
+    it('should respect maxIterations config', async () => {
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 2,
+        targetScore: 8,
+        evaluator: mockEvaluator,
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      const result = await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(result.cyclesCompleted).toBeLessThanOrEqual(2);
+    });
+
+    it('should respect targetScore config', async () => {
+      const highTargetOptimizer = createEvaluatorOptimizer({
+        maxIterations: 5,
+        targetScore: 9,
+        evaluator: new MockEvaluator([6, 7, 8, 8.5, 9]),
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      const result = await highTargetOptimizer.runFeedbackLoop('Content', 'Request');
+
+      expect(result.meetsThreshold).toBe(true);
+      expect(result.finalEvaluation.qualityScore).toBeGreaterThanOrEqual(9);
+    });
+  });
+
+  describe('logging', () => {
+    it('should not log when enableLogging is false', async () => {
+      const consoleSpy = vi.spyOn(console, 'log');
+
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 3,
+        targetScore: 8,
+        evaluator: mockEvaluator,
+        optimizer: mockOptimizer,
+        enableLogging: false,
+      });
+
+      await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(consoleSpy).not.toHaveBeenCalled();
+      consoleSpy.mockRestore();
+    });
+
+    it('should log when enableLogging is true', async () => {
+      const consoleSpy = vi.spyOn(console, 'log');
+
+      const optimizer2 = createEvaluatorOptimizer({
+        maxIterations: 3,
+        targetScore: 8,
+        evaluator: mockEvaluator,
+        optimizer: mockOptimizer,
+        enableLogging: true,
+      });
+
+      await optimizer2.runFeedbackLoop('Content', 'Request');
+
+      expect(consoleSpy).toHaveBeenCalledWith();
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe('edge Cases', () => {
+    it('should handle empty content', async () => {
+      const result = await optimizer.runFeedbackLoop('', 'Request');
+
+      expect(result).toBeDefined();
+      expect(result.finalContent).toBeDefined();
+    });
+
+    it('should handle very long content', async () => {
+      const longContent = 'x'.repeat(10000);
+
+      const result = await optimizer.runFeedbackLoop(longContent, 'Request');
+
+      expect(result).toBeDefined();
+      expect(result.finalContent).toBeDefined();
+    });
+
+    it('should handle special characters in content', async () => {
+      const specialContent = 'Content with émojis 🎉 and spëcial çhars';
+
+      const result = await optimizer.runFeedbackLoop(specialContent, 'Request');
+
+      expect(result).toBeDefined();
+    });
+
+    it('should handle rapid successive calls', async () => {
+      const promises = Array(3)
+        .fill(null)
+        .map(() => optimizer.runFeedbackLoop('Content', 'Request'));
+
+      const results = await Promise.all(promises);
+
+      expect(results).toHaveLength(3);
+      results.forEach(result => {
+        expect(result).toBeDefined();
+      });
+    });
+  });
+});