outcome-cli 1.0.0

package/src/eval/ai-judge.test.ts

@@ -0,0 +1,435 @@
+/**
+ * AI Judge Unit Tests
+ *
+ * Tests for the AI-powered evaluation system.
+ *
+ * @module eval/ai-judge.test
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import {
+  hashArtifact,
+  validateJudgeConfig,
+  AIJudgeError,
+  evaluateWithAIJudge,
+  type JudgeConfig,
+} from './ai-judge.js';
+import {
+  InMemoryJudgeCache,
+  resetJudgeCache,
+  setJudgeCache,
+  type JudgeResult,
+} from './judge-cache.js';
+
+describe('AI Judge', () => {
+  describe('hashArtifact', () => {
+    it('should generate consistent hash for same artifact and rubric', () => {
+      const artifact = { message: 'Hello world', quality: 'high' };
+      const rubric = 'Evaluate for clarity';
+
+      const hash1 = hashArtifact(artifact, rubric);
+      const hash2 = hashArtifact(artifact, rubric);
+
+      expect(hash1).toBe(hash2);
+      expect(hash1).toHaveLength(64); // SHA-256 hex length
+    });
+
+    it('should generate different hash for different artifacts', () => {
+      const artifact1 = { message: 'Hello world' };
+      const artifact2 = { message: 'Goodbye world' };
+      const rubric = 'Evaluate for clarity';
+
+      const hash1 = hashArtifact(artifact1, rubric);
+      const hash2 = hashArtifact(artifact2, rubric);
+
+      expect(hash1).not.toBe(hash2);
+    });
+
+    it('should generate different hash for different rubrics', () => {
+      const artifact = { message: 'Hello world' };
+      const rubric1 = 'Evaluate for clarity';
+      const rubric2 = 'Evaluate for professionalism';
+
+      const hash1 = hashArtifact(artifact, rubric1);
+      const hash2 = hashArtifact(artifact, rubric2);
+
+      expect(hash1).not.toBe(hash2);
+    });
+
+    it('should handle string artifacts', () => {
+      const artifact = 'Simple string artifact';
+      const rubric = 'Evaluate this';
+
+      const hash = hashArtifact(artifact, rubric);
+
+      expect(hash).toHaveLength(64);
+    });
+
+    it('should handle complex nested artifacts', () => {
+      const artifact = {
+        level1: {
+          level2: {
+            level3: ['a', 'b', 'c'],
+          },
+        },
+        numbers: [1, 2, 3],
+      };
+      const rubric = 'Evaluate structure';
+
+      const hash = hashArtifact(artifact, rubric);
+
+      expect(hash).toHaveLength(64);
+    });
+  });
+
+  describe('validateJudgeConfig', () => {
+    it('should accept valid gpt-4o config', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+      };
+
+      expect(validateJudgeConfig(config)).toBe(true);
+    });
+
+    it('should accept valid claude-opus config', () => {
+      const config: JudgeConfig = {
+        model: 'claude-opus',
+        rubric: 'Evaluate for quality',
+        maxScore: 100,
+      };
+
+      expect(validateJudgeConfig(config)).toBe(true);
+    });
+
+    it('should accept config with optional temperature', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        temperature: 0.5,
+      };
+
+      expect(validateJudgeConfig(config)).toBe(true);
+    });
+
+    it('should accept config with optional maxTokens', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        maxTokens: 2048,
+      };
+
+      expect(validateJudgeConfig(config)).toBe(true);
+    });
+
+    it('should reject invalid model', () => {
+      const config = {
+        model: 'invalid-model' as 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('Invalid judge model');
+    });
+
+    it('should reject empty rubric', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: '',
+        maxScore: 10,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('Rubric must be a non-empty string');
+    });
+
+    it('should reject whitespace-only rubric', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: '   ',
+        maxScore: 10,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('Rubric must be a non-empty string');
+    });
+
+    it('should reject zero maxScore', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 0,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('maxScore must be a positive number');
+    });
+
+    it('should reject negative maxScore', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: -5,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('maxScore must be a positive number');
+    });
+
+    it('should reject temperature below 0', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        temperature: -0.1,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('temperature must be between 0 and 2');
+    });
+
+    it('should reject temperature above 2', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        temperature: 2.5,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('temperature must be between 0 and 2');
+    });
+
+    it('should reject maxTokens below 1', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        maxTokens: 0,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('maxTokens must be between 1 and 4096');
+    });
+
+    it('should reject maxTokens above 4096', () => {
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+        maxTokens: 5000,
+      };
+
+      expect(() => validateJudgeConfig(config)).toThrow('maxTokens must be between 1 and 4096');
+    });
+  });
+
+  describe('AIJudgeError', () => {
+    it('should create error with model information', () => {
+      const error = new AIJudgeError('Test error', 'gpt-4o');
+
+      expect(error.message).toBe('Test error');
+      expect(error.model).toBe('gpt-4o');
+      expect(error.name).toBe('AIJudgeError');
+    });
+
+    it('should include cause when provided', () => {
+      const cause = new Error('Original error');
+      const error = new AIJudgeError('Wrapped error', 'claude-opus', cause);
+
+      expect(error.message).toBe('Wrapped error');
+      expect(error.model).toBe('claude-opus');
+      expect(error.cause).toBe(cause);
+    });
+  });
+
+  describe('evaluateWithAIJudge - caching', () => {
+    let cache: InMemoryJudgeCache;
+
+    beforeEach(() => {
+      cache = new InMemoryJudgeCache();
+      setJudgeCache(cache);
+    });
+
+    afterEach(() => {
+      resetJudgeCache();
+    });
+
+    it('should return cached result when available', async () => {
+      const artifact = { message: 'Test message' };
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+      };
+
+      // Pre-populate cache
+      const cacheKey = hashArtifact(artifact, config.rubric);
+      const cachedResult: JudgeResult = {
+        score: 8,
+        normalizedScore: 0.8,
+        reasoning: 'Good quality message',
+        highlights: ['Clear', 'Concise'],
+        model: 'gpt-4o',
+        cached: false,
+        evaluatedAt: new Date().toISOString(),
+      };
+      await cache.set(cacheKey, cachedResult);
+
+      // Should return cached result
+      const result = await evaluateWithAIJudge(artifact, config, cache);
+
+      expect(result.cached).toBe(true);
+      expect(result.score).toBe(8);
+      expect(result.reasoning).toBe('Good quality message');
+    });
+
+    it('should track cache statistics', async () => {
+      const artifact = { message: 'Test message' };
+      const config: JudgeConfig = {
+        model: 'gpt-4o',
+        rubric: 'Evaluate for quality',
+        maxScore: 10,
+      };
+
+      // Pre-populate cache
+      const cacheKey = hashArtifact(artifact, config.rubric);
+      const cachedResult: JudgeResult = {
+        score: 8,
+        normalizedScore: 0.8,
+        reasoning: 'Good quality message',
+        highlights: [],
+        model: 'gpt-4o',
+        cached: false,
+        evaluatedAt: new Date().toISOString(),
+      };
+      await cache.set(cacheKey, cachedResult);
+
+      // First call - cache hit
+      await evaluateWithAIJudge(artifact, config, cache);
+
+      const stats = await cache.stats();
+      expect(stats.hits).toBe(1);
+      expect(stats.misses).toBe(0);
+    });
+  });
+});
+
+describe('InMemoryJudgeCache', () => {
+  let cache: InMemoryJudgeCache;
+
+  beforeEach(() => {
+    cache = new InMemoryJudgeCache();
+  });
+
+  it('should store and retrieve results', async () => {
+    const result: JudgeResult = {
+      score: 8,
+      normalizedScore: 0.8,
+      reasoning: 'Good quality',
+      highlights: ['Clear'],
+      model: 'gpt-4o',
+      cached: false,
+      evaluatedAt: new Date().toISOString(),
+    };
+
+    await cache.set('test-key', result);
+    const retrieved = await cache.get('test-key');
+
+    expect(retrieved).not.toBeNull();
+    expect(retrieved?.score).toBe(8);
+    expect(retrieved?.reasoning).toBe('Good quality');
+  });
+
+  it('should return null for missing keys', async () => {
+    const result = await cache.get('nonexistent-key');
+    expect(result).toBeNull();
+  });
+
+  it('should delete entries', async () => {
+    const result: JudgeResult = {
+      score: 8,
+      normalizedScore: 0.8,
+      reasoning: 'Good quality',
+      highlights: [],
+      model: 'gpt-4o',
+      cached: false,
+      evaluatedAt: new Date().toISOString(),
+    };
+
+    await cache.set('test-key', result);
+    const deleted = await cache.delete('test-key');
+    const retrieved = await cache.get('test-key');
+
+    expect(deleted).toBe(true);
+    expect(retrieved).toBeNull();
+  });
+
+  it('should clear all entries', async () => {
+    const result: JudgeResult = {
+      score: 8,
+      normalizedScore: 0.8,
+      reasoning: 'Good quality',
+      highlights: [],
+      model: 'gpt-4o',
+      cached: false,
+      evaluatedAt: new Date().toISOString(),
+    };
+
+    await cache.set('key1', result);
+    await cache.set('key2', result);
+    await cache.clear();
+
+    const stats = await cache.stats();
+    expect(stats.size).toBe(0);
+  });
+
+  it('should track hit/miss statistics', async () => {
+    const result: JudgeResult = {
+      score: 8,
+      normalizedScore: 0.8,
+      reasoning: 'Good quality',
+      highlights: [],
+      model: 'gpt-4o',
+      cached: false,
+      evaluatedAt: new Date().toISOString(),
+    };
+
+    await cache.set('existing-key', result);
+
+    // One hit
+    await cache.get('existing-key');
+    // Two misses
+    await cache.get('missing-key-1');
+    await cache.get('missing-key-2');
+
+    const stats = await cache.stats();
+    expect(stats.hits).toBe(1);
+    expect(stats.misses).toBe(2);
+    expect(stats.hitRate).toBeCloseTo(1 / 3, 2);
+  });
+
+  it('should expire entries after TTL', async () => {
+    // Create cache with 1 second TTL
+    const shortTtlCache = new InMemoryJudgeCache(1);
+
+    const result: JudgeResult = {
+      score: 8,
+      normalizedScore: 0.8,
+      reasoning: 'Good quality',
+      highlights: [],
+      model: 'gpt-4o',
+      cached: false,
+      evaluatedAt: new Date().toISOString(),
+    };
+
+    await shortTtlCache.set('test-key', result);
+
+    // Should be available immediately
+    const immediate = await shortTtlCache.get('test-key');
+    expect(immediate).not.toBeNull();
+
+    // Wait for TTL to expire
+    await new Promise((resolve) => setTimeout(resolve, 1100));
+
+    // Should be expired now
+    const expired = await shortTtlCache.get('test-key');
+    expect(expired).toBeNull();
+  });
+});