observability-toolkit 2.0.0 → 2.1.1

package/dist/lib/agent-judge/agent-as-judge.test.js

@@ -0,0 +1,839 @@
+/**
+ * Tests for Agent-as-Judge Implementation
+ */
+import { describe, it, beforeEach } from 'vitest';
+import assert from 'node:assert/strict';
+import { z } from 'zod';
+import { FLOAT_TOLERANCE } from '../../test-helpers/test-data-builders.js';
+import { 
+// Constants
+MAX_TRAJECTORY_LENGTH, MAX_CONCURRENT_EVALUATORS, MAX_CONSENSUS_ROUNDS, DEFAULT_CONVERGENCE_THRESHOLD, 
+// Error classes
+AgentEvalTimeoutError, 
+// Utilities
+withAgentTimeout, evaluandSchema, validateStepScore, verifyToolCall, verifyToolCalls, scoreStep, aggregateStepScores, analyzeTrajectory, calculateVariance, calculateMedian, collectiveConsensus, 
+// Classes
+ProceduralJudge, ReactiveJudge, } from './agent-as-judge.js';
+import { InputValidationError } from '../core/input-validator.js';
+import { MAX_STEP_ID_LENGTH } from '../../backends/index.js';
+import { MAX_TOOL_VERIFICATIONS } from '../core/constants.js';
+import { P_VALUE_STRICT, COUNT_FIVE, COUNT_FOUR, COUNT_NINE, COUNT_SEVEN, COUNT_SIX, COUNT_TEN, COUNT_THOUSAND, COUNT_THREE, COUNT_TWO, COUNT_TWO_HUNDRED, VALUE_FORTY_TWO, TEST_TIMEOUT_MS_SHORT, } from '../../test-helpers/test-constants.js';
+import { TEST_SCORE_GOOD, TEST_SCORE_HIGH, TEST_SCORE_LOW, TEST_SCORE_MID, TEST_SCORE_VERY_LOW, } from '../quality/quality-test-constants.js';
+const TEST_TIMEOUT_MS_TINY = 50;
+const TEST_SCORE_STRONG = 0.6;
+const TEST_SCORE_CONSENSUS_MIN = 0.7;
+const STEP_SCORE_NEAR_EXCELLENT = 0.82;
+const STEP_SCORE_NEAR_GOOD = 0.79;
+const WEIGHTED_AVERAGE_EXPECTED = 0.75;
+const MEDIAN_EVEN_EXPECTED = 2.5;
+const TEST_CONVERGENCE_THRESHOLD_TIGHT = 0.01;
+const TEST_CONVERGENCE_THRESHOLD_STRICT = 0.0001;
+const TEST_JUDGE_MEMORY_CAPACITY = COUNT_THOUSAND;
+const TEST_LRU_PREFILL_COUNT = TEST_JUDGE_MEMORY_CAPACITY - COUNT_THREE;
+const VARIANCE_SAMPLE_VALUES = [
+    COUNT_TWO,
+    COUNT_FOUR,
+    COUNT_FOUR,
+    COUNT_FOUR,
+    COUNT_FIVE,
+    COUNT_FIVE,
+    COUNT_SEVEN,
+    COUNT_NINE,
+];
+const VARIANCE_NUMERATOR = 32;
+const VARIANCE_DENOMINATOR = COUNT_SEVEN;
+const VARIANCE_EPSILON = P_VALUE_STRICT;
+const VARIANCE_TWO_POINT_VALUES = [COUNT_TWO, COUNT_SIX];
+const VARIANCE_TWO_POINT_EXPECTED = 8;
+const MEDIAN_SINGLE_VALUE = [COUNT_FIVE];
+const MEDIAN_ODD_VALUES = [1, COUNT_THREE, COUNT_FIVE];
+const MEDIAN_EVEN_VALUES = [1, COUNT_TWO, COUNT_THREE, COUNT_FOUR];
+const MEDIAN_UNSORTED_VALUES = [COUNT_FIVE, 1, COUNT_THREE];
+describe('agent-as-judge', () => {
+    // ============================================================================
+    // Timeout Protection Tests
+    // ============================================================================
+    describe('withAgentTimeout', () => {
+        it('should return result when function completes in time', async () => {
+            const result = await withAgentTimeout(() => Promise.resolve(VALUE_FORTY_TWO), TEST_TIMEOUT_MS_SHORT);
+            assert.equal(result, VALUE_FORTY_TWO);
+        });
+        it('should throw AgentEvalTimeoutError when function times out', async () => {
+            await assert.rejects(withAgentTimeout(() => new Promise((resolve) => setTimeout(resolve, COUNT_TWO_HUNDRED)), TEST_TIMEOUT_MS_TINY), AgentEvalTimeoutError);
+        });
+        it('should include timeout duration in error', async () => {
+            try {
+                await withAgentTimeout(() => new Promise((resolve) => setTimeout(resolve, COUNT_TWO_HUNDRED)), TEST_TIMEOUT_MS_TINY);
+                assert.fail('Should have thrown');
+            }
+            catch (error) {
+                assert.ok(error instanceof AgentEvalTimeoutError);
+                assert.equal(error.timeoutMs, TEST_TIMEOUT_MS_TINY);
+                assert.ok(error.message.includes(`${TEST_TIMEOUT_MS_TINY}`));
+            }
+        });
+        it('should propagate function errors', async () => {
+            await assert.rejects(withAgentTimeout(() => Promise.reject(new Error('Test error')), TEST_TIMEOUT_MS_SHORT), { message: 'Test error' });
+        });
+    });
+    // ============================================================================
+    // Validation Tests
+    // ============================================================================
+    describe('evaluandSchema', () => {
+        it('should pass for valid evaluand', () => {
+            assert.doesNotThrow(() => evaluandSchema.parse({ input: 'test input', output: 'test output' }));
+        });
+        it('should throw for empty input', () => {
+            assert.throws(() => evaluandSchema.parse({ input: '', output: 'test' }), z.ZodError);
+        });
+        it('should throw for whitespace-only input', () => {
+            assert.throws(() => evaluandSchema.parse({ input: '   ', output: 'test' }), z.ZodError);
+        });
+        it('should throw for empty output', () => {
+            assert.throws(() => evaluandSchema.parse({ input: 'test', output: '' }), z.ZodError);
+        });
+        it('should throw for actions exceeding MAX_TRAJECTORY_LENGTH', () => {
+            const actions = Array(MAX_TRAJECTORY_LENGTH + 1).fill({
+                type: 'tool_call',
+                tool: 'test',
+            });
+            assert.throws(() => evaluandSchema.parse({ input: 'test', output: 'test', actions }), z.ZodError);
+        });
+        it('should pass for actions at MAX_TRAJECTORY_LENGTH', () => {
+            const actions = Array(MAX_TRAJECTORY_LENGTH).fill({
+                type: 'tool_call',
+                tool: 'test',
+            });
+            assert.doesNotThrow(() => evaluandSchema.parse({ input: 'test', output: 'test', actions }));
+        });
+    });
+    describe('validateStepScore', () => {
+        it('should pass for valid step score with number step', () => {
+            assert.doesNotThrow(() => validateStepScore({ step: 0, score: TEST_SCORE_MID }));
+        });
+        it('should pass for valid step score with string step', () => {
+            assert.doesNotThrow(() => validateStepScore({ step: 'step_1', score: TEST_SCORE_MID }));
+        });
+        it('should throw for step string exceeding 256 characters', () => {
+            assert.throws(() => validateStepScore({ step: 'a'.repeat(MAX_STEP_ID_LENGTH + 1), score: TEST_SCORE_MID }), InputValidationError);
+        });
+        it('should throw for negative step index', () => {
+            assert.throws(() => validateStepScore({ step: -1, score: TEST_SCORE_MID }), InputValidationError);
+        });
+        it('should throw for non-integer step index', () => {
+            assert.throws(() => validateStepScore({ step: 1.5, score: TEST_SCORE_MID }), InputValidationError);
+        });
+        it('should throw for score below 0', () => {
+            assert.throws(() => validateStepScore({ step: 0, score: -0.1 }), InputValidationError);
+        });
+        it('should throw for score above 1', () => {
+            assert.throws(() => validateStepScore({ step: 0, score: 1.1 }), InputValidationError);
+        });
+        it('should throw for NaN score', () => {
+            assert.throws(() => validateStepScore({ step: 0, score: NaN }), InputValidationError);
+        });
+        it('should throw for Infinity score', () => {
+            assert.throws(() => validateStepScore({ step: 0, score: Infinity }), InputValidationError);
+        });
+    });
+    // ============================================================================
+    // Tool Verification Tests
+    // ============================================================================
+    describe('verifyToolCall', () => {
+        it('should throw for null action (H1)', () => {
+            assert.throws(() => verifyToolCall(null), InputValidationError);
+        });
+        it('should throw for non-object action (H1)', () => {
+            assert.throws(() => verifyToolCall('string'), InputValidationError);
+        });
+        it('should verify correct tool selection', () => {
+            const action = { type: 'tool_call', tool: 'search' };
+            const result = verifyToolCall(action, 'search');
+            assert.equal(result.toolCorrect, true);
+            assert.equal(result.score, 1.0);
+        });
+        it('should detect incorrect tool selection', () => {
+            const action = { type: 'tool_call', tool: 'read' };
+            const result = verifyToolCall(action, 'search');
+            assert.equal(result.toolCorrect, false);
+            assert.equal(result.score, 0);
+        });
+        it('should verify correct arguments', () => {
+            const action = {
+                type: 'tool_call',
+                tool: 'search',
+                arguments: { query: 'test' },
+            };
+            const result = verifyToolCall(action, 'search', { query: 'test' });
+            assert.equal(result.argsCorrect, true);
+            assert.ok(result.score > TEST_SCORE_HIGH); // tool + args correct
+        });
+        it('should detect incorrect arguments', () => {
+            const action = {
+                type: 'tool_call',
+                tool: 'search',
+                arguments: { query: 'wrong' },
+            };
+            const result = verifyToolCall(action, 'search', { query: 'test' });
+            assert.equal(result.argsCorrect, false);
+        });
+        it('should verify result correctness when provided', () => {
+            const action = {
+                type: 'tool_call',
+                tool: 'calc',
+                result: VALUE_FORTY_TWO,
+            };
+            const result = verifyToolCall(action, 'calc', undefined, undefined, VALUE_FORTY_TWO);
+            assert.equal(result.resultCorrect, true);
+        });
+        it('should include evidence in result', () => {
+            const action = {
+                type: 'tool_call',
+                tool: 'test',
+                arguments: { foo: 'bar' },
+            };
+            const result = verifyToolCall(action, 'expected', { baz: 'qux' });
+            assert.ok(result.evidence);
+            assert.deepEqual(result.evidence.actualTool, 'test');
+            assert.deepEqual(result.evidence.expectedTool, 'expected');
+        });
+    });
+    describe('verifyToolCalls', () => {
+        it('should verify multiple tool calls', () => {
+            const actions = [
+                { type: 'tool_call', tool: 'search', toolCallId: 'call_1' },
+                { type: 'reasoning', reasoning: 'thinking...' },
+                { type: 'tool_call', tool: 'read', toolCallId: 'call_2' },
+            ];
+            const expected = new Map([
+                ['call_1', { tool: 'search' }],
+                ['call_2', { tool: 'read' }],
+            ]);
+            const results = verifyToolCalls(actions, expected);
+            assert.equal(results.length, COUNT_TWO); // Only tool_call actions
+            assert.equal(results[0].toolCorrect, true);
+            assert.equal(results[1].toolCorrect, true);
+        });
+        it('should skip non-tool actions', () => {
+            const actions = [
+                { type: 'reasoning', reasoning: 'thinking' },
+                { type: 'response', reasoning: 'responding' },
+            ];
+            const results = verifyToolCalls(actions);
+            assert.equal(results.length, 0);
+        });
+        it('should respect MAX_TOOL_VERIFICATIONS limit', () => {
+            const actions = Array(COUNT_THOUSAND).fill({
+                type: 'tool_call',
+                tool: 'test',
+            });
+            const results = verifyToolCalls(actions);
+            assert.ok(results.length <= MAX_TOOL_VERIFICATIONS);
+        });
+    });
+    // ============================================================================
+    // Step Scoring Tests
+    // ============================================================================
+    describe('scoreStep', () => {
+        it('should create valid step score', () => {
+            const action = { type: 'tool_call', tool: 'search' };
+            const result = scoreStep(action, 0, { score: TEST_SCORE_GOOD, explanation: 'Good' });
+            assert.equal(result.step, 0);
+            assert.equal(result.score, TEST_SCORE_GOOD);
+            assert.equal(result.explanation, 'Good');
+        });
+        it('should clamp score to [0, 1]', () => {
+            const action = { type: 'tool_call', tool: 'test' };
+            const high = scoreStep(action, 0, { score: 1.5 });
+            assert.equal(high.score, 1);
+            const low = scoreStep(action, 0, { score: -0.5 });
+            assert.equal(low.score, 0);
+        });
+        it('should throw for NaN score (H4)', () => {
+            const action = { type: 'tool_call', tool: 'test' };
+            assert.throws(() => scoreStep(action, 0, { score: NaN }), InputValidationError);
+        });
+        it('should throw for string score (H4)', () => {
+            const action = { type: 'tool_call', tool: 'test' };
+            assert.throws(() => scoreStep(action, 0, { score: '0.5' }), InputValidationError);
+        });
+        it('should include action metadata in evidence', () => {
+            const action = {
+                type: 'tool_call',
+                tool: 'search',
+                reasoning: 'searching for data',
+            };
+            const result = scoreStep(action, 0, { score: TEST_SCORE_HIGH });
+            const evidence = result.evidence;
+            assert.equal(evidence.actionType, 'tool_call');
+            assert.equal(evidence.tool, 'search');
+            assert.equal(evidence.reasoning, 'searching for data');
+        });
+    });
+    describe('aggregateStepScores', () => {
+        it('should return 1 for empty array', () => {
+            const result = aggregateStepScores([]);
+            assert.equal(result, 1);
+        });
+        it('should calculate average correctly', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_GOOD },
+                { step: 1, score: TEST_SCORE_STRONG },
+                { step: 2, score: 1 },
+            ];
+            const result = aggregateStepScores(scores, 'average');
+            // Use approximate comparison for floating point
+            assert.ok(Math.abs(result - TEST_SCORE_GOOD) < FLOAT_TOLERANCE);
+        });
+        it('should calculate weighted average correctly', () => {
+            const scores = [
+                { step: 0, score: 1 },
+                { step: 1, score: 0 },
+            ];
+            const result = aggregateStepScores(scores, 'weighted', [COUNT_THREE, 1]);
+            assert.equal(result, WEIGHTED_AVERAGE_EXPECTED);
+        });
+        it('should throw for weighted without weights', () => {
+            const scores = [{ step: 0, score: TEST_SCORE_MID }];
+            assert.throws(() => aggregateStepScores(scores, 'weighted'), Error);
+        });
+        it('should throw for mismatched weights length', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_MID },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [1]), Error);
+        });
+        it('should return 0 for all-zero weights', () => {
+            const scores = [{ step: 0, score: TEST_SCORE_GOOD }];
+            const result = aggregateStepScores(scores, 'weighted', [0]);
+            assert.equal(result, 0);
+        });
+        it('should throw on negative weights (L8)', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [1, -1]), /Invalid weight at index 1: -1\. Weights must be finite non-negative numbers/);
+        });
+        it('should throw on negative weight at index 0', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [-1, 1]), /Invalid weight at index 0: -1\. Weights must be finite non-negative numbers/);
+        });
+        it('should throw on NaN weight (M1)', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [1, NaN]), /Invalid weight at index 1: NaN\. Weights must be finite non-negative numbers/);
+        });
+        it('should throw on Infinity weight', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [1, Infinity]), /Invalid weight at index 1: Infinity\. Weights must be finite non-negative numbers/);
+        });
+        it('should throw on -Infinity weight', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+            ];
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [-Infinity, 1]), /Invalid weight at index 0: -Infinity\. Weights must be finite non-negative numbers/);
+        });
+        it('should fail fast on first invalid weight in array with multiple invalid values', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_MID },
+                { step: 1, score: TEST_SCORE_GOOD },
+                { step: 2, score: TEST_SCORE_STRONG },
+            ];
+            // Should report index 0, not index 1 or 2 — validates fail-fast behavior
+            assert.throws(() => aggregateStepScores(scores, 'weighted', [-1, NaN, Infinity]), /Invalid weight at index 0: -1\. Weights must be finite non-negative numbers/);
+        });
+        it('should calculate min correctly', () => {
+            const scores = [
+                { step: 0, score: TEST_SCORE_HIGH },
+                { step: 1, score: TEST_SCORE_LOW },
+                { step: 2, score: TEST_SCORE_CONSENSUS_MIN },
+            ];
+            const result = aggregateStepScores(scores, 'min');
+            assert.equal(result, TEST_SCORE_LOW);
+        });
+        it('should return 0 for empty array with emptyBehavior=zero', () => {
+            const result = aggregateStepScores([], 'average', undefined, { emptyBehavior: 'zero' });
+            assert.equal(result, 0);
+        });
+        it('should return NaN for empty array with emptyBehavior=NaN', () => {
+            const result = aggregateStepScores([], 'average', undefined, { emptyBehavior: 'NaN' });
+            assert.ok(Number.isNaN(result));
+        });
+        it('should return 1 for empty array with emptyBehavior=vacuous (default)', () => {
+            const result = aggregateStepScores([], 'average', undefined, { emptyBehavior: 'vacuous' });
+            assert.equal(result, 1);
+        });
+    });
+    // ============================================================================
+    // Trajectory Analysis Tests
+    // ============================================================================
+    describe('analyzeTrajectory', () => {
+        it('should calculate basic metrics', () => {
+            const evaluand = {
+                input: 'test',
+                output: 'result',
+                actions: [
+                    { type: 'tool_call', tool: 'search' },
+                    { type: 'reasoning', reasoning: 'thinking' },
+                    { type: 'tool_call', tool: 'read' },
+                ],
+            };
+            const result = analyzeTrajectory(evaluand);
+            assert.equal(result.length, COUNT_THREE);
+            assert.equal(result.toolCallCount, COUNT_TWO);
+            assert.equal(result.uniqueTools, COUNT_TWO);
+        });
+        it('should handle empty actions', () => {
+            const evaluand = { input: 'test', output: 'result' };
+            const result = analyzeTrajectory(evaluand);
+            assert.equal(result.length, 0);
+            assert.equal(result.toolCallCount, 0);
+        });
+        it('should calculate efficiency ratio', () => {
+            const evaluand = {
+                input: 'test',
+                output: 'result',
+                actions: Array(COUNT_TEN).fill({ type: 'tool_call', tool: 'test' }),
+            };
+            const result = analyzeTrajectory(evaluand, COUNT_FIVE);
+            assert.equal(result.efficiencyRatio, TEST_SCORE_MID); // 5/10
+        });
+        it('should cap efficiency ratio at 1', () => {
+            const evaluand = {
+                input: 'test',
+                output: 'result',
+                actions: [{ type: 'tool_call', tool: 'test' }],
+            };
+            const result = analyzeTrajectory(evaluand, COUNT_TEN);
+            assert.equal(result.efficiencyRatio, 1);
+        });
+        it('should detect redundant actions', () => {
+            const evaluand = {
+                input: 'test',
+                output: 'result',
+                actions: [
+                    { type: 'tool_call', tool: 'search', arguments: { q: 'a' } },
+                    { type: 'tool_call', tool: 'search', arguments: { q: 'a' } }, // Duplicate
+                    { type: 'tool_call', tool: 'search', arguments: { q: 'b' } }, // Different args
+                ],
+            };
+            const result = analyzeTrajectory(evaluand);
+            assert.equal(result.redundantActions, 1);
+        });
+    });
+    // ============================================================================
+    // Statistical Functions Tests
+    // ============================================================================
+    describe('calculateVariance', () => {
+        it('should return 0 for empty array', () => {
+            assert.equal(calculateVariance([]), 0);
+        });
+        it('should return 0 for single value', () => {
+            assert.equal(calculateVariance(MEDIAN_SINGLE_VALUE), 0);
+        });
+        it('should return 0 for identical values', () => {
+            assert.equal(calculateVariance([COUNT_THREE, COUNT_THREE, COUNT_THREE, COUNT_THREE]), 0);
+        });
+        it('should calculate sample variance with Bessel correction (M7)', () => {
+            // Values: [2, 4, 4, 4, 5, 5, 7, 9], n=8, mean = 5
+            // Sum of squared diffs = (2-5)^2 + (4-5)^2*3 + (5-5)^2*2 + (7-5)^2 + (9-5)^2
+            //                      = 9 + 3 + 0 + 4 + 16 = 32
+            // Sample variance (Bessel's correction) = 32 / (n-1) = 32/7 ≈ 4.571
+            const result = calculateVariance(VARIANCE_SAMPLE_VALUES);
+            assert.ok(Math.abs(result - VARIANCE_NUMERATOR / VARIANCE_DENOMINATOR) < VARIANCE_EPSILON, `Expected ~4.571, got ${result}`);
+        });
+        it('should calculate variance for two values', () => {
+            // Values: [2, 6], mean = 4
+            // Sum of squared diffs = (2-4)^2 + (6-4)^2 = 4 + 4 = 8
+            // Sample variance = 8 / (2-1) = 8
+            const result = calculateVariance(VARIANCE_TWO_POINT_VALUES);
+            assert.equal(result, VARIANCE_TWO_POINT_EXPECTED);
+        });
+    });
+    describe('calculateMedian', () => {
+        it('should return 0 for empty array', () => {
+            assert.equal(calculateMedian([]), 0);
+        });
+        it('should return value for single element', () => {
+            assert.equal(calculateMedian(MEDIAN_SINGLE_VALUE), COUNT_FIVE);
+        });
+        it('should calculate median for odd count', () => {
+            assert.equal(calculateMedian(MEDIAN_ODD_VALUES), COUNT_THREE);
+        });
+        it('should calculate median for even count', () => {
+            assert.equal(calculateMedian(MEDIAN_EVEN_VALUES), MEDIAN_EVEN_EXPECTED);
+        });
+        it('should handle unsorted input', () => {
+            assert.equal(calculateMedian(MEDIAN_UNSORTED_VALUES), COUNT_THREE);
+        });
+    });
+    // ============================================================================
+    // Consensus Tests
+    // ============================================================================
+    describe('collectiveConsensus', () => {
+        const validEvaluand = { input: 'test', output: 'result' };
+        const config = {
+            rounds: COUNT_THREE,
+            convergenceThreshold: DEFAULT_CONVERGENCE_THRESHOLD,
+        };
+        it('should throw for invalid evaluand', async () => {
+            await assert.rejects(collectiveConsensus({ input: '', output: 'test' }, [], config), z.ZodError);
+        });
+        it('should throw for too many judges (H2)', async () => {
+            const judges = Array(MAX_CONCURRENT_EVALUATORS + 1).fill({
+                id: 'judge',
+                evaluate: async () => TEST_SCORE_MID,
+            });
+            await assert.rejects(collectiveConsensus(validEvaluand, judges, config), InputValidationError);
+        });
+        it('should throw for empty judges array', async () => {
+            await assert.rejects(collectiveConsensus(validEvaluand, [], config), InputValidationError);
+        });
+        it('should throw for rounds < 1 (M9)', async () => {
+            const judges = [{ id: 'j1', evaluate: async () => TEST_SCORE_MID }];
+            await assert.rejects(collectiveConsensus(validEvaluand, judges, { ...config, rounds: 0 }), z.ZodError);
+        });
+        it('should reach consensus with agreeing judges', async () => {
+            const judges = [
+                { id: 'j1', evaluate: async () => TEST_SCORE_GOOD },
+                { id: 'j2', evaluate: async () => STEP_SCORE_NEAR_EXCELLENT },
+                { id: 'j3', evaluate: async () => STEP_SCORE_NEAR_GOOD },
+            ];
+            const result = await collectiveConsensus(validEvaluand, judges, config);
+            assert.ok(result.converged);
+            assert.ok(result.finalScore > TEST_SCORE_CONSENSUS_MIN);
+        });
+        it('should handle disagreeing judges', async () => {
+            const judges = [
+                { id: 'j1', evaluate: async () => TEST_SCORE_VERY_LOW },
+                { id: 'j2', evaluate: async () => TEST_SCORE_GOOD },
+            ];
+            const result = await collectiveConsensus(validEvaluand, judges, {
+                rounds: COUNT_TWO,
+                convergenceThreshold: TEST_CONVERGENCE_THRESHOLD_TIGHT, // Very tight
+            });
+            assert.equal(result.converged, false);
+        });
+        it('should handle judge failures gracefully (H3)', async () => {
+            const judges = [
+                { id: 'j1', evaluate: async () => TEST_SCORE_GOOD },
+                {
+                    id: 'j2',
+                    evaluate: async () => {
+                        throw new Error('Judge failed');
+                    },
+                },
+                { id: 'j3', evaluate: async () => WEIGHTED_AVERAGE_EXPECTED },
+            ];
+            const result = await collectiveConsensus(validEvaluand, judges, config);
+            // Should succeed with 2 working judges
+            assert.ok(result.finalScore > 0);
+        });
+        it('should throw when all judges fail (H3)', async () => {
+            const judges = [
+                {
+                    id: 'j1',
+                    evaluate: async () => {
+                        throw new Error('Failed 1');
+                    },
+                },
+                {
+                    id: 'j2',
+                    evaluate: async () => {
+                        throw new Error('Failed 2');
+                    },
+                },
+            ];
+            await assert.rejects(collectiveConsensus(validEvaluand, judges, config), /All judge evaluations failed/);
+        });
+        it('should respect MAX_CONSENSUS_ROUNDS', async () => {
+            let callCount = 0;
+            const judges = [
+                {
+                    id: 'j1',
+                    evaluate: async () => {
+                        callCount++;
+                        return TEST_SCORE_MID;
+                    },
+                },
+            ];
+            await collectiveConsensus(validEvaluand, judges, {
+                rounds: MAX_CONSENSUS_ROUNDS, // Schema-enforced maximum
+                convergenceThreshold: TEST_CONVERGENCE_THRESHOLD_STRICT,
+            });
+            assert.ok(callCount <= MAX_CONSENSUS_ROUNDS);
+        });
+    });
+    // ============================================================================
+    // ProceduralJudge Tests
+    // ============================================================================
+    describe('ProceduralJudge', () => {
+        it('should execute all stages in order', async () => {
+            const executionOrder = [];
+            const stages = [
+                {
+                    name: 'stage1',
+                    evaluate: async () => {
+                        executionOrder.push('stage1');
+                        return { score: TEST_SCORE_GOOD, explanation: 'Good' };
+                    },
+                },
+                {
+                    name: 'stage2',
+                    evaluate: async () => {
+                        executionOrder.push('stage2');
+                        return { score: TEST_SCORE_HIGH, explanation: 'Great' };
+                    },
+                },
+            ];
+            const judge = new ProceduralJudge(stages);
+            const result = await judge.evaluate({
+                input: 'test',
+                output: 'result',
+            });
+            assert.deepEqual(executionOrder, ['stage1', 'stage2']);
+            assert.equal(result.stepScores.length, COUNT_TWO);
+        });
+        it('should support early termination', async () => {
+            const executionOrder = [];
+            const stages = [
+                {
+                    name: 'safety',
+                    evaluate: async () => {
+                        executionOrder.push('safety');
+                        return { score: TEST_SCORE_VERY_LOW, explanation: 'Failed safety' };
+                    },
+                },
+                {
+                    name: 'quality',
+                    evaluate: async () => {
+                        executionOrder.push('quality');
+                        return { score: TEST_SCORE_HIGH, explanation: 'Good' };
+                    },
+                },
+            ];
+            const judge = new ProceduralJudge(stages, 'safety');
+            const result = await judge.evaluate({
+                input: 'test',
+                output: 'result',
+            });
+            assert.deepEqual(executionOrder, ['safety']);
+            assert.equal(result.overallScore, 0);
+            assert.ok(result.explanation.includes('Early termination'));
+        });
+        it('should pass context between stages', async () => {
+            let stage2Context = {};
+            const stages = [
+                {
+                    name: 'stage1',
+                    evaluate: async (_, ctx) => {
+                        ctx['data'] = 'from_stage1';
+                        return { score: TEST_SCORE_GOOD, explanation: 'OK' };
+                    },
+                },
+                {
+                    name: 'stage2',
+                    evaluate: async (_, ctx) => {
+                        stage2Context = { ...ctx };
+                        return { score: TEST_SCORE_HIGH, explanation: 'OK' };
+                    },
+                },
+            ];
+            const judge = new ProceduralJudge(stages);
+            await judge.evaluate({ input: 'test', output: 'result' });
+            assert.ok(stage2Context['stage1']);
+        });
+        it('should validate evaluand', async () => {
+            const judge = new ProceduralJudge([
+                { name: 'test', evaluate: async () => ({ score: 1, explanation: '' }) },
+            ]);
+            await assert.rejects(judge.evaluate({ input: '', output: 'test' }), z.ZodError);
+        });
+        describe('constructor validation (M8)', () => {
+            it('should throw on empty stages array', () => {
+                assert.throws(() => new ProceduralJudge([]), InputValidationError);
+            });
+            it('should throw on stage with empty name', () => {
+                assert.throws(() => new ProceduralJudge([
+                    { name: '', evaluate: async () => ({ score: 1, explanation: '' }) },
+                ]), InputValidationError);
+            });
+            it('should throw on stage with whitespace-only name', () => {
+                assert.throws(() => new ProceduralJudge([
+                    { name: '   ', evaluate: async () => ({ score: 1, explanation: '' }) },
+                ]), InputValidationError);
+            });
+            it('should throw on stage without evaluate function', () => {
+                assert.throws(
+                // @ts-expect-error - testing runtime validation
+                () => new ProceduralJudge([{ name: 'test' }]), InputValidationError);
+            });
+            it('should throw on invalid earlyTerminationOn stage name', () => {
+                assert.throws(() => new ProceduralJudge([{ name: 'stage1', evaluate: async () => ({ score: 1, explanation: '' }) }], 'nonexistent'), InputValidationError);
+            });
+            it('should accept valid stages with earlyTerminationOn', () => {
+                const judge = new ProceduralJudge([{ name: 'safety', evaluate: async () => ({ score: 1, explanation: '' }) }], 'safety');
+                assert.ok(judge);
+            });
+        });
+    });
+    // ============================================================================
+    // ReactiveJudge Tests
+    // ============================================================================
+    describe('ReactiveJudge', () => {
+        it('should route to selected specialists', async () => {
+            const executedSpecialists = [];
+            const router = async () => ['quality', 'safety'];
+            const specialists = new Map([
+                [
+                    'quality',
+                    async () => {
+                        executedSpecialists.push('quality');
+                        return { score: TEST_SCORE_HIGH, explanation: 'Good quality' };
+                    },
+                ],
+                [
+                    'safety',
+                    async () => {
+                        executedSpecialists.push('safety');
+                        return { score: 1, explanation: 'Safe' };
+                    },
+                ],
+                [
+                    'style',
+                    async () => {
+                        executedSpecialists.push('style');
+                        return { score: TEST_SCORE_GOOD, explanation: 'OK style' };
+                    },
+                ],
+            ]);
+            const judge = new ReactiveJudge(router, specialists);
+            const result = await judge.evaluate({
+                input: 'test',
+                output: 'result',
+            });
+            assert.deepEqual(executedSpecialists, ['quality', 'safety']);
+            assert.equal(result.stepScores.length, COUNT_TWO);
+        });
+        it('should trigger deep dive when needed', async () => {
+            const deepDiveTriggered = { value: false };
+            const router = async () => ['quality'];
+            const specialists = new Map([
+                [
+                    'quality',
+                    async () => ({
+                        score: TEST_SCORE_MID,
+                        explanation: 'Needs investigation',
+                        needsDeepDive: true,
+                    }),
+                ],
+            ]);
+            const deepDiveSpecialists = new Map([
+                [
+                    'quality',
+                    async () => {
+                        deepDiveTriggered.value = true;
+                        return { score: COUNT_FOUR / COUNT_TEN, explanation: 'Deep analysis' };
+                    },
+                ],
+            ]);
+            const judge = new ReactiveJudge(router, specialists, deepDiveSpecialists);
+            const result = await judge.evaluate({
+                input: 'test',
+                output: 'result',
+            });
+            assert.equal(deepDiveTriggered.value, true);
+            assert.equal(result.stepScores.length, COUNT_TWO); // Regular + deep dive
+        });
+        it('should skip missing specialists', async () => {
+            const router = async () => ['missing', 'existing'];
+            const specialists = new Map([
+                ['existing', async () => ({ score: TEST_SCORE_GOOD, explanation: 'OK' })],
+            ]);
+            const judge = new ReactiveJudge(router, specialists);
+            const result = await judge.evaluate({
+                input: 'test',
+                output: 'result',
+            });
+            assert.equal(result.stepScores.length, 1);
+        });
+        it('should validate evaluand', async () => {
+            const judge = new ReactiveJudge(async () => [], new Map());
+            await assert.rejects(judge.evaluate({ input: '', output: 'test' }), z.ZodError);
+        });
+    });
+    // ============================================================================
+    // AgentJudge Memory Tests (H5)
+    // ============================================================================
+    describe('AgentJudge memory management', () => {
+        class TestJudge extends ProceduralJudge {
+            // Expose protected methods for testing
+            testStore(key, value) {
+                this.storeInMemory(key, value);
+            }
+            testGet(key) {
+                return this.getFromMemory(key);
+            }
+            getMemorySize() {
+                return this.memory.size;
+            }
+        }
+        let judge;
+        beforeEach(() => {
+            judge = new TestJudge([
+                { name: 'test', evaluate: async () => ({ score: 1, explanation: '' }) },
+            ]);
+        });
+        it('should store and retrieve values', () => {
+            judge.testStore('key1', 'value1');
+            assert.equal(judge.testGet('key1'), 'value1');
+        });
+        it('should implement LRU - reading moves to end (H5)', () => {
+            // Fill nearly to capacity
+            for (let i = 0; i < TEST_LRU_PREFILL_COUNT; i++) {
+                judge.testStore(`fill_${i}`, i);
+            }
+            // Store 3 more items we care about
+            judge.testStore('key1', 'value1');
+            judge.testStore('key2', 'value2');
+            judge.testStore('key3', 'value3');
+            // Memory is now at 1000 - order is fill_0...fill_996, key1, key2, key3
+            // Read key1 - should move it to end (after key3)
+            judge.testGet('key1');
+            // Order is now: fill_0...fill_996, key2, key3, key1
+            // Store 1 more to trigger eviction of fill_0
+            judge.testStore('new_item', 'new');
+            // Order: fill_1...fill_996, key2, key3, key1, new_item
+            // key1 should still exist (was accessed recently), key2/key3 should exist
+            assert.equal(judge.testGet('key1'), 'value1');
+            assert.equal(judge.testGet('key2'), 'value2');
+            assert.equal(judge.testGet('key3'), 'value3');
+            // fill_0 should be evicted
+            assert.equal(judge.testGet('fill_0'), undefined);
+        });
+        it('should evict oldest on overflow', () => {
+            // Fill to capacity
+            for (let i = 0; i < TEST_JUDGE_MEMORY_CAPACITY; i++) {
+                judge.testStore(`key_${i}`, i);
+            }
+            assert.equal(judge.getMemorySize(), TEST_JUDGE_MEMORY_CAPACITY);
+            // Add one more
+            judge.testStore('new_key', 'new_value');
+            assert.equal(judge.getMemorySize(), TEST_JUDGE_MEMORY_CAPACITY);
+            assert.equal(judge.testGet('key_0'), undefined); // First one evicted
+            assert.equal(judge.testGet('new_key'), 'new_value');
+        });
+    });
+});
+//# sourceMappingURL=agent-as-judge.test.js.map