observability-toolkit 1.8.5 → 2.0.0

package/dist/lib/agent-as-judge.js

@@ -0,0 +1,740 @@
+/**
+ * Agent-as-Judge Implementation
+ *
+ * Provides patterns and utilities for evaluating AI agents using autonomous
+ * judge agents with planning, tool use, memory, and multi-agent collaboration.
+ *
+ * @security
+ * - All user inputs are sanitized via llm-as-judge utilities
+ * - Tool execution should use sandbox isolation
+ * - Memory is bounded to prevent resource exhaustion
+ *
+ * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/
+ * @see docs/quality/agent-as-judge.md
+ */
+import { MAX_STEP_SCORES, MAX_TOOL_VERIFICATIONS, MAX_STEP_ID_LENGTH, } from '../backends/index.js';
+import { InputValidationError } from './input-validator.js';
+// ============================================================================
+// Constants
+// ============================================================================
+/** Maximum trajectory length to analyze for efficiency metrics */
+export const MAX_TRAJECTORY_LENGTH = 1000;
+/** Default timeout for agent evaluation steps (60 seconds) */
+export const DEFAULT_AGENT_EVAL_TIMEOUT_MS = 60000;
+/** Maximum concurrent specialist evaluators */
+export const MAX_CONCURRENT_EVALUATORS = 10;
+/** Maximum consensus rounds for multi-agent evaluation */
+export const MAX_CONSENSUS_ROUNDS = 5;
+/** Default variance threshold for consensus convergence */
+export const DEFAULT_CONVERGENCE_THRESHOLD = 0.1;
+/**
+ * Default threshold for early termination in procedural evaluation.
+ * If a critical stage scores below this threshold, evaluation terminates early.
+ */
+export const DEFAULT_EARLY_TERMINATION_THRESHOLD = 0.3;
+// ============================================================================
+// Timeout Protection (H6)
+// ============================================================================
+/**
+ * Error thrown when an evaluation exceeds the configured timeout.
+ */
+export class AgentEvalTimeoutError extends Error {
+    timeoutMs;
+    constructor(timeoutMs) {
+        super(`Agent evaluation timed out after ${timeoutMs}ms`);
+        this.timeoutMs = timeoutMs;
+        this.name = 'AgentEvalTimeoutError';
+    }
+}
+/**
+ * Execute an async function with timeout protection.
+ *
+ * @param fn - Async function to execute
+ * @param timeoutMs - Timeout in milliseconds (default: DEFAULT_AGENT_EVAL_TIMEOUT_MS)
+ * @returns Result of the function
+ * @throws {AgentEvalTimeoutError} If function times out
+ */
+export async function withAgentTimeout(fn, timeoutMs = DEFAULT_AGENT_EVAL_TIMEOUT_MS) {
+    let timeoutId;
+    const timeoutPromise = new Promise((_, reject) => {
+        timeoutId = setTimeout(() => {
+            reject(new AgentEvalTimeoutError(timeoutMs));
+        }, timeoutMs);
+    });
+    try {
+        const result = await Promise.race([fn(), timeoutPromise]);
+        if (timeoutId !== undefined)
+            clearTimeout(timeoutId);
+        return result;
+    }
+    catch (error) {
+        if (timeoutId !== undefined)
+            clearTimeout(timeoutId);
+        throw error;
+    }
+}
+// ============================================================================
+// Validation Utilities
+// ============================================================================
+/**
+ * Validate an Evaluand object.
+ * @throws {InputValidationError} If validation fails
+ */
+export function validateEvaluand(evaluand) {
+    if (!evaluand.input || evaluand.input.trim().length === 0) {
+        throw new InputValidationError('Evaluand input is required', 'input', 'required');
+    }
+    if (!evaluand.output || evaluand.output.trim().length === 0) {
+        throw new InputValidationError('Evaluand output is required', 'output', 'required');
+    }
+    if (evaluand.actions && evaluand.actions.length > MAX_TRAJECTORY_LENGTH) {
+        throw new InputValidationError(`Actions array exceeds ${MAX_TRAJECTORY_LENGTH} limit`, 'actions', 'maxLength');
+    }
+}
+/**
+ * Validate a StepScore object.
+ * @throws {InputValidationError} If validation fails
+ */
+export function validateStepScore(stepScore) {
+    // Validate step identifier
+    if (typeof stepScore.step === 'string') {
+        if (stepScore.step.length > MAX_STEP_ID_LENGTH) {
+            throw new InputValidationError(`Step identifier exceeds ${MAX_STEP_ID_LENGTH} characters`, 'step', 'maxLength');
+        }
+    }
+    else if (typeof stepScore.step === 'number') {
+        if (stepScore.step < 0 || !Number.isInteger(stepScore.step)) {
+            throw new InputValidationError('Step index must be a non-negative integer', 'step', 'type');
+        }
+    }
+    else {
+        throw new InputValidationError('Step must be a string or number', 'step', 'type');
+    }
+    // Validate score range
+    if (!Number.isFinite(stepScore.score) || stepScore.score < 0 || stepScore.score > 1) {
+        throw new InputValidationError('Score must be in range [0, 1]', 'score', 'range');
+    }
+}
+/**
+ * Validate a ToolVerification object.
+ * @throws {InputValidationError} If validation fails
+ */
+export function validateToolVerification(verification) {
+    if (!verification.toolName || verification.toolName.trim().length === 0) {
+        throw new InputValidationError('Tool name is required', 'toolName', 'required');
+    }
+    if (typeof verification.toolCorrect !== 'boolean') {
+        throw new InputValidationError('toolCorrect must be a boolean', 'toolCorrect', 'type');
+    }
+    if (typeof verification.argsCorrect !== 'boolean') {
+        throw new InputValidationError('argsCorrect must be a boolean', 'argsCorrect', 'type');
+    }
+    if (!Number.isFinite(verification.score) || verification.score < 0 || verification.score > 1) {
+        throw new InputValidationError('Score must be in range [0, 1]', 'score', 'range');
+    }
+}
+// ============================================================================
+// Tool Verification Utilities
+// ============================================================================
+/**
+ * Verify a single tool call against expected behavior.
+ *
+ * @param action - The agent action containing the tool call
+ * @param expectedTool - Expected tool name (optional)
+ * @param expectedArgs - Expected arguments (optional)
+ * @param actualResult - Actual result to compare against expected (optional)
+ * @param expectedResult - Expected result for comparison (optional)
+ * @returns ToolVerification result with weighted score:
+ *   - Tool selection: 40% weight
+ *   - Arguments: 30% weight (if expectedArgs provided)
+ *   - Result: 30% weight (if expectedResult provided)
+ * @throws {InputValidationError} If action is null, undefined, or not an object
+ *
+ * @example
+ * ```typescript
+ * const verification = verifyToolCall(
+ *   { type: 'tool_call', tool: 'search', arguments: { query: 'test' } },
+ *   'search',
+ *   { query: 'test' }
+ * );
+ * console.log(verification.score); // 1.0 (all correct)
+ * ```
+ */
+export function verifyToolCall(action, expectedTool, expectedArgs, actualResult, expectedResult) {
+    // H1: Validate action parameter
+    if (!action || typeof action !== 'object') {
+        throw new InputValidationError('Action is required and must be an object', 'action', 'required');
+    }
+    const toolCorrect = expectedTool ? action.tool === expectedTool : true;
+    const argsCorrect = expectedArgs
+        ? deepEquals(action.arguments, expectedArgs)
+        : true;
+    const resultCorrect = expectedResult !== undefined
+        ? deepEquals(actualResult ?? action.result, expectedResult)
+        : undefined;
+    // Calculate weighted score
+    let score = 0;
+    let weights = 0;
+    // Tool selection: 40% weight
+    score += (toolCorrect ? 0.4 : 0);
+    weights += 0.4;
+    // Arguments: 30% weight
+    if (expectedArgs !== undefined) {
+        score += (argsCorrect ? 0.3 : 0);
+        weights += 0.3;
+    }
+    // Result: 30% weight (only if expected result provided)
+    if (expectedResult !== undefined) {
+        score += (resultCorrect ? 0.3 : 0);
+        weights += 0.3;
+    }
+    // Normalize by weights used
+    const normalizedScore = weights > 0 ? score / weights : (toolCorrect ? 1 : 0);
+    return {
+        toolName: action.tool || 'unknown',
+        toolCallId: action.toolCallId,
+        toolCorrect,
+        argsCorrect,
+        resultCorrect,
+        score: normalizedScore,
+        expectedTool,
+        evidence: {
+            actualTool: action.tool,
+            actualArgs: action.arguments,
+            actualResult: actualResult ?? action.result,
+            expectedTool,
+            expectedArgs,
+            expectedResult,
+        },
+    };
+}
+/**
+ * Verify all tool calls in an agent trajectory.
+ *
+ * @param actions - List of agent actions
+ * @param expectedTools - Map of expected tool calls by index or toolCallId
+ * @returns Array of ToolVerification results
+ */
+export function verifyToolCalls(actions, expectedTools) {
+    const verifications = [];
+    for (let i = 0; i < actions.length && verifications.length < MAX_TOOL_VERIFICATIONS; i++) {
+        const action = actions[i];
+        if (action.type !== 'tool_call' || !action.tool)
+            continue;
+        // Check for expected tool by index or toolCallId
+        const expected = expectedTools?.get(i) ??
+            (action.toolCallId ? expectedTools?.get(action.toolCallId) : undefined);
+        const verification = verifyToolCall(action, expected?.tool, expected?.args, undefined, expected?.result);
+        verifications.push(verification);
+    }
+    return verifications;
+}
+// ============================================================================
+// Step Scoring Utilities
+// ============================================================================
+/**
+ * Score a single step in the agent trajectory.
+ *
+ * @param action - The agent action to score
+ * @param stepIndex - Index of this step (non-negative integer)
+ * @param evaluation - Score and explanation from LLM judge
+ * @returns StepScore result with score clamped to [0, 1]
+ * @throws {InputValidationError} If evaluation.score is not a finite number
+ * @throws {InputValidationError} If stepIndex is invalid (validated via validateStepScore)
+ *
+ * @example
+ * ```typescript
+ * const step = scoreStep(
+ *   { type: 'tool_call', tool: 'search' },
+ *   0,
+ *   { score: 0.85, explanation: 'Correct tool selection' }
+ * );
+ * ```
+ */
+export function scoreStep(action, stepIndex, evaluation) {
+    // H4: Validate score is a finite number before clamping
+    if (typeof evaluation.score !== 'number' || !Number.isFinite(evaluation.score)) {
+        throw new InputValidationError('Evaluation score must be a finite number', 'score', 'type');
+    }
+    const stepScore = {
+        step: stepIndex,
+        score: Math.max(0, Math.min(1, evaluation.score)), // Clamp to [0, 1]
+        explanation: evaluation.explanation,
+        evidence: {
+            actionType: action.type,
+            tool: action.tool,
+            reasoning: action.reasoning,
+        },
+    };
+    validateStepScore(stepScore);
+    return stepScore;
+}
+/**
+ * Calculate aggregate step scores from individual scores.
+ *
+ * @param stepScores - Array of step scores
+ * @param aggregation - Aggregation method ('average', 'weighted', 'min')
+ * @param weights - Weights for weighted aggregation (must match stepScores length, finite non-negative)
+ * @returns Aggregated score (0-1)
+ *   - Empty array returns 1 (no steps to fail = vacuously perfect)
+ *   - This convention matches mathematical definition where empty set satisfies all predicates
+ * @throws {Error} If aggregation is 'weighted' but weights array is missing or wrong length
+ * @throws {Error} If any weight is negative, NaN, or Infinity
+ *
+ * @example
+ * ```typescript
+ * // Simple average
+ * const avg = aggregateStepScores(steps, 'average');
+ *
+ * // Weighted (emphasize later steps)
+ * const weighted = aggregateStepScores(steps, 'weighted', [1, 2, 3]);
+ *
+ * // Minimum score (most strict)
+ * const min = aggregateStepScores(steps, 'min');
+ *
+ * // Empty array returns 1 (vacuously true)
+ * aggregateStepScores([], 'average'); // => 1
+ * ```
+ */
+export function aggregateStepScores(stepScores, aggregation = 'average', weights) {
+    // L7: Empty array returns 1 (vacuously perfect - no steps to fail)
+    if (stepScores.length === 0)
+        return 1;
+    switch (aggregation) {
+        case 'average':
+            return stepScores.reduce((sum, s) => sum + s.score, 0) / stepScores.length;
+        case 'weighted':
+            if (!weights || weights.length !== stepScores.length) {
+                throw new Error('Weights required for weighted aggregation');
+            }
+            // L8/M1: Validate weights are finite non-negative numbers
+            for (let i = 0; i < weights.length; i++) {
+                if (weights[i] < 0 || !Number.isFinite(weights[i])) {
+                    throw new Error(`Invalid weight at index ${i}: ${weights[i]}. Weights must be finite non-negative numbers.`);
+                }
+            }
+            const totalWeight = weights.reduce((sum, w) => sum + w, 0);
+            if (totalWeight === 0)
+                return 0;
+            return stepScores.reduce((sum, s, i) => sum + s.score * weights[i], 0) / totalWeight;
+        case 'min':
+            return Math.min(...stepScores.map(s => s.score));
+    }
+}
+// ============================================================================
+// Trajectory Analysis Utilities
+// ============================================================================
+/**
+ * Calculate trajectory efficiency metrics.
+ *
+ * Uses JSON.stringify for redundancy detection. For large argument objects,
+ * consider implementing hash-based comparison for better performance.
+ *
+ * @param evaluand - The evaluand with actions
+ * @param optimalLength - Expected optimal trajectory length (optional)
+ * @returns Efficiency metrics
+ */
+export function analyzeTrajectory(evaluand, optimalLength) {
+    const actions = evaluand.actions || [];
+    const length = actions.length;
+    const toolCalls = actions.filter(a => a.type === 'tool_call');
+    const toolCallCount = toolCalls.length;
+    const uniqueTools = new Set(toolCalls.map(a => a.tool).filter(Boolean)).size;
+    // Calculate efficiency ratio (if optimal length known)
+    const efficiencyRatio = optimalLength && optimalLength > 0
+        ? Math.min(1, optimalLength / Math.max(length, 1))
+        : 1;
+    // Detect redundant actions (same tool with same args)
+    const seenToolCalls = new Set();
+    let redundantActions = 0;
+    for (const action of toolCalls) {
+        // M2: Wrap in try-catch to handle circular references or non-serializable values
+        try {
+            const key = JSON.stringify({ tool: action.tool, args: action.arguments });
+            if (seenToolCalls.has(key)) {
+                redundantActions++;
+            }
+            seenToolCalls.add(key);
+        }
+        catch {
+            // Skip this action for redundancy detection if serialization fails
+            continue;
+        }
+    }
+    return {
+        length,
+        toolCallCount,
+        uniqueTools,
+        efficiencyRatio,
+        redundantActions,
+    };
+}
+// ============================================================================
+// Multi-Agent Consensus Utilities
+// ============================================================================
+/**
+ * Calculate sample variance of scores for convergence detection.
+ * Uses Bessel's correction (n-1) for unbiased estimation.
+ *
+ * @param scores - Array of score values
+ * @returns Sample variance (0 for empty or single-element arrays)
+ */
+export function calculateVariance(scores) {
+    // Return 0 for empty arrays or single element (no variance measurable)
+    if (scores.length <= 1)
+        return 0;
+    const n = scores.length;
+    const mean = scores.reduce((sum, s) => sum + s, 0) / n;
+    const squaredDiffs = scores.map(s => (s - mean) ** 2);
+    // Bessel's correction: divide by (n-1) for sample variance
+    return squaredDiffs.reduce((sum, d) => sum + d, 0) / (n - 1);
+}
+/**
+ * Calculate median of scores for final consensus.
+ */
+export function calculateMedian(scores) {
+    if (scores.length === 0)
+        return 0;
+    const sorted = [...scores].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0
+        ? (sorted[mid - 1] + sorted[mid]) / 2
+        : sorted[mid];
+}
+/**
+ * Run multi-agent consensus evaluation.
+ *
+ * Uses Promise.allSettled for graceful degradation - partial failures
+ * don't abort the entire consensus. At least one judge must succeed per round.
+ *
+ * @param evaluand - Subject to evaluate
+ * @param judges - Array of judge evaluation functions
+ * @param config - Consensus configuration
+ * @returns ConsensusResult
+ * @throws {InputValidationError} If judges array exceeds MAX_CONCURRENT_EVALUATORS
+ * @throws {InputValidationError} If config.rounds is less than 1
+ * @throws {Error} If all judges fail in any round
+ */
+export async function collectiveConsensus(evaluand, judges, config) {
+    validateEvaluand(evaluand);
+    // H2: Validate judge count to prevent unbounded memory growth
+    if (judges.length > MAX_CONCURRENT_EVALUATORS) {
+        throw new InputValidationError(`Number of judges (${judges.length}) exceeds ${MAX_CONCURRENT_EVALUATORS} limit`, 'judges', 'maxLength');
+    }
+    if (judges.length === 0) {
+        throw new InputValidationError('At least one judge is required', 'judges', 'required');
+    }
+    // M9: Validate rounds is positive
+    if (config.rounds < 1) {
+        throw new InputValidationError('Rounds must be at least 1', 'rounds', 'range');
+    }
+    const rounds = Math.min(config.rounds, MAX_CONSENSUS_ROUNDS);
+    const scores = new Map();
+    // Initialize score arrays
+    for (const judge of judges) {
+        scores.set(judge.id, []);
+    }
+    let convergenceRound = rounds;
+    let converged = false;
+    for (let round = 0; round < rounds; round++) {
+        // H3: Use Promise.allSettled for graceful degradation
+        const roundResults = await Promise.allSettled(judges.map(async (judge) => {
+            const score = await judge.evaluate(evaluand, scores);
+            return { id: judge.id, score };
+        }));
+        // Filter successful results
+        const successfulResults = roundResults
+            .filter((r) => r.status === 'fulfilled')
+            .map(r => r.value);
+        // H3: Require at least one successful judge
+        if (successfulResults.length === 0) {
+            throw new Error(`All judge evaluations failed in round ${round + 1}`);
+        }
+        // Update scores for successful judges only
+        for (const { id, score } of successfulResults) {
+            // H4/M1: Validate score is finite AND in valid range [0, 1] before storing
+            if (Number.isFinite(score) && score >= 0 && score <= 1) {
+                scores.get(id).push(score);
+            }
+        }
+        // Check convergence (only using successful scores)
+        const currentScores = successfulResults
+            .filter(r => Number.isFinite(r.score))
+            .map(r => r.score);
+        if (currentScores.length > 0 && calculateVariance(currentScores) < config.convergenceThreshold) {
+            convergenceRound = round + 1;
+            converged = true;
+            break;
+        }
+    }
+    // Final score is median of last round (only judges with scores)
+    const lastRoundScores = [];
+    for (const judge of judges) {
+        const judgeScores = scores.get(judge.id);
+        if (judgeScores.length > 0) {
+            lastRoundScores.push(judgeScores[judgeScores.length - 1]);
+        }
+    }
+    return {
+        finalScore: lastRoundScores.length > 0 ? calculateMedian(lastRoundScores) : 0,
+        convergenceRound,
+        judgeScores: scores,
+        converged,
+    };
+}
+// ============================================================================
+// Abstract Agent Judge Base Class
+// ============================================================================
+/**
+ * Abstract base class for Agent-as-Judge implementations.
+ *
+ * Provides common infrastructure for memory, tool access, and evaluation flow.
+ * Subclasses implement specific evaluation strategies (procedural, reactive, self-evolving).
+ */
+/** Maximum entries in judge memory before LRU eviction */
+const MAX_JUDGE_MEMORY_SIZE = 1000;
+export class AgentJudge {
+    memory = new Map();
+    /**
+     * Store intermediate state in memory with LRU eviction.
+     * @security Memory is bounded to MAX_JUDGE_MEMORY_SIZE entries
+     */
+    storeInMemory(key, value) {
+        // H5: Proper LRU - delete existing key first to move it to end
+        if (this.memory.has(key)) {
+            this.memory.delete(key);
+        }
+        // Bound memory size to prevent exhaustion
+        if (this.memory.size >= MAX_JUDGE_MEMORY_SIZE) {
+            // Evict least recently used (first entry in insertion order)
+            const firstKey = this.memory.keys().next().value;
+            if (firstKey)
+                this.memory.delete(firstKey);
+        }
+        this.memory.set(key, value);
+    }
+    /**
+     * Retrieve state from memory with LRU update.
+     * @note Access moves item to end (most recently used)
+     */
+    getFromMemory(key) {
+        const value = this.memory.get(key);
+        if (value !== undefined) {
+            // H5: Re-insert to move to end (most recently used)
+            this.memory.delete(key);
+            this.memory.set(key, value);
+        }
+        return value;
+    }
+    /**
+     * Clear all memory.
+     */
+    clearMemory() {
+        this.memory.clear();
+    }
+    /**
+     * Convert AgentEvalResult to OTel-compatible EvaluationResult.
+     */
+    toEvaluationResult(result, evaluand, evaluatorType = 'llm') {
+        return {
+            scoreValue: result.overallScore,
+            explanation: result.explanation,
+            evaluator: this.name,
+            evaluatorType,
+            agentId: evaluand.agentId,
+            agentName: evaluand.agentName,
+            stepScores: result.stepScores.slice(0, MAX_STEP_SCORES),
+            toolVerifications: result.toolVerifications.slice(0, MAX_TOOL_VERIFICATIONS),
+            trajectoryLength: result.trajectoryLength,
+        };
+    }
+}
+/**
+ * Procedural Agent Judge - Fixed evaluation pipeline.
+ *
+ * Executes a predefined sequence of evaluation stages.
+ * Best for domain-specific evaluations with known criteria.
+ */
+export class ProceduralJudge extends AgentJudge {
+    stages;
+    earlyTerminationOn;
+    name = 'procedural-agent-judge';
+    /**
+     * Create a ProceduralJudge with a sequence of evaluation stages.
+     *
+     * @param stages - Array of evaluation stages to execute in order
+     * @param earlyTerminationOn - Optional stage name that triggers early termination on failure
+     * @throws {InputValidationError} If stages array is empty
+     * @throws {InputValidationError} If any stage has empty or missing name
+     * @throws {InputValidationError} If any stage has missing evaluate function
+     * @throws {InputValidationError} If earlyTerminationOn references non-existent stage
+     */
+    constructor(stages, earlyTerminationOn) {
+        super();
+        this.stages = stages;
+        this.earlyTerminationOn = earlyTerminationOn;
+        // M8: Validate stages array
+        if (!stages || stages.length === 0) {
+            throw new InputValidationError('ProceduralJudge requires at least one stage', 'stages', 'required');
+        }
+        // Validate each stage
+        for (let i = 0; i < stages.length; i++) {
+            const stage = stages[i];
+            if (!stage.name || stage.name.trim().length === 0) {
+                throw new InputValidationError(`Stage ${i} has invalid name "${stage.name ?? 'undefined'}" - must be non-empty and not just whitespace`, 'stages', 'required');
+            }
+            if (typeof stage.evaluate !== 'function') {
+                throw new InputValidationError(`Stage ${i} (${stage.name}) must have an evaluate function`, 'stages', 'type');
+            }
+        }
+        // Validate earlyTerminationOn if provided
+        if (earlyTerminationOn !== undefined) {
+            const stageNames = stages.map(s => s.name);
+            if (!stageNames.includes(earlyTerminationOn)) {
+                throw new InputValidationError(`earlyTerminationOn stage '${earlyTerminationOn}' not found in stages: ${stageNames.join(', ')}`, 'earlyTerminationOn', 'invalid');
+            }
+        }
+    }
+    async evaluate(evaluand) {
+        // M3: Wrap entire evaluation in timeout to prevent indefinite hangs
+        return withAgentTimeout(async () => {
+            validateEvaluand(evaluand);
+            const context = {};
+            const stepScores = [];
+            const trajectory = analyzeTrajectory(evaluand);
+            for (let i = 0; i < this.stages.length; i++) {
+                const stage = this.stages[i];
+                const result = await stage.evaluate(evaluand, context);
+                const stepScore = scoreStep({ type: 'evaluation_stage', reasoning: stage.name }, i, result);
+                stepScores.push(stepScore);
+                context[stage.name] = result;
+                // L10: Early termination if configured and stage fails below threshold
+                if (this.earlyTerminationOn === stage.name && result.score < DEFAULT_EARLY_TERMINATION_THRESHOLD) {
+                    return {
+                        overallScore: 0,
+                        stepScores,
+                        toolVerifications: verifyToolCalls(evaluand.actions || []),
+                        trajectoryLength: trajectory.length,
+                        explanation: `Early termination: ${stage.name} failed with score ${result.score}`,
+                        actionableFeedback: [result.explanation],
+                    };
+                }
+            }
+            return {
+                overallScore: aggregateStepScores(stepScores),
+                stepScores,
+                toolVerifications: verifyToolCalls(evaluand.actions || []),
+                trajectoryLength: trajectory.length,
+                explanation: `Procedural evaluation completed ${this.stages.length} stages`,
+                actionableFeedback: stepScores
+                    .filter(s => s.score < 0.7)
+                    .map(s => s.explanation || `Step ${s.step} needs improvement`),
+            };
+        });
+    }
+}
+/**
+ * Reactive Agent Judge - Adaptive evaluation based on content.
+ *
+ * Routes evaluation to appropriate specialists based on initial analysis.
+ * Supports deep-dive evaluation when initial analysis indicates issues.
+ */
+export class ReactiveJudge extends AgentJudge {
+    router;
+    specialists;
+    deepDiveSpecialists;
+    name = 'reactive-agent-judge';
+    constructor(router, specialists, deepDiveSpecialists) {
+        super();
+        this.router = router;
+        this.specialists = specialists;
+        this.deepDiveSpecialists = deepDiveSpecialists;
+    }
+    async evaluate(evaluand) {
+        // M3: Wrap entire evaluation in timeout to prevent indefinite hangs
+        return withAgentTimeout(async () => {
+            validateEvaluand(evaluand);
+            // Route to relevant specialists
+            const relevantSpecialists = await this.router(evaluand);
+            const trajectory = analyzeTrajectory(evaluand);
+            const stepScores = [];
+            for (let i = 0; i < relevantSpecialists.length; i++) {
+                const specialistName = relevantSpecialists[i];
+                const specialist = this.specialists.get(specialistName);
+                if (!specialist)
+                    continue;
+                const result = await specialist(evaluand);
+                const stepScore = scoreStep({ type: 'specialist_evaluation', reasoning: specialistName }, i, result);
+                stepScores.push(stepScore);
+                this.storeInMemory(`eval_${specialistName}`, result);
+                // Trigger deep dive if needed
+                if (result.needsDeepDive && this.deepDiveSpecialists) {
+                    const deepDive = this.deepDiveSpecialists.get(specialistName);
+                    if (deepDive) {
+                        const deepResult = await deepDive(evaluand);
+                        const deepStepScore = scoreStep({ type: 'deep_dive_evaluation', reasoning: `${specialistName}_deep` }, stepScores.length, deepResult);
+                        stepScores.push(deepStepScore);
+                    }
+                }
+            }
+            return {
+                overallScore: aggregateStepScores(stepScores),
+                stepScores,
+                toolVerifications: verifyToolCalls(evaluand.actions || []),
+                trajectoryLength: trajectory.length,
+                explanation: `Reactive evaluation engaged ${relevantSpecialists.length} specialists`,
+                actionableFeedback: stepScores
+                    .filter(s => s.score < 0.7)
+                    .map(s => s.explanation || `${s.step} needs improvement`),
+            };
+        });
+    }
+}
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Deep equality comparison for objects.
+ * Used for comparing tool arguments and results.
+ *
+ * M3: Includes circular reference protection using WeakSet tracking.
+ *
+ * @param a - First value to compare
+ * @param b - Second value to compare
+ * @param seenA - WeakSet tracking visited objects from 'a' (for cycle detection)
+ * @param seenB - WeakSet tracking visited objects from 'b' (for cycle detection)
+ * @returns True if values are deeply equal
+ */
+function deepEquals(a, b, seenA = new WeakSet(), seenB = new WeakSet()) {
+    if (a === b)
+        return true;
+    if (a === null || b === null)
+        return a === b;
+    if (typeof a !== typeof b)
+        return false;
+    if (typeof a !== 'object')
+        return a === b;
+    // M3: Circular reference protection
+    const aObj = a;
+    const bObj = b;
+    if (seenA.has(aObj) || seenB.has(bObj)) {
+        // If we've seen this object before, consider unequal to be safe
+        // (matching circular structures is complex and rarely needed)
+        return false;
+    }
+    seenA.add(aObj);
+    seenB.add(bObj);
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length)
+            return false;
+        return a.every((val, i) => deepEquals(val, b[i], seenA, seenB));
+    }
+    if (Array.isArray(a) || Array.isArray(b))
+        return false;
+    const aKeys = Object.keys(a);
+    const bKeys = Object.keys(b);
+    if (aKeys.length !== bKeys.length)
+        return false;
+    return aKeys.every(key => deepEquals(a[key], b[key], seenA, seenB));
+}
+//# sourceMappingURL=agent-as-judge.js.map