observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/judge/llm-judge-dag.test.js

@@ -0,0 +1,973 @@
+/**
+ * DAGEval Tests — RED phase
+ *
+ * Tests for the not-yet-implemented dagEval function and DAGEvalConfig types.
+ * All tests are expected to fail until llm-judge-dag.ts is implemented.
+ */
+import { describe, it } from 'vitest';
+import assert from 'node:assert';
+import { dagEval, validateDAGConfig, } from './llm-judge-dag.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const DAG_NORMALIZED_MIN = 0;
+const DAG_NORMALIZED_MAX = 1;
+const DAG_SCORE_MID = 5;
+const DAG_NORMALIZED_MID = 0.5;
+const DAG_SCORE_TEN = 10;
+const DAG_SCORE_ZERO = 0;
+const FLOAT_TOLERANCE = 1e-9;
+const SHORT_TIMEOUT_MS = 10;
+const CONDITIONAL_THRESHOLD = 5;
+const EDGE_WEIGHT_HALF = 0.5;
+const MAX_DEPTH_TWO = 2;
+// ============================================================================
+// Mock LLM Provider
+// ============================================================================
+function createMockLLM(responses) {
+    let callIndex = 0;
+    return {
+        async generate(_prompt, _options) {
+            const response = responses[callIndex] ?? responses[responses.length - 1];
+            callIndex++;
+            return { text: response };
+        },
+    };
+}
+function createHangingLLM() {
+    return {
+        generate(_prompt, options) {
+            return new Promise((_resolve, _reject) => {
+                // Respect abort signal if provided
+                if (options?.signal) {
+                    options.signal.addEventListener('abort', () => {
+                        _reject(new Error('AbortError'));
+                    });
+                }
+                // Otherwise hang forever
+            });
+        },
+    };
+}
+// ============================================================================
+// Fixture Builders
+// ============================================================================
+const VERDICT_PASS = {
+    type: 'verdict',
+    score: DAG_SCORE_TEN,
+    label: 'pass',
+};
+const VERDICT_FAIL = {
+    type: 'verdict',
+    score: DAG_SCORE_ZERO,
+    label: 'fail',
+};
+const VERDICT_PARTIAL = {
+    type: 'verdict',
+    score: DAG_SCORE_MID,
+    label: 'partial',
+};
+const BASE_TEST_CASE = {
+    input: 'What is AI?',
+    output: 'AI is artificial intelligence.',
+};
+// ============================================================================
+// Basic Traversal
+// ============================================================================
+describe('dagEval — basic traversal', () => {
+    it('should traverse a single VerdictNode and return normalized score', async () => {
+        const config = {
+            name: 'single-verdict',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX, `score ${result.score} should be in [0, 1]`);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `score 10/10 should normalize to 1.0, got ${result.score}`);
+    });
+    it('should normalize VerdictNode score 0 to 0.0', async () => {
+        const config = {
+            name: 'verdict-zero',
+            nodes: new Map([['root', VERDICT_FAIL]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MIN) < FLOAT_TOLERANCE, `score 0/10 should normalize to 0.0, got ${result.score}`);
+    });
+    it('should normalize VerdictNode score 5 to 0.5', async () => {
+        const config = {
+            name: 'verdict-mid',
+            nodes: new Map([['root', VERDICT_PARTIAL]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MID) < FLOAT_TOLERANCE, `score 5/10 should normalize to 0.5, got ${result.score}`);
+    });
+    it('should follow BinaryJudgementNode true branch when LLM responds true', async () => {
+        const trueChild = { type: 'verdict', score: DAG_SCORE_TEN, label: 'true-branch' };
+        const falseChild = { type: 'verdict', score: DAG_SCORE_ZERO, label: 'false-branch' };
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the response relevant?',
+            trueChild: 'true-node',
+            falseChild: 'false-node',
+        };
+        const config = {
+            name: 'binary-true',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['true-node', trueChild],
+                ['false-node', falseChild],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['true']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `true branch should yield score 1.0, got ${result.score}`);
+    });
+    it('should follow BinaryJudgementNode false branch when LLM responds false', async () => {
+        const trueChild = { type: 'verdict', score: DAG_SCORE_TEN, label: 'true-branch' };
+        const falseChild = { type: 'verdict', score: DAG_SCORE_ZERO, label: 'false-branch' };
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the response relevant?',
+            trueChild: 'true-node',
+            falseChild: 'false-node',
+        };
+        const config = {
+            name: 'binary-false',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['true-node', trueChild],
+                ['false-node', falseChild],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['false']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MIN) < FLOAT_TOLERANCE, `false branch should yield score 0.0, got ${result.score}`);
+    });
+    it('should follow NonBinaryJudgementNode to matching verdict child', async () => {
+        const excellentVerdict = { type: 'verdict', score: DAG_SCORE_TEN, label: 'excellent' };
+        const poorVerdict = { type: 'verdict', score: DAG_SCORE_ZERO, label: 'poor' };
+        const nonBinaryNode = {
+            type: 'non_binary_judgement',
+            criteria: 'Rate the quality',
+            verdicts: new Map([
+                ['excellent', 'excellent-node'],
+                ['poor', 'poor-node'],
+            ]),
+        };
+        const config = {
+            name: 'non-binary',
+            nodes: new Map([
+                ['root', nonBinaryNode],
+                ['excellent-node', excellentVerdict],
+                ['poor-node', poorVerdict],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['excellent']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `'excellent' verdict should yield score 1.0, got ${result.score}`);
+    });
+    it('should use TaskNode output as transformed input for next node', async () => {
+        const taskNode = {
+            type: 'task',
+            instruction: 'Extract the main topic from the output',
+            next: 'judge-node',
+        };
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the topic relevant?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'task-then-judge',
+            nodes: new Map([
+                ['root', taskNode],
+                ['judge-node', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // First LLM call: TaskNode transformation. Second: BinaryJudgementNode.
+        const result = await dagEval(createMockLLM(['artificial intelligence', 'true']), config, BASE_TEST_CASE);
+        assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX);
+    });
+    it('should traverse a multi-hop DAG and reach the correct VerdictNode', async () => {
+        const firstJudge = {
+            type: 'binary_judgement',
+            criteria: 'Is the output non-empty?',
+            trueChild: 'second-judge',
+            falseChild: 'fail-node',
+        };
+        const secondJudge = {
+            type: 'binary_judgement',
+            criteria: 'Is the output accurate?',
+            trueChild: 'pass-node',
+            falseChild: 'partial-node',
+        };
+        const config = {
+            name: 'multi-hop',
+            nodes: new Map([
+                ['root', firstJudge],
+                ['second-judge', secondJudge],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+                ['partial-node', VERDICT_PARTIAL],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['true', 'true']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `two true branches should reach pass verdict (1.0), got ${result.score}`);
+    });
+});
+// ============================================================================
+// Explanation Trail
+// ============================================================================
+describe('dagEval — explanation trail', () => {
+    it('should return a non-empty reason string', async () => {
+        const config = {
+            name: 'reason-check',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(typeof result.reason === 'string' && result.reason.length > 0, 'reason should be a non-empty string');
+    });
+    it('should include the verdict label in the reason', async () => {
+        const config = {
+            name: 'label-in-reason',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(result.reason.includes('pass'), `reason should include verdict label 'pass', got: ${result.reason}`);
+    });
+    it('should include each traversed node ID in the reason', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the output relevant?',
+            trueChild: 'verdict-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'trail-test',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['verdict-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['true']), config, BASE_TEST_CASE);
+        assert.ok(result.reason.includes('root') && result.reason.includes('verdict-node'), `reason should include traversed node IDs, got: ${result.reason}`);
+    });
+    it('should include each node decision in the reason', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the response correct?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'decision-trail',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM(['true']), config, BASE_TEST_CASE);
+        // Reason should capture the binary decision made
+        assert.ok(result.reason.length > 0);
+        assert.ok(typeof result.reason === 'string');
+    });
+});
+// ============================================================================
+// Score Normalization
+// ============================================================================
+describe('dagEval — score normalization', () => {
+    it('should always return score in [0, 1] range regardless of VerdictNode score', async () => {
+        const scores = [DAG_SCORE_ZERO, 1, 3, DAG_SCORE_MID, 7, DAG_SCORE_TEN];
+        for (const rawScore of scores) {
+            const config = {
+                name: `score-${rawScore}`,
+                nodes: new Map([['root', { type: 'verdict', score: rawScore, label: 'test' }]]),
+                rootId: 'root',
+            };
+            const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+            assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX, `raw score ${rawScore} normalized to ${result.score}, expected in [0, 1]`);
+        }
+    });
+    it('should produce monotonically increasing normalized scores for increasing raw scores', async () => {
+        const rawScores = [DAG_SCORE_ZERO, DAG_SCORE_MID, DAG_SCORE_TEN];
+        const normalizedScores = [];
+        for (const rawScore of rawScores) {
+            const config = {
+                name: `monotone-${rawScore}`,
+                nodes: new Map([['root', { type: 'verdict', score: rawScore, label: 'test' }]]),
+                rootId: 'root',
+            };
+            const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+            normalizedScores.push(result.score);
+        }
+        assert.ok(normalizedScores[0] < normalizedScores[1], `0 → ${normalizedScores[0]} should be less than 5 → ${normalizedScores[1]}`);
+        assert.ok(normalizedScores[1] < normalizedScores[2], `5 → ${normalizedScores[1]} should be less than 10 → ${normalizedScores[2]}`);
+    });
+});
+// ============================================================================
+// Cycle Detection
+// ============================================================================
+describe('dagEval — cycle detection', () => {
+    it('should throw when a node references itself as a child', async () => {
+        const selfRefNode = {
+            type: 'binary_judgement',
+            criteria: 'Is it valid?',
+            trueChild: 'root', // self-reference
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'self-cycle',
+            nodes: new Map([
+                ['root', selfRefNode],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /cycle/i, 'should throw an error mentioning cycle');
+    });
+    it('should throw when two nodes form a mutual cycle', async () => {
+        const nodeA = {
+            type: 'binary_judgement',
+            criteria: 'Check A',
+            trueChild: 'node-b',
+            falseChild: 'node-b',
+        };
+        const nodeB = {
+            type: 'binary_judgement',
+            criteria: 'Check B',
+            trueChild: 'node-a', // cycle back
+            falseChild: 'node-a',
+        };
+        const config = {
+            name: 'mutual-cycle',
+            nodes: new Map([
+                ['root', nodeA],
+                ['node-a', nodeA],
+                ['node-b', nodeB],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /cycle/i);
+    });
+    it('should throw when validateDAGConfig is called with a cyclic config', () => {
+        const selfRefNode = {
+            type: 'binary_judgement',
+            criteria: 'Cycle check',
+            trueChild: 'root',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'validate-cycle',
+            nodes: new Map([
+                ['root', selfRefNode],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        assert.throws(() => validateDAGConfig(config), /cycle/i);
+    });
+});
+// ============================================================================
+// Config Validation
+// ============================================================================
+describe('dagEval — config validation', () => {
+    it('should throw when rootId does not exist in nodes map', async () => {
+        const config = {
+            name: 'missing-root',
+            nodes: new Map([['other-node', VERDICT_PASS]]),
+            rootId: 'nonexistent',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /root/i, 'should throw an error mentioning root');
+    });
+    it('should throw when a child node reference does not exist in nodes map', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is it valid?',
+            trueChild: 'pass-node',
+            falseChild: 'dangling-node', // does not exist
+        };
+        const config = {
+            name: 'dangling-ref',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                // 'dangling-node' intentionally omitted
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /node/i);
+    });
+    it('should throw when no VerdictNode is reachable from root', async () => {
+        // Non-binary node pointing to non-existent verdict keys with no matching responses
+        const nonBinaryNode = {
+            type: 'non_binary_judgement',
+            criteria: 'Classify quality',
+            verdicts: new Map([
+                ['good', 'good-verdict'],
+            ]),
+        };
+        const config = {
+            name: 'unreachable-verdict',
+            nodes: new Map([
+                ['root', nonBinaryNode],
+                ['good-verdict', VERDICT_PASS],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns a verdict key that doesn't exist in the map
+        await assert.rejects(dagEval(createMockLLM(['unknown-label']), config, BASE_TEST_CASE), /verdict/i);
+    });
+    it('should throw when nodes map is empty', async () => {
+        const config = {
+            name: 'empty-nodes',
+            nodes: new Map(),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /root/i);
+    });
+    it('should throw when validateDAGConfig is called with missing rootId', () => {
+        const config = {
+            name: 'missing-root-validate',
+            nodes: new Map([['other', VERDICT_PASS]]),
+            rootId: 'missing',
+        };
+        assert.throws(() => validateDAGConfig(config), /root/i);
+    });
+    it('should throw when validateDAGConfig is called with a dangling child reference', () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Check',
+            trueChild: 'exists',
+            falseChild: 'does-not-exist',
+        };
+        const config = {
+            name: 'dangling-validate',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['exists', VERDICT_PASS],
+            ]),
+            rootId: 'root',
+        };
+        assert.throws(() => validateDAGConfig(config), /node/i);
+    });
+    it('should not throw when validateDAGConfig is called with a valid config', () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is it good?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'valid-config',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        assert.doesNotThrow(() => validateDAGConfig(config));
+    });
+});
+// ============================================================================
+// Timeout Behavior
+// ============================================================================
+describe('dagEval — timeout', () => {
+    it('should reject with a timeout error when LLM hangs and timeoutMs is exceeded', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the output relevant?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'timeout-test',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createHangingLLM(), config, BASE_TEST_CASE, SHORT_TIMEOUT_MS), (err) => {
+            const msg = err.message.toLowerCase();
+            return msg.includes('timeout') || msg.includes('abort') || err.constructor.name === 'LLMTimeoutError';
+        });
+    });
+    it('should complete without error when LLM responds before timeout', async () => {
+        const config = {
+            name: 'no-timeout',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        // VerdictNode requires no LLM call; any reasonable timeout should pass
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE, 5000);
+        assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX);
+    });
+});
+// ============================================================================
+// Return Shape
+// ============================================================================
+describe('dagEval — return shape', () => {
+    it('should return an object with score and reason fields', async () => {
+        const config = {
+            name: 'shape-test',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok('score' in result, 'result should have score field');
+        assert.ok('reason' in result, 'result should have reason field');
+        assert.ok(typeof result.score === 'number', 'score should be a number');
+        assert.ok(typeof result.reason === 'string', 'reason should be a string');
+    });
+    it('should include the config name somewhere in the result metadata or reason', async () => {
+        const config = {
+            name: 'unique-eval-name-xyz',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        // Either reason includes the name, or rawResponse does — flexible check
+        const serialized = JSON.stringify(result);
+        assert.ok(serialized.includes('unique-eval-name-xyz') || result.reason.length > 0, 'result should reference the evaluation config name or have a non-empty reason');
+    });
+});
+// ============================================================================
+// ConditionalJudgementNode
+// ============================================================================
+describe('dagEval — ConditionalJudgementNode', () => {
+    it('should route to aboveChild when LLM score >= threshold', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-above',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns 7, which is >= threshold of 5
+        const result = await dagEval(createMockLLM(['7']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `score >= threshold should route to pass (1.0), got ${result.score}`);
+    });
+    it('should route to belowChild when LLM score < threshold', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-below',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns 3, which is < threshold of 5
+        const result = await dagEval(createMockLLM(['3']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MIN) < FLOAT_TOLERANCE, `score < threshold should route to fail (0.0), got ${result.score}`);
+    });
+    it('should throw when scoreThreshold is out of [0, 10] range', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: 11, // invalid
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'invalid-threshold',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM([]), config, BASE_TEST_CASE), /scoreThreshold/i);
+    });
+    it('should throw when LLM returns non-numeric response for conditional node', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-bad-response',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM(['not-a-number']), config, BASE_TEST_CASE), /score/i);
+    });
+    it('should parse score from multi-line response when number is not on first line', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-multiline',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns text then number on second line — should extract 8 and route above threshold
+        const result = await dagEval(createMockLLM(['The quality is high.\n8']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `multi-line response with score 8 should route above threshold (5) to 1.0, got ${result.score}`);
+    });
+    it('should not parse partial-numeric lines like "1.5x" as a score', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-partial-numeric',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // "Score: 1.5x" is not a fully-numeric line — should throw rather than silently parse 1.5
+        await assert.rejects(dagEval(createMockLLM(['Score: 1.5x (out of 10)']), config, BASE_TEST_CASE), /Expected a line containing only a number/i);
+    });
+    it('should throw with descriptive message when all lines are non-numeric', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-all-text',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        await assert.rejects(dagEval(createMockLLM(['I cannot provide a number for this.']), config, BASE_TEST_CASE), /Expected a line containing only a number/i);
+    });
+    it('should clamp negative LLM score to 0 and route to belowChild', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-negative-clamp',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns -3: clamped to 0, which is < threshold (5), so routes to belowChild (fail)
+        const result = await dagEval(createMockLLM(['-3']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MIN) < FLOAT_TOLERANCE, `negative score -3 should clamp to 0, route below threshold (5) to 0.0, got ${result.score}`);
+    });
+    it('should clamp LLM score to [0, 10] before comparing to threshold', async () => {
+        const conditionalNode = {
+            type: 'conditional_judgement',
+            criteria: 'Rate the quality',
+            scoreThreshold: CONDITIONAL_THRESHOLD,
+            aboveChild: 'pass-node',
+            belowChild: 'fail-node',
+        };
+        const config = {
+            name: 'conditional-clamp',
+            nodes: new Map([
+                ['root', conditionalNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+        };
+        // LLM returns 100 (clamped to 10 >= 5 → above)
+        const result = await dagEval(createMockLLM(['100']), config, BASE_TEST_CASE);
+        assert.ok(result.score > 0, `out-of-range score 100 should clamp to 10, route above threshold, got ${result.score}`);
+    });
+});
+// ============================================================================
+// maxDepth
+// ============================================================================
+describe('dagEval — maxDepth', () => {
+    it('should throw when traversal depth exceeds maxDepth', async () => {
+        const judgeA = {
+            type: 'binary_judgement',
+            criteria: 'First check',
+            trueChild: 'judge-b',
+            falseChild: 'fail-node',
+        };
+        const judgeB = {
+            type: 'binary_judgement',
+            criteria: 'Second check',
+            trueChild: 'judge-c',
+            falseChild: 'fail-node',
+        };
+        const judgeC = {
+            type: 'binary_judgement',
+            criteria: 'Third check',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'depth-exceeded',
+            nodes: new Map([
+                ['root', judgeA],
+                ['judge-b', judgeB],
+                ['judge-c', judgeC],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+            maxDepth: MAX_DEPTH_TWO, // allows visiting ≤2 nodes; 3-hop chain exceeds this
+        };
+        await assert.rejects(dagEval(createMockLLM(['true', 'true', 'true']), config, BASE_TEST_CASE), /maxDepth/i);
+    });
+    it('should complete normally when traversal stays within maxDepth', async () => {
+        const config = {
+            name: 'depth-ok',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+            maxDepth: MAX_DEPTH_TWO,
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX);
+    });
+});
+// ============================================================================
+// edgeWeights
+// ============================================================================
+describe('dagEval — edgeWeights', () => {
+    it('should apply edge weight to final verdict score', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the response relevant?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'weighted-edge',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+            edgeWeights: new Map([['root:pass-node', EDGE_WEIGHT_HALF]]),
+        };
+        // Routes to pass-node (score=10), but edge weight=0.5 → final 1.0 * 0.5 = 0.5
+        const result = await dagEval(createMockLLM(['true']), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MID) < FLOAT_TOLERANCE, `edge weight 0.5 on pass path should yield 0.5, got ${result.score}`);
+    });
+    it('should use default weight 1.0 for edges not in edgeWeights map', async () => {
+        const config = {
+            name: 'default-weight',
+            nodes: new Map([['root', VERDICT_PASS]]),
+            rootId: 'root',
+            edgeWeights: new Map(), // empty map
+        };
+        const result = await dagEval(createMockLLM([]), config, BASE_TEST_CASE);
+        assert.ok(Math.abs(result.score - DAG_NORMALIZED_MAX) < FLOAT_TOLERANCE, `default weight should not affect score, expected 1.0, got ${result.score}`);
+    });
+    it('should throw when an edge weight is zero', async () => {
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: 'Is the response relevant?',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'zero-weight',
+            nodes: new Map([
+                ['root', judgeNode],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+            edgeWeights: new Map([['root:pass-node', 0]]),
+        };
+        await assert.rejects(dagEval(createMockLLM(['true']), config, BASE_TEST_CASE), /edge weight/i);
+    });
+    it('should accumulate weights across multiple hops', async () => {
+        const judgeA = {
+            type: 'binary_judgement',
+            criteria: 'First check',
+            trueChild: 'judge-b',
+            falseChild: 'fail-node',
+        };
+        const judgeB = {
+            type: 'binary_judgement',
+            criteria: 'Second check',
+            trueChild: 'pass-node',
+            falseChild: 'fail-node',
+        };
+        const config = {
+            name: 'multi-hop-weights',
+            nodes: new Map([
+                ['root', judgeA],
+                ['judge-b', judgeB],
+                ['pass-node', VERDICT_PASS],
+                ['fail-node', VERDICT_FAIL],
+            ]),
+            rootId: 'root',
+            // Both edges weighted at 0.5 → accumulated = 0.25
+            edgeWeights: new Map([
+                ['root:judge-b', EDGE_WEIGHT_HALF],
+                ['judge-b:pass-node', EDGE_WEIGHT_HALF],
+            ]),
+        };
+        const result = await dagEval(createMockLLM(['true', 'true']), config, BASE_TEST_CASE);
+        // 1.0 * 0.5 * 0.5 = 0.25
+        assert.ok(Math.abs(result.score - 0.25) < FLOAT_TOLERANCE, `two 0.5 weights should accumulate to 0.25, got ${result.score}`);
+    });
+});
+// ============================================================================
+// Stress Tests (ST-R6)
+// ============================================================================
+const ST_R6_DEEP_TREE_SIZE = 52; // 50+ nodes
+const ST_R6_CYCLE_SIZE = 20;
+/**
+ * Build a linear chain DAG: root → n1 → n2 → ... → verdict.
+ * Each intermediate node is a binary_judgement that always takes trueChild.
+ */
+function buildDeepLinearDAG(depth) {
+    const nodes = new Map();
+    const verdictNode = { type: 'verdict', score: 8, label: 'deep-pass' };
+    nodes.set('verdict', verdictNode);
+    for (let i = depth - 1; i >= 0; i--) {
+        const nextId = i === depth - 1 ? 'verdict' : `node-${i + 1}`;
+        const judgeNode = {
+            type: 'binary_judgement',
+            criteria: `Check step ${i}`,
+            trueChild: nextId,
+            falseChild: 'verdict',
+        };
+        nodes.set(`node-${i}`, judgeNode);
+    }
+    return {
+        name: 'deep-linear',
+        nodes,
+        rootId: 'node-0',
+    };
+}
+describe('dagEval stress (ST-R6)', () => {
+    it('deep tree (50+ nodes): validateDAGConfig succeeds without throwing', () => {
+        const config = buildDeepLinearDAG(ST_R6_DEEP_TREE_SIZE);
+        assert.doesNotThrow(() => validateDAGConfig(config), 'validateDAGConfig should not throw for a valid deep tree');
+    });
+    it('deep tree (50+ nodes): traversal completes and score is in [0, 1]', async () => {
+        const config = buildDeepLinearDAG(ST_R6_DEEP_TREE_SIZE);
+        // Always return 'true' so traversal takes all trueChild paths
+        const responses = Array.from({ length: ST_R6_DEEP_TREE_SIZE }, () => 'true');
+        const result = await dagEval(createMockLLM(responses), config, BASE_TEST_CASE);
+        assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX, `score ${result.score} out of [0,1] for deep tree`);
+    });
+    it('maxDepth enforcement: passes with limit = total nodes, throws with limit = total nodes - 1', async () => {
+        // buildDeepLinearDAG(N) produces N judgment nodes + 1 verdict node = N+1 total nodes visited.
+        // The check is trail.length > maxDepth, so maxDepth must be >= N+1 to avoid throwing.
+        const totalNodes = ST_R6_DEEP_TREE_SIZE + 1; // 52 judgment + 1 verdict
+        // With maxDepth === totalNodes, trail.length never exceeds it → completes successfully
+        const configPass = { ...buildDeepLinearDAG(ST_R6_DEEP_TREE_SIZE), maxDepth: totalNodes };
+        const responsesPass = Array.from({ length: ST_R6_DEEP_TREE_SIZE }, () => 'true');
+        const resultPass = await dagEval(createMockLLM(responsesPass), configPass, BASE_TEST_CASE);
+        assert.ok(resultPass.score >= DAG_NORMALIZED_MIN && resultPass.score <= DAG_NORMALIZED_MAX, `score ${resultPass.score} out of [0,1] at exact maxDepth boundary`);
+        // With maxDepth === totalNodes - 1 (= ST_R6_DEEP_TREE_SIZE), the verdict push makes trail.length = N+1 > N → throws
+        const configFail = { ...buildDeepLinearDAG(ST_R6_DEEP_TREE_SIZE), maxDepth: ST_R6_DEEP_TREE_SIZE };
+        const responsesFail = Array.from({ length: ST_R6_DEEP_TREE_SIZE }, () => 'true');
+        await assert.rejects(dagEval(createMockLLM(responsesFail), configFail, BASE_TEST_CASE), /maxDepth/i, 'should throw when traversal exceeds maxDepth by exactly 1');
+    });
+    it('cyclic graph at scale: validateDAGConfig throws for a cycle in a large graph', () => {
+        // Build a long chain, then create a back-edge to simulate a cycle at scale
+        const nodes = new Map();
+        for (let i = 0; i < ST_R6_CYCLE_SIZE; i++) {
+            const nextId = i < ST_R6_CYCLE_SIZE - 1 ? `node-${i + 1}` : 'node-0'; // back-edge at end
+            const judgeNode = {
+                type: 'binary_judgement',
+                criteria: `Check ${i}`,
+                trueChild: nextId,
+                falseChild: nextId, // both branches continue chain
+            };
+            nodes.set(`node-${i}`, judgeNode);
+        }
+        const config = { name: 'cyclic-large', nodes, rootId: 'node-0' };
+        assert.throws(() => validateDAGConfig(config), /cycle/i, 'should reject cyclic graph at scale');
+    });
+    it('score normalization: all verdict scores are clamped to [0, 1] regardless of raw score value', async () => {
+        // Verdict with score=10 → normalized 1.0
+        const verdictMax = { type: 'verdict', score: 10, label: 'max' };
+        // Verdict with score=0 → normalized 0.0
+        const verdictMin = { type: 'verdict', score: 0, label: 'min' };
+        const rootNode = {
+            type: 'binary_judgement',
+            criteria: 'check',
+            trueChild: 'verdict-max',
+            falseChild: 'verdict-min',
+        };
+        const configMax = {
+            name: 'score-max',
+            nodes: new Map([['root', rootNode], ['verdict-max', verdictMax], ['verdict-min', verdictMin]]),
+            rootId: 'root',
+        };
+        const resultMax = await dagEval(createMockLLM(['true']), configMax, BASE_TEST_CASE);
+        const resultMin = await dagEval(createMockLLM(['false']), configMax, BASE_TEST_CASE);
+        assert.ok(Math.abs(resultMax.score - 1.0) < FLOAT_TOLERANCE, `score should be 1.0, got ${resultMax.score}`);
+        assert.ok(Math.abs(resultMin.score - 0.0) < FLOAT_TOLERANCE, `score should be 0.0, got ${resultMin.score}`);
+    });
+    it('score normalization stability: 50 traversals on a 50+-node tree all produce scores in [0,1]', async () => {
+        const config = buildDeepLinearDAG(ST_R6_DEEP_TREE_SIZE);
+        for (let trial = 0; trial < 50; trial++) {
+            // Random binary responses — each traversal takes a different path through the tree
+            const responses = Array.from({ length: ST_R6_DEEP_TREE_SIZE }, () => Math.random() > 0.5 ? 'true' : 'false');
+            const result = await dagEval(createMockLLM(responses), config, BASE_TEST_CASE);
+            assert.ok(result.score >= DAG_NORMALIZED_MIN && result.score <= DAG_NORMALIZED_MAX, `trial ${trial}: score ${result.score} out of [0,1]`);
+        }
+    });
+});
+//# sourceMappingURL=llm-judge-dag.test.js.map