@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/test_agents/test_synthesis_agent.py

@@ -1,1195 +0,0 @@
-"""Tests for SynthesisAgent."""
-
-from __future__ import annotations
-
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-from vds_audit_orchestrator.agents.base import AgentResult
-from vds_audit_orchestrator.agents.synthesis_agent import (
-    ExecutiveSummaryResult,
-    MaturityLevel,
-    RecommendationsResult,
-    SynthesisAgent,
-)
-
-
-@pytest.fixture
-def mock_agent_results():
-    """Create sample agent results."""
-    return [
-        AgentResult(
-            agent_name="CodeAnalysisAgent",
-            success=True,
-            findings=[
-                {
-                    "type": "technical_debt",
-                    "severity": "medium",
-                    "description": "TODOs found",
-                    "source_agent": "CodeAnalysisAgent",
-                }
-            ],
-        ),
-        AgentResult(
-            agent_name="SecurityAgent",
-            success=True,
-            findings=[
-                {
-                    "type": "security_vulnerability",
-                    "severity": "critical",
-                    "description": "Hardcoded secret",
-                    "source_agent": "SecurityAgent",
-                }
-            ],
-        ),
-        AgentResult(
-            agent_name="DocumentationAgent",
-            success=True,
-            findings=[
-                {
-                    "type": "doc_quality",
-                    "severity": "low",
-                    "description": "Minor formatting issue",
-                    "source_agent": "DocumentationAgent",
-                }
-            ],
-        ),
-    ]
-
-
-class TestSynthesisAgent:
-    """Tests for SynthesisAgent."""
-
-    @pytest.mark.asyncio
-    async def test_analyze_no_context(self):
-        """Test handling of missing context."""
-        agent = SynthesisAgent()
-        # Analyze without specific context should return failure
-        result = await agent.analyze({})
-        assert result.success is False
-        assert result.error and "Evidence bundle missing" in result.error
-
-    @pytest.mark.asyncio
-    async def test_synthesize_aggregation(self, mock_agent_results):
-        """Test aggregation of findings from multiple agents."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False  # Disable LLM for deterministic test
-        agent = SynthesisAgent(settings=mock_settings)
-
-        # Synthesize expects repo name and list of AgentResult objects
-        synthesis = await agent.synthesize("test-repo", mock_agent_results)
-
-        # Check aggregated findings (via agent scores which reflects all agents processed)
-        assert len(synthesis.agent_scores) == 3
-
-        # Check critical gaps (1 critical from security)
-        assert len(synthesis.critical_gaps) == 1
-        assert synthesis.critical_gaps[0]["type"] == "security_vulnerability"
-
-    @pytest.mark.asyncio
-    async def test_calculate_score_and_maturity(self, mock_agent_results):
-        """Test score calculation and maturity level determination."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        synthesis = await agent.synthesize("test-repo", mock_agent_results)
-
-        # 1 Critical finding should heavily impact score
-        score = synthesis.overall_score
-        assert score < 100
-
-        # With a critical finding, score usually drops significantly
-        maturity = synthesis.maturity_level
-        # Critical findings generally prevent high maturity
-        assert maturity
-
-    @pytest.mark.asyncio
-    async def test_llm_summary(self, mock_agent_results):
-        """Test LLM-based executive summary generation."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = True
-
-        agent = SynthesisAgent(settings=mock_settings)
-        agent._run_pydantic_agent = AsyncMock(
-            return_value=ExecutiveSummaryResult(summary="Executive Summary: The project has critical security issues.")
-        )
-
-        synthesis = await agent.synthesize("test-repo", mock_agent_results)
-
-        assert synthesis.executive_summary == "Executive Summary: The project has critical security issues."
-
-    @pytest.mark.asyncio
-    async def test_incoherent_low_score_summary_falls_back_to_template(self):
-        """Low-score positive LLM summary should be replaced with coherent template output."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = True
-        agent = SynthesisAgent(settings=mock_settings)
-        agent._run_pydantic_agent = AsyncMock(
-            side_effect=[
-                RecommendationsResult(recommendations=[]),
-                ExecutiveSummaryResult(summary="Continue current practices while monitoring for issues."),
-            ]
-        )
-
-        low_score_results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": f"Critical security finding {idx}",
-                        "source_agent": "SecurityAgent",
-                    }
-                    for idx in range(5)
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", low_score_results)
-
-        assert synthesis.overall_score <= 30
-        assert "Continue current practices while monitoring for issues." not in synthesis.executive_summary
-        assert "Prioritize addressing security and high-severity issues." in synthesis.executive_summary
-
-    @pytest.mark.asyncio
-    async def test_fallback_summary(self, mock_agent_results):
-        """Test fallback summary generation when LLM is disabled."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        synthesis = await agent.synthesize("test-repo", mock_agent_results)
-
-        summary = synthesis.executive_summary
-        assert "Audit Summary for test-repo" in summary
-        assert "scored" in summary
-        assert "critical gap(s) identified" in summary
-
-    def test_template_summary_avoids_positive_recommendation_for_low_score_without_critical_gaps(self):
-        """Low-score runs should not emit positive maintenance-only recommendations."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        summary = agent._get_template_summary(
-            repository="test-repo",
-            overall_score=20.0,
-            maturity_level=MaturityLevel.CRITICAL.value,
-            critical_gaps=[],
-            aligned_practices=[],
-        )
-
-        assert "Continue current practices" not in summary
-        assert "Investigate low-score drivers" in summary
-
-    @pytest.mark.asyncio
-    async def test_empty_agent_results_list(self):
-        """Test synthesis with empty agent results list."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        synthesis = await agent.synthesize("test-repo", [])
-
-        # Empty results should yield zero score and critical maturity
-        assert synthesis.overall_score == 0.0
-        assert synthesis.maturity_level == MaturityLevel.CRITICAL.value
-        assert len(synthesis.agent_scores) == 0
-        assert len(synthesis.critical_gaps) == 0
-        assert len(synthesis.aligned_practices) == 0
-        assert "test-repo" in synthesis.executive_summary
-
-    @pytest.mark.asyncio
-    async def test_all_agents_failed(self):
-        """Test synthesis when all agents failed."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        failed_results = [
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=False,
-                error="Analysis failed",
-                findings=[],
-            ),
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=False,
-                error="Security scan timeout",
-                findings=[],
-            ),
-            AgentResult(
-                agent_name="DocumentationAgent",
-                success=False,
-                error="Docs processing error",
-                findings=[],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", failed_results)
-
-        # All failed agents should get zero scores
-        assert len(synthesis.agent_scores) == 3
-        assert all(score == 0.0 for score in synthesis.agent_scores.values())
-        assert synthesis.overall_score == 0.0
-        assert synthesis.maturity_level == MaturityLevel.CRITICAL.value
-        assert len(synthesis.critical_gaps) == 0
-        assert len(synthesis.aligned_practices) == 0
-
-    @pytest.mark.asyncio
-    async def test_mixed_success_failure_agent_results(self):
-        """Test synthesis with mixed successful and failed agents."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        mixed_results = [
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "technical_debt",
-                        "description": "Medium TODO",
-                        "source_agent": "CodeAnalysisAgent",
-                    }
-                ],
-            ),
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=False,
-                error="Scan failed",
-                findings=[],
-            ),
-            AgentResult(
-                agent_name="DocumentationAgent",
-                success=True,
-                findings=[],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", mixed_results)
-
-        # Failed agent gets zero, successful agents get scores
-        assert synthesis.agent_scores["SecurityAgent"] == 0.0
-        assert synthesis.agent_scores["CodeAnalysisAgent"] > 0
-        assert synthesis.agent_scores["CodeAnalysisAgent"] < 100  # Has finding
-        assert synthesis.agent_scores["DocumentationAgent"] == 100.0  # No findings
-        # Both successful agents have no critical/high findings
-        assert len(synthesis.aligned_practices) == 2
-        aligned_domains = {p["domain"] for p in synthesis.aligned_practices}
-        assert "Documentation" in aligned_domains
-        assert "CodeAnalysis" in aligned_domains
-        # No critical gaps since medium severity is not critical/high
-        assert len(synthesis.critical_gaps) == 0
-
-    @pytest.mark.asyncio
-    async def test_conflict_resolution_between_agent_findings(self):
-        """Test handling of conflicting findings from multiple agents."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        # Create conflicting findings for the same location
-        conflicting_results = [
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "technical_debt",
-                        "severity": "medium",
-                        "description": "Complex function needs refactoring",
-                        "location": "src/auth.py",
-                        "source_agent": "CodeAnalysisAgent",
-                    }
-                ],
-            ),
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Authentication bypass vulnerability",
-                        "location": "src/auth.py",
-                        "source_agent": "SecurityAgent",
-                    }
-                ],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", conflicting_results)
-
-        # Should identify cross-cutting concerns
-        assert len(synthesis.cross_cutting_concerns) > 0
-        concern = synthesis.cross_cutting_concerns[0]
-        assert concern["location"] == "src/auth.py"
-        assert len(concern["affected_domains"]) == 2
-        assert "CodeAnalysisAgent" in concern["affected_domains"]
-        assert "SecurityAgent" in concern["affected_domains"]
-
-        # Critical finding should be in critical_gaps
-        assert any(g["location"] == "src/auth.py" and g["severity"] == "critical" for g in synthesis.critical_gaps)
-
-    @pytest.mark.asyncio
-    async def test_maturity_level_determinations(self):
-        """Test different maturity level determinations based on scores."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        test_cases = [
-            # (expected_level, findings_config) - designed to produce specific score ranges
-            # Critical: 5 critical findings = 5 * 25 = 125 deduction capped at 100 -> score 0
-            (MaturityLevel.CRITICAL, [{"type": "security_vulnerability"}] * 5),
-            # Developing: Need score 26-50, use ~4 high severity (4 * 15 = 60 deduction -> score 40)
-            (MaturityLevel.DEVELOPING, [{"type": "security_risk"}] * 4),
-            # Established: Need score 51-75, use ~2 high severity (2 * 15 = 30 deduction -> score 70)
-            (MaturityLevel.ESTABLISHED, [{"type": "security_risk"}] * 2),
-            # Optimizing: Need score 76-90, use multiple medium findings
-            # 3 medium = 3 * 8 = 24 deduction -> score 76
-            (MaturityLevel.OPTIMIZING, [{"type": "technical_debt"}] * 3),
-            # Leading: No findings -> score 100
-            (MaturityLevel.LEADING, []),
-        ]
-
-        for expected_level, findings_config in test_cases:
-            results = [
-                AgentResult(
-                    agent_name="TestAgent",
-                    success=True,
-                    findings=[
-                        {"type": f["type"], "description": f"Test finding {i}"} for i, f in enumerate(findings_config)
-                    ],
-                )
-            ]
-
-            synthesis = await agent.synthesize("test-repo", results)
-            # For high findings, verify we're in expected range
-            if expected_level == MaturityLevel.CRITICAL:
-                assert synthesis.maturity_level == MaturityLevel.CRITICAL.value, (
-                    f"Expected {expected_level.value}, got {synthesis.maturity_level} (score: {synthesis.overall_score}) for findings: {findings_config}"
-                )
-            elif expected_level == MaturityLevel.DEVELOPING:
-                assert synthesis.maturity_level in [MaturityLevel.CRITICAL.value, MaturityLevel.DEVELOPING.value], (
-                    f"Expected CRITICAL or DEVELOPING, got {synthesis.maturity_level} (score: {synthesis.overall_score}) for findings: {findings_config}"
-                )
-            elif expected_level == MaturityLevel.ESTABLISHED:
-                # Verify the score is in the expected range
-                score = synthesis.overall_score
-                assert 51 <= score <= 75, f"Score {score} not in ESTABLISHED range for findings: {findings_config}"
-            elif expected_level == MaturityLevel.OPTIMIZING:
-                # Verify the score is in the expected range
-                score = synthesis.overall_score
-                assert 76 <= score <= 90, f"Score {score} not in OPTIMIZING range for findings: {findings_config}"
-            elif expected_level == MaturityLevel.LEADING:
-                assert synthesis.maturity_level == MaturityLevel.LEADING.value, (
-                    f"Expected {expected_level.value}, got {synthesis.maturity_level} (score: {synthesis.overall_score}) for findings: {findings_config}"
-                )
-
-    @pytest.mark.asyncio
-    async def test_edge_case_scores_zero(self):
-        """Test synthesis with zero score (maximum findings)."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        # Create enough critical findings to drive score to zero
-        many_critical_findings = [
-            {
-                "type": "security_vulnerability",
-                "severity": "critical",
-                "description": f"Critical vulnerability {i}",
-                "source_agent": "SecurityAgent",
-            }
-            for i in range(10)
-        ]
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=many_critical_findings,
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        assert synthesis.overall_score == 0.0
-        assert synthesis.maturity_level == MaturityLevel.CRITICAL.value
-
-    @pytest.mark.asyncio
-    async def test_edge_case_scores_perfect(self):
-        """Test synthesis with perfect score (no findings)."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        perfect_results = [
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[],
-            ),
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[],
-            ),
-            AgentResult(
-                agent_name="DocumentationAgent",
-                success=True,
-                findings=[],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", perfect_results)
-
-        assert synthesis.overall_score == 100.0
-        assert synthesis.maturity_level == MaturityLevel.LEADING.value
-        assert len(synthesis.critical_gaps) == 0
-        assert len(synthesis.aligned_practices) == 3
-
-    @pytest.mark.asyncio
-    async def test_edge_case_scores_boundary_values(self):
-        """Test synthesis with boundary score values (25, 50, 75, 90).
-
-        Note: FR-6 changes affect score calculation:
-        - HIGH severity findings are only elevated to critical_gaps when score < 40
-        - This affects the penalty calculation for overall score
-        """
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        # Test specific maturity level scenarios
-        # Critical: score <= 25
-        critical_results = AgentResult(
-            agent_name="TestAgent",
-            success=True,
-            findings=[{"type": "security_vulnerability"}] * 4,  # 4 * 25 = 100 deduction -> 0 score
-        )
-        synthesis = await agent.synthesize("test-repo", [critical_results])
-        assert synthesis.maturity_level == MaturityLevel.CRITICAL.value
-
-        # Developing: 26 <= score <= 50
-        # FR-6 change: HIGH findings only elevated when score < 40
-        # Use 4 high findings = 60 deduction -> 40 agent score
-        # With 4 high gaps elevated (score < 40 initially): 40 - 20 = 20 overall -> Critical
-        # Actually need to use a mix to get Developing range
-        # Use 2 critical findings = 50 deduction -> 50 agent score, minus 20 penalty = 30 -> Developing
-        developing_results = AgentResult(
-            agent_name="TestAgent",
-            success=True,
-            findings=[{"type": "security_vulnerability"}] * 2,  # 2 * 25 = 50 deduction -> 50 agent score
-        )
-        synthesis = await agent.synthesize("test-repo", [developing_results])
-        # With 2 critical gaps (20 penalty): 50 - 20 = 30 overall score -> Developing
-        assert synthesis.maturity_level == MaturityLevel.DEVELOPING.value
-
-        # Established: 51 <= score <= 75
-        # Use 1 critical finding = 25 deduction -> 75 agent score, minus 10 penalty = 65 -> Established
-        established_results = AgentResult(
-            agent_name="TestAgent",
-            success=True,
-            findings=[{"type": "security_vulnerability"}] * 1,  # 1 * 25 = 25 deduction -> 75 agent score
-        )
-        synthesis = await agent.synthesize("test-repo", [established_results])
-        # With 1 critical gap (10 penalty): 75 - 10 = 65 overall score -> Established
-        assert synthesis.maturity_level == MaturityLevel.ESTABLISHED.value
-
-        # Optimizing: 76 <= score <= 90 (use 3 medium findings = 24 deduction -> 76)
-        optimizing_results = AgentResult(
-            agent_name="TestAgent",
-            success=True,
-            findings=[{"type": "technical_debt"}] * 3,  # 3 * 8 = 24 deduction -> 76 score
-        )
-        synthesis = await agent.synthesize("test-repo", [optimizing_results])
-        # Score should be 76 (at the lower boundary of Optimizing)
-        assert synthesis.maturity_level == MaturityLevel.OPTIMIZING.value
-
-        # Leading: score >= 91 (no findings)
-        leading_results = AgentResult(
-            agent_name="TestAgent",
-            success=True,
-            findings=[],
-        )
-        synthesis = await agent.synthesize("test-repo", [leading_results])
-        assert synthesis.maturity_level == MaturityLevel.LEADING.value
-
-    @pytest.mark.asyncio
-    async def test_large_number_of_findings_aggregation(self):
-        """Test synthesis with a large number of findings from multiple agents."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        # Create findings grouped by agent with overlapping locations for cross-cutting
-        large_results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "description": f"Critical finding {i}",
-                        "location": f"src/shared_module_{i % 3}.py",  # Reuse locations
-                    }
-                    for i in range(10)
-                ],
-            ),
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "technical_debt",
-                        "description": f"Debt finding {i}",
-                        "location": f"src/shared_module_{i % 3}.py",  # Same locations as SecurityAgent
-                    }
-                    for i in range(10)
-                ],
-            ),
-            AgentResult(
-                agent_name="DocumentationAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "doc_quality",
-                        "description": f"Doc finding {i}",
-                        "location": f"docs/doc_{i}.md",
-                    }
-                    for i in range(5)
-                ],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", large_results)
-
-        # Verify aggregation worked correctly
-        assert len(synthesis.agent_scores) == 3
-        # All agents should have varied scores due to many findings
-        # SecurityAgent has 10 critical findings -> score 0
-        assert synthesis.agent_scores["SecurityAgent"] == 0.0
-        # CodeAnalysisAgent has 10 medium findings -> 100 - (10 * 8) = 20
-        assert synthesis.agent_scores["CodeAnalysisAgent"] < 50
-        # DocumentationAgent has 5 low findings -> 100 - (5 * 3) = 85
-        assert synthesis.agent_scores["DocumentationAgent"] > 50
-        # Should identify cross-cutting concerns (security + debt overlap)
-        assert len(synthesis.cross_cutting_concerns) > 0
-        # Should have security_debt_overlap pattern
-        overlap = [c for c in synthesis.cross_cutting_concerns if c.get("pattern") == "security_debt_overlap"]
-        assert len(overlap) > 0
-
-        # Verify cross-cutting concerns were identified (locations with multiple agents)
-        assert len(synthesis.cross_cutting_concerns) > 0
-
-        # Verify critical_gaps contains high-priority findings
-        critical_and_high = [g for g in synthesis.critical_gaps if g["severity"] in ("critical", "high")]
-        assert len(critical_and_high) > 0
-
-    @pytest.mark.asyncio
-    async def test_executive_summary_with_llm_failures(self):
-        """Test executive summary generation when LLM fails."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = True
-
-        agent = SynthesisAgent(settings=mock_settings)
-        agent._run_pydantic_agent = AsyncMock(side_effect=Exception("LLM service unavailable"))
-
-        results = [
-            AgentResult(
-                agent_name="TestAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Test finding",
-                        "source_agent": "TestAgent",
-                    }
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # Should fall back to template summary
-        assert "Audit Summary for test-repo" in synthesis.executive_summary
-        assert "scored" in synthesis.executive_summary
-        assert synthesis.overall_score < 100  # Should have calculated score
-
-    @pytest.mark.asyncio
-    async def test_cross_cutting_security_debt_pattern(self):
-        """Test identification of security + technical debt overlap pattern."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "SQL injection",
-                        "location": "src/database.py",
-                        "source_agent": "SecurityAgent",
-                    }
-                ],
-            ),
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "technical_debt",
-                        "severity": "medium",
-                        "description": "Complex SQL query",
-                        "location": "src/database.py",
-                        "source_agent": "CodeAnalysisAgent",
-                    }
-                ],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # Should identify security_debt_overlap pattern
-        overlap_concerns = [c for c in synthesis.cross_cutting_concerns if c.get("pattern") == "security_debt_overlap"]
-        assert len(overlap_concerns) > 0
-        assert "src/database.py" in overlap_concerns[0]["locations"]
-
-    @pytest.mark.asyncio
-    async def test_recommendations_with_cross_cutting_concerns(self):
-        """Test recommendation generation includes cross-cutting concerns."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Issue 1",
-                        "location": "shared.py",
-                        "source_agent": "SecurityAgent",
-                    }
-                ],
-            ),
-            AgentResult(
-                agent_name="CodeAnalysisAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "technical_debt",
-                        "severity": "high",
-                        "description": "Issue 2",
-                        "location": "shared.py",
-                        "source_agent": "CodeAnalysisAgent",
-                    }
-                ],
-            ),
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # Should have recommendations including cross-cutting
-        cross_cutting_recs = [r for r in synthesis.recommendations if r.get("category") == "cross_cutting"]
-        assert len(cross_cutting_recs) > 0
-
-    @pytest.mark.asyncio
-    async def test_critical_gaps_deduplicate_same_finding(self):
-        """Test duplicate critical findings are deduplicated.
-
-        FR-77 adds semantic normalization before synthesis aggregation.
-        Duplicate security findings for the same location should collapse.
-        """
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Hardcoded secret",
-                        "location": "src/auth.py",
-                    },
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Hardcoded secret",
-                        "location": "src/auth.py",
-                    },
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        assert len(synthesis.critical_gaps) == 1
-        assert all(g["location"] == "src/auth.py" for g in synthesis.critical_gaps)
-
-    @pytest.mark.asyncio
-    async def test_code_agent_quality_floor_blocks_low_confidence_or_groundedness(self):
-        """FR-76: low-quality code-agent output is blocked pre-synthesis."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-        warning_mock = MagicMock()
-        agent.logger.warning = warning_mock
-
-        low_quality_code = AgentResult(
-            agent_name="CodeAnalysisAgent",
-            success=True,
-            findings=[
-                {
-                    "type": "security_vulnerability",
-                    "severity": "critical",
-                    "description": "Untrusted claim without file grounding",
-                    "location": "unknown",
-                }
-            ],
-            metadata={"confidence": 0.2},
-        )
-        security_result = AgentResult(
-            agent_name="SecurityAgent",
-            success=True,
-            findings=[],
-        )
-
-        synthesis = await agent.synthesize("test-repo", [low_quality_code, security_result])
-
-        # CodeAnalysisAgent should be blocked by quality floor and scored as failed.
-        assert synthesis.agent_scores["CodeAnalysisAgent"] == 0.0
-        quality_floor_calls = [
-            call
-            for call in warning_mock.call_args_list
-            if call.args and call.args[0] == "synthesis_code_quality_floor_blocked"
-        ]
-        assert quality_floor_calls, "expected quality-floor warning event"
-        payload = quality_floor_calls[-1].kwargs
-        assert payload["agent_name"] == "CodeAnalysisAgent"
-        assert payload["confidence"] == 0.2
-        assert payload["groundedness"] == 0.0
-        assert "min_confidence" in payload
-        assert "min_groundedness" in payload
-
-    @pytest.mark.asyncio
-    async def test_semantic_security_dedup_collapses_reworded_findings(self):
-        """FR-77: semantically identical security findings are normalized."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_sql_string_concat",
-                        "severity": "critical",
-                        "description": "Potential SQL injection via string concatenation",
-                        "location": "src/db.py:12",
-                    },
-                    {
-                        "type": "security_sql_string_concat",
-                        "severity": "critical",
-                        "description": "Possible SQL injection risk from concatenated query",
-                        "location": "src/db.py:48",
-                    },
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-        assert len(synthesis.critical_gaps) == 1
-
-    @pytest.mark.asyncio
-    async def test_recommendations_deduplicate_llm_entries(self):
-        """Test duplicate LLM recommendations are deduplicated.
-
-        Note: FR-6 adds coherent recommendations that are merged with LLM recommendations.
-        The deduplication happens on the combined list, so we may have:
-        - Coherent recommendations (based on score/gaps)
-        - LLM recommendations (from _get_llm_recommendations)
-
-        This test verifies that duplicate LLM entries are deduplicated,
-        but coherent recommendations may add additional entries.
-        """
-        mock_settings = MagicMock()
-        mock_settings.enabled = True
-        agent = SynthesisAgent(settings=mock_settings)
-
-        agent._get_llm_recommendations = AsyncMock(
-            return_value=[
-                {
-                    "priority": "high",
-                    "title": "Rotate credentials",
-                    "description": "Rotate leaked credentials and secrets.",
-                    "category": "security",
-                    "effort": "medium",
-                },
-                {
-                    "priority": "high",
-                    "title": "Rotate credentials",
-                    "description": "Rotate leaked credentials and secrets.",
-                    "category": "security",
-                    "effort": "medium",
-                },
-            ]
-        )
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Hardcoded secret",
-                    }
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # FR-6: Coherent recommendations are merged with LLM recommendations
-        # We expect:
-        # 1. Coherent recommendation (based on score/gaps) - e.g., "MEDIUM: Address security vulnerabilities"
-        # 2. LLM recommendation (deduplicated) - "Rotate credentials"
-        # The duplicate LLM entry should be removed
-        assert len(synthesis.recommendations) >= 1
-        # Verify the LLM recommendation is present (deduplicated)
-        llm_rec = [r for r in synthesis.recommendations if "Rotate credentials" in r.get("title", "")]
-        assert len(llm_rec) == 1  # Only one "Rotate credentials" entry
-
-    @pytest.mark.asyncio
-    async def test_llm_empty_recommendations_fallback_to_rules(self):
-        """Test empty LLM recommendations fall back to rule-based."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = True
-        agent = SynthesisAgent(settings=mock_settings)
-
-        agent._get_llm_recommendations = AsyncMock(return_value=[])
-
-        results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "description": "Hardcoded secret",
-                    }
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        assert len(synthesis.recommendations) > 0
-        assert any(r["category"] == "security" for r in synthesis.recommendations)
-
-    @pytest.mark.asyncio
-    async def test_severity_determination_for_unknown_types(self):
-        """Test severity determination for unknown finding types."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="TestAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "unknown_finding_type",
-                        "description": "Unknown type finding",
-                        "source_agent": "TestAgent",
-                    }
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # Unknown types should default to "medium" severity
-        assert len(synthesis.critical_gaps) == 0  # Medium severity not in critical gaps
-        # Agent score should reflect medium severity deduction
-        assert synthesis.agent_scores["TestAgent"] < 100
-
-    @pytest.mark.asyncio
-    async def test_agent_scoring_with_mixed_severity_findings(self):
-        """Test agent scoring with mixed severity findings."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="TestAgent",
-                success=True,
-                findings=[
-                    {"type": "security_vulnerability", "description": "Critical issue"},
-                    {"type": "security_risk", "description": "High issue"},
-                    {"type": "technical_debt", "description": "Medium issue"},
-                    {"type": "doc_quality", "description": "Low issue"},
-                ],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        # Score should reflect cumulative deductions
-        # Agent score: 100 - 25 (critical) -15 (high) -8 (medium) -3 (low) = 49
-        # Overall score has additional -10 penalty for critical finding
-        assert synthesis.agent_scores["TestAgent"] == 49.0
-        # Overall score should be agent score minus critical penalty
-        assert synthesis.overall_score < synthesis.agent_scores["TestAgent"]
-
-    @pytest.mark.asyncio
-    async def test_synthesis_result_to_dict(self):
-        """Test SynthesisResult serialization to dictionary."""
-        mock_settings = MagicMock()
-        mock_settings.enabled = False
-        agent = SynthesisAgent(settings=mock_settings)
-
-        results = [
-            AgentResult(
-                agent_name="TestAgent",
-                success=True,
-                findings=[],
-            )
-        ]
-
-        synthesis = await agent.synthesize("test-repo", results)
-
-        result_dict = synthesis.to_dict()
-
-        # Verify all fields are present
-        assert "overall_score" in result_dict
-        assert "maturity_level" in result_dict
-        assert "critical_gaps" in result_dict
-        assert "aligned_practices" in result_dict
-        assert "recommendations" in result_dict
-        assert "executive_summary" in result_dict
-        assert "agent_scores" in result_dict
-        assert "cross_cutting_concerns" in result_dict
-
-        # Verify values match
-        assert result_dict["overall_score"] == synthesis.overall_score
-        assert result_dict["maturity_level"] == synthesis.maturity_level
-
-
-# =============================================================================
-# P166-C: Synthesis Fallback Telemetry (TSK-166.7, TSK-166.8)
-# =============================================================================
-
-
-class TestSynthesisFallbackTelemetry:
-    """Tests for structured fallback cause telemetry (Phase 166)."""
-
-    def _make_agent(self, *, enabled: bool = True) -> SynthesisAgent:
-        mock_settings = MagicMock()
-        mock_settings.enabled = enabled
-        return SynthesisAgent(settings=mock_settings)
-
-    # -------------------------------------------------------------------------
-    # _classify_synthesis_fallback_cause unit tests
-    # -------------------------------------------------------------------------
-
-    def test_classify_pydantic_validation_error_returns_parse_failed(self):
-        """pydantic.ValidationError must map to PARSE_VALIDATION_FAILED via isinstance."""
-        from pydantic import ValidationError
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        # Build a minimal ValidationError via model_validate
-        from pydantic import BaseModel
-
-        class _M(BaseModel):
-            x: int
-
-        try:
-            _M.model_validate({"x": "not-an-int-oops"})
-        except ValidationError as exc:
-            cause = agent._classify_synthesis_fallback_cause(exc)
-            assert cause == SynthesisFallbackCause.PARSE_VALIDATION_FAILED
-
-    def test_classify_auth_error_returns_terminal_auth(self):
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        exc = RuntimeError("401 unauthorized — invalid api key")
-        assert agent._classify_synthesis_fallback_cause(exc) == SynthesisFallbackCause.TERMINAL_AUTH
-
-    def test_classify_failover_exhausted_returns_failover_exhausted(self):
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        exc = RuntimeError("prompt_backend_provider: timeout_failover_exhausted")
-        assert agent._classify_synthesis_fallback_cause(exc) == SynthesisFallbackCause.FAILOVER_EXHAUSTED
-
-    def test_classify_no_credentials_returns_failover_exhausted(self):
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        exc = RuntimeError("No credentials for provider: anthropic")
-        assert agent._classify_synthesis_fallback_cause(exc) == SynthesisFallbackCause.FAILOVER_EXHAUSTED
-
-    def test_classify_retry_exhausted_returns_retry_exhausted(self):
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        exc = RuntimeError("provider_retry_exhausted after 3 attempts")
-        assert agent._classify_synthesis_fallback_cause(exc) == SynthesisFallbackCause.RETRY_EXHAUSTED
-
-    def test_classify_unknown_error_defaults_to_retry_exhausted(self):
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        exc = RuntimeError("some completely unknown error")
-        assert agent._classify_synthesis_fallback_cause(exc) == SynthesisFallbackCause.RETRY_EXHAUSTED
-
-    # -------------------------------------------------------------------------
-    # TSK-166.7: _get_llm_recommendations fallback cause (the gap I fixed)
-    # -------------------------------------------------------------------------
-
-    @pytest.mark.asyncio
-    async def test_get_llm_recommendations_logs_fallback_cause_on_exception(self):
-        """llm_parse_failed log must include structured fallback_cause (TSK-166.7)."""
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        agent._run_pydantic_agent = AsyncMock(side_effect=RuntimeError("provider_retry_exhausted"))
-
-        logged_calls: list[dict] = []
-
-        def capture_warning(event, **kw):
-            logged_calls.append({"event": event, **kw})
-
-        agent.logger = MagicMock()
-        agent.logger.warning.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-
-        result = await agent._get_llm_recommendations(
-            repository="test-repo",
-            critical_gaps=[],
-            cross_cutting=[],
-            maturity_level="Developing",
-        )
-
-        assert result == []
-        parse_failed_calls = [c for c in logged_calls if c["event"] == "llm_parse_failed"]
-        assert len(parse_failed_calls) == 1
-        assert "fallback_cause" in parse_failed_calls[0]
-        assert parse_failed_calls[0]["fallback_cause"] == SynthesisFallbackCause.RETRY_EXHAUSTED.value
-
-    # -------------------------------------------------------------------------
-    # TSK-166.7: empty LLM output emits EMPTY_LLM_OUTPUT cause
-    # -------------------------------------------------------------------------
-
-    @pytest.mark.asyncio
-    async def test_generate_recommendations_empty_llm_logs_empty_cause(self):
-        """Empty LLM output path must log EMPTY_LLM_OUTPUT as fallback_cause."""
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        agent._get_llm_recommendations = AsyncMock(return_value=[])
-
-        logged_calls: list[dict] = []
-        agent.logger = MagicMock()
-        agent.logger.info.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-        agent.logger.warning.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-
-        await agent._generate_recommendations(
-            repository="test-repo",
-            critical_gaps=[{"type": "security_vulnerability", "title": "SQL injection"}],
-            cross_cutting=[],
-            maturity_level="Critical",
-        )
-
-        empty_calls = [c for c in logged_calls if c["event"] == "llm_recommendations_empty"]
-        assert len(empty_calls) == 1
-        assert empty_calls[0].get("fallback_cause") == SynthesisFallbackCause.EMPTY_LLM_OUTPUT.value
-
-    # -------------------------------------------------------------------------
-    # TSK-166.8: executive summary fallback cause
-    # -------------------------------------------------------------------------
-
-    @pytest.mark.asyncio
-    async def test_generate_executive_summary_logs_fallback_cause_on_llm_failure(self):
-        """llm_summary_failed must include fallback_cause (TSK-166.8)."""
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        agent._run_pydantic_agent = AsyncMock(side_effect=RuntimeError("authentication failed — unauthorized"))
-
-        logged_calls: list[dict] = []
-        agent.logger = MagicMock()
-        agent.logger.warning.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-        agent.logger.info.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-
-        summary = await agent._generate_executive_summary(
-            repository="test-repo",
-            overall_score=45.0,
-            maturity_level="Developing",
-            critical_gaps=[],
-            aligned_practices=[],
-            agent_results=[],
-        )
-
-        # Template fallback should still return a summary
-        assert "test-repo" in summary or "45" in summary
-
-        summary_failed_calls = [c for c in logged_calls if c["event"] == "llm_summary_failed"]
-        assert len(summary_failed_calls) == 1
-        assert summary_failed_calls[0].get("fallback_cause") == SynthesisFallbackCause.TERMINAL_AUTH.value
-
-    @pytest.mark.asyncio
-    async def test_generate_executive_summary_fallback_cause_propagates_to_completion_log(self):
-        """executive_summary_generated log must reflect the fallback_cause when template is used."""
-        from vds_audit_orchestrator.engine.provider_failure_classifier import SynthesisFallbackCause
-
-        agent = self._make_agent()
-        agent._run_pydantic_agent = AsyncMock(side_effect=RuntimeError("timeout_failover_exhausted"))
-
-        logged_calls: list[dict] = []
-        agent.logger = MagicMock()
-        agent.logger.warning.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-        agent.logger.info.side_effect = lambda event, **kw: logged_calls.append({"event": event, **kw})
-
-        await agent._generate_executive_summary(
-            repository="test-repo",
-            overall_score=60.0,
-            maturity_level="Established",
-            critical_gaps=[],
-            aligned_practices=[],
-            agent_results=[],
-        )
-
-        completion_calls = [c for c in logged_calls if c["event"] == "executive_summary_generated"]
-        assert len(completion_calls) == 1
-        assert completion_calls[0].get("fallback_cause") == SynthesisFallbackCause.FAILOVER_EXHAUSTED.value
-        assert completion_calls[0].get("source") == "template"