@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/integration/test_refined_workflow.py

@@ -1,1180 +0,0 @@
-"""Integration Tests for Refined Audit Workflow (Phase 23).
-
-This module contains comprehensive integration tests for the audit-llm-analysis-refinement
-spec (AUDIT-LLM-001). Tests verify the integration between components:
-
-1. Security Pattern Scanner (scoped results) - FR-1, FR-5
-2. Evidence Matcher (targeted context) - FR-2
-3. Batch Processor (resume capability) - FR-3
-4. Evidence Readiness Gate (fail fast) - FR-4
-5. Grounding Validation (anchor verification) - FR-7
-6. Synthesis Coherence (score vs recommendations) - FR-6
-7. Debug Mode Verification - NFR-3
-
-Requirements validated:
-- FR-1.1, FR-1.2, FR-1.3: No grepai in security workflow
-- FR-2.2: Evidence matching limits
-- FR-3.1, FR-3.3: Batch processing and checkpoint/resume
-- FR-4.1, FR-4.2: Evidence readiness gate
-- FR-6.1, FR-6.2, FR-6.3: Synthesis coherence
-- FR-7.1, FR-7.2: Grounding validation
-- NFR-3: Debug output
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import shutil
-from datetime import UTC, datetime
-from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-from structlog import get_logger
-from vds_audit_orchestrator.agents.base import AgentResult
-from vds_audit_orchestrator.agents.security_scanner import (
-    SecurityPatternScanner,
-)
-from vds_audit_orchestrator.agents.synthesis_agent import SynthesisAgent
-from vds_audit_orchestrator.engine.batch_processor import (
-    BatchConfig,
-    BatchRowProcessor,
-)
-from vds_audit_orchestrator.engine.grounding_validator import (
-    GroundingConfig,
-    GroundingValidator,
-)
-from vds_audit_orchestrator.engine.row_evaluator_types import RowEvaluationResult
-from vds_audit_orchestrator.evidence.corpus import EvidenceCorpus, EvidenceDocument
-from vds_audit_orchestrator.evidence.matcher import (
-    EvidenceMatcher,
-)
-from vds_audit_orchestrator.evidence.readiness import check_evidence_readiness
-from vds_audit_orchestrator.models.checklist import EvidenceAnchor, RowProvenance, RowStatus, ScoreBreakdown
-
-logger = get_logger()
-
-# =============================================================================
-# Test Fixtures
-# =============================================================================
-
-FIXTURES_DIR = Path(__file__).parent / "fixtures"
-
-
-@pytest.fixture
-def mock_repo_dir(tmp_path: Path) -> Path:
-    """Create a mock repository with known security issues.
-
-    This fixture copies the mock_audit_repo fixtures to a temp directory
-    for isolated testing.
-    """
-    repo_dir = tmp_path / "mock_repo"
-    source_dir = FIXTURES_DIR / "mock_audit_repo"
-
-    if source_dir.exists():
-        shutil.copytree(source_dir, repo_dir)
-    else:
-        # Fallback: create minimal mock repo inline
-        repo_dir.mkdir()
-        (repo_dir / "config.py").write_text(
-            'DATABASE_PASSWORD = "super_secret_password_123!"\nAPI_KEY = "sk-1234567890abcdef"\n'
-        )
-        (repo_dir / "db.py").write_text('query = "SELECT * FROM users WHERE name = " + user_input\n')
-        (repo_dir / "auth.py").write_text("SSL_VERIFY = False\nDEBUG = True\n# TODO: bypass auth for testing\n")
-        (repo_dir / "main.py").write_text('print("Hello World")\n')
-        (repo_dir / "src").mkdir()
-        (repo_dir / "src" / "utils.py").write_text("# Safe code\n")
-
-    return repo_dir
-
-
-@pytest.fixture
-def mock_evidence_dir(tmp_path: Path) -> Path:
-    """Create a mock evidence directory with known docs.
-
-    This fixture copies the mock_evidence fixtures to a temp directory
-    for isolated testing.
-    """
-    evidence_dir = tmp_path / "evidence"
-    source_dir = FIXTURES_DIR / "mock_evidence"
-
-    if source_dir.exists():
-        shutil.copytree(source_dir, evidence_dir)
-        legacy_code_pack = evidence_dir / "code_evidence_pack.json"
-        if legacy_code_pack.exists():
-            code_dir = evidence_dir / "code"
-            code_dir.mkdir(parents=True, exist_ok=True)
-            legacy_code_pack.replace(code_dir / "code_evidence_pack.json")
-    else:
-        # Fallback: create minimal evidence inline
-        evidence_dir.mkdir()
-        manifest = {
-            "project": {
-                "project_name": "mock-project",
-                "page_id": "12345",
-                "page_url": "http://confluence/page/12345",
-                "documents": [
-                    {
-                        "title": "BRD",
-                        "source_path": "brd.md",
-                        "doc_type": "BRD",
-                        "role": "PO",
-                        "url": "http://confluence/brd",
-                        "source": "confluence",
-                    },
-                    {
-                        "title": "SRS",
-                        "source_path": "srs.md",
-                        "doc_type": "SRS",
-                        "role": "BA",
-                        "url": "http://confluence/srs",
-                        "source": "confluence",
-                    },
-                    {
-                        "title": "Design",
-                        "source_path": "design.md",
-                        "doc_type": "DESIGN",
-                        "role": "SA",
-                        "url": "http://confluence/design",
-                        "source": "confluence",
-                    },
-                    {
-                        "title": "Test",
-                        "source_path": "test.md",
-                        "doc_type": "TEST",
-                        "role": "QA",
-                        "url": "http://confluence/test",
-                        "source": "confluence",
-                    },
-                    {
-                        "title": "Security",
-                        "source_path": "security.md",
-                        "doc_type": "OTHER",
-                        "role": "SA",
-                        "url": "http://confluence/security",
-                        "source": "confluence",
-                    },
-                ],
-            }
-        }
-        (evidence_dir / "manifest.json").write_text(json.dumps(manifest))
-
-        code_pack = {
-            "repo_path": "mock-repo",
-            "files": [
-                {
-                    "file_path": "config.py",
-                    "content": "password = 'secret'",
-                    "original_lines": 1,
-                    "original_bytes": 20,
-                    "priority": 1,
-                },
-                {
-                    "file_path": "main.py",
-                    "content": "print('hello')",
-                    "original_lines": 1,
-                    "original_bytes": 14,
-                    "priority": 2,
-                },
-            ],
-        }
-        code_dir = evidence_dir / "code"
-        code_dir.mkdir(parents=True, exist_ok=True)
-        (code_dir / "code_evidence_pack.json").write_text(json.dumps(code_pack))
-
-        proj_dir = evidence_dir / "docs" / "mock-project"
-        proj_dir.mkdir(parents=True)
-        (proj_dir / "brd.md").write_text("# BRD\nAuthentication requirements")
-        (proj_dir / "srs.md").write_text("# SRS\nFunctional requirements")
-        (proj_dir / "design.md").write_text("# Design\nArchitecture overview")
-        (proj_dir / "test.md").write_text("# Test Plan\nTest strategy")
-        (proj_dir / "security.md").write_text("# Security\nSecurity guidelines")
-
-    return evidence_dir
-
-
-@pytest.fixture
-def empty_evidence_dir(tmp_path: Path) -> Path:
-    """Create an empty evidence directory for fail-fast testing."""
-    evidence_dir = tmp_path / "empty_evidence"
-    evidence_dir.mkdir()
-    return evidence_dir
-
-
-@pytest.fixture
-def multi_project_evidence_dir(tmp_path: Path) -> Path:
-    """Create evidence directory with multiple projects for scope testing."""
-    evidence_dir = tmp_path / "multi_project"
-    evidence_dir.mkdir()
-
-    # Create code evidence pack
-    code_pack = {
-        "repo_path": "repo",
-        "files": [
-            {"file_path": "a.py", "content": "a", "original_lines": 1, "original_bytes": 1, "priority": 1},
-            {"file_path": "b.py", "content": "b", "original_lines": 1, "original_bytes": 1, "priority": 1},
-            {"file_path": "c.py", "content": "c", "original_lines": 1, "original_bytes": 1, "priority": 1},
-        ],
-    }
-    code_dir = evidence_dir / "code"
-    code_dir.mkdir(parents=True, exist_ok=True)
-    (code_dir / "code_evidence_pack.json").write_text(json.dumps(code_pack))
-
-    # Create Project A manifest
-    (evidence_dir / "proj_a").mkdir()
-    manifest_a = {
-        "project": {
-            "project_name": "ProjectA",
-            "page_id": "1",
-            "page_url": "http://confluence/page/1",
-            "documents": [
-                {"title": "BRD A", "source_path": "brd.md", "doc_type": "BRD"},
-                {"title": "SRS A", "source_path": "srs.md", "doc_type": "SRS"},
-                {"title": "Design A", "source_path": "design.md", "doc_type": "DESIGN"},
-                {"title": "Test A", "source_path": "test.md", "doc_type": "TEST"},
-                {"title": "BRD A2", "source_path": "brd2.md", "doc_type": "BRD"},
-            ],
-        }
-    }
-    (evidence_dir / "proj_a" / "manifest.json").write_text(json.dumps(manifest_a))
-
-    # Create Project B manifest
-    (evidence_dir / "proj_b").mkdir()
-    manifest_b = {
-        "project": {
-            "project_name": "ProjectB",
-            "page_id": "2",
-            "page_url": "http://confluence/page/2",
-            "documents": [
-                {"title": "BRD B", "source_path": "brd.md", "doc_type": "BRD"},
-            ],
-        }
-    }
-    (evidence_dir / "proj_b" / "manifest.json").write_text(json.dumps(manifest_b))
-
-    return evidence_dir
-
-
-# =============================================================================
-# Task 7.2: Test Full Audit Without grepai (FR-1.1, FR-1.2, FR-1.3)
-# =============================================================================
-
-
-class TestSecurityScannerNoGrepai:
-    """Tests verifying security scanner works without grepai.
-
-    Requirements: FR-1.1, FR-1.2, FR-1.3
-    """
-
-    def test_scanner_finds_security_issues_in_mock_repo(self, mock_repo_dir: Path):
-        """Verify scanner finds all expected security issues in mock repo."""
-        scanner = SecurityPatternScanner(mock_repo_dir)
-
-        # Force Python fallback to ensure consistent behavior
-        with patch.object(scanner, "_check_rg_available", return_value=False):
-            results = scanner.scan()
-
-        # Verify we found security issues
-        assert len(results) > 0, "Scanner should find security issues"
-
-        # Check for hardcoded password (CRITICAL)
-        passwords = [r for r in results if r.pattern_name == "hardcoded_password"]
-        assert len(passwords) >= 1, "Should find hardcoded password"
-        assert any(r.file_path == "config.py" for r in passwords)
-
-        # Check for SQL injection (HIGH)
-        sql_issues = [r for r in results if r.pattern_name == "sql_string_concat"]
-        assert len(sql_issues) >= 1, "Should find SQL injection"
-        assert any(r.file_path == "db.py" for r in sql_issues)
-
-    def test_scanner_results_scoped_to_repo_only(self, mock_repo_dir: Path, tmp_path: Path):
-        """Verify all scanner results are from target repo only (FR-1.3)."""
-        # Create another directory with security issues outside repo
-        other_dir = tmp_path / "other_repo"
-        other_dir.mkdir()
-        (other_dir / "secrets.py").write_text('password = "leaked_password"')
-
-        scanner = SecurityPatternScanner(mock_repo_dir)
-
-        with patch.object(scanner, "_check_rg_available", return_value=False):
-            results = scanner.scan()
-
-        # All results should be relative to mock_repo_dir
-        for result in results:
-            full_path = mock_repo_dir / result.file_path
-            assert full_path.exists(), f"Result path should exist in repo: {result.file_path}"
-            # Should not contain paths from other_repo
-            assert "other_repo" not in result.file_path
-            assert "secrets.py" not in result.file_path
-
-    def test_scanner_no_grepai_calls(self, mock_repo_dir: Path):
-        """Verify scanner does not call grepai (FR-1.1, FR-1.2)."""
-        scanner = SecurityPatternScanner(mock_repo_dir)
-
-        # Mock grepai to track if it's called
-        with patch("subprocess.run") as mock_run:
-            # Configure mock to return empty results for rg
-            mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
-
-            # Force Python fallback
-            with patch.object(scanner, "_check_rg_available", return_value=False):
-                scanner.scan()
-
-            # Verify no grepai calls were made
-            for call in mock_run.call_args_list:
-                args = call[0][0] if call[0] else []
-                assert "grepai" not in str(args), "Should not call grepai"
-
-
-# =============================================================================
-# Task 7.3: Test Batch Processing with Resume (FR-3.3)
-# =============================================================================
-
-
-class TestBatchProcessingResume:
-    """Tests for batch processing with checkpoint/resume capability.
-
-    Requirements: FR-3.1, FR-3.3
-    """
-
-    def test_checkpoint_created_after_each_batch(self, tmp_path: Path):
-        """Verify checkpoint file is created with full row results after each batch."""
-        checkpoint_dir = tmp_path / "checkpoints"
-        checkpoint_dir.mkdir()
-
-        # Create mock template with 10 checks
-        template = MagicMock()
-        checks = [MagicMock(id=f"chk_{i}", name=f"Check {i}", description=f"Desc {i}") for i in range(10)]
-        template.sections = [MagicMock(id="SEC-1", checks=checks)]
-
-        # Create mock evaluator
-        evaluator = AsyncMock()
-
-        async def mock_evaluate(
-            check, evidence, row_id, row_index, evidence_context, evidence_refs, project_profile, **kwargs
-        ):
-            return RowEvaluationResult(
-                row_id=row_id,
-                check_id=check.id,
-                status=RowStatus.PASS,
-                score=85.0,
-                score_breakdown=MagicMock(model_dump=lambda: {"score": 85.0}),
-                reason="Test passed",
-                finding="No issues found",
-                evidence_anchors=[],
-                provenance=MagicMock(model_dump=lambda: {}),
-            )
-
-        evaluator.aevaluate_row.side_effect = mock_evaluate
-        evaluator.template_hash = "test_hash_123"
-        evaluator.evidence_hash = "evidence_hash_456"
-
-        # Create mock matcher
-        matcher = MagicMock()
-        matcher.match_row.return_value = MagicMock(
-            format_for_llm=lambda: "Mock evidence context", evidence_refs=["http://confluence/doc1", "src/main.py"]
-        )
-
-        config = BatchConfig(batch_size=2, checkpoint_dir=checkpoint_dir, resume_from_checkpoint=False)
-
-        processor = BatchRowProcessor(template, evaluator, matcher, config)
-
-        # Run processing
-        asyncio.run(processor.process_all("test_thread_001"))
-
-        # Verify checkpoint exists
-        checkpoint_path = checkpoint_dir / "batch-checkpoint-test_thread_001.json"
-        assert checkpoint_path.exists(), "Checkpoint file should be created"
-
-        # Verify checkpoint content
-        checkpoint_data = json.loads(checkpoint_path.read_text())
-        assert checkpoint_data["thread_id"] == "test_thread_001"
-        assert checkpoint_data["total_rows"] == 10
-        assert len(checkpoint_data["completed_batches"]) == 5  # 10 rows / 2 batch_size
-        assert len(checkpoint_data["results_by_row_id"]) == 10  # All rows stored
-
-        # Verify full row results are stored (FR-3.3)
-        for _row_id, row_data in checkpoint_data["results_by_row_id"].items():
-            assert "status" in row_data
-            assert "score" in row_data
-            assert "reason" in row_data
-
-    def test_resume_skips_completed_batches(self, tmp_path: Path):
-        """Verify resume skips already completed batches and uses stored results."""
-        checkpoint_dir = tmp_path / "checkpoints"
-        checkpoint_dir.mkdir()
-
-        # Create a checkpoint with 2 batches completed (4 rows)
-        existing_checkpoint = {
-            "thread_id": "resume_test",
-            "total_rows": 10,
-            "template_hash": "test_hash",
-            "evidence_hash": "ev_hash",
-            "completed_batches": [0, 1],  # First 2 batches done
-            "results_by_row_id": {
-                f"chk_{i}:row_{i}": {
-                    "row_id": f"chk_{i}:row_{i}",
-                    "check_id": f"chk_{i}",
-                    "status": "PASS",
-                    "score": 90.0,
-                    "reason": "Previously evaluated",
-                    "finding": "No issues",
-                    "evidence_anchors": [],
-                    "provenance": {},
-                    "score_breakdown": {},
-                }
-                for i in range(4)  # 4 rows from 2 batches
-            },
-            "last_updated": datetime.now(UTC).isoformat(),
-        }
-        checkpoint_path = checkpoint_dir / "batch-checkpoint-resume_test.json"
-        checkpoint_path.write_text(json.dumps(existing_checkpoint))
-
-        # Create mock template
-        template = MagicMock()
-        checks = [MagicMock(id=f"chk_{i}", name=f"Check {i}", description=f"Desc {i}") for i in range(10)]
-        template.sections = [MagicMock(id="SEC-1", checks=checks)]
-
-        # Track which rows are evaluated
-        evaluated_rows = []
-
-        async def mock_evaluate(
-            check, evidence, row_id, row_index, evidence_context, evidence_refs, project_profile, **kwargs
-        ):
-            evaluated_rows.append(row_index)
-            return RowEvaluationResult(
-                row_id=row_id,
-                check_id=check.id,
-                status=RowStatus.PASS,
-                score=85.0,
-                score_breakdown=MagicMock(model_dump=lambda: {}),
-                reason="Newly evaluated",
-                finding="No issues",
-                evidence_anchors=[],
-                provenance=MagicMock(model_dump=lambda: {}),
-            )
-
-        evaluator = AsyncMock()
-        evaluator.aevaluate_row.side_effect = mock_evaluate
-        evaluator.template_hash = "test_hash"
-        evaluator.evidence_hash = "ev_hash"
-
-        matcher = MagicMock()
-        matcher.match_row.return_value = MagicMock(format_for_llm=lambda: "context", evidence_refs=[])
-
-        config = BatchConfig(batch_size=2, checkpoint_dir=checkpoint_dir, resume_from_checkpoint=True)
-
-        processor = BatchRowProcessor(template, evaluator, matcher, config)
-        results = asyncio.run(processor.process_all("resume_test"))
-
-        # Should have all 10 results
-        assert len(results) == 10
-
-        # Should only evaluate rows 4-9 (batches 2-4)
-        assert 0 not in evaluated_rows, "Row 0 should be skipped (from checkpoint)"
-        assert 1 not in evaluated_rows, "Row 1 should be skipped (from checkpoint)"
-        assert 2 not in evaluated_rows, "Row 2 should be skipped (from checkpoint)"
-        assert 3 not in evaluated_rows, "Row 3 should be skipped (from checkpoint)"
-        assert 4 in evaluated_rows, "Row 4 should be evaluated"
-        assert 9 in evaluated_rows, "Row 9 should be evaluated"
-
-    def test_no_duplicate_rows_after_resume(self, tmp_path: Path):
-        """Verify no duplicate rows in final results after resume."""
-        checkpoint_dir = tmp_path / "checkpoints"
-        checkpoint_dir.mkdir()
-
-        # Create checkpoint with partial completion
-        existing_checkpoint = {
-            "thread_id": "dedup_test",
-            "total_rows": 6,
-            "template_hash": "hash",
-            "evidence_hash": "ev",
-            "completed_batches": [0],  # Only first batch
-            "results_by_row_id": {
-                f"chk_{i}:row_{i}": {
-                    "row_id": f"chk_{i}:row_{i}",
-                    "check_id": f"chk_{i}",
-                    "status": "PASS",
-                    "score": 90.0,
-                    "reason": "From checkpoint",
-                    "finding": "No issues",
-                    "evidence_anchors": [],
-                    "provenance": {},
-                    "score_breakdown": {},
-                }
-                for i in range(2)
-            },
-            "last_updated": datetime.now(UTC).isoformat(),
-        }
-        checkpoint_path = checkpoint_dir / "batch-checkpoint-dedup_test.json"
-        checkpoint_path.write_text(json.dumps(existing_checkpoint))
-
-        template = MagicMock()
-        checks = [MagicMock(id=f"chk_{i}", name=f"Check {i}", description=f"Desc {i}") for i in range(6)]
-        template.sections = [MagicMock(id="SEC-1", checks=checks)]
-
-        async def mock_evaluate(
-            check, evidence, row_id, row_index, evidence_context, evidence_refs, project_profile, **kwargs
-        ):
-            return RowEvaluationResult(
-                row_id=row_id,
-                check_id=check.id,
-                status=RowStatus.PASS,
-                score=85.0,
-                score_breakdown=MagicMock(model_dump=lambda: {}),
-                reason="New evaluation",
-                finding="No issues",
-                evidence_anchors=[],
-                provenance=MagicMock(model_dump=lambda: {}),
-            )
-
-        evaluator = AsyncMock()
-        evaluator.aevaluate_row.side_effect = mock_evaluate
-        evaluator.template_hash = "hash"
-        evaluator.evidence_hash = "ev"
-
-        matcher = MagicMock()
-        matcher.match_row.return_value = MagicMock(format_for_llm=lambda: "context", evidence_refs=[])
-
-        config = BatchConfig(batch_size=2, checkpoint_dir=checkpoint_dir, resume_from_checkpoint=True)
-
-        processor = BatchRowProcessor(template, evaluator, matcher, config)
-        results = asyncio.run(processor.process_all("dedup_test"))
-
-        # Verify no duplicates
-        row_ids = [r.row_id for r in results]
-        assert len(row_ids) == len(set(row_ids)), "Should have no duplicate row IDs"
-        assert len(results) == 6, "Should have exactly 6 results"
-
-
-# =============================================================================
-# Task 7.4: Test Evidence Readiness Gate (FR-4.1, FR-4.2)
-# =============================================================================
-
-
-class TestEvidenceReadinessGate:
-    """Tests for evidence readiness gate functionality.
-
-    Requirements: FR-4.1, FR-4.2
-    """
-
-    def test_empty_evidence_fails_fast(self, empty_evidence_dir: Path):
-        """Verify empty evidence directory triggers fail-fast (FR-4.2)."""
-        result = check_evidence_readiness(min_docs=1, min_code_files=1)
-
-        assert not result.ready, "Should fail with empty evidence"
-        assert len(result.missing) > 0, "Should have missing items"
-        # Should mention what's missing
-        missing_text = " ".join(result.missing).lower()
-        assert "code" in missing_text or "confluence" in missing_text
-
-    def test_multi_project_without_selection_fails(self, multi_project_evidence_dir: Path):
-        """Verify multi-project evidence without --evidence-project fails (FR-4.1)."""
-        manifest_payloads = [
-            {
-                "projects": [
-                    {
-                        "project_name": "ProjectA",
-                        "page_id": "1",
-                        "documents": [
-                            {"title": "BRD A", "doc_type": "BRD", "source": "confluence"},
-                        ],
-                    },
-                    {
-                        "project_name": "ProjectB",
-                        "page_id": "2",
-                        "documents": [
-                            {"title": "BRD B", "doc_type": "BRD", "source": "confluence"},
-                        ],
-                    },
-                ]
-            }
-        ]
-        code_pack_path = multi_project_evidence_dir / "code" / "code_evidence_pack.json"
-        result = check_evidence_readiness(
-            min_docs=1,
-            min_code_files=1,
-            evidence_project=None,  # No project selected
-            manifest_payloads=manifest_payloads,
-            code_pack_path=code_pack_path,
-        )
-
-        assert not result.ready, "Should fail without project selection"
-        assert len(result.projects_found) > 1, "Should detect multiple projects"
-
-    def test_skip_evidence_check_proceeds_with_warning(self, empty_evidence_dir: Path):
-        """Verify --skip-evidence-check allows proceeding with warning.
-
-        Note: The skip is handled at CLI level, but we test that low thresholds
-        effectively allow proceeding.
-        """
-        # With zero thresholds, should "pass" even with empty evidence
-        result = check_evidence_readiness(
-            min_docs=0,
-            min_code_files=0,
-            manifest_payloads=[],
-            code_pack_path=Path("/nonexistent"),
-        )
-
-        # Should report actual counts (0) but not fail on thresholds
-        assert result.confluence_docs_count == 0
-        assert result.code_files_count == 0
-
-    def test_valid_evidence_passes(self, mock_evidence_dir: Path):
-        """Verify valid evidence directory passes readiness check."""
-        manifest_payloads = [
-            {
-                "project": {
-                    "project_name": "mock-project",
-                    "page_id": "12345",
-                    "documents": [
-                        {"title": "BRD", "doc_type": "BRD", "source": "confluence", "url": "http://confluence/brd"},
-                        {"title": "SRS", "doc_type": "SRS", "source": "confluence", "url": "http://confluence/srs"},
-                        {
-                            "title": "Design",
-                            "doc_type": "DESIGN",
-                            "source": "confluence",
-                            "url": "http://confluence/design",
-                        },
-                        {"title": "Test", "doc_type": "TEST", "source": "confluence", "url": "http://confluence/test"},
-                    ],
-                }
-            }
-        ]
-        code_pack_path = mock_evidence_dir / "code" / "code_evidence_pack.json"
-        result = check_evidence_readiness(
-            min_docs=1,
-            min_code_files=1,
-            manifest_payloads=manifest_payloads,
-            code_pack_path=code_pack_path,
-        )
-
-        assert result.ready, f"Should pass with valid evidence. Missing: {result.missing}"
-        assert result.confluence_docs_count >= 1
-        assert result.code_files_count >= 1
-
-
-# =============================================================================
-# Task 7.5: Test Grounding Validation (FR-7.1, FR-7.2)
-# =============================================================================
-
-
-class TestGroundingValidation:
-    """Tests for grounding validation of LLM evidence anchors.
-
-    Requirements: FR-7.1, FR-7.2
-    """
-
-    def test_invalid_anchors_removed(self):
-        """Verify invalid anchors are removed from results (FR-7.1)."""
-        validator = GroundingValidator(GroundingConfig(strict_mode=False))
-
-        # Create result with mix of valid and invalid anchors
-        dummy_provenance = RowProvenance(
-            row_llm_mode="test",
-            protocol="test",
-            model="test",
-            template_hash="hash",
-            rubric_version="1.0",
-            evidence_hash="hash",
-            evaluated_at=datetime.now(UTC),
-        )
-
-        result = RowEvaluationResult(
-            row_id="test_row",
-            check_id="test_check",
-            status=RowStatus.PASS,
-            score=80.0,
-            score_breakdown=ScoreBreakdown.compute(80.0),
-            reason="Test",
-            finding="Test finding",
-            evidence_anchors=[
-                EvidenceAnchor(ref_type="url", ref_value="http://confluence/valid_doc", excerpt="Valid content"),
-                EvidenceAnchor(ref_type="url", ref_value="http://hallucinated.com/fake", excerpt="Fake content"),
-                EvidenceAnchor(ref_type="file", ref_value="src/valid.py", excerpt="Valid code"),
-            ],
-            provenance=dummy_provenance,
-        )
-
-        allowed_refs = ["http://confluence/valid_doc", "src/valid.py", "http://confluence/other_doc"]
-
-        validated = validator.validate(result, allowed_refs, "Valid content Valid code")
-
-        # Should have only 2 valid anchors
-        assert len(validated.evidence_anchors) == 2
-        ref_values = [a.ref_value for a in validated.evidence_anchors]
-        assert "http://confluence/valid_doc" in ref_values
-        assert "src/valid.py" in ref_values
-        assert "http://hallucinated.com/fake" not in ref_values
-
-    def test_strict_mode_error_on_zero_valid_anchors(self):
-        """Verify strict mode marks ERROR when no valid anchors (FR-7.2)."""
-        validator = GroundingValidator(GroundingConfig(strict_mode=True))
-
-        dummy_provenance = RowProvenance(
-            row_llm_mode="test",
-            protocol="test",
-            model="test",
-            template_hash="hash",
-            rubric_version="1.0",
-            evidence_hash="hash",
-            evaluated_at=datetime.now(UTC),
-        )
-
-        result = RowEvaluationResult(
-            row_id="strict_test",
-            check_id="check_1",
-            status=RowStatus.PASS,
-            score=75.0,
-            score_breakdown=ScoreBreakdown.compute(75.0),
-            reason="Test",
-            finding="Finding",
-            evidence_anchors=[
-                EvidenceAnchor(ref_type="url", ref_value="http://invalid.com/fake1", excerpt="Fake 1"),
-                EvidenceAnchor(ref_type="url", ref_value="http://invalid.com/fake2", excerpt="Fake 2"),
-            ],
-            provenance=dummy_provenance,
-        )
-
-        allowed_refs = ["http://valid.com/real"]
-
-        validated = validator.validate(result, allowed_refs, "context")
-
-        # Should be marked as ERROR
-        assert validated.status == RowStatus.ERROR
-        assert validated.error_message and "Grounding validation failed" in validated.error_message
-        assert len(validated.evidence_anchors) == 0
-
-    def test_strict_mode_no_error_when_no_anchors_initially(self):
-        """Verify strict mode doesn't error if input had no anchors."""
-        validator = GroundingValidator(GroundingConfig(strict_mode=True))
-
-        dummy_provenance = RowProvenance(
-            row_llm_mode="test",
-            protocol="test",
-            model="test",
-            template_hash="hash",
-            rubric_version="1.0",
-            evidence_hash="hash",
-            evaluated_at=datetime.now(UTC),
-        )
-
-        result = RowEvaluationResult(
-            row_id="no_anchors",
-            check_id="check_1",
-            status=RowStatus.PASS,
-            score=90.0,
-            score_breakdown=ScoreBreakdown.compute(90.0),
-            reason="Test",
-            finding="Finding",
-            evidence_anchors=[],  # No anchors
-            provenance=dummy_provenance,
-        )
-
-        validated = validator.validate(result, ["http://valid.com"], "context")
-
-        # Should stay PASS (no anchors to validate)
-        assert validated.status == RowStatus.PASS
-
-
-# =============================================================================
-# Task 7.6: Test Synthesis Coherence (FR-6.1, FR-6.2, FR-6.3)
-# =============================================================================
-
-
-class TestSynthesisCoherence:
-    """Tests for synthesis agent coherence between findings and recommendations.
-
-    Requirements: FR-6.1, FR-6.2, FR-6.3
-    """
-
-    @pytest.fixture
-    def synthesis_agent(self):
-        """Create synthesis agent with LLM disabled."""
-        return SynthesisAgent(settings=MagicMock(enabled=False))
-
-    def test_critical_findings_in_critical_gaps(self, synthesis_agent):
-        """Verify CRITICAL findings appear in critical_gaps (FR-6.1)."""
-        findings = [
-            {
-                "type": "security_vulnerability",
-                "severity": "critical",
-                "title": "SQL Injection",
-                "location": "db.py",
-                "source_agent": "SecurityAgent",
-            },
-            {
-                "type": "security_vulnerability",
-                "severity": "critical",
-                "title": "Hardcoded Password",
-                "location": "config.py",
-                "source_agent": "SecurityAgent",
-            },
-        ]
-
-        prioritized = synthesis_agent._prioritize_findings(findings)
-        elevated = synthesis_agent._elevate_critical_findings(prioritized, overall_score=50.0)
-
-        assert len(elevated) == 2
-        titles = {f["title"] for f in elevated}
-        assert "SQL Injection" in titles
-        assert "Hardcoded Password" in titles
-
-    def test_high_findings_elevated_when_score_low(self, synthesis_agent):
-        """Verify HIGH findings elevated when score < 40 (FR-6.2)."""
-        findings = [
-            {
-                "type": "security_risk",
-                "severity": "high",
-                "title": "SSL Disabled",
-                "location": "auth.py",
-                "source_agent": "SecurityAgent",
-            },
-        ]
-
-        prioritized = synthesis_agent._prioritize_findings(findings)
-
-        # Score 30 (< 40) should elevate HIGH
-        elevated_low = synthesis_agent._elevate_critical_findings(prioritized, overall_score=30.0)
-        assert len(elevated_low) == 1
-
-        # Score 50 (>= 40) should NOT elevate HIGH
-        elevated_high = synthesis_agent._elevate_critical_findings(prioritized, overall_score=50.0)
-        assert len(elevated_high) == 0
-
-    def test_recommendations_match_severity(self, synthesis_agent):
-        """Verify recommendations match severity of findings (FR-6.3)."""
-        gaps = [{"type": "security_vulnerability", "title": "Critical Gap"}]
-
-        # Low score should get CRITICAL prefix
-        prefix_low, recs_low = synthesis_agent._generate_coherent_recommendations(20.0, gaps)
-        assert prefix_low == "CRITICAL"
-        assert any(r["priority"] == "critical" for r in recs_low)
-
-        # High score should get MAINTENANCE prefix
-        prefix_high, recs_high = synthesis_agent._generate_coherent_recommendations(80.0, gaps)
-        assert prefix_high == "MAINTENANCE"
-        assert any(r["priority"] == "low" for r in recs_high)
-
-    @pytest.mark.asyncio
-    async def test_full_synthesis_with_critical_findings(self, synthesis_agent):
-        """Integration test: full synthesis with critical findings."""
-        agent_results = [
-            AgentResult(
-                agent_name="SecurityAgent",
-                success=True,
-                findings=[
-                    {
-                        "type": "security_vulnerability",
-                        "severity": "critical",
-                        "title": "Hardcoded Password",
-                        "location": "config.py",
-                        "source_agent": "SecurityAgent",
-                    },
-                ],
-            ),
-        ]
-
-        synthesis = await synthesis_agent.synthesize("test-repo", agent_results)
-
-        # Should have gaps
-        assert len(synthesis.gaps) >= 1
-
-        # Score should be affected by critical finding
-        assert synthesis.overall_score < 100
-
-        # Should have recommendations
-        assert len(synthesis.recommendations) >= 1
-
-
-# =============================================================================
-# Task 7.7: Full Debug Mode Verification (NFR-3)
-# =============================================================================
-
-
-class TestDebugModeVerification:
-    """Tests for debug mode output and artifacts.
-
-    Requirements: NFR-3
-    """
-
-    def test_debug_evidence_context_output(self, tmp_path: Path):
-        """Verify debug evidence context is written when enabled."""
-        report_dir = tmp_path / "reports"
-        report_dir.mkdir()
-
-        # Create mock evidence context data
-        evidence_context_data = [
-            {
-                "row_id": "row_1",
-                "evidence_refs": ["http://confluence/doc1", "src/main.py"],
-                "matched_docs_count": 3,
-                "matched_code_count": 2,
-                "total_chars": 5000,
-            },
-            {
-                "row_id": "row_2",
-                "evidence_refs": ["http://confluence/doc2"],
-                "matched_docs_count": 1,
-                "matched_code_count": 0,
-                "total_chars": 1500,
-            },
-        ]
-
-        # Write debug artifact
-        evidence_context_path = report_dir / "evidence_context.jsonl"
-        with evidence_context_path.open("w") as f:
-            for item in evidence_context_data:
-                f.write(json.dumps(item) + "\n")
-
-        # Verify artifact exists and is valid
-        assert evidence_context_path.exists()
-
-        lines = evidence_context_path.read_text().strip().split("\n")
-        assert len(lines) == 2
-
-        for line in lines:
-            data = json.loads(line)
-            assert "row_id" in data
-            assert "evidence_refs" in data
-
-    def test_debug_batch_timing_output(self, tmp_path: Path):
-        """Verify debug batch timing is written when enabled."""
-        report_dir = tmp_path / "reports"
-        report_dir.mkdir()
-
-        # Create mock batch timing data
-        batch_timing_data = {
-            "thread_id": "debug_test",
-            "total_batches": 5,
-            "total_rows": 10,
-            "batches": [
-                {"batch_index": 0, "duration_ms": 1200, "success_count": 2, "error_count": 0},
-                {"batch_index": 1, "duration_ms": 1500, "success_count": 2, "error_count": 0},
-                {"batch_index": 2, "duration_ms": 1100, "success_count": 2, "error_count": 0},
-                {"batch_index": 3, "duration_ms": 1300, "success_count": 2, "error_count": 0},
-                {"batch_index": 4, "duration_ms": 1400, "success_count": 2, "error_count": 0},
-            ],
-            "total_duration_ms": 6500,
-            "avg_batch_duration_ms": 1300,
-        }
-
-        # Write debug artifact
-        batch_timing_path = report_dir / "batch_timing.json"
-        batch_timing_path.write_text(json.dumps(batch_timing_data, indent=2))
-
-        # Verify artifact exists and is valid
-        assert batch_timing_path.exists()
-
-        loaded = json.loads(batch_timing_path.read_text())
-        assert loaded["total_batches"] == 5
-        assert loaded["total_rows"] == 10
-        assert len(loaded["batches"]) == 5
-
-    def test_structured_logging_events(self, caplog):
-        """Verify expected log events are present during processing."""
-        import logging
-
-        # Configure logging to capture structlog events
-        caplog.set_level(logging.DEBUG)
-
-        # Create a simple scanner and run it
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="")
-
-            # This should log events
-            import tempfile
-            from pathlib import Path
-
-            from vds_audit_orchestrator.agents.security_scanner import SecurityPatternScanner
-
-            with tempfile.TemporaryDirectory() as tmpdir:
-                repo_path = Path(tmpdir)
-                (repo_path / "test.py").write_text("password = 'secret'")
-
-                scanner = SecurityPatternScanner(repo_path)
-                with patch.object(scanner, "_check_rg_available", return_value=False):
-                    scanner.scan()
-
-        # Log events should be captured (structlog integration varies)
-        # This is a basic check that logging doesn't crash
-        assert True  # If we get here, logging worked
-
-
-# =============================================================================
-# Evidence Matcher Integration Tests (FR-2.2)
-# =============================================================================
-
-
-class TestEvidenceMatcherIntegration:
-    """Integration tests for evidence matcher.
-
-    Requirements: FR-2.2
-    """
-
-    def test_evidence_matcher_links_requirements_to_docs(self):
-        """Verify evidence matcher links requirements to relevant docs."""
-        # Create corpus with known documents
-        corpus = EvidenceCorpus()
-        corpus.add_document(
-            EvidenceDocument(
-                title="BRD - Authentication",
-                content="Users must authenticate using JWT tokens. Password policy enforced.",
-                source_path="brd/auth.md",
-                doc_type="BRD",
-                project_name="test-project",
-            )
-        )
-        corpus.add_document(
-            EvidenceDocument(
-                title="SRS - Security",
-                content="System shall validate all inputs. SQL injection prevention required.",
-                source_path="srs/security.md",
-                doc_type="SRS",
-                project_name="test-project",
-            )
-        )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project")
-
-        # Match authentication-related requirement
-        context = matcher.match_row(
-            row_id="row_1", requirement_text="Users must authenticate via login with JWT tokens"
-        )
-
-        # Should match BRD - Authentication
-        assert len(context.matched_docs) > 0
-        titles = [d.title for d in context.matched_docs]
-        assert any("Authentication" in t for t in titles)
-
-        # Should have evidence_refs for grounding
-        assert len(context.evidence_refs) > 0
-
-    def test_evidence_matcher_respects_limits(self):
-        """Verify evidence matcher respects max 5 docs/code limit (FR-2.2)."""
-        # Create corpus with many documents
-        corpus = EvidenceCorpus()
-        for i in range(20):
-            corpus.add_document(
-                EvidenceDocument(
-                    title=f"Doc {i}",
-                    content=f"Content about authentication and security {i}",
-                    source_path=f"doc{i}.md",
-                    doc_type="BRD",
-                    project_name="test-project",
-                )
-            )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project")
-
-        context = matcher.match_row(row_id="row_1", requirement_text="Authentication and security requirements")
-
-        # Should have max 5 matched docs
-        assert len(context.matched_docs) <= 5
-
-    def test_synonym_expansion_improves_matching(self):
-        """Verify synonym expansion improves evidence matching (FR-2.1)."""
-        corpus = EvidenceCorpus()
-        corpus.add_document(
-            EvidenceDocument(
-                title="Login Flow",
-                content="User login process with session management and JWT tokens.",
-                source_path="login.md",
-                doc_type="DESIGN",
-                project_name="test-project",
-            )
-        )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project")
-
-        # Search for "auth" which should expand to include "login", "jwt", etc.
-        context = matcher.match_row(row_id="row_1", requirement_text="Authentication requirements")
-
-        # Should match Login Flow due to synonym expansion
-        # "auth" -> ["login", "jwt", "session", ...]
-        assert len(context.matched_docs) > 0 or "auth" in matcher.SYNONYM_MAP
-
-    def test_retrieval_trace_payload_suppressed_by_default(self):
-        """Retrieval debug payload should not be emitted unless debug mode is enabled."""
-        corpus = EvidenceCorpus()
-        corpus.add_document(
-            EvidenceDocument(
-                title="Authentication Design",
-                content="Authentication and JWT login design.",
-                source_path="design/auth.md",
-                doc_type="DESIGN",
-                project_name="test-project",
-            )
-        )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project")
-        context = matcher.match_row(row_id="row_1", requirement_text="Authentication design")
-
-        assert len(context.matched_docs) > 0
-        assert context.retrieval_trace is None
-
-    def test_retrieval_trace_payload_emitted_in_debug_mode(self):
-        """Retrieval debug payload should be emitted when debug mode is enabled."""
-        corpus = EvidenceCorpus()
-        corpus.add_document(
-            EvidenceDocument(
-                title="Authentication Design",
-                content="Authentication and JWT login design.",
-                source_path="design/auth.md",
-                doc_type="DESIGN",
-                project_name="test-project",
-            )
-        )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project", emit_retrieval_trace=True)
-        context = matcher.match_row(row_id="row_1", requirement_text="Authentication design")
-
-        assert len(context.matched_docs) > 0
-        assert context.retrieval_trace is not None
-        assert context.retrieval_trace.get("mode") == "lexical"
-        assert "docs" in context.retrieval_trace
-        assert "code" in context.retrieval_trace
-
-    def test_evidence_matcher_excludes_generated_audit_artifacts(self):
-        """Generated checklist/workflow attachments must not be used as docs evidence."""
-        corpus = EvidenceCorpus()
-        corpus.add_document(
-            EvidenceDocument(
-                title="att88717084_audit-checklist",
-                content="Checklist attachment content mentioning authentication and security",
-                source_path="projects/X/attachments/att88717084_audit-checklist.md",
-                doc_type="OTHER",
-                project_name="test-project",
-            )
-        )
-        corpus.add_document(
-            EvidenceDocument(
-                title="att88717085_workflow-summary",
-                content="Workflow summary attachment content mentioning authentication",
-                source_path="projects/X/attachments/att88717085_workflow-summary.md",
-                doc_type="OTHER",
-                project_name="test-project",
-            )
-        )
-        corpus.add_document(
-            EvidenceDocument(
-                title="BRD - Authentication",
-                content="Users authenticate via JWT; interfaces are contract-based.",
-                source_path="projects/X/brd/auth.md",
-                doc_type="BRD",
-                project_name="test-project",
-            )
-        )
-
-        matcher = EvidenceMatcher(corpus, project_name="test-project")
-        context = matcher.match_row(
-            row_id="row_1",
-            requirement_text="Authentication and interface contract requirements",
-        )
-
-        assert len(context.matched_docs) > 0
-        assert all(
-            "audit-checklist" not in doc.source_path.lower() and "workflow-summary" not in doc.source_path.lower()
-            for doc in context.matched_docs
-        )