@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/test_engine/test_scorer.py

@@ -1,597 +0,0 @@
-"""Tests for audit scoring functionality.
-
-Tests score_section, score_repository, and Scorer class.
-"""
-
-from __future__ import annotations
-
-import pytest
-from vds_audit_orchestrator.engine.scorer import (
-    SEVERITY_FALLBACK_WEIGHTS,
-    Scorer,
-    SectionScore,
-    score_repository,
-    score_section,
-)
-from vds_audit_orchestrator.models.enums import (
-    CheckStatus,
-    Priority,
-    Severity,
-)
-from vds_audit_orchestrator.models.findings import (
-    CheckFinding,
-    compute_grade,
-)
-from vds_audit_orchestrator.models.template import (
-    AuditCheck,
-    AuditSection,
-    AuditTemplate,
-)
-
-
-class TestSeverityFallbackWeights:
-    """Tests for severity-based weight fallback."""
-
-    def test_severity_weights_defined(self) -> None:
-        """Test all severity levels have fallback weights."""
-        assert Severity.CRITICAL in SEVERITY_FALLBACK_WEIGHTS
-        assert Severity.HIGH in SEVERITY_FALLBACK_WEIGHTS
-        assert Severity.MEDIUM in SEVERITY_FALLBACK_WEIGHTS
-        assert Severity.LOW in SEVERITY_FALLBACK_WEIGHTS
-        assert Severity.INFO in SEVERITY_FALLBACK_WEIGHTS
-
-    def test_severity_weights_are_positive(self) -> None:
-        """Test all severity weights are positive."""
-        for severity, weight in SEVERITY_FALLBACK_WEIGHTS.items():
-            assert weight > 0, f"{severity} has non-positive weight: {weight}"
-
-    def test_critical_has_highest_weight(self) -> None:
-        """Test Critical severity has highest weight."""
-        critical_weight = SEVERITY_FALLBACK_WEIGHTS[Severity.CRITICAL]
-        for severity, weight in SEVERITY_FALLBACK_WEIGHTS.items():
-            if severity != Severity.CRITICAL:
-                assert critical_weight > weight, f"Critical weight ({critical_weight}) not > {severity} ({weight})"
-
-    def test_weight_ordering_makes_sense(self) -> None:
-        """Test weights decrease with lower severity."""
-        assert (
-            SEVERITY_FALLBACK_WEIGHTS[Severity.CRITICAL]
-            > SEVERITY_FALLBACK_WEIGHTS[Severity.HIGH]
-            > SEVERITY_FALLBACK_WEIGHTS[Severity.MEDIUM]
-            > SEVERITY_FALLBACK_WEIGHTS[Severity.LOW]
-        )
-
-
-class TestScoreSection:
-    """Tests for score_section function."""
-
-    def _make_finding(
-        self,
-        check_id: str,
-        status: CheckStatus,
-        score: float = 100.0,
-    ) -> CheckFinding:
-        """Helper to create a test finding."""
-        return CheckFinding(
-            check_id=check_id,
-            status=status,
-            score_percentage=score,
-            points_awarded=score,
-            max_points=100,
-            execution_time_ms=0,
-            message="test",
-            severity=Severity.MEDIUM,
-            priority=Priority.P3,
-            migration_effort="Medium",
-        )
-
-    def test_score_section_all_pass(self) -> None:
-        """Test scoring section with all passing checks."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0, max_points=100),
-            AuditCheck(id="c2", description="C2", weight=50.0, max_points=100),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            self._make_finding("c2", CheckStatus.PASS, 100.0),
-        ]
-
-        result = score_section(section, findings)
-        assert result.score_percent == 100.0
-        assert result.passed_checks == 2
-        assert result.failed_checks == 0
-        assert result.status == CheckStatus.PASS
-
-    def test_score_section_all_fail(self) -> None:
-        """Test scoring section with all failing checks."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.FAIL, 0.0),
-            self._make_finding("c2", CheckStatus.FAIL, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        assert result.score_percent == 0.0
-        assert result.passed_checks == 0
-        assert result.failed_checks == 2
-        assert result.status == CheckStatus.FAIL
-
-    def test_score_section_mixed_results(self) -> None:
-        """Test scoring section with mixed results."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            self._make_finding("c2", CheckStatus.FAIL, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        assert result.score_percent == 50.0
-        assert result.passed_checks == 1
-        assert result.failed_checks == 1
-
-    def test_score_section_with_na_checks(self) -> None:
-        """Test N/A checks are excluded from scoring."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            self._make_finding("c2", CheckStatus.NA, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        # Only c1 is applicable
-        assert result.score_percent == 100.0
-        assert result.na_checks == 1
-
-    def test_score_section_with_skipped_checks(self) -> None:
-        """Test skipped checks are excluded from scoring."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=100.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.SKIPPED, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        assert result.skipped_checks == 1
-
-    def test_score_section_with_error_checks(self) -> None:
-        """Test ERROR checks reduce coverage factor."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            self._make_finding("c2", CheckStatus.ERROR, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        # Coverage factor should be reduced
-        assert result.coverage_factor < 1.0
-        assert result.error_checks == 1
-
-    def test_score_section_missing_findings(self) -> None:
-        """Test missing findings are treated as ERROR."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            # c2 finding is missing
-        ]
-
-        result = score_section(section, findings)
-        assert "c2" in result.missing_findings
-        assert result.error_checks == 1
-
-    def test_score_section_empty(self) -> None:
-        """Test scoring section with no checks."""
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=[])
-        findings = []
-
-        result = score_section(section, findings)
-        assert result.total_checks == 0
-        assert result.score_percent == 0.0
-
-    def test_score_section_weighted_scoring(self) -> None:
-        """Test checks are scored by weight."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=75.0),
-            AuditCheck(id="c2", description="C2", weight=25.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 100.0),
-            self._make_finding("c2", CheckStatus.FAIL, 0.0),
-        ]
-
-        result = score_section(section, findings)
-        # c1 (75%) passed, c2 (25%) failed = 75%
-        assert result.score_percent == 75.0
-
-    def test_score_section_severity_fallback_weights(self) -> None:
-        """Test severity-based fallback weights when check.weight is None."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=None, severity=Severity.CRITICAL),
-            AuditCheck(id="c2", description="C2", weight=None, severity=Severity.LOW),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.FAIL, 0.0),
-            self._make_finding("c2", CheckStatus.PASS, 100.0),
-        ]
-
-        result = score_section(section, findings)
-        # Critical (10) failed, Low (2.5) passed
-        # Normalized: Critical 10/(10+2.5) = 0.8, Low 2.5/12.5 = 0.2
-        # Score = 0.8 * 0 + 0.2 * 100 = 20
-        assert 20.0 <= result.score_percent <= 25.0
-
-    def test_score_section_partial_scores(self) -> None:
-        """Test checks with partial scores."""
-        checks = [
-            AuditCheck(id="c1", description="C1", weight=50.0),
-            AuditCheck(id="c2", description="C2", weight=50.0),
-        ]
-        section = AuditSection(id="s1", name="Section", weight=100.0, checks=checks)
-        findings = [
-            self._make_finding("c1", CheckStatus.PASS, 50.0),  # Partial score
-            self._make_finding("c2", CheckStatus.PASS, 100.0),
-        ]
-
-        result = score_section(section, findings)
-        # (50% * 50% + 100% * 50%) = 75%
-        assert result.score_percent == 75.0
-
-
-class TestScoreRepository:
-    """Tests for score_repository function."""
-
-    def test_score_repository_single_section(self) -> None:
-        """Test scoring repository with single section."""
-        sections = [
-            AuditSection(
-                id="s1",
-                name="Section 1",
-                description="S1",
-                weight=100.0,
-                checks=[
-                    AuditCheck(id="c1", description="C1", weight=100.0),
-                ],
-            ),
-        ]
-        template = AuditTemplate(name="Test", sections=sections)
-
-        section_score = SectionScore(
-            section_id="s1",
-            score_percent=80.0,
-            weighted_score=80.0,
-            status=CheckStatus.PASS,
-            coverage_factor=1.0,
-            confidence_factor=1.0,
-            total_checks=1,
-            passed_checks=1,
-            failed_checks=0,
-            error_checks=0,
-            skipped_checks=0,
-            na_checks=0,
-        )
-
-        result = score_repository(template, [section_score])
-        assert result.total_score == 80.0
-        assert len(result.section_scores) == 1
-
-    def test_score_repository_multiple_sections(self) -> None:
-        """Test scoring repository with multiple sections."""
-        sections = [
-            AuditSection(id="s1", name="S1", description="S1", weight=50.0),
-            AuditSection(id="s2", name="S2", description="S2", weight=30.0),
-            AuditSection(id="s3", name="S3", description="S3", weight=20.0),
-        ]
-        template = AuditTemplate(name="Test", sections=sections)
-
-        section_scores = [
-            SectionScore(
-                section_id="s1",
-                score_percent=100.0,
-                weighted_score=50.0,
-                status=CheckStatus.PASS,
-                coverage_factor=1.0,
-                confidence_factor=1.0,
-                total_checks=1,
-                passed_checks=1,
-                failed_checks=0,
-                error_checks=0,
-                skipped_checks=0,
-                na_checks=0,
-            ),
-            SectionScore(
-                section_id="s2",
-                score_percent=50.0,
-                weighted_score=15.0,
-                status=CheckStatus.WARNING,
-                coverage_factor=1.0,
-                confidence_factor=1.0,
-                total_checks=1,
-                passed_checks=0,
-                failed_checks=1,
-                error_checks=0,
-                skipped_checks=0,
-                na_checks=0,
-            ),
-            SectionScore(
-                section_id="s3",
-                score_percent=0.0,
-                weighted_score=0.0,
-                status=CheckStatus.FAIL,
-                coverage_factor=1.0,
-                confidence_factor=1.0,
-                total_checks=1,
-                passed_checks=0,
-                failed_checks=1,
-                error_checks=0,
-                skipped_checks=0,
-                na_checks=0,
-            ),
-        ]
-
-        result = score_repository(template, section_scores)
-        # 100*0.5 + 50*0.3 + 0*0.2 = 50 + 15 + 0 = 65
-        assert result.total_score == 65.0
-
-    def test_score_repository_excludes_na_sections(self) -> None:
-        """Test NA sections are excluded from scoring."""
-        sections = [
-            AuditSection(id="s1", name="S1", description="S1", weight=50.0),
-            AuditSection(id="s2", name="S2", description="S2", weight=50.0),
-        ]
-        template = AuditTemplate(name="Test", sections=sections)
-
-        section_scores = [
-            SectionScore(
-                section_id="s1",
-                score_percent=100.0,
-                weighted_score=100.0,
-                status=CheckStatus.PASS,
-                coverage_factor=1.0,
-                confidence_factor=1.0,
-                total_checks=1,
-                passed_checks=1,
-                failed_checks=0,
-                error_checks=0,
-                skipped_checks=0,
-                na_checks=0,
-            ),
-            SectionScore(
-                section_id="s2",
-                score_percent=0.0,
-                weighted_score=0.0,
-                status=CheckStatus.NA,
-                coverage_factor=0.0,
-                confidence_factor=1.0,
-                total_checks=1,
-                passed_checks=0,
-                failed_checks=0,
-                error_checks=0,
-                skipped_checks=0,
-                na_checks=1,
-            ),
-        ]
-
-        result = score_repository(template, section_scores)
-        # Only s1 contributes, weight renormalized
-        assert result.total_score == 100.0
-
-
-class TestScorer:
-    """Tests for Scorer class."""
-
-    def test_scorer_calculate_complete_result(self) -> None:
-        """Test Scorer.calculate produces complete result."""
-        template = AuditTemplate(
-            name="Test",
-            sections=[
-                AuditSection(
-                    id="security",
-                    name="Security",
-                    description="Sec",
-                    weight=100.0,
-                    checks=[
-                        AuditCheck(
-                            id="sec-001",
-                            description="Branch protection",
-                            weight=100.0,
-                            severity=Severity.CRITICAL,
-                        ),
-                    ],
-                ),
-            ],
-        )
-
-        findings = {
-            "sec-001": CheckFinding(
-                check_id="sec-001",
-                check_name="Branch Protection",
-                status=CheckStatus.PASS,
-                score=100.0,
-                message="Passed",
-                severity=Severity.CRITICAL,
-                priority=Priority.P1,
-                migration_effort="Low",
-            ),
-        }
-
-        scorer = Scorer()
-        result = scorer.calculate(
-            template=template,
-            repo_name="test-repo",
-            findings=findings,
-            duration=60.0,
-        )
-
-        assert result.repo_name == "test-repo"
-        assert result.total_score == 100.0
-        assert result.total_findings == 0
-        assert len(result.sections) == 1
-
-    def test_scorer_handles_missing_findings(self) -> None:
-        """Test Scorer creates ERROR findings for missing checks."""
-        template = AuditTemplate(
-            name="Test",
-            sections=[
-                AuditSection(
-                    id="s1",
-                    name="S1",
-                    description="S1",
-                    weight=100.0,
-                    checks=[
-                        AuditCheck(id="c1", description="C1", weight=100.0),
-                    ],
-                ),
-            ],
-        )
-
-        scorer = Scorer()
-        result = scorer.calculate(
-            template=template,
-            repo_name="test-repo",
-            findings={},  # No findings provided
-            duration=10.0,
-        )
-
-        # Should create ERROR finding
-        assert result.sections[0].total_checks == 1
-        assert result.sections[0].error_checks == 1
-
-    def test_scorer_counts_issues_by_severity(self) -> None:
-        """Test Scorer counts issues by severity."""
-        template = AuditTemplate(
-            name="Test",
-            sections=[
-                AuditSection(
-                    id="s1",
-                    name="S1",
-                    description="S1",
-                    weight=100.0,
-                    checks=[
-                        AuditCheck(
-                            id="c1",
-                            description="C1",
-                            weight=33.33,
-                            severity=Severity.CRITICAL,
-                        ),
-                        AuditCheck(
-                            id="c2",
-                            description="C2",
-                            weight=33.33,
-                            severity=Severity.HIGH,
-                        ),
-                        AuditCheck(
-                            id="c3",
-                            description="C3",
-                            weight=33.34,
-                            severity=Severity.MEDIUM,
-                        ),
-                    ],
-                ),
-            ],
-        )
-
-        findings = {
-            "c1": CheckFinding(
-                check_id="c1",
-                check_name="C1",
-                status=CheckStatus.FAIL,
-                score=0.0,
-                message="Failed",
-                severity=Severity.CRITICAL,
-                priority=Priority.P1,
-                migration_effort="High",
-            ),
-            "c2": CheckFinding(
-                check_id="c2",
-                check_name="C2",
-                status=CheckStatus.FAIL,
-                score=0.0,
-                message="Failed",
-                severity=Severity.HIGH,
-                priority=Priority.P2,
-                migration_effort="Medium",
-            ),
-            "c3": CheckFinding(
-                check_id="c3",
-                check_name="C3",
-                status=CheckStatus.FAIL,
-                score=0.0,
-                message="Failed",
-                severity=Severity.MEDIUM,
-                priority=Priority.P3,
-                migration_effort="Low",
-            ),
-        }
-
-        scorer = Scorer()
-        result = scorer.calculate(
-            template=template,
-            repo_name="test-repo",
-            findings=findings,
-            duration=30.0,
-        )
-
-        assert result.critical_issues == 1
-        assert result.high_issues == 1
-        assert result.medium_issues == 1
-        assert result.low_issues == 0
-
-
-class TestComputeGrade:
-    """Tests for compute_grade function."""
-
-    @pytest.mark.parametrize(
-        ("score", "expected_grade"),
-        [
-            (100, "A"),
-            (95, "A"),
-            (90, "A"),
-            (89, "B"),
-            (85, "B"),
-            (80, "B"),
-            (79, "C"),
-            (75, "C"),
-            (70, "C"),
-            (69, "D"),
-            (65, "D"),
-            (60, "D"),
-            (59, "F"),
-            (50, "F"),
-            (0, "F"),
-        ],
-    )
-    def test_compute_grade_boundaries(self, score: float, expected_grade: str) -> None:
-        """Test grade computation at score boundaries."""
-        assert compute_grade(score) == expected_grade
-
-    def test_compute_grade_exact_boundaries(self) -> None:
-        """Test exact grade boundary values."""
-        assert compute_grade(90.0) == "A"
-        assert compute_grade(80.0) == "B"
-        assert compute_grade(70.0) == "C"
-        assert compute_grade(60.0) == "D"
-        assert compute_grade(59.99) == "F"