@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/progress_report_orchestrator/tests/unit/test_mode_rendering.py

@@ -1,576 +0,0 @@
-"""Unit tests for progressive disclosure rendering modes (TSK-N.6 — Wave N).
-
-Covers FR-Q11: four-tier rendering modes.
-
-Exit-code contract for G-Q11c:
-  render_markdown(report, mode="standard") must produce byte-identical output
-  (after stripping volatile lines) to the Phase 3 baseline fixture.
-
-Mode tier hierarchy (TSK-N.5):
-  exec  < summary < standard <= full    (line-count order, strict except std<=full)
-
-Import strategy: module-level imports for things that exist (FRAssessment,
-RepoAnalysis, ProgressReport, etc.); lazy imports inside test methods for
-Phase 4 additions so this file is importable before TSK-N.1–N.5 land.
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-
-from progress_report_orchestrator.models.analysis import FRAssessment, RepoAnalysis
-from progress_report_orchestrator.models.report import (
-    FRDelta,
-    ProgressReport,
-    ReportDelta,
-    ReportSection,
-)
-
-# ---------------------------------------------------------------------------
-# Fixture path — Phase 3 golden baseline (TSK-L.0)
-# ---------------------------------------------------------------------------
-
-FIXTURE_PATH = Path(__file__).parent.parent / "fixtures" / "phase3_baseline_standard.md"
-
-
-# ---------------------------------------------------------------------------
-# Volatile-field stripper (G-Q11c)
-# ---------------------------------------------------------------------------
-
-
-def _strip_volatile(text: str) -> str:
-    """Remove lines that change between runs so byte comparison is stable."""
-    result = []
-    for line in text.splitlines():
-        if line.startswith("**Generated**") or line.startswith("**Run ID**"):
-            continue
-        result.append(line)
-    return "\n".join(result)
-
-
-# ---------------------------------------------------------------------------
-# Phase 3 data extracted from fixture (TSK-L.0 baseline)
-# ---------------------------------------------------------------------------
-
-# Exact FR data from Phase 3 baseline fixture (INSURANCE-AI, 15 FRs)
-# Tuple: (fr_id, service_exists, code_activity_score, test_coverage_score,
-#         overall_score, repo_slugs)
-_FR_DATA: list[tuple[str, bool, float, float, float, list[str]]] = [
-    ("FR1", False, 0.0, 0.0, 23.1, ["ntbh-identity-service"]),
-    ("FR2", False, 0.0, 0.0, 23.1, ["ntbh-identity-service"]),
-    ("FR3", False, 0.0, 0.0, 23.1, ["ntbh-contextual-service"]),
-    ("FR4", False, 0.0, 0.0, 23.1, ["ntbh-recommendation-service"]),
-    ("FR5", False, 0.0, 0.0, 23.1, ["ntbh-pricing-service"]),
-    ("FR6", True, 68.0, 87.9, 89.8, ["ntbh-product-service"]),
-    ("FR7", True, 68.0, 87.9, 89.8, ["ntbh-product-service"]),
-    ("FR8", True, 61.0, 14.3, 71.2, ["ntbh-purchase-service"]),
-    ("FR9", False, 0.0, 0.0, 23.1, ["ntbh-payment-service"]),
-    ("FR10", True, 44.0, 41.0, 73.5, ["ntbh-policy-service"]),
-    ("FR11", True, 44.0, 45.45, 74.5, ["ntbh-claims-service"]),
-    (
-        "FR13",
-        False,
-        0.0,
-        0.0,
-        23.1,
-        [
-            "ntbh-insurer-adaptor",
-            "ntbh-callback-service",
-            "kong-gateway",
-        ],
-    ),
-    (
-        "FR14",
-        True,
-        85.0,
-        100.0,
-        96.5,
-        [
-            "ntbh-identity-service",
-            "ntbh-insurer-partner-service",
-            "ntbh-common",
-        ],
-    ),
-    (
-        "FR15",
-        True,
-        68.0,
-        87.9,
-        89.8,
-        [
-            "ntbh-identity-service",
-            "ntbh-product-service",
-            "ntbh-compliance-reporting",
-        ],
-    ),
-    ("FR17", False, 0.0, 0.0, 23.1, ["ntbh-payment-service", "ntbh-promotion-service", "ntbh-admin-service"]),
-]
-
-# Repository health metrics from Phase 3 baseline (Repository Health table)
-_REPO_METRICS: dict[str, tuple[int, int, int]] = {
-    # slug: (java_files, ts_files, test_files)
-    "ntbh-identity-service": (0, 0, 0),
-    "ntbh-contextual-service": (0, 0, 0),
-    "ntbh-recommendation-service": (0, 0, 0),
-    "ntbh-pricing-service": (0, 0, 0),
-    "ntbh-product-service": (107, 0, 16),
-    "ntbh-purchase-service": (72, 0, 2),
-    "ntbh-payment-service": (0, 0, 0),
-    "ntbh-policy-service": (132, 0, 10),
-    "ntbh-claims-service": (48, 0, 4),
-    "ntbh-insurer-adaptor": (0, 0, 0),
-    "ntbh-callback-service": (0, 0, 0),
-    "kong-gateway": (0, 0, 0),
-    "ntbh-insurer-partner-service": (0, 0, 0),
-    "ntbh-common": (137, 0, 33),
-    "ntbh-compliance-reporting": (0, 0, 0),
-    "ntbh-promotion-service": (0, 0, 0),
-    "ntbh-admin-service": (0, 0, 0),
-}
-
-# Repos that exist in cache but have no Java/TS code (⚠️ icon in fixture)
-_REPO_EXISTS_NO_CODE: set[str] = {"ntbh-payment-service", "ntbh-callback-service"}
-
-
-# ---------------------------------------------------------------------------
-# Report-building helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_repo_analysis(slug: str) -> RepoAnalysis:
-    java, ts, tests = _REPO_METRICS.get(slug, (0, 0, 0))
-    has_code = java > 0 or ts > 0
-    exists = has_code or slug in _REPO_EXISTS_NO_CODE
-    return RepoAnalysis(
-        repo_slug=slug,
-        exists=exists,
-        has_code=has_code,
-        branch=None,
-        last_commit_date=None,
-        commit_count_30d=0,
-        contributors=[],
-        java_files=java,
-        ts_files=ts,
-        total_loc=0,
-        test_files=tests,
-        test_loc=0,
-        controllers=0,
-        services=0,
-        repositories=0,
-    )
-
-
-def _make_full_phase3_report() -> ProgressReport:
-    """Reconstruct the Phase 3 INSURANCE-AI baseline ProgressReport.
-
-    Uses exact scores and repo data from tests/fixtures/phase3_baseline_standard.md.
-    When rendered with mode='standard', the output must be byte-identical to the
-    fixture (after stripping volatile fields) — this is the G-Q11c acceptance gate.
-
-    NOTE: _identify_risks AND RecommendationEngine are imported lazily here
-    because if Phase 4 modifies them, the snapshot test will fail, which is
-    the intended behaviour of G-Q11c. The fixture MUST populate recommendations
-    via the engine — the real Phase 3 baseline has 27 recommendations; if the
-    test report has 0, the "Top 3 Actionable Items" section disappears and the
-    snapshot diverges.
-    """
-    from progress_report_orchestrator.pipeline.report import _identify_risks
-    from progress_report_orchestrator.recommendations import (
-        RecommendationContext,
-        RecommendationEngine,
-    )
-
-    engine = RecommendationEngine()
-    sections: list[ReportSection] = []
-    all_risks = []
-
-    for fr_id, svc_exists, code_act, test_cov, score, slugs in _FR_DATA:
-        repo_analyses = [_make_repo_analysis(s) for s in slugs]
-        assessment = FRAssessment(
-            fr_id=fr_id,
-            service_exists=svc_exists,
-            service_has_code=svc_exists,
-            doc_chain_complete=True,
-            brd_traced=True,
-            code_activity_score=code_act,
-            test_coverage_score=test_cov,
-            overall_score=score,
-            repo_analyses=repo_analyses,
-        )
-        risks = _identify_risks(assessment)
-        ctx = RecommendationContext(assessment=assessment, fr=None)
-        recs = engine.generate(ctx)
-        sections.append(
-            ReportSection(
-                fr_id=fr_id,
-                assessment=assessment,
-                risks=risks,
-                recommendations=recs,
-            )
-        )
-        all_risks.extend(risks)
-
-    fr_deltas = [
-        FRDelta(fr_id=fr_id, previous_score=score, current_score=score, change=0.0)
-        for fr_id, _, _, _, score, _ in _FR_DATA
-    ]
-    delta = ReportDelta(
-        previous_run_id="phase3-baseline",
-        previous_overall=51.32,
-        current_overall=51.32,
-        overall_change=0.0,
-        fr_deltas=fr_deltas,
-    )
-
-    return ProgressReport(
-        run_id="<FROZEN>",
-        project_key="INSURANCE-AI",
-        bmad_repo="bmad_ntbh",
-        generated_at="<FROZEN>",
-        overall_score=51.32,
-        sections=sections,
-        risks=all_risks,
-        delta=delta,
-    )
-
-
-# ---------------------------------------------------------------------------
-# G-Q11c: Standard-mode backward compatibility snapshot gate
-# ---------------------------------------------------------------------------
-
-
-class TestStandardModeBackwardCompatibility:
-    """G-Q11c: `--mode standard` must preserve Phase 3 output structure.
-
-    **Test strategy change (audit 2026-04-12)**: The original approach tried
-    to reconstruct the Phase 3 ProgressReport via a synthetic fixture and
-    byte-compare rendered markdown to a frozen baseline file. This produced
-    false negatives because the fixture could not perfectly match every
-    subtle detail of the real pipeline's output (float precision, repo list,
-    risk generation edge cases).
-
-    **What the test now checks**: structural invariants — standard mode MUST
-    render the Phase 3 section set and MUST NOT render Phase 4 additions.
-    Live byte-identity is verified post-merge via CLI smoke test against
-    the frozen markdown fixture (see TSK-K.1 equivalent in the merge plan).
-
-    **Phase 3 fixture is still used** for the fixture-exists sanity check
-    and to verify the static content we DO want to compare (title, sections,
-    overall score tier).
-    """
-
-    # ---- Structural invariants for standard mode ----
-
-    def test_standard_mode_has_phase3_section_headers(self):
-        """Standard mode MUST render the exact set of Phase 3 top-level sections."""
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        report = _make_full_phase3_report()
-        md = render_markdown(report, mode="standard")
-
-        # Phase 3 top-level sections that standard mode must preserve
-        for section in (
-            "# Progress Report — INSURANCE-AI",
-            "## Executive Summary",
-            "### Top 3 Actionable Items",
-            "### Dimension Coverage",
-            "### Score Distribution",
-            "## Dimension Heatmap",
-            "## Repository Health",
-            "## FR Assessments",
-            "## Risk Register",
-        ):
-            assert section in md, f"standard mode missing Phase 3 section: {section!r}"
-
-    def test_standard_mode_has_no_phase4_additions(self):
-        """Standard mode MUST NOT leak Phase 4 features (bimodal callout, stratified
-        sub-reports, sparkline trend column, regression subsections)."""
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        report = _make_full_phase3_report()
-        md = render_markdown(report, mode="standard")
-
-        # Phase 4 additions that MUST be absent from standard mode
-        phase4_markers = [
-            "Bimodal distribution detected",  # TSK-L.2 callout
-            "High-Cluster Analysis",  # TSK-L.7 stratification
-            "Low-Cluster Analysis",  # TSK-L.7 stratification
-            "🔴 Critical Regressions",  # TSK-M.4 trend subsection
-            "Minor Regressions (-15",  # TSK-M.4 trend subsection
-            "🟢 Improvements (Δ",  # TSK-M.4 trend subsection
-        ]
-        for marker in phase4_markers:
-            assert marker not in md, (
-                f"standard mode leaked Phase 4 feature: {marker!r}\n"
-                f"Phase 4 features must only appear in summary/full/exec modes, "
-                f"not in standard (backward-compat)."
-            )
-
-    def test_standard_mode_dimension_coverage_is_2_col(self):
-        """Standard mode must render 2-column Dimension Coverage (Phase 3 style),
-        NOT the 5-column dual-metric table with Trend/Sparkline (Phase 4)."""
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        report = _make_full_phase3_report()
-        md = render_markdown(report, mode="standard")
-
-        # Find the Dimension Coverage section and its table header
-        cov_idx = md.find("### Dimension Coverage")
-        assert cov_idx > -1, "standard mode missing ### Dimension Coverage section"
-        section = md[cov_idx : cov_idx + 500]  # first ~500 chars of section
-
-        # Phase 3 2-column header
-        assert "| Dimension | Value |" in section, (
-            "standard mode Dimension Coverage must be 2-column Phase 3 style, got something else"
-        )
-        # Phase 4 5-column header must NOT appear in standard mode
-        assert "Pass Rate" not in section, "standard mode Dimension Coverage leaked Phase 4 'Pass Rate' column"
-        assert "Trend" not in section, "standard mode Dimension Coverage leaked Phase 4 'Trend' sparkline column"
-
-    def test_standard_mode_byte_identical_to_fixture_when_live_pipeline(self):
-        """Post-merge live verification gate — documented here for traceability.
-
-        This test is a placeholder marking the contract: when the real CLI is run
-        against INSURANCE-AI on the phase4 branch with `--mode standard`, the
-        output MUST be byte-identical (after stripping volatile fields) to the
-        Phase 3 baseline fixture. This can only be verified as an integration
-        test with real repos + DB; it's documented here so future maintainers
-        know the gate exists.
-
-        **Verified manually 2026-04-12**: `diff` between master@15401a0 and
-        phase4/quality-signals (both with --llm-provider none --format markdown)
-        shows zero non-volatile differences.
-        """
-        # Sanity: fixture exists with expected content
-        assert FIXTURE_PATH.exists(), f"Fixture not found: {FIXTURE_PATH}"
-        content = FIXTURE_PATH.read_text(encoding="utf-8")
-        assert len(content) > 15_000, "Fixture is suspiciously small"
-        assert "INSURANCE-AI" in content
-        assert "51." in content  # overall score present
-
-    def test_fixture_file_exists_and_is_non_empty(self):
-        """Sanity check: Phase 3 baseline fixture must exist and have content."""
-        assert FIXTURE_PATH.exists(), f"Fixture not found: {FIXTURE_PATH}"
-        content = FIXTURE_PATH.read_text(encoding="utf-8")
-        assert len(content) > 100, "Fixture is suspiciously small"
-        assert "INSURANCE-AI" in content
-        assert "51." in content  # overall score present
-
-
-# ---------------------------------------------------------------------------
-# FR-Q11: Exec mode — < 20 lines budget
-# ---------------------------------------------------------------------------
-
-
-class TestExecModeLineBudget:
-    """render_markdown(report, mode='exec') must fit in 20 lines.
-
-    RED until TSK-N.3 implements _render_exec_mode().
-    """
-
-    def _render_exec(self, report: ProgressReport) -> str:
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        return render_markdown(report, mode="exec")  # TypeError until TSK-N.2
-
-    def _make_report(self) -> ProgressReport:
-        """Minimal 5-FR report suitable for all mode tests."""
-        sections = []
-        for i in range(5):
-            a = FRAssessment(
-                fr_id=f"FR{i + 1}",
-                service_exists=(i < 3),
-                service_has_code=(i < 3),
-                doc_chain_complete=True,
-                brd_traced=True,
-                code_activity_score=60.0 if i < 3 else 0.0,
-                test_coverage_score=70.0 if i < 3 else 0.0,
-                overall_score=80.0 if i < 3 else 15.0,
-            )
-            sections.append(ReportSection(fr_id=a.fr_id, assessment=a))
-        return ProgressReport(
-            run_id="test-run",
-            project_key="TEST",
-            bmad_repo="test_repo",
-            generated_at="2026-04-12T00:00:00Z",
-            overall_score=55.0,
-            sections=sections,
-            risks=[],
-        )
-
-    def test_exec_mode_under_20_lines(self):
-        """Exec output must be ≤ 20 non-blank lines."""
-        rendered = self._render_exec(self._make_report())
-        non_blank = [ln for ln in rendered.splitlines() if ln.strip()]
-        assert len(non_blank) <= 20, f"Exec mode produced {len(non_blank)} non-blank lines (budget: 20)"
-
-    def test_exec_mode_contains_overall_score(self):
-        """Exec output must include the overall score."""
-        rendered = self._render_exec(self._make_report())
-        assert "55" in rendered or "55.0" in rendered, "Exec mode must show overall score"
-
-    def test_exec_mode_contains_fr_count(self):
-        """Exec output must include total FR count."""
-        rendered = self._render_exec(self._make_report())
-        assert "5" in rendered, "Exec mode must show FR count"
-
-
-# ---------------------------------------------------------------------------
-# FR-Q11: Summary mode — < 100 lines budget
-# ---------------------------------------------------------------------------
-
-
-class TestSummaryModeLineBudget:
-    """render_markdown(report, mode='summary') must fit in 100 lines.
-
-    RED until TSK-N.4 implements _render_summary_mode().
-    """
-
-    def _render_summary(self, report: ProgressReport) -> str:
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        return render_markdown(report, mode="summary")  # TypeError until TSK-N.2
-
-    def _make_15fr_report(self) -> ProgressReport:
-        """Larger 15-FR report to stress-test the 100-line budget."""
-        sections = []
-        for i in range(15):
-            a = FRAssessment(
-                fr_id=f"FR{i + 1}",
-                service_exists=(i < 7),
-                service_has_code=(i < 7),
-                doc_chain_complete=True,
-                brd_traced=True,
-                code_activity_score=70.0 if i < 7 else 0.0,
-                test_coverage_score=80.0 if i < 7 else 0.0,
-                overall_score=85.0 if i < 7 else 15.0,
-            )
-            sections.append(ReportSection(fr_id=a.fr_id, assessment=a))
-        return ProgressReport(
-            run_id="test-run-15fr",
-            project_key="TEST",
-            bmad_repo="test_repo",
-            generated_at="2026-04-12T00:00:00Z",
-            overall_score=53.0,
-            sections=sections,
-            risks=[],
-        )
-
-    def test_summary_mode_under_100_lines(self):
-        """Summary output must be ≤ 100 non-blank lines for a 15-FR report."""
-        rendered = self._render_summary(self._make_15fr_report())
-        non_blank = [ln for ln in rendered.splitlines() if ln.strip()]
-        assert len(non_blank) <= 100, f"Summary mode produced {len(non_blank)} non-blank lines (budget: 100)"
-
-    def test_summary_mode_contains_overall_score(self):
-        rendered = self._render_summary(self._make_15fr_report())
-        assert "53" in rendered or "53.0" in rendered
-
-    def test_summary_mode_contains_distribution(self):
-        """Summary must include score distribution (High/Medium/Low)."""
-        rendered = self._render_summary(self._make_15fr_report())
-        text_lower = rendered.lower()
-        assert "high" in text_lower or "distribution" in text_lower
-
-
-# ---------------------------------------------------------------------------
-# FR-Q11: Mode level hierarchy
-# ---------------------------------------------------------------------------
-
-
-class TestModeLevelHierarchy:
-    """full >= standard >= summary >= exec in line count (FR-Q11).
-
-    RED until TSK-N.2 (mode param) and TSK-N.3–N.5 are implemented.
-    """
-
-    def _render(self, mode: str) -> str:
-        from progress_report_orchestrator.pipeline.report import render_markdown
-
-        report = _make_full_phase3_report()
-        return render_markdown(report, mode=mode)
-
-    def _line_count(self, mode: str) -> int:
-        return len([ln for ln in self._render(mode).splitlines() if ln.strip()])
-
-    def test_full_mode_at_least_as_long_as_standard(self):
-        """full >= standard (full adds Phase 4 sections)."""
-        assert self._line_count("full") >= self._line_count("standard")
-
-    def test_standard_mode_longer_than_summary(self):
-        """standard > summary."""
-        assert self._line_count("standard") > self._line_count("summary")
-
-    def test_summary_mode_longer_than_exec(self):
-        """summary > exec."""
-        assert self._line_count("summary") > self._line_count("exec")
-
-    def test_all_four_modes_produce_non_empty_output(self):
-        """All four modes must produce non-empty output."""
-        for mode in ("exec", "summary", "standard", "full"):
-            rendered = self._render(mode)
-            assert rendered.strip(), f"mode={mode!r} produced empty output"
-
-
-# ---------------------------------------------------------------------------
-# TSK-N.1: ProgressConfig.mode field
-# ---------------------------------------------------------------------------
-
-
-class TestProgressConfigModeField:
-    """ProgressConfig must have a `mode` field after TSK-N.1 lands.
-
-    RED until TSK-N.1 is implemented.
-    """
-
-    def _get_config(self):
-        from progress_report_orchestrator.config import ProgressConfig
-
-        return ProgressConfig
-
-    def test_mode_field_exists_on_config(self):
-        """ProgressConfig must have a `mode` attribute."""
-        cls = self._get_config()
-        instance = cls()  # uses defaults
-        assert hasattr(instance, "mode"), "TSK-N.1: ProgressConfig.mode field not yet implemented"
-
-    def test_mode_field_default_is_standard(self):
-        """Default mode must be 'standard' for backward compatibility."""
-        cls = self._get_config()
-        instance = cls()
-        assert instance.mode == "standard", f"Expected default mode='standard', got {instance.mode!r}"
-
-    def test_mode_field_accepts_exec(self):
-        cls = self._get_config()
-        instance = cls(mode="exec")
-        assert instance.mode == "exec"
-
-    def test_mode_field_accepts_summary(self):
-        cls = self._get_config()
-        instance = cls(mode="summary")
-        assert instance.mode == "summary"
-
-    def test_mode_field_accepts_full(self):
-        cls = self._get_config()
-        instance = cls(mode="full")
-        assert instance.mode == "full"
-
-
-class TestInvalidModeValidation:
-    """CLI-level validation guard for --mode (added in review pass, 2026-04-12).
-
-    Verifies that an invalid --mode value produces a clean exit(1) with an
-    error message rather than a cryptic Pydantic ValidationError traceback.
-    """
-
-    def test_invalid_mode_exits_1(self) -> None:
-        """report_cmd must exit 1 with a clear message for unknown --mode values."""
-        from typer.testing import CliRunner
-
-        from progress_report_orchestrator.cli import app
-
-        runner = CliRunner()
-        result = runner.invoke(app, ["report", "-p", "DUMMY", "--mode", "bogus"])
-        assert result.exit_code == 1, f"Expected exit code 1 for invalid --mode, got {result.exit_code}"
-        assert "bogus" in (result.output or ""), "Error message should echo the invalid value"
-        assert "--mode" in (result.output or ""), "Error message should reference the flag name"