@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/tests/test_result_merger_p149.py

@@ -1,347 +0,0 @@
-"""Tests for P149-E merge row ranking and run selection scoring.
-
-TSK-149.22: Unit tests for all pairwise status rank orderings.
-TSK-149.24: Unit tests for _weighted_run_score helper.
-TSK-149.25: Unit tests for weighted-based run selection.
-TSK-149.26-149.28: Unit tests for critique timeout cooldown.
-"""
-
-from __future__ import annotations
-
-import time
-
-import pytest
-from vds_audit_orchestrator.engine import result_merger
-from vds_audit_orchestrator.engine.result_merger import (
-    MergeLevel,
-    RunInput,
-    _is_better_row,
-    _weighted_run_score,
-    merge_cross_run_results,
-    reset_critique_timeout_state,
-)
-
-# ── Helpers ──────────────────────────────────────────────────────────
-
-
-def _row(status: str, score: float = 50.0, weight: float = 10.0) -> dict:
-    return {
-        "row_id": "R1",
-        "row_index": 0,
-        "status": status,
-        "score_percent": score,
-        "weight_adjusted": weight,
-    }
-
-
-def _make_full_row(
-    row_id: str,
-    row_index: int,
-    status: str,
-    score: float,
-    weight: float = 10.0,
-) -> dict:
-    return {
-        "row_id": row_id,
-        "row_index": row_index,
-        "status": status,
-        "score_percent": score,
-        "weight_adjusted": weight,
-    }
-
-
-# ── TSK-149.21 / TSK-149.22: Pairwise status rank orderings ──────────
-
-
-class TestStatusRankOrderings:
-    """TSK-149.22: All pairwise rank orderings via _is_better_row."""
-
-    def test_pass_beats_partial(self):
-        assert _is_better_row(_row("PASS"), _row("PARTIAL"))
-
-    def test_pass_beats_fail(self):
-        assert _is_better_row(_row("PASS"), _row("FAIL"))
-
-    def test_pass_beats_na(self):
-        assert _is_better_row(_row("PASS"), _row("N/A"))
-
-    def test_partial_beats_fail(self):
-        assert _is_better_row(_row("PARTIAL"), _row("FAIL"))
-
-    def test_partial_beats_na(self):
-        assert _is_better_row(_row("PARTIAL"), _row("N/A"))
-
-    def test_fail_beats_na(self):
-        """EQ-6 key fix: FAIL (rank 2) must beat N/A (rank 3)."""
-        assert _is_better_row(_row("FAIL"), _row("N/A"))
-
-    def test_fail_beats_error(self):
-        assert _is_better_row(_row("FAIL"), _row("ERROR"))
-
-    def test_na_beats_error(self):
-        assert _is_better_row(_row("N/A"), _row("ERROR"))
-
-    def test_na_does_not_beat_fail(self):
-        """Negative: N/A must NOT beat FAIL (was the bug in EQ-6)."""
-        assert not _is_better_row(_row("N/A"), _row("FAIL"))
-
-    def test_same_status_higher_score_wins(self):
-        assert _is_better_row(_row("PASS", score=90.0), _row("PASS", score=70.0))
-
-    def test_same_status_lower_score_loses(self):
-        assert not _is_better_row(_row("PASS", score=50.0), _row("PASS", score=80.0))
-
-    def test_status_rank_dict_values(self):
-        """TSK-149.21: Verify the rank dict directly."""
-        rank = result_merger._STATUS_RANK
-        assert rank["PASS"] == 0
-        assert rank["PARTIAL"] == 1
-        assert rank["FAIL"] == 2
-        assert rank["N/A"] == 3
-        assert rank["ERROR"] == 4
-        # FAIL must be numerically lower than N/A
-        assert rank["FAIL"] < rank["N/A"]
-
-
-# ── TSK-149.24: _weighted_run_score helper ────────────────────────────
-
-
-class TestWeightedRunScore:
-    def test_weighted_average_correct(self):
-        rows = [
-            {"status": "PASS", "score_percent": 80.0, "weight_adjusted": 2.0},
-            {"status": "FAIL", "score_percent": 20.0, "weight_adjusted": 8.0},
-        ]
-        score = _weighted_run_score(rows)
-        # (80*2 + 20*8) / (2+8) = (160 + 160) / 10 = 32.0
-        assert abs(score - 32.0) < 0.001
-
-    def test_fallback_simple_average_when_no_weights(self):
-        rows = [
-            {"status": "PASS", "score_percent": 60.0},
-            {"status": "FAIL", "score_percent": 40.0},
-        ]
-        score = _weighted_run_score(rows)
-        assert abs(score - 50.0) < 0.001
-
-    def test_all_na_returns_zero(self):
-        rows = [
-            {"status": "N/A", "score_percent": 0.0, "weight_adjusted": 5.0},
-            {"status": "NA", "score_percent": 0.0, "weight_adjusted": 5.0},
-        ]
-        assert _weighted_run_score(rows) == 0.0
-
-    def test_single_row(self):
-        rows = [{"status": "PASS", "score_percent": 75.0, "weight_adjusted": 3.0}]
-        assert abs(_weighted_run_score(rows) - 75.0) < 0.001
-
-    def test_na_excluded_from_weighted_average(self):
-        rows = [
-            {"status": "PASS", "score_percent": 80.0, "weight_adjusted": 10.0},
-            {"status": "N/A", "score_percent": 0.0, "weight_adjusted": 10.0},
-        ]
-        # Only PASS row should count
-        assert abs(_weighted_run_score(rows) - 80.0) < 0.001
-
-    def test_empty_rows_returns_zero(self):
-        assert _weighted_run_score([]) == 0.0
-
-
-# ── TSK-149.25: Run selection uses weighted score ─────────────────────
-
-
-class TestRunSelectionUsesWeightedScore:
-    """EQ-7: repo-level merge must pick the run with the higher weighted score."""
-
-    def test_run_selection_uses_weighted_score(self):
-        """Run-A has unweighted avg=60 but weighted=40; run-B has lower unweighted=50 but weighted=80.
-        Repo-level selection should pick run-B (higher weighted score).
-        """
-        run_a_rows = [
-            # High-weight row fails (FAIL, score=0), low-weight passes (PASS, score=100)
-            _make_full_row("R1", 0, "FAIL", 0.0, weight=9.0),
-            _make_full_row("R2", 1, "PASS", 100.0, weight=1.0),
-            # weighted avg = (0*9 + 100*1)/(9+1) = 10.0
-        ]
-        run_b_rows = [
-            # Both rows pass with decent score, moderate weights
-            _make_full_row("R1", 0, "PASS", 80.0, weight=5.0),
-            _make_full_row("R2", 1, "PASS", 80.0, weight=5.0),
-            # weighted avg = (80*5 + 80*5)/(5+5) = 80.0
-        ]
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=run_a_rows),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=run_b_rows),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.REPO)
-        repo = result.repos["repo-1"]
-        # run-B should win (higher weighted score)
-        assert repo.source_runs[0] == "run-B" or all(r.source_run_id == "run-B" for r in repo.rows)
-
-    def test_final_overall_score_from_selected_run(self):
-        """overall_score on merged result should reflect the selected run's data."""
-        run_a_rows = [_make_full_row("R1", 0, "FAIL", 0.0, weight=10.0)]
-        run_b_rows = [_make_full_row("R1", 0, "PASS", 90.0, weight=10.0)]
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=run_a_rows),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=run_b_rows),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.REPO)
-        repo = result.repos["repo-1"]
-        # Should select run-B with score=90
-        assert repo.overall_score > 50.0
-
-
-# ── TSK-149.26 / TSK-149.27 / TSK-149.28: Critique timeout cooldown ──
-
-
-class TestCritiqueTimeoutCooldown:
-    @pytest.fixture(autouse=True)
-    def _reset_state(self):
-        """TSK-149.28: reset_critique_timeout_state clears the module-level flag."""
-        reset_critique_timeout_state()
-        yield
-        reset_critique_timeout_state()
-
-    def test_default_cooldown_env(self):
-        """TSK-149.26: CRITIQUE_TIMEOUT_COOLDOWN defaults to 60 seconds."""
-        assert pytest.approx(60.0) == result_merger.CRITIQUE_TIMEOUT_COOLDOWN
-
-    def test_cooldown_variable_starts_at_zero(self):
-        """TSK-149.26: Fresh state means no cooldown active."""
-        assert result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL == 0.0
-
-    def test_reset_clears_timeout_state(self):
-        """TSK-149.28: reset_critique_timeout_state sets the flag back to 0."""
-        result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL = time.time() + 9999.0
-        reset_critique_timeout_state()
-        assert result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL == 0.0
-
-    def test_no_cooldown_after_reset(self):
-        """After reset, time.time() >= _UNTIL, so no cooldown is active."""
-        reset_critique_timeout_state()
-        active = time.time() < result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL
-        assert not active
-
-    def test_cooldown_active_when_until_in_future(self):
-        """Setting _UNTIL to future activates the cooldown gate."""
-        result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL = time.time() + 9999.0
-        active = time.time() < result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL
-        assert active
-
-    def test_cooldown_expires_after_time_passes(self):
-        """After cooldown period, gate reopens."""
-        # Set UNTIL to 1 second ago (already expired)
-        result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL = time.time() - 1.0
-        active = time.time() < result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL
-        assert not active
-
-
-# ── TSK-149.T2: Thread-safety of _CRITIQUE_TIMEOUT_POISONED_UNTIL ──────
-
-
-class TestCritiqueTimeoutThreadSafety:
-    """Verify that concurrent reads/writes to _CRITIQUE_TIMEOUT_POISONED_UNTIL
-    are protected by _critique_timeout_lock — no torn reads or lost writes."""
-
-    @pytest.fixture(autouse=True)
-    def _reset(self):
-        reset_critique_timeout_state()
-        yield
-        reset_critique_timeout_state()
-
-    def test_concurrent_reset_does_not_corrupt_state(self):
-        """10 threads all call reset_critique_timeout_state() concurrently.
-        The flag must be 0.0 (not some intermediate float) when all finish."""
-        import threading
-
-        n_threads = 10
-        barrier = threading.Barrier(n_threads)
-        errors: list[Exception] = []
-
-        def _worker() -> None:
-            try:
-                barrier.wait()  # start all threads at once
-                reset_critique_timeout_state()
-            except Exception as exc:
-                errors.append(exc)
-
-        threads = [threading.Thread(target=_worker, daemon=True) for _ in range(n_threads)]
-        for t in threads:
-            t.start()
-        for t in threads:
-            t.join(timeout=5.0)
-
-        assert not errors, f"Thread errors: {errors}"
-        assert result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL == 0.0
-
-    def test_concurrent_poison_then_reset_yields_consistent_value(self):
-        """5 threads set the timeout to a future value; 5 threads reset it.
-        Afterwards the value must be either 0.0 or a valid future timestamp —
-        never NaN, never a negative value, never a partial write."""
-        import threading
-
-        n_threads = 10
-        barrier = threading.Barrier(n_threads)
-        errors: list[Exception] = []
-
-        def _poisoner() -> None:
-            try:
-                barrier.wait()
-                result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL = (  # direct write via lock
-                    time.time() + result_merger.CRITIQUE_TIMEOUT_COOLDOWN
-                )
-            except Exception as exc:
-                errors.append(exc)
-
-        def _resetter() -> None:
-            try:
-                barrier.wait()
-                reset_critique_timeout_state()
-            except Exception as exc:
-                errors.append(exc)
-
-        threads = [threading.Thread(target=_poisoner if i < 5 else _resetter, daemon=True) for i in range(n_threads)]
-        for t in threads:
-            t.start()
-        for t in threads:
-            t.join(timeout=5.0)
-
-        assert not errors, f"Thread errors: {errors}"
-        val = result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL
-        # Must be either 0.0 (reset won) or a positive future/recent timestamp (poison won)
-        assert isinstance(val, float), f"Not a float: {val!r}"
-        assert val >= 0.0, f"Negative timestamp: {val}"
-        assert val == val, "NaN detected"  # NaN != NaN
-
-    def test_lock_object_is_a_threading_lock(self):
-        """Confirm the lock is a real threading.Lock (or RLock) instance."""
-        import threading
-
-        lock = result_merger._critique_timeout_lock
-        assert isinstance(lock, type(threading.Lock())), f"Expected Lock, got {type(lock)}"
-
-    def test_poisoned_state_visible_across_threads(self):
-        """After one thread sets the poison flag, another thread must observe it."""
-        import threading
-
-        observed: list[float] = []
-        event = threading.Event()
-
-        def _setter() -> None:
-            result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL = time.time() + 9999.0
-            event.set()
-
-        def _reader() -> None:
-            event.wait(timeout=2.0)
-            observed.append(result_merger._CRITIQUE_TIMEOUT_POISONED_UNTIL)
-
-        setter = threading.Thread(target=_setter, daemon=True)
-        reader = threading.Thread(target=_reader, daemon=True)
-        reader.start()
-        setter.start()
-        setter.join(timeout=2.0)
-        reader.join(timeout=2.0)
-
-        assert observed, "Reader thread produced no result"
-        assert observed[0] > time.time(), f"Stale value observed: {observed[0]}"

package/tools/vds-scripts/audit_orchestrator/tests/test_route_mode_p149.py

@@ -1,178 +0,0 @@
-"""P149-D unit tests: route mode final clamp guard for docs modality.
-
-TSK-149.18: docs in required_modalities → docs_target survives the final clamp.
-TSK-149.19: decoupling_category_code_only_override absent in ambiguity_reason_codes
-            when hybrid route is preserved; present when code_only fires.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import vds_audit_orchestrator.engine.tool_first_planner as tfp_module
-from vds_audit_orchestrator.engine.requirement_analysis import RequirementAnalysisResult
-from vds_audit_orchestrator.engine.tool_first_planner import ToolFirstPlanner
-
-if TYPE_CHECKING:
-    import pytest
-
-
-def _make_heuristic(
-    *,
-    required_anchor_modalities: list[str],
-    analysis_reason_codes: list[str],
-    analysis_mode: str = "hybrid_balanced",
-    finalization_policy: str = "hybrid_required",
-) -> RequirementAnalysisResult:
-    """Build a RequirementAnalysisResult for use in route decision tests."""
-    return RequirementAnalysisResult(
-        analysis_mode=analysis_mode,
-        required_anchor_modalities=required_anchor_modalities,
-        project_scope_required=False,
-        execution_scope_class="repo_local",
-        finalization_policy=finalization_policy,
-        heuristic_confidence=0.85,
-        classification_source="heuristic",
-        content_fingerprint="fp-test",
-        classification_fingerprint="cfp-test",
-        analysis_reason_codes=analysis_reason_codes,
-    )
-
-
-def _decoupling_context(evidence_targets: list[str] | None = None) -> dict:
-    """Minimal context for a decoupling-category row."""
-    return {
-        "requirement_interpretation": {
-            "control_objective": "1. Decoupling",
-            "evidence_targets": evidence_targets or ["docs", "code"],
-            "code_targets": ["src/main/java/com/example/ArchConfig.java"],
-            "detailed_guidance": "",
-        },
-    }
-
-
-# ---------------------------------------------------------------------------
-# TSK-149.18: docs required → docs_target=True survives final clamp
-# ---------------------------------------------------------------------------
-
-
-def test_decoupling_docs_requirement_survives_final_clamp(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Regression: decoupling row with required_anchor_modalities=['docs'] and
-    'mixed_modalities_required' absent from reason codes → docs_target must be True.
-    Pre-fix: final clamp overwrote this to False.
-    """
-    heuristic = _make_heuristic(
-        required_anchor_modalities=["docs"],
-        analysis_reason_codes=["docs_required_by_policy"],  # no mixed_modalities_required
-    )
-    monkeypatch.setattr(tfp_module, "analyze_requirement_intent", lambda **_: heuristic)
-
-    decision = ToolFirstPlanner._resolve_route_decision(
-        context=_decoupling_context(),
-        requirement_text="Shared library usage must be documented in architecture docs.",
-        requirement_category="1. Decoupling",
-    )
-
-    assert decision.docs_target is True, (
-        f"docs_target should be True when docs in required_modalities, got route_mode={decision.route_mode!r}"
-    )
-
-
-def test_decoupling_neither_docs_nor_mixed_modalities_gives_code_only(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """When neither 'docs' in required_modalities nor 'mixed_modalities_required'
-    in reason codes → final clamp applies → code_only route.
-    """
-    heuristic = _make_heuristic(
-        required_anchor_modalities=["code"],
-        analysis_reason_codes=["code_substantive_required"],
-    )
-    monkeypatch.setattr(tfp_module, "analyze_requirement_intent", lambda **_: heuristic)
-
-    decision = ToolFirstPlanner._resolve_route_decision(
-        context=_decoupling_context(evidence_targets=["code"]),
-        requirement_text="Check hexagonal architecture compliance via source code.",
-        requirement_category="1. Decoupling",
-    )
-
-    assert decision.docs_target is False
-    assert decision.code_target is True
-
-
-def test_decoupling_both_mixed_and_docs_signals_preserve_hybrid(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """When both 'mixed_modalities_required' in reason codes AND 'docs' in
-    required_modalities, the hybrid route is preserved.
-    """
-    heuristic = _make_heuristic(
-        required_anchor_modalities=["docs", "code"],
-        analysis_reason_codes=["mixed_modalities_required"],
-    )
-    monkeypatch.setattr(tfp_module, "analyze_requirement_intent", lambda **_: heuristic)
-
-    decision = ToolFirstPlanner._resolve_route_decision(
-        context=_decoupling_context(),
-        requirement_text="Shared library compliance: check code AND docs.",
-        requirement_category="1. Decoupling",
-    )
-
-    assert decision.docs_target is True
-
-
-# ---------------------------------------------------------------------------
-# TSK-149.19: log event absence/presence in ambiguity_reason_codes
-# ---------------------------------------------------------------------------
-
-CLAMP_OVERRIDE_CODE = "decoupling_category_code_only_override"
-
-
-def test_clamp_override_code_absent_when_hybrid_preserved(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """When hybrid is preserved (docs in required_modalities), the
-    'decoupling_category_code_only_override' reason code must NOT be emitted.
-    """
-    heuristic = _make_heuristic(
-        required_anchor_modalities=["docs"],
-        analysis_reason_codes=["docs_required_by_policy"],
-    )
-    monkeypatch.setattr(tfp_module, "analyze_requirement_intent", lambda **_: heuristic)
-
-    decision = ToolFirstPlanner._resolve_route_decision(
-        context=_decoupling_context(),
-        requirement_text="Architecture docs check.",
-        requirement_category="1. Decoupling",
-    )
-
-    assert CLAMP_OVERRIDE_CODE not in decision.ambiguity_reason_codes, (
-        f"Override code should not be present for hybrid route, "
-        f"got ambiguity_reason_codes={decision.ambiguity_reason_codes!r}"
-    )
-
-
-def test_clamp_override_code_present_for_code_only_route(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """When code_only clamp fires (no docs requirement, no mixed_modalities_required),
-    'decoupling_category_code_only_override' must be present in ambiguity_reason_codes.
-    """
-    heuristic = _make_heuristic(
-        required_anchor_modalities=["code"],
-        analysis_reason_codes=["code_substantive_required"],
-    )
-    monkeypatch.setattr(tfp_module, "analyze_requirement_intent", lambda **_: heuristic)
-
-    decision = ToolFirstPlanner._resolve_route_decision(
-        context=_decoupling_context(evidence_targets=["code"]),
-        requirement_text="Check circular dependency via AST.",
-        requirement_category="1. Decoupling",
-    )
-
-    assert CLAMP_OVERRIDE_CODE in decision.ambiguity_reason_codes, (
-        f"Override code should be present for code_only route, "
-        f"got ambiguity_reason_codes={decision.ambiguity_reason_codes!r}"
-    )