@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/integration/test_merge_ranking_p149.py

@@ -1,98 +0,0 @@
-"""Integration tests for P149-E merge row ranking.
-
-TSK-149.23: test_fail_preferred_over_na_in_row_merge — merge two runs where
-one has N/A and the other has FAIL for the same check; FAIL must survive.
-"""
-
-from __future__ import annotations
-
-from vds_audit_orchestrator.engine.result_merger import (
-    MergeLevel,
-    RunInput,
-    merge_cross_run_results,
-)
-
-
-def _make_row(
-    row_id: str,
-    row_index: int,
-    status: str,
-    score: float,
-    *,
-    weight: float = 10.0,
-) -> dict:
-    return {
-        "row_id": row_id,
-        "row_key": row_id,
-        "row_index": row_index,
-        "check_id": f"CL-{row_id}",
-        "status": status,
-        "score_percent": score,
-        "weight_adjusted": weight,
-        "finding": f"Finding for {status}",
-        "reason": f"Reason for {status}",
-    }
-
-
-class TestFailPreferredOverNaInMerge:
-    """TSK-149.23: FAIL must beat N/A in row-level cross-run merge (EQ-6 fix)."""
-
-    def test_fail_preferred_over_na_in_row_merge(self):
-        """Named regression test: run-A has N/A, run-B has FAIL — result must be FAIL."""
-        run_a_rows = [_make_row("001", 0, "N/A", 0.0)]
-        run_b_rows = [_make_row("001", 0, "FAIL", 0.0)]
-
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=run_a_rows),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=run_b_rows),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.ROW)
-        repo = result.repos["repo-1"]
-
-        assert len(repo.rows) == 1
-        merged_row = repo.rows[0]
-        assert merged_row.status == "FAIL", (
-            f"Expected FAIL to win over N/A, got {merged_row.status!r} (source run: {merged_row.source_run_id!r})"
-        )
-        assert merged_row.source_run_id == "run-B"
-
-    def test_na_does_not_override_fail_when_na_arrives_first(self):
-        """Order independence: N/A in run-A and FAIL in run-B — FAIL must still win."""
-        # Same as above but explicit about ordering
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=[_make_row("R1", 0, "N/A", 0.0)]),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=[_make_row("R1", 0, "FAIL", 0.0)]),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.ROW)
-        row = result.repos["repo-1"].rows[0]
-        assert row.status == "FAIL"
-
-    def test_pass_still_beats_fail(self):
-        """Sanity: PASS > FAIL ordering preserved after rank fix."""
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=[_make_row("R1", 0, "PASS", 90.0)]),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=[_make_row("R1", 0, "FAIL", 0.0)]),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.ROW)
-        row = result.repos["repo-1"].rows[0]
-        assert row.status == "PASS"
-
-    def test_multi_row_merge_fail_beats_na_selectively(self):
-        """Multi-row: per-row independence — FAIL/N/A decision is per row_id."""
-        run_a_rows = [
-            _make_row("R1", 0, "N/A", 0.0),  # run-A: N/A for R1
-            _make_row("R2", 1, "PASS", 95.0),  # run-A: PASS for R2
-        ]
-        run_b_rows = [
-            _make_row("R1", 0, "FAIL", 0.0),  # run-B: FAIL for R1
-            _make_row("R2", 1, "PARTIAL", 50.0),  # run-B: PARTIAL for R2
-        ]
-        inputs = [
-            RunInput(run_id="run-A", repo_key="repo-1", rows=run_a_rows),
-            RunInput(run_id="run-B", repo_key="repo-1", rows=run_b_rows),
-        ]
-        result = merge_cross_run_results(inputs, merge_level=MergeLevel.ROW)
-        rows_by_id = {r.row_id: r for r in result.repos["repo-1"].rows}
-
-        assert rows_by_id["R1"].status == "FAIL"  # FAIL beats N/A
-        assert rows_by_id["R2"].status == "PASS"  # PASS beats PARTIAL

package/tools/vds-scripts/audit_orchestrator/tests/integration/test_modality_mismatch_p149.py

@@ -1,171 +0,0 @@
-"""P149-B integration tests: modality allowlist / grounding alignment (TSK-149.11).
-
-Full row evaluation for a CL-016 row (docs finalization_policy) with mixed-modality
-LLM anchors. The grounding pass is mocked by including anchor refs in
-fallback_evidence_refs — anchors whose ref_value appears there are treated as
-grounding-verified by the P149-B recovery path.
-
-All 3 tests assert that final status ∈ {"PARTIAL", "FAIL"} (not "ERROR") when
-at least one anchor is grounding-verified, and that ERROR is preserved when none are.
-"""
-
-from __future__ import annotations
-
-from types import SimpleNamespace
-
-from vds_audit_orchestrator.engine.llm_row_evaluator import LLMRowEvaluator
-from vds_audit_orchestrator.llm.audit_schemas import RowEvaluationResponse
-from vds_audit_orchestrator.models.checklist import RowStatus
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Shared fixtures
-# ─────────────────────────────────────────────────────────────────────────────
-
-# Docs-modality ref that keeps allowed_anchor_set non-empty after the filter
-_DOCS_REF = "http://confluence.internal/finalization-policy-docs"
-
-# Code-modality refs the LLM cites (wrong modality for docs-primary finalization)
-_CODE_REFS = [
-    "src/main/java/com/vds/finalization/FinalizationPolicy.java",
-    "src/main/java/com/vds/finalization/PolicyConfig.java",
-    "src/main/java/com/vds/finalization/PolicyRegistry.java",
-    "src/main/java/com/vds/finalization/DefaultPolicy.java",
-    "src/main/java/com/vds/finalization/PolicyValidator.java",
-    "src/main/java/com/vds/finalization/PolicyLoader.java",
-    "src/main/java/com/vds/finalization/PolicyService.java",
-]
-
-# CL-016: finalization_policy docs check — requires docs-primary grounding
-_CL016_INTERP = {
-    "finalization_policy": "docs_primary",
-    "required_anchor_modalities": ["docs"],
-    "check_category": "dependency_management",
-}
-
-
-def _make_evaluator() -> LLMRowEvaluator:
-    return LLMRowEvaluator(
-        config=SimpleNamespace(
-            protocol=SimpleNamespace(value="anthropic"),
-            model="claude-haiku-4-5-20251001",
-            mode=SimpleNamespace(value="prompt_evaluator"),
-            use_dspy=False,
-        ),
-        template_hash="integration-tpl",
-        evidence_hash="integration-hash",
-    )
-
-
-def _run_cl016(
-    anchor_refs: list[str],
-    *,
-    grounding_refs: list[str],
-    score: float = 65.0,
-    confidence: float = 0.75,
-):
-    """Simulate CL-016 row evaluation with the given anchor setup.
-
-    fallback_evidence_refs = [_DOCS_REF] + grounding_refs
-    The docs ref ensures allowed_anchor_set is non-empty (one docs-modality ref).
-    grounding_refs are the refs "verified" by the mock grounding pass (present in fallback).
-    anchor_refs are what the LLM claims to cite.
-    """
-    evaluator = _make_evaluator()
-    response = RowEvaluationResponse(
-        reasoning=(
-            "The finalization policy is implemented in "
-            "src/main/java/com/vds/finalization/. "
-            "FinalizationPolicy.java defines the policy contract."
-        ),
-        reason="Finalization policy found in code — cited 7 source files.",
-        score=score,
-        confidence=confidence,
-        finding="Finalization policy implemented in Java package com.vds.finalization.",
-        evidence_anchors=[
-            {
-                "ref_type": "code_path",
-                "ref_value": ref,
-                "excerpt": "",
-                "excerpt_unavailable_reason": "no_excerpt",
-                "verified": False,
-            }
-            for ref in anchor_refs
-        ],
-    )
-    fallback_evidence_refs = [_DOCS_REF, *grounding_refs]
-    return evaluator._convert_response(
-        response=response,
-        check=SimpleNamespace(id="CL-016"),
-        row_id="CL-016-integration-test",
-        fallback_evidence_refs=fallback_evidence_refs,
-        requirement_interpretation=_CL016_INTERP,
-        evaluation_duration_ms=500,
-    )
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Test 1: All anchors grounding-verified → not ERROR (PARTIAL or FAIL)
-# This is the core regression case.
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-def test_cl016_all_anchors_grounding_verified_not_error() -> None:
-    """All 7 code anchors are grounding-verified via mock grounding pass.
-
-    Final status must NOT be ERROR — downgraded to PARTIAL or FAIL with penalty.
-    """
-    result = _run_cl016(
-        anchor_refs=_CODE_REFS,
-        grounding_refs=_CODE_REFS,  # mock: all anchors verified by grounding
-    )
-    assert result.status != RowStatus.ERROR, f"P149-B: all-verified anchors must not produce ERROR; got {result.status}"
-    assert result.status in {RowStatus.PARTIAL, RowStatus.FAIL}
-    # Mismatch trace must be recorded
-    assert result.retrieval_trace is not None
-    assert result.retrieval_trace.get("allowlist_grounding_mismatch", {}).get("detected") is True
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Test 2: No anchors grounding-verified → ERROR preserved (pre-fix behavior)
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-def test_cl016_no_grounding_verified_anchors_preserves_error() -> None:
-    """When mock grounding pass returns no verified anchors, ERROR is preserved.
-
-    This validates that the recovery path is only taken when grounding verification
-    actually succeeded — not as a blanket softening of the allowlist check.
-    """
-    result = _run_cl016(
-        anchor_refs=_CODE_REFS,
-        grounding_refs=[],  # mock: grounding pass found nothing → not verified
-    )
-    assert result.status == RowStatus.ERROR, (
-        f"No grounding-verified anchors must still produce ERROR; got {result.status}"
-    )
-    assert result.error_message == "anchor_allowlist_violation"
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# Test 3: Mixed — some grounding-verified, some not → not ERROR, partial recovery
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-def test_cl016_mixed_grounding_verified_not_error_partial_recovery() -> None:
-    """3/7 code anchors are grounding-verified; recovery path uses those 3.
-
-    Final status must be in {PARTIAL, FAIL} (not ERROR).
-    Evidence anchors in result must equal the grounding-verified subset (3).
-    """
-    grounding_verified_refs = _CODE_REFS[:3]
-    result = _run_cl016(
-        anchor_refs=_CODE_REFS,
-        grounding_refs=grounding_verified_refs,  # mock: only 3 are verified
-    )
-    assert result.status != RowStatus.ERROR, (
-        f"Partial grounding verification must not produce ERROR; got {result.status}"
-    )
-    assert result.status in {RowStatus.PARTIAL, RowStatus.FAIL}
-    assert len(result.evidence_anchors) == 3, (
-        f"Expected 3 evidence anchors (grounding-verified subset), got {len(result.evidence_anchors)}"
-    )

package/tools/vds-scripts/audit_orchestrator/tests/integration/test_phase117_118_storage.py

@@ -1,350 +0,0 @@
-"""Integration coverage for Phase 117/118 storage paths using Postgres testcontainer.
-
-These tests exercise the real Postgres-backed persistence contract for:
-
-- Phase 117: project evidence discoveries
-- Phase 118: project knowledge storage and TTL cleanup
-
-The shared fixtures in `tests/integration/conftest.py` provide a session-scoped
-Postgres instance backed by testcontainers (or an externally supplied DSN).
-"""
-
-from __future__ import annotations
-
-from datetime import UTC, datetime, timedelta
-from uuid import uuid4
-
-import pytest
-from vds_audit_orchestrator.engine.project_evidence_context import (
-    EvidenceDiscovery,
-    EvidenceDiscoveryType,
-    ProjectEvidenceContext,
-    ProjectEvidenceStore,
-    publish_evidence_discovery,
-    query_evidence_discoveries,
-)
-from vds_audit_orchestrator.engine.project_knowledge_query_service import (
-    ProjectKnowledgeQueryService,
-)
-from vds_audit_orchestrator.engine.project_knowledge_store import (
-    ProjectKnowledgeEntry,
-    ProjectKnowledgeStore,
-)
-from vds_audit_orchestrator.state.store import PostgresStateStore
-
-pytestmark = [pytest.mark.serial, pytest.mark.integration]
-
-
-def _unique_project_key() -> str:
-    return f"phase117-118-{uuid4().hex[:12]}"
-
-
-def _unique_thread_id() -> str:
-    return f"thread-{uuid4().hex[:12]}"
-
-
-def _unique_context_id() -> str:
-    return f"ctx-{uuid4().hex[:12]}"
-
-
-def test_project_evidence_round_trip_persists_and_excludes_self(
-    initialized_pg,
-    pg_dsn: str,
-    db_connection,
-) -> None:
-    store = PostgresStateStore(pg_dsn)
-    try:
-        project_key = _unique_project_key()
-        context = ProjectEvidenceContext.create(
-            project_key=project_key,
-            thread_id=_unique_thread_id(),
-            context_id=_unique_context_id(),
-        )
-
-        repo_a = publish_evidence_discovery(
-            context,
-            EvidenceDiscovery(
-                source_repo="repo-a",
-                discovery_type=EvidenceDiscoveryType.API_CONTRACT,
-                ref="services/a/openapi.yaml",
-                excerpt="GET /health returns 200",
-                confidence=0.82,
-                requirement_type_tag="api_contract",
-                metadata={"lane": "primary"},
-            ),
-            state_store=store,
-        )
-        repo_b = publish_evidence_discovery(
-            context,
-            EvidenceDiscovery(
-                source_repo="repo-b",
-                discovery_type=EvidenceDiscoveryType.API_CONTRACT,
-                ref="services/b/openapi.yaml",
-                excerpt="POST /claims creates a claim",
-                confidence=0.91,
-                requirement_type_tag="api_contract",
-                metadata={"lane": "secondary"},
-            ),
-            state_store=store,
-        )
-
-        assert repo_a.source_repo == "repo-a"
-        assert repo_b.source_repo == "repo-b"
-
-        visible_to_repo_a = query_evidence_discoveries(
-            context,
-            repo_key="repo-a",
-            state_store=store,
-            discovery_type=EvidenceDiscoveryType.API_CONTRACT,
-            requirement_type_tag="api_contract",
-            limit=10,
-        )
-        assert [item.source_repo for item in visible_to_repo_a] == ["repo-b"]
-        assert visible_to_repo_a[0].ref == "services/b/openapi.yaml"
-
-        visible_to_repo_c = query_evidence_discoveries(
-            context,
-            repo_key="repo-c",
-            state_store=store,
-            discovery_type=EvidenceDiscoveryType.API_CONTRACT,
-            requirement_type_tag="api_contract",
-            limit=10,
-        )
-        assert {item.source_repo for item in visible_to_repo_c} == {"repo-a", "repo-b"}
-
-        with db_connection.cursor() as cur:
-            cur.execute(
-                """
-                SELECT COUNT(*) AS count
-                FROM audit_state.project_evidence_discoveries
-                WHERE context_id = %s
-                """,
-                (context.context_id,),
-            )
-            row = cur.fetchone()
-
-        assert int(row[0]) == 2
-    finally:
-        store.close()
-
-
-def test_project_evidence_deduplicates_same_ref_and_merges_confidence_and_metadata(
-    initialized_pg,
-    pg_dsn: str,
-    db_connection,
-) -> None:
-    store = PostgresStateStore(pg_dsn)
-    try:
-        project_key = _unique_project_key()
-        context = ProjectEvidenceContext.create(
-            project_key=project_key,
-            thread_id=_unique_thread_id(),
-            context_id=_unique_context_id(),
-        )
-
-        publish_evidence_discovery(
-            context,
-            EvidenceDiscovery(
-                source_repo="repo-dup",
-                discovery_type=EvidenceDiscoveryType.ARCHITECTURE_BOUNDARY,
-                ref="src/architecture.md",
-                excerpt="Ports and adapters isolate domain from framework",
-                confidence=0.41,
-                requirement_type_tag="architecture",
-                metadata={"first_seen": True},
-            ),
-            state_store=store,
-        )
-        merged = publish_evidence_discovery(
-            context,
-            EvidenceDiscovery(
-                source_repo="repo-dup",
-                discovery_type=EvidenceDiscoveryType.ARCHITECTURE_BOUNDARY,
-                ref="src/architecture.md",
-                excerpt="Ports and adapters isolate domain from framework",
-                confidence=0.91,
-                requirement_type_tag="architecture",
-                metadata={"validated": "yes"},
-            ),
-            state_store=store,
-        )
-
-        assert merged.source_repo == "repo-dup"
-        assert merged.discovery_type is EvidenceDiscoveryType.ARCHITECTURE_BOUNDARY
-
-        with db_connection.cursor() as cur:
-            cur.execute(
-                """
-                SELECT confidence, metadata
-                FROM audit_state.project_evidence_discoveries
-                WHERE context_id = %s
-                  AND source_repo = %s
-                  AND ref = %s
-                """,
-                (context.context_id, "repo-dup", "src/architecture.md"),
-            )
-            rows = cur.fetchall()
-
-        assert len(rows) == 1
-        assert float(rows[0][0]) == pytest.approx(0.91)
-        metadata = dict(rows[0][1] or {})
-        assert metadata["first_seen"] is True
-        assert metadata["validated"] == "yes"
-        assert metadata["thread_id"] == context.thread_id
-        assert "published_at" in metadata
-    finally:
-        store.close()
-
-
-def test_project_evidence_auto_accumulates_knowledge_and_query_service_respects_scope(
-    initialized_pg,
-    pg_dsn: str,
-) -> None:
-    store = PostgresStateStore(pg_dsn)
-    try:
-        project_key = _unique_project_key()
-        context = ProjectEvidenceContext.create(
-            project_key=project_key,
-            thread_id=_unique_thread_id(),
-            context_id=_unique_context_id(),
-        )
-        knowledge_store = ProjectKnowledgeStore(state_store=store)
-        evidence_store = ProjectEvidenceStore(
-            state_store=store,
-            knowledge_store=knowledge_store,
-            knowledge_ttl_days=7,
-            confidence_threshold=0.8,
-        )
-
-        published = evidence_store.publish(
-            context,
-            EvidenceDiscovery(
-                source_repo="repo-a",
-                discovery_type=EvidenceDiscoveryType.ARCHITECTURE_BOUNDARY,
-                ref="repo-a/src/architecture.md",
-                excerpt="Inbound adapters call application services through ports",
-                confidence=0.93,
-                requirement_type_tag="architecture",
-                metadata={"source_section": "hexagonal"},
-            ),
-        )
-
-        project_level_entry = ProjectKnowledgeEntry.with_default_ttl(
-            project_key=project_key,
-            repo_key=None,
-            knowledge_type="architecture_pattern",
-            content="Project-wide architecture policy requires ports/adapters boundaries.",
-            confidence=0.88,
-            source_run_id=context.thread_id,
-            requirement_type_tag="architecture",
-            metadata={"scope": "project"},
-        )
-        knowledge_store.accumulate_knowledge(project_level_entry)
-
-        service = ProjectKnowledgeQueryService(state_store=store, ttl_days=30)
-        entries = service.query_knowledge(
-            project_key,
-            repo_key="repo-a",
-            knowledge_type="architecture_pattern",
-            requirement_type_tag="architecture",
-            execution_scope_class="cross_repo_code_required",
-            limit=10,
-        )
-
-        assert len(entries) == 2
-        by_repo = {entry.repo_key: entry for entry in entries}
-
-        auto_entry = by_repo["repo-a"]
-        assert auto_entry.content == published.excerpt
-        assert auto_entry.knowledge_type == "architecture_pattern"
-        assert auto_entry.execution_scope_class == "cross_repo_code_required"
-        assert auto_entry.source_context_id == context.context_id
-        assert auto_entry.metadata["ref"] == "repo-a/src/architecture.md"
-        assert auto_entry.metadata["discovery_type"] == "architecture_boundary"
-        assert auto_entry.metadata["source_section"] == "hexagonal"
-        assert auto_entry.expires_at is not None
-        assert auto_entry.expires_at > datetime.now(UTC)
-
-        project_entry = by_repo[None]
-        assert project_entry.metadata["scope"] == "project"
-        assert project_entry.execution_scope_class is None
-    finally:
-        store.close()
-
-
-def test_project_knowledge_cleanup_soft_retires_expired_rows(
-    initialized_pg,
-    pg_dsn: str,
-    db_connection,
-) -> None:
-    store = PostgresStateStore(pg_dsn)
-    try:
-        project_key = _unique_project_key()
-        knowledge_store = ProjectKnowledgeStore(state_store=store)
-        query_service = ProjectKnowledgeQueryService(state_store=store, ttl_days=30)
-
-        expired_entry = ProjectKnowledgeEntry.create(
-            project_key=project_key,
-            repo_key="repo-expired",
-            knowledge_type="dependency_relationship",
-            content="Expired dependency observation",
-            confidence=0.72,
-            source_run_id=_unique_thread_id(),
-            requirement_type_tag="dependency_relationship",
-            execution_scope_class="cross_repo_code_required",
-            metadata={"kind": "expired"},
-            expires_at=datetime.now(UTC) - timedelta(days=2),
-        )
-        active_entry = ProjectKnowledgeEntry.create(
-            project_key=project_key,
-            repo_key="repo-active",
-            knowledge_type="dependency_relationship",
-            content="Active dependency observation",
-            confidence=0.84,
-            source_run_id=_unique_thread_id(),
-            requirement_type_tag="dependency_relationship",
-            execution_scope_class="cross_repo_code_required",
-            metadata={"kind": "active"},
-            expires_at=datetime.now(UTC) + timedelta(days=10),
-        )
-
-        persisted_expired = knowledge_store.accumulate_knowledge(expired_entry)
-        persisted_active = knowledge_store.accumulate_knowledge(active_entry)
-
-        before_cleanup = query_service.query_knowledge(
-            project_key,
-            knowledge_type="dependency_relationship",
-            requirement_type_tag="dependency_relationship",
-            execution_scope_class="cross_repo_code_required",
-            limit=10,
-        )
-        assert [entry.id for entry in before_cleanup] == [persisted_active.id]
-
-        deleted = knowledge_store.cleanup_expired_knowledge(project_key)
-        assert deleted == 1
-
-        with db_connection.cursor() as cur:
-            cur.execute(
-                """
-                SELECT id, retired_at
-                FROM audit_state.project_knowledge
-                WHERE id IN (%s, %s)
-                ORDER BY id
-                """,
-                (persisted_expired.id, persisted_active.id),
-            )
-            rows = {row[0]: row for row in cur.fetchall()}
-
-        assert rows[persisted_expired.id][1] is not None
-        assert rows[persisted_active.id][1] is None
-
-        after_cleanup = query_service.query_knowledge(
-            project_key,
-            knowledge_type="dependency_relationship",
-            requirement_type_tag="dependency_relationship",
-            execution_scope_class="cross_repo_code_required",
-            limit=10,
-        )
-        assert [entry.id for entry in after_cleanup] == [persisted_active.id]
-    finally:
-        store.close()