@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_resume_manifest_p167b.py

@@ -1,268 +0,0 @@
-"""Unit tests for P167-B: Resume/manifest/lane identity hardening (FR-167.5).
-
-Verifies:
-- _run_matches_dispatch_job: dispatch_job_id is primary match (profile-agnostic).
-- _run_matches_dispatch_job: late-bound jobs (no executed_profile) reject legacy fallback.
-- _run_matches_dispatch_job: non-late-bound jobs still resume via lane_thread_id fallback.
-- _dispatch_job_identity_payload: late-bound jobs produce lane_thread_id=None.
-- lane_thread_ids list is cleared when all jobs are late-bound.
-"""
-
-from __future__ import annotations
-
-from vds_audit_orchestrator import workflow_cmds
-
-# ---------------------------------------------------------------------------
-# Helper
-# ---------------------------------------------------------------------------
-
-
-def _make_identity(
-    *,
-    thread_id: str = "phase167",
-    repo_key: str = "test-repo",
-    profile: str | None = "profile-a",
-    origin_profile: str | None = None,
-    stage: str = "default",
-    extra_target: str | None = None,
-    row_shard_index: int | None = None,
-    row_shard_count: int | None = None,
-) -> dict:
-    job: dict = {
-        "repo_key": repo_key,
-        "repo_path": f"/tmp/{repo_key}",
-        "stage": stage,
-    }
-    if profile is not None:
-        job["profile"] = profile
-    if origin_profile is not None:
-        job["origin_profile"] = origin_profile
-    if extra_target is not None:
-        job["extra_target"] = extra_target
-    if row_shard_index is not None:
-        job["row_shard_index"] = row_shard_index
-    if row_shard_count is not None:
-        job["row_shard_count"] = row_shard_count
-    return workflow_cmds._dispatch_job_identity_payload(thread_id=thread_id, job=job)
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.3 — dispatch_job_id primary match (profile-agnostic)
-# ---------------------------------------------------------------------------
-
-
-class TestRunMatchesDispatchJobPrimaryMatch:
-    """FR-167.5: dispatch_job_id is the authoritative primary key for resume matching."""
-
-    def test_dispatch_job_id_match_returns_true(self) -> None:
-        """Primary path: run payload has dispatch_job_id that matches identity."""
-        identity = _make_identity(profile="profile-a")
-        run_row = {
-            "payload": {
-                "dispatch_job_id": identity["dispatch_job_id"],
-                "thread_id": "should-not-matter",
-            }
-        }
-        assert workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-    def test_dispatch_job_id_mismatch_returns_false(self) -> None:
-        identity = _make_identity(profile="profile-a")
-        run_row = {
-            "payload": {
-                "dispatch_job_id": "0000000000000000",  # wrong hash
-            }
-        }
-        assert not workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-    def test_dispatch_job_id_in_run_parameters_also_matches(self) -> None:
-        """dispatch_job_id may be nested in run_parameters."""
-        identity = _make_identity(profile="profile-b")
-        run_row = {
-            "payload": {
-                "thread_id": "wrong-thread-id",
-                "run_parameters": {"dispatch_job_id": identity["dispatch_job_id"]},
-            }
-        }
-        assert workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-    def test_profile_change_same_dispatch_job_id_matches(self) -> None:
-        """After profile failover, the dispatch_job_id is unchanged — resume must succeed."""
-        identity_original = _make_identity(profile="zai-openai")
-        identity_retry = _make_identity(profile="claude-sonnet")
-        # Both identities have the same dispatch_job_id (P167-A guarantee)
-        assert identity_original["dispatch_job_id"] == identity_retry["dispatch_job_id"]
-
-        # A run from the original profile should be found by the retry identity
-        run_row = {
-            "payload": {
-                "dispatch_job_id": identity_original["dispatch_job_id"],
-                "executed_profile": "zai-openai",
-            }
-        }
-        assert workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity_retry)
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.3 — late-bound jobs rejected by legacy fallback path
-# ---------------------------------------------------------------------------
-
-
-class TestRunMatchesDispatchJobLatebound:
-    """FR-167.5: Late-bound jobs (no executed_profile) must not match via legacy fallback."""
-
-    def test_late_bound_job_no_dispatch_job_id_returns_false(self) -> None:
-        """Late-bound job identity: executed_profile is empty → legacy fallback blocked."""
-        # Simulate a late-bound job identity (profile=None before submit)
-        late_bound_identity = _make_identity(profile=None)
-        # Run payload with matching lane_thread_id but NO dispatch_job_id
-        run_row = {
-            "payload": {
-                "thread_id": late_bound_identity["lane_thread_id"] or "phase167-",
-                "origin_profile": None,
-                "executed_profile": None,
-                "dispatch_stage": "default",
-            }
-        }
-        assert not workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=late_bound_identity)
-
-    def test_late_bound_job_with_matching_dispatch_job_id_returns_true(self) -> None:
-        """Late-bound job: dispatch_job_id match still works even without executed_profile."""
-        late_bound_identity = _make_identity(profile=None)
-        run_row = {
-            "payload": {
-                "dispatch_job_id": late_bound_identity["dispatch_job_id"],
-            }
-        }
-        assert workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=late_bound_identity)
-
-    def test_late_bound_job_prevents_false_positive_match(self) -> None:
-        """
-        Guard against false positive: an unrelated run with matching thread_id
-        must not be claimed as a match for a late-bound job.
-        """
-        late_bound_identity = workflow_cmds._dispatch_job_identity_payload(
-            thread_id="phase167-test",
-            job={"repo_key": "billing", "stage": "row_sharded", "row_shard_index": 0},
-        )
-        # A different repo's run that happened to produce a similar thread_id
-        # (no dispatch_job_id in run — legacy run) should NOT match
-        unrelated_run = {
-            "payload": {
-                "thread_id": "phase167-test-",  # malformed but similar
-                "origin_profile": None,
-                "executed_profile": None,
-                "dispatch_stage": "row_sharded",
-            }
-        }
-        assert not workflow_cmds._run_matches_dispatch_job(run_row=unrelated_run, job_identity=late_bound_identity)
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.3 — non-late-bound jobs still use legacy lane_thread_id fallback
-# ---------------------------------------------------------------------------
-
-
-class TestRunMatchesDispatchJobLegacyFallback:
-    """Non-late-bound jobs still resume correctly via lane_thread_id + profile fallback."""
-
-    def test_non_late_bound_legacy_match_succeeds(self) -> None:
-        """Standard row-sharded job: legacy fallback works when no dispatch_job_id."""
-        identity = _make_identity(
-            profile="profile-a",
-            origin_profile="profile-a",
-            row_shard_index=1,
-            row_shard_count=3,
-        )
-        run_row = {
-            "payload": {
-                "thread_id": "phase167-profile-a-shard-1",
-                "origin_profile": "profile-a",
-                "executed_profile": "profile-a",
-                "dispatch_stage": "default",
-            }
-        }
-        assert workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-    def test_non_late_bound_wrong_profile_legacy_mismatch(self) -> None:
-        """Legacy fallback: different origin_profile → no match for non-late-bound jobs."""
-        identity = _make_identity(
-            profile="profile-a",
-            origin_profile="profile-a",
-        )
-        run_row = {
-            "payload": {
-                "thread_id": "phase167-profile-a",
-                "origin_profile": "profile-b",  # different origin
-                "executed_profile": "profile-a",
-                "dispatch_stage": "default",
-            }
-        }
-        assert not workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-    def test_non_late_bound_wrong_thread_id_mismatch(self) -> None:
-        """Legacy fallback: different thread_id → no match."""
-        identity = _make_identity(profile="profile-a")
-        run_row = {
-            "payload": {
-                "thread_id": "completely-different-thread",
-                "origin_profile": "profile-a",
-                "executed_profile": "profile-a",
-                "dispatch_stage": "default",
-            }
-        }
-        assert not workflow_cmds._run_matches_dispatch_job(run_row=run_row, job_identity=identity)
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.4 — manifest: lane_thread_id=None for late-bound jobs
-# ---------------------------------------------------------------------------
-
-
-class TestDispatchJobIdentityLaneThreadIdLatebound:
-    """FR-167.5 (TSK-167.4): Late-bound jobs have lane_thread_id=None in manifest."""
-
-    def test_late_bound_job_has_null_lane_thread_id(self) -> None:
-        """profile=None (late-bound) → lane_thread_id is None in the identity payload."""
-        identity = workflow_cmds._dispatch_job_identity_payload(
-            thread_id="phase167",
-            job={"repo_key": "billing", "stage": "row_sharded", "row_shard_index": 0},
-        )
-        assert identity["lane_thread_id"] is None
-
-    def test_explicit_profile_job_has_non_null_lane_thread_id(self) -> None:
-        """Non-late-bound jobs still get a profile-derived lane_thread_id."""
-        identity = _make_identity(profile="profile-a", row_shard_index=0, row_shard_count=2)
-        assert identity["lane_thread_id"] == "phase167-profile-a-shard-0"
-
-    def test_late_bound_job_with_shard_index_still_has_null_lane_thread_id(self) -> None:
-        """Even with row_shard_index, a null profile produces null lane_thread_id."""
-        identity = workflow_cmds._dispatch_job_identity_payload(
-            thread_id="phase167",
-            job={
-                "repo_key": "billing",
-                "stage": "row_sharded",
-                "row_shard_index": 2,
-                "row_shard_count": 5,
-                # no "profile" key at all → late-bound
-            },
-        )
-        assert identity["lane_thread_id"] is None
-
-    def test_late_bound_has_no_trailing_dash_artifact(self) -> None:
-        """Regression: previously profile=None would create lane_thread_id='phase167-'."""
-        identity = workflow_cmds._dispatch_job_identity_payload(
-            thread_id="phase167-test",
-            job={"repo_key": "billing"},
-        )
-        # Must be None, NOT "phase167-test-" (trailing dash artifact)
-        assert identity["lane_thread_id"] is None
-        assert identity["lane_thread_id"] != "phase167-test-"
-
-    def test_late_bound_dispatch_job_id_is_still_computable(self) -> None:
-        """dispatch_job_id must still be valid even when lane_thread_id is None."""
-        identity = workflow_cmds._dispatch_job_identity_payload(
-            thread_id="phase167",
-            job={"repo_key": "billing"},
-        )
-        assert identity["dispatch_job_id"]
-        assert len(identity["dispatch_job_id"]) == 16

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_risk_mitigations_p160e1.py

@@ -1,348 +0,0 @@
-"""TSK-160.27 / TSK-160.28: Risk mitigation tests for P160-E1.
-
-Covers:
-- TSK-160.27: Row failure counter skips remaining jobs for same row after N failures
-- TSK-160.28: Resume with >100 completed runs per repo correctly identifies all
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any
-from unittest.mock import MagicMock, patch
-
-from vds_audit_orchestrator.workflow_cmds import (
-    _build_row_sharded_jobs,
-    _query_completed_dispatch_runs,
-)
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_repos(tmp_path: Path, names: list[str]) -> list[tuple[str, Path]]:
-    repos = []
-    for name in names:
-        p = tmp_path / name
-        p.mkdir(parents=True, exist_ok=True)
-        repos.append((name, p))
-    return repos
-
-
-# ---------------------------------------------------------------------------
-# TSK-160.27: Row failure counter skips remaining jobs after N failures
-# ---------------------------------------------------------------------------
-
-
-class TestRowFailureCounter:
-    """Verify the per-row failure counter logic matches dispatch loop behavior."""
-
-    @staticmethod
-    def _simulate_failure_counting(
-        pending_jobs: list[dict[str, Any]],
-        failure_sequence: list[dict[str, str]],
-        threshold: int = 3,
-    ) -> tuple[dict[str, int], list[dict[str, Any]], list[str]]:
-        """Simulate the dispatch loop failure counting logic.
-
-        Args:
-            pending_jobs: Mutable list of pending jobs (will be modified).
-            failure_sequence: List of {"extra_target": "CL-XXX", "status": "error|timeout"}.
-            threshold: Failure threshold before eviction.
-
-        Returns:
-            (row_failure_counts, remaining_pending_jobs, evicted_rows)
-        """
-        row_failure_counts: dict[str, int] = {}
-        evicted_rows: list[str] = []
-
-        for result in failure_sequence:
-            failed_row = str(result.get("extra_target") or "").strip()
-            if not failed_row:
-                continue
-            row_failure_counts[failed_row] = row_failure_counts.get(failed_row, 0) + 1
-            if row_failure_counts[failed_row] >= threshold:
-                evicted = [pj for pj in pending_jobs if str(pj.get("extra_target") or "").strip() == failed_row]
-                if evicted:
-                    for ej in evicted:
-                        pending_jobs.remove(ej)
-                    evicted_rows.append(failed_row)
-
-        return row_failure_counts, pending_jobs, evicted_rows
-
-    def test_row_evicted_after_three_failures(self, tmp_path: Path) -> None:
-        repos = _make_repos(tmp_path, ["repo-a"])
-        row_keys = ["CL-001", "CL-002", "CL-003"]
-        repo_row_keys = {"repo-a": row_keys}
-
-        jobs, _assignment, _idle = _build_row_sharded_jobs(
-            profiles=["p1", "p2", "p3"],
-            repos=repos,
-            repo_row_keys=repo_row_keys,
-            rows_per_shard=1,
-        )
-
-        # CL-002 fails 3 times across different profiles
-        failures = [
-            {"extra_target": "CL-002", "status": "error"},
-            {"extra_target": "CL-002", "status": "timeout"},
-            {"extra_target": "CL-002", "status": "error"},
-        ]
-
-        counts, remaining, evicted = self._simulate_failure_counting(list(jobs), failures)
-
-        assert "CL-002" in evicted
-        assert counts["CL-002"] == 3
-        # CL-002 jobs removed from pending
-        assert not any(j["extra_target"] == "CL-002" for j in remaining)
-
-    def test_row_not_evicted_below_threshold(self, tmp_path: Path) -> None:
-        repos = _make_repos(tmp_path, ["repo-a"])
-        row_keys = ["CL-001", "CL-002"]
-        repo_row_keys = {"repo-a": row_keys}
-
-        jobs, _assignment, _idle = _build_row_sharded_jobs(
-            profiles=["p1", "p2", "p3"],
-            repos=repos,
-            repo_row_keys=repo_row_keys,
-            rows_per_shard=1,
-        )
-
-        # CL-001 fails only 2 times — below threshold
-        failures = [
-            {"extra_target": "CL-001", "status": "error"},
-            {"extra_target": "CL-001", "status": "timeout"},
-        ]
-
-        counts, remaining, evicted = self._simulate_failure_counting(list(jobs), failures)
-
-        assert "CL-001" not in evicted
-        assert counts["CL-001"] == 2
-        # CL-001 jobs still in pending
-        assert any(j["extra_target"] == "CL-001" for j in remaining)
-
-    def test_multiple_rows_independent_counting(self, tmp_path: Path) -> None:
-        repos = _make_repos(tmp_path, ["repo-a"])
-        row_keys = ["CL-001", "CL-002", "CL-003"]
-        repo_row_keys = {"repo-a": row_keys}
-
-        jobs, _assignment, _idle = _build_row_sharded_jobs(
-            profiles=["p1", "p2", "p3"],
-            repos=repos,
-            repo_row_keys=repo_row_keys,
-            rows_per_shard=1,
-        )
-
-        # CL-001 fails 3x (evicted), CL-002 fails 1x (kept), CL-003 never fails
-        failures = [
-            {"extra_target": "CL-001", "status": "error"},
-            {"extra_target": "CL-002", "status": "error"},
-            {"extra_target": "CL-001", "status": "timeout"},
-            {"extra_target": "CL-001", "status": "error"},
-        ]
-
-        counts, _remaining, evicted = self._simulate_failure_counting(list(jobs), failures)
-
-        assert "CL-001" in evicted
-        assert "CL-002" not in evicted
-        assert "CL-003" not in evicted
-        assert counts.get("CL-003") is None  # never failed
-
-    def test_no_extra_target_ignored(self) -> None:
-        """Jobs without extra_target (non-row-sharded) are not tracked."""
-        pending: list[dict[str, Any]] = [{"repo_key": "repo-a", "stage": "default"}]
-        failures = [{"extra_target": "", "status": "error"}]
-
-        counts, remaining, evicted = self._simulate_failure_counting(pending, failures)
-
-        assert not evicted
-        assert not counts
-        assert len(remaining) == 1
-
-    def test_eviction_only_removes_matching_target(self, tmp_path: Path) -> None:
-        repos = _make_repos(tmp_path, ["repo-a"])
-        row_keys = ["CL-001", "CL-002"]
-        repo_row_keys = {"repo-a": row_keys}
-
-        jobs, _assignment, _idle = _build_row_sharded_jobs(
-            profiles=["p1", "p2", "p3"],
-            repos=repos,
-            repo_row_keys=repo_row_keys,
-            rows_per_shard=1,
-        )
-
-        initial_count = len(jobs)
-        cl001_count = sum(1 for j in jobs if j["extra_target"] == "CL-001")
-        cl002_count = sum(1 for j in jobs if j["extra_target"] == "CL-002")
-
-        # Evict CL-001 only
-        failures = [
-            {"extra_target": "CL-001", "status": "error"},
-            {"extra_target": "CL-001", "status": "error"},
-            {"extra_target": "CL-001", "status": "error"},
-        ]
-
-        _counts, remaining, evicted = self._simulate_failure_counting(list(jobs), failures)
-
-        assert "CL-001" in evicted
-        # Only CL-001 jobs removed
-        assert len(remaining) == initial_count - cl001_count
-        # CL-002 jobs untouched
-        assert sum(1 for j in remaining if j["extra_target"] == "CL-002") == cl002_count
-
-
-# ---------------------------------------------------------------------------
-# TSK-160.28: Resume with >100 completed runs correctly identifies all
-# ---------------------------------------------------------------------------
-
-
-class TestResumeDynamicLimit:
-    """Verify _query_completed_dispatch_runs uses dynamic limit for >100 jobs."""
-
-    def test_dynamic_limit_scales_with_job_count(self) -> None:
-        """With 150 jobs for one repo, limit should be max(200, 300) = 300."""
-        jobs: list[dict[str, Any]] = [
-            {"repo_key": "repo-a", "dispatch_job_id": f"job-{i}", "extra_target": f"CL-{i:03d}"} for i in range(150)
-        ]
-
-        mock_store_instance = MagicMock()
-        mock_store_instance.list_runs.return_value = []
-        mock_store_instance.close.return_value = None
-
-        with patch(
-            "vds_audit_orchestrator.workflow_cmds.PostgresStateStore",
-            return_value=mock_store_instance,
-        ):
-            _completed, _stats = _query_completed_dispatch_runs(
-                state_dsn="postgresql://test:test@localhost/test",
-                project_storage_key="test-project",
-                thread_id="test-thread",
-                jobs=jobs,
-            )
-
-        # Verify list_runs was called with limit=300 (150 * 2)
-        call_args = mock_store_instance.list_runs.call_args
-        assert call_args.kwargs.get("limit") == 300
-
-    def test_minimum_limit_200(self) -> None:
-        """Even with few jobs, limit should be at least 200."""
-        jobs: list[dict[str, Any]] = [
-            {"repo_key": "repo-a", "dispatch_job_id": "job-1", "extra_target": "CL-001"},
-            {"repo_key": "repo-a", "dispatch_job_id": "job-2", "extra_target": "CL-002"},
-        ]
-
-        mock_store_instance = MagicMock()
-        mock_store_instance.list_runs.return_value = []
-        mock_store_instance.close.return_value = None
-
-        with patch(
-            "vds_audit_orchestrator.workflow_cmds.PostgresStateStore",
-            return_value=mock_store_instance,
-        ):
-            _completed, _stats = _query_completed_dispatch_runs(
-                state_dsn="postgresql://test:test@localhost/test",
-                project_storage_key="test-project",
-                thread_id="test-thread",
-                jobs=jobs,
-            )
-
-        call_args = mock_store_instance.list_runs.call_args
-        assert call_args.kwargs.get("limit") == 200  # max(200, 2*2=4) = 200
-
-    def test_multi_repo_independent_limits(self) -> None:
-        """Different repos get different limits based on their job counts."""
-        jobs: list[dict[str, Any]] = [
-            {"repo_key": "repo-a", "dispatch_job_id": f"a-{i}", "extra_target": f"CL-{i:03d}"} for i in range(120)
-        ] + [{"repo_key": "repo-b", "dispatch_job_id": f"b-{i}", "extra_target": f"CL-{i:03d}"} for i in range(50)]
-
-        mock_store_instance = MagicMock()
-        mock_store_instance.list_runs.return_value = []
-        mock_store_instance.close.return_value = None
-
-        with patch(
-            "vds_audit_orchestrator.workflow_cmds.PostgresStateStore",
-            return_value=mock_store_instance,
-        ):
-            _completed, _stats = _query_completed_dispatch_runs(
-                state_dsn="postgresql://test:test@localhost/test",
-                project_storage_key="test-project",
-                thread_id="test-thread",
-                jobs=jobs,
-            )
-
-        # Should be called twice — once per repo
-        assert mock_store_instance.list_runs.call_count == 2
-
-        # Extract limits from both calls
-        calls = mock_store_instance.list_runs.call_args_list
-        limits = {c.kwargs["repo_key"]: c.kwargs["limit"] for c in calls}
-        assert limits["repo-a"] == 240  # max(200, 120*2)
-        assert limits["repo-b"] == 200  # max(200, 50*2)
-
-    def test_completed_runs_matched_correctly(self) -> None:
-        """Completed runs are correctly matched to job identities."""
-        jobs: list[dict[str, Any]] = [
-            {
-                "repo_key": "repo-a",
-                "dispatch_job_id": "job-1",
-                "extra_target": "CL-001",
-                "profile": "p1",
-            },
-            {
-                "repo_key": "repo-a",
-                "dispatch_job_id": "job-2",
-                "extra_target": "CL-002",
-                "profile": "p1",
-            },
-        ]
-
-        # Mock a completed run that matches job-1
-        mock_run = {
-            "run_id": "run-abc",
-            "repo_key": "repo-a",
-            "thread_id": "test-thread",
-            "status": "completed",
-            "payload": {"target": "CL-001", "profile": "p1"},
-        }
-
-        mock_store_instance = MagicMock()
-        mock_store_instance.list_runs.return_value = [mock_run]
-        mock_store_instance.close.return_value = None
-
-        with (
-            patch(
-                "vds_audit_orchestrator.workflow_cmds.PostgresStateStore",
-                return_value=mock_store_instance,
-            ),
-            patch(
-                "vds_audit_orchestrator.workflow_cmds._run_matches_dispatch_job",
-                side_effect=lambda run_row, job_identity: (
-                    str(job_identity.get("extra_target") or "")
-                    == str((run_row.get("payload") or {}).get("target") or "")
-                ),
-            ),
-        ):
-            completed, stats = _query_completed_dispatch_runs(
-                state_dsn="postgresql://test:test@localhost/test",
-                project_storage_key="test-project",
-                thread_id="test-thread",
-                jobs=jobs,
-            )
-
-        assert stats["total_planned"] == 2
-        assert stats["already_completed"] == 1
-        assert "job-1" in completed
-        assert "job-2" not in completed
-
-    def test_empty_jobs_returns_empty(self) -> None:
-        completed, stats = _query_completed_dispatch_runs(
-            state_dsn="postgresql://test:test@localhost/test",
-            project_storage_key="test-project",
-            thread_id="test-thread",
-            jobs=[],
-        )
-
-        assert completed == {}
-        assert stats["total_planned"] == 0
-        assert stats["already_completed"] == 0