@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_junction_table_phase95.py

@@ -1,259 +0,0 @@
-"""Unit tests for project_bitbucket_links junction table CRUD (Phase 95, FR-263/FR-265)."""
-
-from __future__ import annotations
-
-import inspect
-from pathlib import Path
-
-from vds_audit_orchestrator.state.store import (
-    _PHASE47_SCHEMA_VERSION,
-    ProjectBitbucketLinkRow,
-)
-
-# P4 / TSK-4.1.8: BOOTSTRAP_SQL constants were retired when init_schema() moved
-# to Alembic.  Schema CONTENT tests now read the squash migration source.
-_SQUASH_PATH = Path(__file__).resolve().parents[2] / "alembic" / "versions" / "0001_initial_audit_state_schema.py"
-_SQUASH_SOURCE = _SQUASH_PATH.read_text(encoding="utf-8")
-
-
-class TestSchemaVersionBump:
-    """TSK-940.28: Schema version bumped for junction table."""
-
-    def test_schema_version_is_52(self) -> None:
-        assert _PHASE47_SCHEMA_VERSION == 52
-
-
-class TestProjectBitbucketLinkRowShape:
-    """TSK-940.28: ProjectBitbucketLinkRow TypedDict is importable."""
-
-    def test_typed_dict_has_required_fields(self) -> None:
-        annotations = ProjectBitbucketLinkRow.__annotations__
-        assert "id" in annotations
-        assert "project_id" in annotations
-        assert "bitbucket_project_id" in annotations
-        assert "source" in annotations
-        assert "metadata" in annotations
-        assert "created_at" in annotations
-
-
-class TestJunctionTableDDL:
-    """TSK-940.28: Junction table DDL is present in bootstrap SQL."""
-
-    def test_junction_table_in_bootstrap_sql(self) -> None:
-        _BOOTSTRAP_TABLES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        assert "project_bitbucket_links" in _BOOTSTRAP_TABLES_SQL
-
-    def test_junction_table_has_unique_constraint(self) -> None:
-        _BOOTSTRAP_TABLES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        assert "UNIQUE(project_id, bitbucket_project_id)" in _BOOTSTRAP_TABLES_SQL
-
-    def test_junction_table_has_cascade_deletes(self) -> None:
-        _BOOTSTRAP_TABLES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        # Find the junction table section
-        start = _BOOTSTRAP_TABLES_SQL.index("project_bitbucket_links")
-        end = _BOOTSTRAP_TABLES_SQL.index(");", start)
-        junction_ddl = _BOOTSTRAP_TABLES_SQL[start:end]
-        assert "ON DELETE CASCADE" in junction_ddl
-
-    def test_junction_table_in_required_tables(self) -> None:
-        from vds_audit_orchestrator.state.store import _PHASE47_REQUIRED_TABLES
-
-        assert "project_bitbucket_links" in _PHASE47_REQUIRED_TABLES
-
-    def test_junction_indexes_in_required_indexes(self) -> None:
-        from vds_audit_orchestrator.state.store import _PHASE47_REQUIRED_INDEXES
-
-        assert "idx_audit_state_project_bitbucket_links_project_id" in _PHASE47_REQUIRED_INDEXES
-        assert "idx_audit_state_project_bitbucket_links_bitbucket_project_id" in _PHASE47_REQUIRED_INDEXES
-
-    def test_junction_indexes_in_bootstrap_indexes_sql(self) -> None:
-        _BOOTSTRAP_INDEXES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        assert "idx_audit_state_project_bitbucket_links_project_id" in _BOOTSTRAP_INDEXES_SQL
-        assert "idx_audit_state_project_bitbucket_links_bitbucket_project_id" in _BOOTSTRAP_INDEXES_SQL
-
-
-class TestJunctionCRUDMethodsExist:
-    """TSK-940.29: CRUD methods exist on PostgresStateStore."""
-
-    def test_upsert_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "upsert_project_bitbucket_link")
-        assert callable(PostgresStateStore.upsert_project_bitbucket_link)
-
-    def test_resolve_links_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "resolve_project_bitbucket_links")
-        assert callable(PostgresStateStore.resolve_project_bitbucket_links)
-
-    def test_resolve_from_bitbucket_key_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "resolve_project_from_bitbucket_key")
-        assert callable(PostgresStateStore.resolve_project_from_bitbucket_key)
-
-
-class TestBackfillMethodsExist:
-    """TSK-940.30/31: Backfill methods exist on PostgresStateStore."""
-
-    def test_backfill_junction_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "backfill_project_bitbucket_links")
-        assert callable(PostgresStateStore.backfill_project_bitbucket_links)
-
-    def test_backfill_corpus_project_ids_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "backfill_corpus_document_project_ids")
-        assert callable(PostgresStateStore.backfill_corpus_document_project_ids)
-
-
-class TestJunctionDDLSourceColumn:
-    """TSK-940.28: Junction table source column has correct default."""
-
-    def test_source_default_is_manual(self) -> None:
-        _BOOTSTRAP_TABLES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        start = _BOOTSTRAP_TABLES_SQL.index("project_bitbucket_links")
-        end = _BOOTSTRAP_TABLES_SQL.index(");", start)
-        junction_ddl = _BOOTSTRAP_TABLES_SQL[start:end]
-        assert "DEFAULT 'manual'" in junction_ddl
-
-    def test_metadata_default_is_empty_jsonb(self) -> None:
-        _BOOTSTRAP_TABLES_SQL = _SQUASH_SOURCE  # P4 / TSK-4.1.8 alias
-
-        start = _BOOTSTRAP_TABLES_SQL.index("project_bitbucket_links")
-        end = _BOOTSTRAP_TABLES_SQL.index(");", start)
-        junction_ddl = _BOOTSTRAP_TABLES_SQL[start:end]
-        assert "'{}'::jsonb" in junction_ddl
-
-
-class TestUpsertByKeysMethodExists:
-    """TSK-940.33: Convenience upsert_project_bitbucket_link_by_keys method exists."""
-
-    def test_upsert_by_keys_method_exists(self) -> None:
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        assert hasattr(PostgresStateStore, "upsert_project_bitbucket_link_by_keys")
-        assert callable(PostgresStateStore.upsert_project_bitbucket_link_by_keys)
-
-    def test_upsert_by_keys_signature_has_project_key(self) -> None:
-        import inspect
-
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        sig = inspect.signature(PostgresStateStore.upsert_project_bitbucket_link_by_keys)
-        param_names = list(sig.parameters.keys())
-        assert "project_key" in param_names
-        assert "bitbucket_project_key" in param_names
-        assert "source" in param_names
-
-    def test_upsert_by_keys_returns_link_row_annotation(self) -> None:
-        import inspect
-
-        from vds_audit_orchestrator.state.store import PostgresStateStore
-
-        sig = inspect.signature(PostgresStateStore.upsert_project_bitbucket_link_by_keys)
-        assert sig.return_annotation is not inspect.Signature.empty
-
-
-class TestSyncCmdsJunctionLookup:
-    """TSK-940.32: sync_cmds.py enforces junction lookup hard-cut resolution."""
-
-    def test_sync_repos_imports_postgres_state_store(self) -> None:
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert "PostgresStateStore" in source
-
-    def test_sync_repos_has_junction_resolution_logging(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert "sync_repos_project_resolution" in source
-        assert "linked_existing" in source
-        assert "missing_link_requires_page_id" in source
-
-    def test_sync_repos_has_resolve_project_from_bitbucket_key_call(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert "resolve_project_from_bitbucket_key" in source
-
-
-class TestSyncCmdsJunctionUpsert:
-    """TSK-940.33: sync_cmds.py upserts junction link after sync."""
-
-    def test_sync_repos_has_upsert_junction_link(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert "upsert_project_bitbucket_link_by_keys" in source
-
-    def test_sync_repos_junction_upsert_uses_sync_repos_source(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert 'source="sync_repos"' in source
-
-    def test_sync_repos_has_junction_link_failed_warning(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.sync_cmds as mod
-
-        source = inspect.getsource(mod)
-        assert "sync_repos_junction_link_failed" in source
-
-
-class TestPrepareCmdJunctionBind:
-    """TSK-940.34: prepare command upserts junction links for discovered sync targets."""
-
-    def test_prepare_has_junction_upsert_call(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.workflow_cmds as mod
-
-        source = inspect.getsource(mod.prepare)
-        assert "upsert_project_bitbucket_link_by_keys" in source
-
-    def test_prepare_junction_upsert_uses_prepare_source(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.workflow_cmds as mod
-
-        source = inspect.getsource(mod.prepare)
-        assert 'source="prepare"' in source
-
-    def test_prepare_junction_upsert_failure_is_non_fatal(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.workflow_cmds as mod
-
-        source = inspect.getsource(mod.prepare)
-        assert "prepare_junction_link_upsert_failed" in source
-
-
-class TestPrepareCmdCanonicalKeyPassthrough:
-    """TSK-940.35: prepare passes canonical project_storage_key to downstream sync_repos."""
-
-    def test_prepare_passes_target_project_storage_key(self) -> None:
-        import inspect
-
-        import vds_audit_orchestrator.workflow_cmds as mod
-
-        source = inspect.getsource(mod.prepare)
-        assert "target_project_storage_key=project_key" in source

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_late_binding_assignment_p167c.py

@@ -1,286 +0,0 @@
-"""Unit tests for P167-C: Late-binding profile assignment (FR-167.2).
-
-Verifies:
-- _build_row_sharded_jobs: rows_per_shard==1 emits profile=None jobs (late-bound).
-- _build_row_sharded_jobs: multi-row sharding modes retain explicit profile assignment.
-- _build_row_sharded_jobs: idle_profiles is empty [] for rows_per_shard==1.
-- Late-bound profile selection: least-loaded profile wins; rotating cursor breaks ties.
-- Circuit-breaker integration: tripped profiles excluded from late-bound selection.
-- No idle profiles while rows remain pending (all profiles stay busy).
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any
-
-from vds_audit_orchestrator import workflow_cmds
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_repos(keys: list[str]) -> list[tuple[str, Path]]:
-    return [(k, Path(f"/tmp/{k}")) for k in keys]
-
-
-def _make_row_keys(repo_keys: list[str], rows_each: int) -> dict[str, list[str]]:
-    return {rk: [f"CL-{i:03d}" for i in range(rows_each)] for rk in repo_keys}
-
-
-def _least_loaded_profile(
-    available: list[str],
-    active_per_profile: dict[str, int],
-    cursor: int,
-) -> tuple[str, int]:
-    """Mirror of _fill_capacity's profile selection logic (TSK-167.7)."""
-    min_load = min(active_per_profile[p] for p in available)
-    least_loaded = [p for p in available if active_per_profile[p] == min_load]
-    chosen = least_loaded[cursor % len(least_loaded)]
-    return chosen, cursor + 1
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.6 — _build_row_sharded_jobs emits late-bound jobs
-# ---------------------------------------------------------------------------
-
-
-class TestBuildRowShardedJobsLateBinding:
-    """FR-167.2: rows_per_shard==1 must emit profile=None for all row jobs."""
-
-    def test_rows_per_shard_1_emits_null_profiles(self) -> None:
-        """Every row-sharded job has profile=None when rows_per_shard==1."""
-        jobs, _, _ = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a", "profile-b"],
-            repos=_make_repos(["repo-x"]),
-            repo_row_keys=_make_row_keys(["repo-x"], rows_each=4),
-            rows_per_shard=1,
-        )
-        row_jobs = [j for j in jobs if j.get("stage") == "row_sharded"]
-        assert len(row_jobs) == 4
-        for job in row_jobs:
-            assert job["profile"] is None, f"Expected profile=None, got {job['profile']!r}"
-            assert job["origin_profile"] is None
-
-    def test_rows_per_shard_1_shard_assignment_is_late_bound(self) -> None:
-        """shard_assignment must reflect late_bound sentinel, not a profile name."""
-        jobs, _, _ = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a"],
-            repos=_make_repos(["repo-y"]),
-            repo_row_keys=_make_row_keys(["repo-y"], rows_each=2),
-            rows_per_shard=1,
-        )
-        for job in jobs:
-            assert "late_bound" in job["shard_assignment"], job["shard_assignment"]
-            assert "profile-a" not in job["shard_assignment"]
-
-    def test_rows_per_shard_1_idle_profiles_is_empty(self) -> None:
-        """idle_profiles must be [] for rows_per_shard==1 (TSK-167.6 removal)."""
-        _, _, idle = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a", "profile-b", "profile-c"],
-            repos=_make_repos(["repo-z"]),
-            repo_row_keys=_make_row_keys(["repo-z"], rows_each=5),
-            rows_per_shard=1,
-        )
-        assert idle == [], f"Expected empty idle_profiles, got {idle!r}"
-
-    def test_rows_per_shard_1_correct_row_count(self) -> None:
-        """One job per row across all repos."""
-        repos = _make_repos(["repo-a", "repo-b"])
-        row_keys = {
-            "repo-a": ["CL-001", "CL-002", "CL-003"],
-            "repo-b": ["CL-010", "CL-011"],
-        }
-        jobs, _, _ = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a", "profile-b"],
-            repos=repos,
-            repo_row_keys=row_keys,
-            rows_per_shard=1,
-        )
-        row_jobs = [j for j in jobs if j.get("stage") == "row_sharded"]
-        assert len(row_jobs) == 5
-
-    def test_multi_row_sharding_retains_explicit_profiles(self) -> None:
-        """rows_per_shard==0 (default) still assigns explicit profiles."""
-        jobs, _, _ = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a", "profile-b"],
-            repos=_make_repos(["repo-m"]),
-            repo_row_keys=_make_row_keys(["repo-m"], rows_each=4),
-            rows_per_shard=0,
-        )
-        row_jobs = [j for j in jobs if j.get("stage") == "row_sharded"]
-        for job in row_jobs:
-            assert job["profile"] is not None, "Non-late-bound sharding must have explicit profile"
-            assert job["profile"] in ("profile-a", "profile-b")
-
-    def test_assignment_dict_is_late_bound_for_rows_per_shard_1(self) -> None:
-        """sharding_assignment dict reflects 'late_bound' sentinel per repo."""
-        _, assignment, _ = workflow_cmds._build_row_sharded_jobs(
-            profiles=["profile-a", "profile-b"],
-            repos=_make_repos(["billing", "payments"]),
-            repo_row_keys=_make_row_keys(["billing", "payments"], rows_each=2),
-            rows_per_shard=1,
-        )
-        assert assignment["billing"] == "late_bound"
-        assert assignment["payments"] == "late_bound"
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.7 — least-loaded profile selection with rotating cursor
-# ---------------------------------------------------------------------------
-
-
-class TestLeastLoadedProfileSelection:
-    """FR-167.2: _fill_capacity must pick least-loaded profile; rotate on ties."""
-
-    def test_least_loaded_wins(self) -> None:
-        """Profile with fewest active jobs is selected."""
-        available = ["profile-a", "profile-b", "profile-c"]
-        active = {"profile-a": 2, "profile-b": 0, "profile-c": 1}
-        chosen, _ = _least_loaded_profile(available, active, cursor=0)
-        assert chosen == "profile-b"
-
-    def test_tie_broken_by_cursor_position_0(self) -> None:
-        """Tie: cursor=0 selects first equally-loaded profile."""
-        available = ["profile-a", "profile-b", "profile-c"]
-        active = {"profile-a": 1, "profile-b": 1, "profile-c": 1}
-        chosen, next_cursor = _least_loaded_profile(available, active, cursor=0)
-        assert chosen == "profile-a"
-        assert next_cursor == 1
-
-    def test_tie_broken_by_cursor_position_1(self) -> None:
-        """Tie: cursor=1 selects second equally-loaded profile."""
-        available = ["profile-a", "profile-b", "profile-c"]
-        active = {"profile-a": 1, "profile-b": 1, "profile-c": 1}
-        chosen, _ = _least_loaded_profile(available, active, cursor=1)
-        assert chosen == "profile-b"
-
-    def test_cursor_wraps_around(self) -> None:
-        """Cursor wraps: cursor=3 with 3 tied profiles selects profile-a again."""
-        available = ["profile-a", "profile-b", "profile-c"]
-        active = {"profile-a": 0, "profile-b": 0, "profile-c": 0}
-        chosen, _ = _least_loaded_profile(available, active, cursor=3)
-        assert chosen == "profile-a"
-
-    def test_single_profile_always_chosen(self) -> None:
-        """Single available profile is always chosen regardless of load."""
-        available = ["profile-a"]
-        active = {"profile-a": 5}
-        chosen, next_cursor = _least_loaded_profile(available, active, cursor=0)
-        assert chosen == "profile-a"
-        assert next_cursor == 1
-
-    def test_rotation_distributes_evenly_over_10_jobs(self) -> None:
-        """Simulate 10 jobs with 2 equally loaded profiles → ~5 jobs each."""
-        available = ["profile-a", "profile-b"]
-        # Start with equal load; simulate adding jobs one at a time.
-        active: dict[str, int] = {"profile-a": 0, "profile-b": 0}
-        cursor = 0
-        selections: list[str] = []
-        for _ in range(10):
-            chosen, cursor = _least_loaded_profile(available, active, cursor)
-            active[chosen] += 1
-            selections.append(chosen)
-        count_a = selections.count("profile-a")
-        count_b = selections.count("profile-b")
-        assert count_a == 5 and count_b == 5, f"Expected 5/5 distribution, got {count_a}/{count_b}"
-
-    def test_3_profiles_10_rows_no_profile_idles(self) -> None:
-        """3 profiles with 10 late-bound jobs → all profiles get at least 1 job."""
-        available = ["profile-a", "profile-b", "profile-c"]
-        active: dict[str, int] = dict.fromkeys(available, 0)
-        cursor = 0
-        selections: list[str] = []
-        for _ in range(10):
-            chosen, cursor = _least_loaded_profile(available, active, cursor)
-            active[chosen] += 1
-            selections.append(chosen)
-        assert "profile-a" in selections
-        assert "profile-b" in selections
-        assert "profile-c" in selections
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.7 — circuit-breaker exclusion for late-bound selection
-# ---------------------------------------------------------------------------
-
-
-class TestCircuitBreakerExclusionLatebound:
-    """Tripped profiles must be excluded from late-bound profile selection."""
-
-    def test_tripped_profile_excluded(self) -> None:
-        """When profile-a is tripped, late-bound jobs only get profile-b or profile-c."""
-        all_profiles = ["profile-a", "profile-b", "profile-c"]
-        # Simulate health_tracker.healthy_profiles() excluding profile-a
-        healthy = [p for p in all_profiles if p != "profile-a"]
-        active: dict[str, int] = dict.fromkeys(healthy, 0)
-        cursor = 0
-        for _ in range(6):
-            chosen, cursor = _least_loaded_profile(healthy, active, cursor)
-            active[chosen] += 1
-            assert chosen != "profile-a", "Tripped profile must never be selected"
-
-    def test_only_one_profile_healthy_all_jobs_go_there(self) -> None:
-        """When only one profile is healthy, all late-bound jobs use it."""
-        healthy = ["profile-c"]
-        active: dict[str, int] = {"profile-c": 0}
-        cursor = 0
-        for _ in range(5):
-            chosen, cursor = _least_loaded_profile(healthy, active, cursor)
-            active[chosen] += 1
-            assert chosen == "profile-c"
-
-
-# ---------------------------------------------------------------------------
-# TSK-167.8 — _requeue_pending_jobs skips late-bound jobs
-# ---------------------------------------------------------------------------
-
-
-class TestRequeuePendingJobsLatebound:
-    """Late-bound jobs (profile=None) must not be requeued on circuit-breaker trip."""
-
-    def _make_job(self, profile: str | None, repo_key: str = "repo-x") -> dict[str, Any]:
-        return {
-            "profile": profile,
-            "origin_profile": profile,
-            "repo_path": Path(f"/tmp/{repo_key}"),
-            "repo_key": repo_key,
-            "stage": "row_sharded" if profile is None else "horizontal",
-            "extra_target": "CL-001",
-            "row_shard_index": 0,
-            "row_shard_count": 3,
-        }
-
-    def test_late_bound_job_not_in_requeue_targets(self) -> None:
-        """profile=None jobs must not be caught by tripped-profile filter."""
-        late_bound = self._make_job(profile=None)
-        explicit = self._make_job(profile="profile-a")
-
-        # The filter: job.get("profile") == tripped_profile
-        tripped = "profile-a"
-        requeue_targets = [j for j in [late_bound, explicit] if j.get("profile") == tripped]
-        assert explicit in requeue_targets
-        assert late_bound not in requeue_targets
-
-    def test_late_bound_job_stays_in_pending(self) -> None:
-        """profile=None jobs remain in pending queue after a circuit-breaker trip."""
-        late_bound = self._make_job(profile=None)
-        explicit = self._make_job(profile="profile-a")
-
-        tripped = "profile-a"
-        # After requeue: pending_jobs = [j for j if j.get("profile") != tripped]
-        remaining = [j for j in [late_bound, explicit] if j.get("profile") != tripped]
-        assert late_bound in remaining
-        assert explicit not in remaining
-
-    def test_explicit_profile_job_gets_requeued(self) -> None:
-        """Explicit-profile jobs (horizontal/cascade) are still requeued correctly."""
-        jobs = [
-            self._make_job(profile="profile-a", repo_key="repo-1"),
-            self._make_job(profile="profile-a", repo_key="repo-2"),
-            self._make_job(profile=None, repo_key="repo-3"),
-        ]
-        tripped = "profile-a"
-        targets = [j for j in jobs if j.get("profile") == tripped]
-        assert len(targets) == 2, "Both explicit profile-a jobs should be requeue targets"