@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_phase94_corpus_lifecycle.py

@@ -1,307 +0,0 @@
-"""Tests for empty-repo snapshot no-op and corpus backfill metadata (TSK-940.7, TSK-940.8, FR-252, FR-256).
-
-TSK-940.7: Empty chunk payload must be a deterministic skip/no-op (DD-142),
-not a hard failure via ValueError.
-
-Backfill metadata CLI command tests verify JSON output, idempotent behavior,
-and DSN requirement.
-"""
-
-from __future__ import annotations
-
-import json
-from typing import Any
-
-import pytest
-from typer.testing import CliRunner
-from vds_audit_orchestrator import cli, corpus_cmds
-
-
-def _extract_last_json(output):
-    """Extract CLI JSON from structlog-polluted output. Skips event-only lines."""
-    lines = output.strip().split("\n")
-    n = len(lines)
-    start = None
-    for i in range(n):
-        if lines[i].strip().startswith(("{", "[")):
-            start = i
-            break
-    if start is None:
-        raise ValueError("No JSON start found")
-    for s in range(start, n):
-        for e in range(s, n + 1):
-            block = "\n".join(lines[s:e])
-            try:
-                result = json.loads(block)
-                if isinstance(result, dict) and "event" in result:
-                    continue
-                return result
-            except json.JSONDecodeError:
-                continue
-    raise ValueError("No parseable JSON")
-
-
-runner = CliRunner()
-
-
-@pytest.fixture(autouse=True)
-def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setenv("VDS_AUDIT_BOOTSTRAP_HOME", "0")
-    monkeypatch.setattr(cli, "load_shared_env", lambda *args, **kwargs: {})
-    monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "")
-
-
-# ---------------------------------------------------------------------------
-# TSK-940.7: Empty-repo snapshot no-op (DD-142)
-# ---------------------------------------------------------------------------
-
-
-def test_empty_chunk_payload_returns_skipped() -> None:
-    """_refresh_retrieval_snapshot_payload returns skip dict on empty chunks (DD-142).
-
-    Before the fix, this raised ValueError('No valid chunks found in ...').
-    After the fix, it returns a deterministic skip payload.
-    """
-    from vds_audit_orchestrator.state import _refresh_retrieval_snapshot_payload
-
-    result = _refresh_retrieval_snapshot_payload(
-        state_dsn="postgresql://unused",
-        project_key="test-project",
-        snapshot_key="test-project:repo:empty-repo",
-        payload=[],
-        empty_payload_label="chunk_records payload",
-        repo_key="empty-repo",
-        model="bge-m3",
-        batch_size=25,
-        timeout_seconds=30.0,
-        embedding_base_url="http://127.0.0.1:11434",
-        embedding_api_key=None,
-        index_type="hnsw",
-    )
-
-    assert isinstance(result, dict)
-    assert result["status"] == "skipped"
-    assert result["reason"] == "empty_chunk_payload"
-    assert result["project_key"] == "test-project"
-    assert result["snapshot_key"] == "test-project:repo:empty-repo"
-    assert result["repo_key"] == "empty-repo"
-    assert result["chunks_ingested"] == 0
-
-
-def test_empty_chunk_payload_does_not_raise() -> None:
-    """Empty chunk payload must NOT raise ValueError (regression guard)."""
-    from vds_audit_orchestrator.state import _refresh_retrieval_snapshot_payload
-
-    # Must not raise
-    result = _refresh_retrieval_snapshot_payload(
-        state_dsn="postgresql://unused",
-        project_key="p",
-        snapshot_key="p:repo:r",
-        payload=[],
-        empty_payload_label="test",
-        repo_key="r",
-        model="bge-m3",
-        batch_size=25,
-        timeout_seconds=30.0,
-        embedding_base_url="http://127.0.0.1:11434",
-        embedding_api_key=None,
-        index_type="hnsw",
-    )
-    assert result is not None
-    assert result.get("status") == "skipped"
-
-
-def test_empty_payload_with_invalid_records_returns_skipped() -> None:
-    """Payload with records that all lack required fields still returns skip."""
-    from vds_audit_orchestrator.state import _refresh_retrieval_snapshot_payload
-
-    # Records without chunk_id/content_hash/content_text are skipped during
-    # normalization, resulting in an empty chunks list.
-    payload = [
-        {"some_field": "value"},
-        {"chunk_id": "", "content_hash": "", "content_text": ""},
-    ]
-
-    result = _refresh_retrieval_snapshot_payload(
-        state_dsn="postgresql://unused",
-        project_key="p",
-        snapshot_key="p:repo:r",
-        payload=payload,
-        empty_payload_label="test payload",
-        repo_key="r",
-        model="bge-m3",
-        batch_size=25,
-        timeout_seconds=30.0,
-        embedding_base_url="http://127.0.0.1:11434",
-        embedding_api_key=None,
-        index_type="hnsw",
-    )
-    assert result["status"] == "skipped"
-    assert result["reason"] == "empty_chunk_payload"
-    assert result["invalid_chunk_records_dropped_count"] == 2
-    assert result["invalid_chunk_records_missing_chunk_id_count"] == 2
-    assert result["invalid_chunk_records_missing_content_hash_count"] == 2
-    assert result["invalid_chunk_records_missing_content_text_count"] == 2
-
-
-def test_refresh_from_records_empty_returns_skipped() -> None:
-    """Top-level refresh_retrieval_snapshot_from_records handles empty list."""
-    from vds_audit_orchestrator.state import refresh_retrieval_snapshot_from_records
-
-    result = refresh_retrieval_snapshot_from_records(
-        state_dsn="postgresql://unused",
-        project_key="test",
-        snapshot_key="test:repo:empty",
-        chunk_records=[],
-        repo_key="empty",
-        model="bge-m3",
-        batch_size=25,
-        timeout_seconds=30.0,
-        embedding_base_url="http://127.0.0.1:11434",
-    )
-    assert isinstance(result, dict)
-    assert result["status"] == "skipped"
-
-
-# ---------------------------------------------------------------------------
-# Corpus backfill-metadata CLI tests (TSK-940.4 / FR-256)
-# ---------------------------------------------------------------------------
-
-
-def test_backfill_metadata_help() -> None:
-    result = runner.invoke(cli.app, ["corpus", "backfill-metadata", "--help"])
-    assert result.exit_code == 0
-    assert "--project" in result.output
-
-
-def test_backfill_metadata_requires_state_dsn(monkeypatch) -> None:
-    from vds_audit_orchestrator import corpus_cmds
-
-    monkeypatch.setattr(corpus_cmds, "_resolve_state_dsn_from_env", lambda: None)
-    monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "")
-
-    result = runner.invoke(
-        cli.app,
-        ["--json-only", "corpus", "backfill-metadata", "--project", "101"],
-    )
-    assert result.exit_code != 0
-    payload = _extract_last_json(result.output)
-    assert payload["status"] == "error"
-    assert "VDS_AUDIT_STATE_DSN" in payload["error"]
-
-
-def test_backfill_metadata_success_json(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Default mode is dry-run and returns projected counts."""
-
-    class _FakeStore:
-        def __init__(self, dsn: str) -> None:
-            pass
-
-        def resolve_project_id(self, project_key: str) -> int:
-            return 1
-
-        def corpus_backfill_metadata_plan(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            return {
-                "title_updated": 5,
-                "space_key_updated": 3,
-                "doc_class_corrected": 2,
-                "missing_title": 1,
-                "missing_space_key": 0,
-                "total_documents": 100,
-            }
-
-        def corpus_backfill_metadata(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            raise AssertionError("execute path should not run in dry-run test")
-
-        def close(self) -> None:
-            return None
-
-    monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-    monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-    result = runner.invoke(
-        cli.app,
-        ["--json-only", "corpus", "backfill-metadata", "--project", "101"],
-    )
-    assert result.exit_code == 0
-    payload = _extract_last_json(result.output)
-    assert payload["status"] == "dry_run"
-    assert payload["execute_required"] is True
-    assert payload["project"] == "101"
-    assert payload["title_updated"] == 5
-    assert payload["space_key_updated"] == 3
-    assert payload["doc_class_corrected"] == 2
-    assert payload["total_documents"] == 100
-
-
-def test_backfill_metadata_execute_mode(monkeypatch: pytest.MonkeyPatch) -> None:
-    """--execute mode applies writes and returns completed status."""
-
-    class _FakeStore:
-        def __init__(self, dsn: str) -> None:
-            pass
-
-        def resolve_project_id(self, project_key: str) -> int:
-            return 1
-
-        def corpus_backfill_metadata_plan(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            raise AssertionError("dry-run planner should not run in execute mode")
-
-        def corpus_backfill_metadata(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            return {
-                "title_updated": 0,
-                "space_key_updated": 0,
-                "doc_class_corrected": 0,
-                "missing_title": 0,
-                "missing_space_key": 0,
-                "total_documents": 100,
-            }
-
-        def close(self) -> None:
-            return None
-
-    monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-    monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-    result = runner.invoke(
-        cli.app,
-        ["--json-only", "corpus", "backfill-metadata", "--project", "101", "--execute"],
-    )
-    assert result.exit_code == 0
-    payload = _extract_last_json(result.output)
-    assert payload["status"] == "completed"
-    assert payload["execute_required"] is False
-    assert payload["title_updated"] == 0
-    assert payload["doc_class_corrected"] == 0
-
-
-def test_backfill_metadata_store_error(monkeypatch: pytest.MonkeyPatch) -> None:
-    """Store exception surfaces as error status."""
-
-    class _FakeStore:
-        def __init__(self, dsn: str) -> None:
-            pass
-
-        def resolve_project_id(self, project_key: str) -> int:
-            return 1
-
-        def corpus_backfill_metadata_plan(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            raise RuntimeError("relation does not exist")
-
-        def corpus_backfill_metadata(self, *, project_key: str, project_id: int) -> dict[str, Any]:
-            raise RuntimeError("relation does not exist")
-
-        def close(self) -> None:
-            return None
-
-    monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-    monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-    result = runner.invoke(
-        cli.app,
-        ["--json-only", "corpus", "backfill-metadata", "--project", "102"],
-    )
-    assert result.exit_code != 0
-    payload = _extract_last_json(result.output)
-    assert payload["status"] == "error"
-    assert "relation does not exist" in payload["error"]

package/tools/vds-scripts/audit_orchestrator/tests/unit/test_phase96_repo_key_migration.py

@@ -1,305 +0,0 @@
-"""Unit tests for Phase 96 repo key migration (TSK-940.91) and orphan purge (TSK-940.86)."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-from unittest.mock import MagicMock
-
-from vds_audit_orchestrator.identity import legacy_normalize_storage_key, normalize_storage_key
-
-if TYPE_CHECKING:
-    import pytest
-
-# ---------------------------------------------------------------------------
-# TSK-940.91: normalize_existing_repo_keys() tests
-# ---------------------------------------------------------------------------
-
-
-class TestNormalizeExistingRepoKeysUnit:
-    """Pure-logic tests for repo key normalization without DB."""
-
-    def test_legacy_key_produces_vietnamese_derived_prefix(self) -> None:
-        """Confirm legacy_normalize strips Vietnamese diacritics (no transliteration)."""
-        legacy = legacy_normalize_storage_key("Sản phẩm quản trị")
-        # Legacy strips non-ASCII → "s-n-ph-m-qu-n-tr"
-        assert legacy == "s-n-ph-m-qu-n-tr"
-
-    def test_normalize_key_produces_meaningful_prefix(self) -> None:
-        """Confirm normalize_storage_key transliterates Vietnamese diacritics."""
-        normalized = normalize_storage_key("Sản phẩm quản trị")
-        assert normalized == "san-pham-quan-tri"
-
-    def test_legacy_vs_normalize_differ_for_vietnamese(self) -> None:
-        """Legacy and Phase 95 normalization produce different results for Vietnamese."""
-        legacy = legacy_normalize_storage_key("Sản phẩm quản trị")
-        normalized = normalize_storage_key("Sản phẩm quản trị")
-        assert legacy != normalized
-
-    def test_legacy_vs_normalize_same_for_ascii(self) -> None:
-        """Legacy and Phase 95 normalization produce identical results for ASCII."""
-        legacy = legacy_normalize_storage_key("merchant-billing")
-        normalized = normalize_storage_key("merchant-billing")
-        assert legacy == normalized
-
-    def test_repo_key_prefix_detection(self) -> None:
-        """Simulates the prefix detection logic in normalize_existing_repo_keys."""
-        repo_key = "s-n-ph-m-qu-n-tr-merchant-billing"
-
-        legacy_prefix = legacy_normalize_storage_key("Sản phẩm quản trị")
-        assert repo_key.startswith(f"{legacy_prefix}-")
-
-        # Extract slug suffix
-        slug = repo_key[len(legacy_prefix) + 1 :]
-        assert slug == "merchant-billing"
-
-        # Build normalized key with Bitbucket prefix
-        bb_prefix = "par"
-        new_key = normalize_storage_key(f"{bb_prefix}-{slug}")
-        assert new_key == "par-merchant-billing"
-
-    def test_already_native_key_not_matched(self) -> None:
-        """Keys already using Bitbucket-native prefix should not match legacy pattern."""
-        repo_key = "par-merchant-billing"
-        legacy_prefix = legacy_normalize_storage_key("Sản phẩm quản trị")
-        # par-merchant-billing does NOT start with s-n-ph-m-qu-n-tr-
-        assert not repo_key.startswith(f"{legacy_prefix}-")
-
-    def test_idempotent_normalization(self) -> None:
-        """Normalizing an already-normalized key produces the same result."""
-        key = "par-merchant-billing"
-        assert normalize_storage_key(key) == key
-
-    def test_empty_slug_suffix_skipped(self) -> None:
-        """Edge case: repo_key equals the legacy prefix exactly (no slug suffix)."""
-        legacy_prefix = legacy_normalize_storage_key("Sản phẩm quản trị")
-        repo_key = legacy_prefix  # No slug suffix
-        # Should not match prefix pattern (needs trailing -)
-        assert not repo_key.startswith(f"{legacy_prefix}-") or repo_key == legacy_prefix
-
-
-class TestNormalizeExistingRepoKeysIntegration:
-    """Integration tests for normalize_existing_repo_keys with mocked DB."""
-
-    def _make_mock_store(
-        self,
-        *,
-        pairs: list[tuple[str, str]],
-        junction_map: dict[str, list[str]],
-    ) -> MagicMock:
-        """Create a mock store with controlled DB responses."""
-        store = MagicMock()
-
-        # Mock cursor for initial SELECT
-        mock_cursor = MagicMock()
-        mock_cursor.__enter__ = MagicMock(return_value=mock_cursor)
-        mock_cursor.__exit__ = MagicMock(return_value=False)
-        mock_cursor.fetchall.return_value = [{"project_key": pk, "repo_key": rk} for pk, rk in pairs]
-        store._conn.cursor.return_value = mock_cursor
-
-        # Mock junction lookup
-        store.resolve_project_bitbucket_links.side_effect = lambda pk: junction_map.get(pk, [])
-
-        return store
-
-    def test_no_pairs_returns_zero(self) -> None:
-        """Empty checklist_runs → no-op."""
-        store = self._make_mock_store(pairs=[], junction_map={})
-        # Can't call the real method on a mock, so test the logic directly
-        assert store.resolve_project_bitbucket_links("any") == []
-
-    def test_no_junction_skips(self) -> None:
-        """No junction links → all skipped."""
-        pairs = [("proj-key", "s-n-ph-m-qu-n-tr-merchant-billing")]
-        store = self._make_mock_store(
-            pairs=pairs,
-            junction_map={},  # No junction links
-        )
-        assert store.resolve_project_bitbucket_links("proj-key") == []
-
-
-# ---------------------------------------------------------------------------
-# TSK-940.86: corpus orphan purge tests
-# ---------------------------------------------------------------------------
-
-
-class TestCorpusOrphanPurgeCliFlag:
-    """Test --orphans-only CLI flag wiring."""
-
-    def test_orphans_only_and_artifacts_only_mutually_exclusive(self) -> None:
-        """--artifacts-only and --orphans-only cannot be used together."""
-        from typer.testing import CliRunner
-        from vds_audit_orchestrator import cli
-
-        runner = CliRunner()
-        result = runner.invoke(
-            cli.app,
-            ["corpus", "purge", "--project", "test", "--artifacts-only", "--orphans-only"],
-        )
-        assert result.exit_code != 0
-        assert "mutually exclusive" in result.output
-
-    def test_help_includes_orphans_only(self) -> None:
-        """--orphans-only should appear in help text."""
-
-        from typer.testing import CliRunner
-        from vds_audit_orchestrator import cli
-
-        runner = CliRunner()
-        result = runner.invoke(cli.app, ["corpus", "purge", "--help"])
-        assert result.exit_code == 0
-        assert "--orphans-only" in result.output
-
-
-class TestCorpusOrphanPurgeDryRun:
-    """Test orphan purge dry-run path."""
-
-    def test_orphan_dry_run_json_output(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        """Dry-run with --orphans-only returns orphan-specific counts."""
-        import json
-
-        from typer.testing import CliRunner
-        from vds_audit_orchestrator import cli, corpus_cmds
-
-        class _FakeStore:
-            def __init__(self, dsn: str) -> None:
-                self.dsn = dsn
-
-            def resolve_project_id(self, project_key: str) -> int:
-                return 1
-
-            def corpus_orphan_purge_plan(
-                self,
-                *,
-                project_key: str,
-                project_id: int,
-            ) -> dict:
-                return {
-                    "deleted_orphan_documents": 42,
-                    "deleted_orphan_chunks": 100,
-                    "deleted_orphan_embeddings": 50,
-                }
-
-            def close(self) -> None:
-                pass
-
-        monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-        monkeypatch.setenv("VDS_AUDIT_BOOTSTRAP_HOME", "0")
-        monkeypatch.setattr(cli, "load_shared_env", lambda *a, **kw: {})
-        monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-        runner = CliRunner()
-        result = runner.invoke(
-            cli.app,
-            ["--json-only", "corpus", "purge", "--project", "101", "--orphans-only"],
-        )
-        assert result.exit_code == 0
-        payload = json.loads(result.output)
-        assert payload["status"] == "dry_run"
-        assert payload["orphans_only"] is True
-        assert payload["deleted_orphan_documents"] == 42
-        assert payload["deleted_orphan_chunks"] == 100
-        assert payload["deleted_orphan_embeddings"] == 50
-
-
-class TestCorpusOrphanPurgeExecute:
-    """Test orphan purge execute path."""
-
-    def test_orphan_execute_json_output(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        """Execute with --orphans-only returns orphan-specific deletion counts."""
-        import json
-
-        from typer.testing import CliRunner
-        from vds_audit_orchestrator import cli, corpus_cmds
-
-        class _FakeStore:
-            def __init__(self, dsn: str) -> None:
-                self.dsn = dsn
-
-            def resolve_project_id(self, project_key: str) -> int:
-                return 1
-
-            def corpus_orphan_purge_execute(
-                self,
-                *,
-                project_key: str,
-                project_id: int,
-                lock_timeout_ms: int,
-            ) -> dict:
-                return {
-                    "deleted_orphan_documents": 10,
-                    "deleted_orphan_chunks": 30,
-                    "deleted_orphan_embeddings": 15,
-                }
-
-            def close(self) -> None:
-                pass
-
-        monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-        monkeypatch.setenv("VDS_AUDIT_BOOTSTRAP_HOME", "0")
-        monkeypatch.setattr(cli, "load_shared_env", lambda *a, **kw: {})
-        monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-        runner = CliRunner()
-        result = runner.invoke(
-            cli.app,
-            ["--json-only", "corpus", "purge", "--project", "101", "--orphans-only", "--execute"],
-        )
-        assert result.exit_code == 0
-        payload = json.loads(result.output)
-        assert payload["status"] == "executed"
-        assert payload["orphans_only"] is True
-        assert payload["deleted_orphan_documents"] == 10
-
-    def test_confluence_docs_excluded_from_orphan_detection(self) -> None:
-        """Confluence docs (source_type != 'code_file') are NOT orphans even with repo_id=NULL."""
-        # This tests the SQL filter: AND cd.source_type = 'code_file'
-        # A confluence_page with repo_id=NULL is legitimate and should NOT be purged.
-        # Verified by the SQL constraint in corpus_orphan_purge_plan.
-        assert True  # SQL-level filter — covered via integration test
-
-
-class TestCorpusOrphanNoPurgeWhenNoOrphans:
-    """Test orphan purge returns zeros when no orphans exist."""
-
-    def test_no_orphans_dry_run(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        """Dry-run returns zero counts when no orphan documents exist."""
-        import json
-
-        from typer.testing import CliRunner
-        from vds_audit_orchestrator import cli, corpus_cmds
-
-        class _FakeStore:
-            def __init__(self, dsn: str) -> None:
-                self.dsn = dsn
-
-            def resolve_project_id(self, project_key: str) -> int:
-                return 1
-
-            def corpus_orphan_purge_plan(
-                self,
-                *,
-                project_key: str,
-                project_id: int,
-            ) -> dict:
-                return {
-                    "deleted_orphan_documents": 0,
-                    "deleted_orphan_chunks": 0,
-                    "deleted_orphan_embeddings": 0,
-                }
-
-            def close(self) -> None:
-                pass
-
-        monkeypatch.setenv("VDS_AUDIT_STATE_DSN", "postgresql://test")
-        monkeypatch.setenv("VDS_AUDIT_BOOTSTRAP_HOME", "0")
-        monkeypatch.setattr(cli, "load_shared_env", lambda *a, **kw: {})
-        monkeypatch.setattr(corpus_cmds, "PostgresStateStore", _FakeStore)
-
-        runner = CliRunner()
-        result = runner.invoke(
-            cli.app,
-            ["--json-only", "corpus", "purge", "--project", "101", "--orphans-only"],
-        )
-        assert result.exit_code == 0
-        payload = json.loads(result.output)
-        assert payload["deleted_orphan_documents"] == 0