@ngocsangairvds/vsaf 3.1.27 → 3.2.1

package/tools/vds-scripts/git_orchestrator/src/vds_git_orchestrator/reporting.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from datetime import datetime
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from .orchestrator import RepoResult, RunSummary
+
+
+def print_summary(summary: RunSummary) -> None:
+    console = Console()
+
+    header = (
+        f"Total: {summary.total} | "
+        f"Successes: {summary.successes} | "
+        f"Updated: {summary.updates} | "
+        f"Failures: {summary.failures}"
+    )
+    console.print(f"[bold cyan]Git operation summary[/bold cyan] :: {header}")
+
+    if summary.failures == 0:
+        return
+
+    console.print("[bold red]Failures:[/bold red]")
+    table = Table("Repository", "Project", "Message", show_lines=False)
+    for result in _failed(summary.results.values()):
+        table.add_row(result.repo.name, result.repo.project, result.message or "")
+    console.print(table)
+
+
+def _failed(results: Iterable[RepoResult]) -> list[RepoResult]:
+    return [result for result in results if not result.success]
+
+
+def write_json_report(
+    summary: RunSummary,
+    report_dir: Path,
+    metadata: dict | None = None,
+) -> Path:
+    report_dir.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
+    payload = {
+        "timestamp": timestamp,
+        "metadata": metadata or {},
+        "summary": summary.to_dict(),
+    }
+    path = report_dir / f"git-run-{timestamp}.json"
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
+    return path

package/tools/vds-scripts/git_orchestrator/tests/test_cli_settings.py

@@ -0,0 +1,21 @@
+import os
+
+from vds_git_orchestrator.cli import _default_concurrency
+
+
+def test_default_concurrency_from_env(monkeypatch):
+    monkeypatch.setenv("VDS_GIT_CONCURRENCY_DEFAULT", "12")
+    assert _default_concurrency() == 12
+
+
+def test_default_concurrency_cpu_fallback(monkeypatch):
+    monkeypatch.delenv("VDS_GIT_CONCURRENCY_DEFAULT", raising=False)
+    monkeypatch.setattr(os, "cpu_count", lambda: 6)
+    assert _default_concurrency() == 3  # max(2, 6//2)=3 within cap
+
+
+def test_default_concurrency_invalid_env(monkeypatch):
+    monkeypatch.setenv("VDS_GIT_CONCURRENCY_DEFAULT", "bad")
+    monkeypatch.setattr(os, "cpu_count", lambda: 32)
+    # cpu fallback -> max(2, 16)=16, capped at 8
+    assert _default_concurrency() == 8

package/tools/vds-scripts/git_orchestrator/tests/test_integration.py

@@ -0,0 +1,74 @@
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+from vds_git_orchestrator.manifest import Manifest, ManifestFilters, Repository
+from vds_git_orchestrator.orchestrator import GitOperation, orchestrate_repositories
+
+
+def _create_remote_repo(base: Path, project: str, name: str) -> None:
+    remote_path = base / project / f"{name}.git"
+    remote_path.mkdir(parents=True, exist_ok=True)
+    subprocess.run(["git", "init", "--bare", str(remote_path)], check=True, stdout=subprocess.DEVNULL)
+
+    seed = base.parent / "seed"
+    subprocess.run(["git", "init", "-b", "release", str(seed)], check=True, stdout=subprocess.DEVNULL)
+    readme = seed / "README.md"
+    readme.write_text("seed repo")
+
+    env = {
+        **os.environ,
+        "GIT_AUTHOR_NAME": "VDS Bot",
+        "GIT_AUTHOR_EMAIL": "vds@example.com",
+        "GIT_COMMITTER_NAME": "VDS Bot",
+        "GIT_COMMITTER_EMAIL": "vds@example.com",
+    }
+
+    subprocess.run(["git", "-C", str(seed), "add", "README.md"], check=True, env=env, stdout=subprocess.DEVNULL)
+    subprocess.run(["git", "-C", str(seed), "commit", "-m", "seed"], check=True, env=env, stdout=subprocess.DEVNULL)
+    subprocess.run(["git", "-C", str(seed), "remote", "add", "origin", str(remote_path)], check=True, stdout=subprocess.DEVNULL)
+    subprocess.run(["git", "-C", str(seed), "push", "origin", "release"], check=True, env=env, stdout=subprocess.DEVNULL)
+
+
+@pytest.mark.asyncio
+async def test_clone_against_local_bare_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    remote_root = tmp_path / "remote"
+    _create_remote_repo(remote_root, "lep", "service-a")
+
+    monkeypatch.setenv("VDS_BITBUCKET_HTTP_BASE", remote_root.as_uri())
+    monkeypatch.delenv("VDS_USERNAME", raising=False)
+    monkeypatch.delenv("VDS_PASSWORD", raising=False)
+
+    workspace = tmp_path / "workspace"
+    repo = Repository(
+        name="service-a",
+        path=workspace / "service-a",
+        project="lep",
+        branch="release",
+        tags=set(),
+    )
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.CLONE,
+        filters=ManifestFilters(),
+        concurrency=2,
+        dry_run=False,
+        run_id="integration",
+    )
+
+    assert summary.total == 1
+    assert summary.successes == 1
+    assert summary.updates == 1
+    assert (workspace / "service-a/.git").exists()
+
+    # Ensure the release branch was checked out
+    result = subprocess.run(
+        ["git", "-C", str(workspace / "service-a"), "rev-parse", "--abbrev-ref", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert result.stdout.strip() == "release"

package/tools/vds-scripts/git_orchestrator/tests/test_manifest.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+from vds_git_orchestrator.manifest import Manifest, ManifestFilters, Repository, load_manifest
+
+
+def test_load_manifest_roundtrip(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "repos.yaml"
+    manifest_path.write_text(
+        """
+repositories:
+  - name: service-a
+    path: services/service-a
+    project: lep
+    branch: release
+    tags: ["core", "backend"]
+    hooks:
+      pre_pull:
+        - "./scripts/check.sh"
+  - name: web-ui
+    path: web/web-ui
+    project: insurance
+    branch: main
+"""
+    )
+
+    manifest = load_manifest(manifest_path)
+
+    assert len(manifest.repositories) == 2
+    repo_a = manifest.repositories[0]
+    assert repo_a.name == "service-a"
+    assert repo_a.path == Path("services/service-a")
+    assert repo_a.project == "lep"
+    assert repo_a.branch == "release"
+    assert repo_a.tags == {"core", "backend"}
+    assert repo_a.hooks["pre_pull"] == ["./scripts/check.sh"]
+
+    repo_b = manifest.repositories[1]
+    assert repo_b.name == "web-ui"
+    assert repo_b.branch == "main"
+    assert repo_b.tags == set()
+
+
+def test_manifest_filtering_by_project_tag_and_name(tmp_path: Path) -> None:
+    repo_root = tmp_path / "workspace"
+    repo_root.mkdir()
+
+    manifest = Manifest(
+        repositories=[
+            Repository(
+                name="service-a",
+                path=repo_root / "service-a",
+                project="lep",
+                branch="release",
+                tags={"core", "backend"},
+            ),
+            Repository(
+                name="service-b",
+                path=repo_root / "service-b",
+                project="insurance",
+                branch="release",
+                tags={"event"},
+            ),
+            Repository(
+                name="web-ui",
+                path=repo_root / "web-ui",
+                project="insurance",
+                branch="main",
+                tags={"frontend"},
+            ),
+        ]
+    )
+
+    filters = ManifestFilters(project="insurance", include_tags={"frontend"})
+    results = manifest.filter(filters)
+    assert [repo.name for repo in results] == ["web-ui"]
+
+    filters = ManifestFilters(name_match="service-*", exclude_tags={"event"})
+    results = manifest.filter(filters)
+    assert [repo.name for repo in results] == ["service-a"]

package/tools/vds-scripts/git_orchestrator/tests/test_orchestrator.py

@@ -0,0 +1,204 @@
+from pathlib import Path
+
+import pytest
+from vds_git_orchestrator.manifest import Manifest, ManifestFilters, Repository
+from vds_git_orchestrator.orchestrator import GitOperation, orchestrate_repositories
+
+
+@pytest.mark.asyncio
+async def test_orchestrate_repositories_dry_run_skips_hooks(tmp_path: Path) -> None:
+    repo_dir = tmp_path / "dry-run"
+    repo_dir.mkdir()
+    repo = Repository(
+        name="dry-run-repo",
+        path=repo_dir,
+        project="lep",
+        branch="release",
+        tags=set(),
+        hooks={
+            "pre_pull": ["touch pre.txt"],
+            "post_pull": ["touch post.txt"],
+        },
+    )
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.PULL,
+        filters=ManifestFilters(),
+        concurrency=2,
+        dry_run=True,
+        run_id="test",
+    )
+
+    assert summary.total == 1
+    assert summary.successes == 1
+    assert summary.failures == 0
+    assert summary.serial_retries == 0
+    assert summary.results["dry-run-repo"].message == "dry-run"
+    assert not (repo_dir / "pre.txt").exists()
+    assert not (repo_dir / "post.txt").exists()
+
+
+def _success_git_result() -> tuple[int, str, str]:
+    return 0, "Fast-forward", ""
+
+
+@pytest.mark.asyncio
+async def test_orchestrate_repositories_runs_hooks_on_success(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    repo_dir = tmp_path / "real-run"
+    repo_dir.mkdir()
+
+    pre_marker = repo_dir / "pre.marker"
+    post_marker = repo_dir / "post.marker"
+
+    repo = Repository(
+        name="real-run-repo",
+        path=repo_dir,
+        project="insurance",
+        branch="release",
+        tags=set(),
+        hooks={
+            "pre_pull": [f"touch {pre_marker.name}"],
+            "post_pull": [f"touch {post_marker.name}"],
+        },
+    )
+
+    # Patch _run_git to avoid invoking the real git binary.
+    from vds_git_orchestrator import orchestrator as orchestrator_module
+
+    async def fake_run_git(*args, **kwargs):  # type: ignore[no-untyped-def]
+        return _success_git_result()
+
+    monkeypatch.setattr(orchestrator_module, "_run_git", fake_run_git)
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.PULL,
+        filters=ManifestFilters(),
+        concurrency=1,
+        dry_run=False,
+        run_id="run",
+        retries=0,
+    )
+
+    assert summary.total == 1
+    assert summary.successes == 1
+    assert summary.updates == 1
+    assert summary.serial_retries == 0
+    assert pre_marker.exists()
+    assert post_marker.exists()
+
+
+@pytest.mark.asyncio
+async def test_orchestrate_repositories_serial_retry(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    repo_dir = tmp_path / "net"
+    repo_dir.mkdir()
+
+    repo = Repository(
+        name="net-repo",
+        path=repo_dir,
+        project="lep",
+        branch="release",
+        tags=set(),
+    )
+
+    from vds_git_orchestrator import orchestrator as orchestrator_module
+
+    attempts = {repo.name: 0}
+
+    async def fake_run_git(target_repo: Repository, args: list[str]):  # type: ignore[no-untyped-def]
+        idx = attempts.setdefault(target_repo.name, 0)
+        attempts[target_repo.name] = idx + 1
+        if idx == 0:
+            return 1, "", "Failed to connect to server"
+        return 0, "Already up to date", ""
+
+    monkeypatch.setattr(orchestrator_module, "_run_git", fake_run_git)
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.PULL,
+        filters=ManifestFilters(),
+        concurrency=2,
+        dry_run=False,
+        run_id="serial",
+        retries=0,
+    )
+
+    assert summary.total == 1
+    assert summary.successes == 1
+    assert summary.failures == 0
+    assert summary.serial_retries == 1
+    # two attempts: first parallel, second serial fallback
+    assert attempts[repo.name] == 2
+
+
+@pytest.mark.asyncio
+async def test_clone_dry_run(tmp_path: Path) -> None:
+    repo = Repository(
+        name="service-a",
+        path=tmp_path / "service-a",
+        project="lep",
+        branch="release",
+        tags=set(),
+    )
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.CLONE,
+        filters=ManifestFilters(),
+        concurrency=2,
+        dry_run=True,
+        run_id="clone-dry",
+    )
+
+    assert summary.total == 1
+    assert summary.successes == 1
+    assert summary.results["service-a"].message == "dry-run"
+    assert summary.serial_retries == 0
+    assert not repo.absolute_path.exists()
+
+
+@pytest.mark.asyncio
+async def test_clone_executes_git(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    repo = Repository(
+        name="service-a",
+        path=tmp_path / "service-a",
+        project="lep",
+        branch="release",
+        tags=set(),
+    )
+
+    calls: list[tuple[list[str], Path]] = []
+
+    from vds_git_orchestrator import orchestrator as orchestrator_module
+
+    async def fake_run_process(command: list[str], cwd: Path):  # type: ignore[no-untyped-def]
+        calls.append((command, cwd))
+        repo.absolute_path.mkdir(parents=True, exist_ok=True)
+        return 0, "cloned", ""
+
+    monkeypatch.setattr(orchestrator_module, "_run_process", fake_run_process)
+
+    manifest = Manifest([repo])
+    summary = await orchestrate_repositories(
+        manifest=manifest,
+        operation=GitOperation.CLONE,
+        filters=ManifestFilters(),
+        concurrency=1,
+        dry_run=False,
+        run_id="clone",
+    )
+
+    assert summary.successes == 1
+    assert summary.updates == 1
+    assert summary.serial_retries == 0
+    assert repo.absolute_path.parent.exists()
+    # first argument should be git clone
+    assert calls
+    assert calls[0][0][0] == "git"
+    assert "clone" in calls[0][0]

package/tools/vds-scripts/git_orchestrator/tests/test_public_api.py

@@ -0,0 +1,236 @@
+"""
+Tests for vds_git_orchestrator public API exports (TSK-221).
+
+Validates FR-21.1.2: Export Repository, Manifest, ManifestFilters, GitOperation,
+RunSummary, orchestrate_repositories from vds_git_orchestrator.__init__
+
+These tests verify that:
+1. All public symbols can be imported from the package root
+2. The __all__ list contains all expected symbols
+3. Imported symbols are the correct types/classes
+4. Basic functionality works when using package-level imports
+"""
+
+from pathlib import Path
+
+import pytest
+
+
+class TestPublicAPIExports:
+    """Test that all public symbols are exported from package root."""
+
+    def test_import_repository_from_package_root(self) -> None:
+        """Verify Repository can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import Repository
+
+        # Verify it's the correct class by checking it's a dataclass
+        assert hasattr(Repository, "__dataclass_fields__")
+        assert "name" in Repository.__dataclass_fields__
+        assert "path" in Repository.__dataclass_fields__
+        assert "project" in Repository.__dataclass_fields__
+        assert "branch" in Repository.__dataclass_fields__
+
+    def test_import_manifest_from_package_root(self) -> None:
+        """Verify Manifest can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import Manifest
+
+        # Verify it's the correct class
+        assert hasattr(Manifest, "__dataclass_fields__")
+        assert "repositories" in Manifest.__dataclass_fields__
+        assert hasattr(Manifest, "merge")
+        assert hasattr(Manifest, "filter")
+
+    def test_import_manifest_filters_from_package_root(self) -> None:
+        """Verify ManifestFilters can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import ManifestFilters
+
+        # Verify it's the correct class
+        assert hasattr(ManifestFilters, "__dataclass_fields__")
+        assert "project" in ManifestFilters.__dataclass_fields__
+        assert "name_match" in ManifestFilters.__dataclass_fields__
+        assert "include_tags" in ManifestFilters.__dataclass_fields__
+        assert "exclude_tags" in ManifestFilters.__dataclass_fields__
+
+    def test_import_git_operation_from_package_root(self) -> None:
+        """Verify GitOperation can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import GitOperation
+
+        # Verify it's an Enum with expected values
+        assert hasattr(GitOperation, "FETCH")
+        assert hasattr(GitOperation, "PULL")
+        assert hasattr(GitOperation, "CLONE")
+        assert hasattr(GitOperation, "STATUS")
+        assert GitOperation.PULL.value == "pull"
+
+    def test_import_run_summary_from_package_root(self) -> None:
+        """Verify RunSummary can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import RunSummary
+
+        # Verify it's the correct class
+        assert hasattr(RunSummary, "__dataclass_fields__")
+        assert "total" in RunSummary.__dataclass_fields__
+        assert "successes" in RunSummary.__dataclass_fields__
+        assert "failures" in RunSummary.__dataclass_fields__
+        assert hasattr(RunSummary, "from_results")
+        assert hasattr(RunSummary, "to_dict")
+
+    def test_import_orchestrate_repositories_from_package_root(self) -> None:
+        """Verify orchestrate_repositories can be imported from vds_git_orchestrator."""
+        from vds_git_orchestrator import orchestrate_repositories
+
+        # Verify it's a callable (async function)
+        import inspect
+
+        assert callable(orchestrate_repositories)
+        assert inspect.iscoroutinefunction(orchestrate_repositories)
+
+
+class TestAllList:
+    """Test that __all__ contains all expected public symbols."""
+
+    def test_all_list_contains_manifest_symbols(self) -> None:
+        """Verify __all__ includes Repository, Manifest, ManifestFilters."""
+        import vds_git_orchestrator
+
+        assert "Repository" in vds_git_orchestrator.__all__
+        assert "Manifest" in vds_git_orchestrator.__all__
+        assert "ManifestFilters" in vds_git_orchestrator.__all__
+
+    def test_all_list_contains_orchestrator_symbols(self) -> None:
+        """Verify __all__ includes GitOperation, RunSummary, orchestrate_repositories."""
+        import vds_git_orchestrator
+
+        assert "GitOperation" in vds_git_orchestrator.__all__
+        assert "RunSummary" in vds_git_orchestrator.__all__
+        assert "orchestrate_repositories" in vds_git_orchestrator.__all__
+
+    def test_all_list_has_exactly_six_symbols(self) -> None:
+        """Verify __all__ contains exactly the 6 expected public symbols."""
+        import vds_git_orchestrator
+
+        expected_symbols = {
+            "Repository",
+            "Manifest",
+            "ManifestFilters",
+            "GitOperation",
+            "RunSummary",
+            "orchestrate_repositories",
+        }
+        assert set(vds_git_orchestrator.__all__) == expected_symbols
+
+
+class TestImportEquivalence:
+    """Test that package-level imports are equivalent to module-level imports."""
+
+    def test_repository_import_equivalence(self) -> None:
+        """Verify package import equals module import for Repository."""
+        from vds_git_orchestrator import Repository as PkgRepository
+        from vds_git_orchestrator.manifest import Repository as ModRepository
+
+        assert PkgRepository is ModRepository
+
+    def test_manifest_import_equivalence(self) -> None:
+        """Verify package import equals module import for Manifest."""
+        from vds_git_orchestrator import Manifest as PkgManifest
+        from vds_git_orchestrator.manifest import Manifest as ModManifest
+
+        assert PkgManifest is ModManifest
+
+    def test_manifest_filters_import_equivalence(self) -> None:
+        """Verify package import equals module import for ManifestFilters."""
+        from vds_git_orchestrator import ManifestFilters as PkgFilters
+        from vds_git_orchestrator.manifest import ManifestFilters as ModFilters
+
+        assert PkgFilters is ModFilters
+
+    def test_git_operation_import_equivalence(self) -> None:
+        """Verify package import equals module import for GitOperation."""
+        from vds_git_orchestrator import GitOperation as PkgOp
+        from vds_git_orchestrator.orchestrator import GitOperation as ModOp
+
+        assert PkgOp is ModOp
+
+    def test_run_summary_import_equivalence(self) -> None:
+        """Verify package import equals module import for RunSummary."""
+        from vds_git_orchestrator import RunSummary as PkgSummary
+        from vds_git_orchestrator.orchestrator import RunSummary as ModSummary
+
+        assert PkgSummary is ModSummary
+
+    def test_orchestrate_repositories_import_equivalence(self) -> None:
+        """Verify package import equals module import for orchestrate_repositories."""
+        from vds_git_orchestrator import orchestrate_repositories as pkg_func
+        from vds_git_orchestrator.orchestrator import orchestrate_repositories as mod_func
+
+        assert pkg_func is mod_func
+
+
+class TestBasicFunctionality:
+    """Test basic functionality using package-level imports."""
+
+    def test_create_repository_from_package_import(self, tmp_path: Path) -> None:
+        """Verify Repository can be instantiated using package import."""
+        from vds_git_orchestrator import Repository
+
+        repo = Repository(
+            name="test-repo",
+            path=tmp_path / "test-repo",
+            project="insurance",
+            branch="release",
+            tags={"core", "backend"},
+        )
+
+        assert repo.name == "test-repo"
+        assert repo.project == "insurance"
+        assert repo.branch == "release"
+        assert repo.tags == {"core", "backend"}
+
+    def test_create_manifest_from_package_import(self, tmp_path: Path) -> None:
+        """Verify Manifest can be instantiated using package import."""
+        from vds_git_orchestrator import Manifest, Repository
+
+        repo = Repository(
+            name="test-repo",
+            path=tmp_path / "test-repo",
+            project="insurance",
+            branch="release",
+        )
+        manifest = Manifest(repositories=[repo])
+
+        assert len(manifest.repositories) == 1
+        assert manifest.repositories[0].name == "test-repo"
+
+    def test_filter_manifest_from_package_import(self, tmp_path: Path) -> None:
+        """Verify ManifestFilters works with package imports."""
+        from vds_git_orchestrator import Manifest, ManifestFilters, Repository
+
+        repos = [
+            Repository(
+                name="service-a",
+                path=tmp_path / "service-a",
+                project="insurance",
+                branch="release",
+            ),
+            Repository(
+                name="service-b",
+                path=tmp_path / "service-b",
+                project="lep",
+                branch="release",
+            ),
+        ]
+        manifest = Manifest(repositories=repos)
+        filters = ManifestFilters(project="insurance")
+
+        filtered = manifest.filter(filters)
+
+        assert len(filtered) == 1
+        assert filtered[0].name == "service-a"
+
+    def test_git_operation_enum_from_package_import(self) -> None:
+        """Verify GitOperation enum works with package import."""
+        from vds_git_orchestrator import GitOperation
+
+        assert GitOperation.PULL.value == "pull"
+        assert GitOperation.FETCH.value == "fetch"
+        assert GitOperation.CLONE.value == "clone"
+        assert GitOperation.STATUS.value == "status"