@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/sync/repo_sync.py

@@ -1,409 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import os
-from base64 import b64encode
-from pathlib import Path
-from typing import TYPE_CHECKING
-from urllib.parse import urlparse
-
-import structlog
-from vds_git_orchestrator import GitOperation, Manifest, ManifestFilters, RunSummary, orchestrate_repositories
-
-if TYPE_CHECKING:
-    from vds_audit_orchestrator.sources.base import RepoInfo
-
-logger = structlog.get_logger(__name__)
-
-_BRANCH_RESOLUTION_CANDIDATES = ("main", "release", "master", "staging")
-_BRANCH_RESOLUTION_EXHAUSTED = "branch_resolution_exhausted"
-_BRANCH_HINT_SAMPLE_LIMIT = 5
-_BRANCH_MISSING_MARKERS = (
-    "no matching remote branch found in origin for candidates",
-    "no such ref was fetched",
-    "couldn't find remote ref",
-    "remote ref did not resolve",
-    "remote branch",
-    "not found in upstream origin",
-)
-
-
-def _is_branch_missing_failure(message: str) -> bool:
-    marker = str(message or "").strip().casefold()
-    if not marker:
-        return False
-    return any(token in marker for token in _BRANCH_MISSING_MARKERS)
-
-
-def _build_ls_remote_command(
-    *,
-    remote_url: str,
-    branch_candidates: tuple[str, ...] | None = None,
-    include_symref: bool = False,
-) -> list[str]:
-    command: list[str] = ["git"]
-    username = os.environ.get("VDS_USERNAME")
-    password = os.environ.get("VDS_PASSWORD")
-    if username and password:
-        token = b64encode(f"{username}:{password}".encode()).decode()
-        command.extend(["-c", f"http.extraHeader=Authorization: Basic {token}"])
-    command.append("ls-remote")
-    if include_symref:
-        command.append("--symref")
-    command.extend(["--heads", remote_url])
-    if branch_candidates:
-        refs = [f"refs/heads/{branch}" for branch in branch_candidates]
-        command.extend(refs)
-    return command
-
-
-async def _probe_remote_branch_candidates(*, remote_url: str, branch_candidates: tuple[str, ...]) -> dict[str, object]:
-    normalized_remote = str(remote_url or "").strip()
-    if not normalized_remote:
-        return {
-            "deterministic": False,
-            "available_branches": [],
-            "error": "missing_remote_url",
-        }
-
-    parsed_remote = urlparse(normalized_remote)
-    if parsed_remote.scheme not in {"http", "https", "ssh", "git"}:
-        return {
-            "deterministic": False,
-            "available_branches": [],
-            "error": f"unsupported_remote_scheme:{parsed_remote.scheme or 'none'}",
-        }
-
-    command = _build_ls_remote_command(remote_url=normalized_remote, branch_candidates=branch_candidates)
-    process = await asyncio.create_subprocess_exec(
-        *command,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    try:
-        stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=20.0)
-    except TimeoutError:
-        process.kill()
-        await process.communicate()
-        return {
-            "deterministic": False,
-            "available_branches": [],
-            "error": "ls_remote_timeout",
-        }
-
-    returncode = process.returncode or 0
-    stdout = stdout_bytes.decode(errors="replace")
-    stderr = stderr_bytes.decode(errors="replace")
-    if returncode != 0:
-        return {
-            "deterministic": False,
-            "available_branches": [],
-            "error": (stderr.strip() or stdout.strip() or f"ls_remote_failed:{returncode}"),
-        }
-
-    available_branches: list[str] = []
-    branch_set = set(branch_candidates)
-    for line in stdout.splitlines():
-        ref = str(line).strip().split()[-1] if line.strip() else ""
-        if not ref.startswith("refs/heads/"):
-            continue
-        branch_name = ref.removeprefix("refs/heads/").strip()
-        if branch_name in branch_set and branch_name not in available_branches:
-            available_branches.append(branch_name)
-    return {
-        "deterministic": True,
-        "available_branches": available_branches,
-        "error": None,
-    }
-
-
-async def _probe_remote_branch_hints(
-    *,
-    remote_url: str,
-    branch_sample_limit: int = _BRANCH_HINT_SAMPLE_LIMIT,
-) -> dict[str, object]:
-    normalized_remote = str(remote_url or "").strip()
-    if not normalized_remote:
-        return {
-            "deterministic": False,
-            "default_branch": None,
-            "discovered_branches": [],
-            "discovered_branch_count": 0,
-            "error": "missing_remote_url",
-        }
-
-    parsed_remote = urlparse(normalized_remote)
-    if parsed_remote.scheme not in {"http", "https", "ssh", "git"}:
-        return {
-            "deterministic": False,
-            "default_branch": None,
-            "discovered_branches": [],
-            "discovered_branch_count": 0,
-            "error": f"unsupported_remote_scheme:{parsed_remote.scheme or 'none'}",
-        }
-
-    command = _build_ls_remote_command(
-        remote_url=normalized_remote,
-        branch_candidates=None,
-        include_symref=True,
-    )
-    process = await asyncio.create_subprocess_exec(
-        *command,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    try:
-        stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=20.0)
-    except TimeoutError:
-        process.kill()
-        await process.communicate()
-        return {
-            "deterministic": False,
-            "default_branch": None,
-            "discovered_branches": [],
-            "discovered_branch_count": 0,
-            "error": "ls_remote_timeout",
-        }
-
-    returncode = process.returncode or 0
-    stdout = stdout_bytes.decode(errors="replace")
-    stderr = stderr_bytes.decode(errors="replace")
-    if returncode != 0:
-        return {
-            "deterministic": False,
-            "default_branch": None,
-            "discovered_branches": [],
-            "discovered_branch_count": 0,
-            "error": (stderr.strip() or stdout.strip() or f"ls_remote_failed:{returncode}"),
-        }
-
-    discovered_branches: list[str] = []
-    default_branch: str | None = None
-    for line in stdout.splitlines():
-        stripped = str(line).strip()
-        if not stripped:
-            continue
-        parts = stripped.split()
-        if len(parts) >= 3 and parts[0] == "ref:" and parts[-1] == "HEAD":
-            ref = str(parts[1] or "").strip()
-            if ref.startswith("refs/heads/"):
-                parsed_default = ref.removeprefix("refs/heads/").strip()
-                if parsed_default:
-                    default_branch = parsed_default
-            continue
-        ref = parts[-1] if parts else ""
-        if not str(ref).startswith("refs/heads/"):
-            continue
-        branch_name = str(ref).removeprefix("refs/heads/").strip()
-        if branch_name and branch_name not in discovered_branches:
-            discovered_branches.append(branch_name)
-
-    safe_sample_limit = max(int(branch_sample_limit), 1)
-    return {
-        "deterministic": True,
-        "default_branch": default_branch,
-        "discovered_branches": discovered_branches[:safe_sample_limit],
-        "discovered_branch_count": len(discovered_branches),
-        "error": None,
-    }
-
-
-class RepoSynchronizer:
-    """
-    Handles cloning and pulling of repositories using vds_git_orchestrator.
-    """
-
-    def __init__(self, workspace_path: Path):
-        self.workspace_path = workspace_path
-
-    async def synchronize_repos(
-        self,
-        repos_to_sync: list[RepoInfo],
-        operation: GitOperation = GitOperation.PULL,
-        concurrency: int = 5,
-        dry_run: bool = False,
-        overwrite_local_with_remote: bool = False,
-        run_id: str = "audit-sync",
-    ) -> RunSummary:
-        """
-        Synchronizes a list of repositories (clones if missing, pulls if exists).
-
-        Args:
-            repos_to_sync: A list of RepoInfo objects to synchronize.
-            operation: The GitOperation to perform (default: PULL).
-            concurrency: Number of concurrent git operations.
-            dry_run: If True, simulate operations without making changes.
-            overwrite_local_with_remote: If True and operation is pull, hard-reset local branch
-                to origin/<branch> to follow remote state.
-            run_id: Identifier for the synchronization run.
-
-        Returns:
-            A RunSummary object detailing the results of the operations.
-        """
-        if not repos_to_sync:
-            logger.info("no_repos_to_sync", run_id=run_id)
-            return RunSummary(total=0, successes=0, updates=0, failures=0, results={})
-
-        # Convert RepoInfo objects to Repository objects using the to_repository() method
-        git_orchestrator_repos = [
-            repo_info.to_repository(workspace_path=self.workspace_path) for repo_info in repos_to_sync
-        ]
-
-        # Create a Manifest object for the orchestrator
-        manifest = Manifest(repositories=git_orchestrator_repos)
-
-        # ManifestFilters is used by orchestrate_repositories to select repos from manifest.
-        # Since we explicitly construct the manifest from repos_to_sync, we use a filter
-        # that allows all of them through.
-        all_tags = set().union(*[repo.tags for repo in repos_to_sync])
-
-        orchestrator_filters = ManifestFilters(
-            project=None,  # Operate on all projects in the manifest
-            name_match="*",  # Match all names
-            include_tags=all_tags if all_tags else set(),  # Include all tags if present
-            exclude_tags=set(),
-        )
-
-        logger.info(
-            "starting_repo_synchronization",
-            num_repos=len(repos_to_sync),
-            operation=operation.value,
-            concurrency=concurrency,
-            dry_run=dry_run,
-            overwrite_local_with_remote=overwrite_local_with_remote,
-            run_id=run_id,
-        )
-
-        summary = await orchestrate_repositories(
-            manifest=manifest,
-            operation=operation,
-            filters=orchestrator_filters,
-            concurrency=concurrency,
-            dry_run=dry_run,
-            overwrite_local_with_remote=overwrite_local_with_remote,
-            run_id=run_id,
-        )
-
-        await self._annotate_branch_resolution_failures(
-            repos_to_sync=repos_to_sync,
-            summary=summary,
-            operation=operation,
-            dry_run=dry_run,
-            run_id=run_id,
-        )
-
-        logger.info(
-            "repo_synchronization_complete",
-            num_repos=summary.total,
-            successes=summary.successes,
-            failures=summary.failures,
-            run_id=run_id,
-        )
-        return summary
-
-    async def _annotate_branch_resolution_failures(
-        self,
-        *,
-        repos_to_sync: list[RepoInfo],
-        summary: RunSummary,
-        operation: GitOperation,
-        dry_run: bool,
-        run_id: str,
-    ) -> None:
-        if dry_run or operation not in {GitOperation.PULL, GitOperation.CLONE}:
-            return
-
-        results = getattr(summary, "results", {})
-        if not isinstance(results, dict) or not results:
-            return
-
-        repo_lookup: dict[tuple[str, str], RepoInfo] = {}
-        for repo_info in repos_to_sync:
-            repo_lookup[(str(repo_info.project or "").casefold(), str(repo_info.name or "").casefold())] = repo_info
-
-        probe_cache: dict[str, dict[str, object]] = {}
-        hint_cache: dict[str, dict[str, object]] = {}
-        branch_list_label = ", ".join(_BRANCH_RESOLUTION_CANDIDATES)
-
-        for fallback_repo_name, result in results.items():
-            if bool(getattr(result, "success", False)):
-                continue
-            message = str(getattr(result, "message", "") or "").strip()
-            if not _is_branch_missing_failure(message):
-                continue
-
-            repository = getattr(result, "repo", None) or getattr(result, "repository", None)
-            repo_name = str(getattr(repository, "name", "") or fallback_repo_name or "").strip()
-            repo_project = str(getattr(repository, "project", "") or "").strip()
-            repo_info = repo_lookup.get((repo_project.casefold(), repo_name.casefold()))
-            if repo_info is None:
-                continue
-
-            remote_url = str(getattr(repo_info, "url", "") or "").strip()
-            if not remote_url:
-                continue
-
-            probe = probe_cache.get(remote_url)
-            if probe is None:
-                probe = await _probe_remote_branch_candidates(
-                    remote_url=remote_url,
-                    branch_candidates=_BRANCH_RESOLUTION_CANDIDATES,
-                )
-                probe_cache[remote_url] = probe
-
-            if not bool(probe.get("deterministic")):
-                continue
-            available_branches = [str(item) for item in (probe.get("available_branches") or []) if str(item).strip()]
-            if available_branches:
-                continue
-
-            branch_hints = hint_cache.get(remote_url)
-            if branch_hints is None:
-                branch_hints = await _probe_remote_branch_hints(remote_url=remote_url)
-                hint_cache[remote_url] = branch_hints
-            discovered_branches = [
-                str(item) for item in (branch_hints.get("discovered_branches") or []) if str(item).strip()
-            ]
-            default_branch_hint = str(branch_hints.get("default_branch") or "").strip() or None
-            discovered_branch_count = int(branch_hints.get("discovered_branch_count") or 0)
-            hint_segments: list[str] = []
-            if default_branch_hint:
-                hint_segments.append(f"Remote default branch hint: {default_branch_hint}.")
-            if discovered_branches:
-                branch_sample_text = ", ".join(discovered_branches)
-                extra_count = max(discovered_branch_count - len(discovered_branches), 0)
-                if extra_count > 0:
-                    branch_sample_text = f"{branch_sample_text} (+{extra_count} more)"
-                hint_segments.append(f"Discovered remote branches: {branch_sample_text}.")
-
-            hint_text = (" ".join(hint_segments) + " ") if hint_segments else ""
-            operator_diagnostic = (
-                "deterministic_branch_resolution_exhausted: "
-                f"none of [{branch_list_label}] exists on remote. "
-                f"{hint_text}"
-                "Set --branch <existing-branch> or restore a canonical branch."
-            )
-            branch_hint_probe_error = str(branch_hints.get("error") or "").strip() or None
-            result.status = "failed"
-            result.status_classification = _BRANCH_RESOLUTION_EXHAUSTED
-            result.operator_diagnostic = operator_diagnostic
-            result.remote_default_branch_hint = default_branch_hint
-            result.discovered_remote_branches = discovered_branches
-            result.discovered_remote_branch_count = discovered_branch_count
-            if branch_hint_probe_error:
-                result.branch_hint_probe_error = branch_hint_probe_error
-            if operator_diagnostic not in message:
-                augmented_message = f"{message} | {operator_diagnostic}" if message else operator_diagnostic
-                result.message = augmented_message
-
-            logger.warning(
-                "repo_sync_branch_resolution_exhausted",
-                repo=repo_name,
-                project=repo_project or str(repo_info.project or ""),
-                run_id=run_id,
-                branch_candidates=list(_BRANCH_RESOLUTION_CANDIDATES),
-                remote_url=remote_url,
-                remote_default_branch_hint=default_branch_hint,
-                discovered_remote_branches=discovered_branches,
-                branch_hint_probe_error=branch_hint_probe_error,
-            )