@ngocsangairvds/vsaf 3.1.26 → 3.2.1

package/tools/vds-scripts/git_orchestrator/src/vds_git_orchestrator/orchestrator.py

@@ -0,0 +1,819 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from base64 import b64encode
+from dataclasses import dataclass
+from enum import Enum
+
+from .manifest import Manifest, ManifestFilters, Repository
+
+logger = logging.getLogger(__name__)
+
+
+async def _run_hook(repo: Repository, command: str) -> tuple[int, str, str]:
+    process = await asyncio.create_subprocess_shell(
+        command,
+        cwd=str(repo.absolute_path),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await process.communicate()
+    return process.returncode or 0, stdout.decode(), stderr.decode()
+
+
+class GitOperation(str, Enum):
+    FETCH = "fetch"
+    PULL = "pull"
+    STATUS = "status"
+    CLONE = "clone"
+    BRANCH = "branch"
+    CHECKOUT = "checkout"
+    ADD = "add"
+    COMMIT = "commit"
+    PUSH = "push"
+    LOG = "log"
+    DIFF = "diff"
+    STASH = "stash"
+    TAG = "tag"
+    REMOTE = "remote"
+
+
+@dataclass
+class RepoResult:
+    repo: Repository
+    success: bool
+    updated: bool = False
+    message: str | None = None
+
+
+@dataclass
+class RunSummary:
+    total: int
+    successes: int
+    updates: int
+    failures: int
+    results: dict[str, RepoResult]
+    serial_retries: int = 0
+
+    @classmethod
+    def from_results(
+        cls, results: dict[str, RepoResult], serial_retries: int = 0
+    ) -> RunSummary:
+        successes = sum(1 for result in results.values() if result.success)
+        updates = sum(1 for result in results.values() if result.updated)
+        failures = len(results) - successes
+        return cls(
+            total=len(results),
+            successes=successes,
+            updates=updates,
+            failures=failures,
+            results=results,
+            serial_retries=serial_retries,
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "total": self.total,
+            "successes": self.successes,
+            "updates": self.updates,
+            "failures": self.failures,
+            "serial_retries": self.serial_retries,
+            "results": {
+                name: {
+                    "project": result.repo.project,
+                    "path": str(result.repo.absolute_path),
+                    "branch": result.repo.branch,
+                    "success": result.success,
+                    "updated": result.updated,
+                    "message": result.message,
+                }
+                for name, result in self.results.items()
+            },
+        }
+
+
+def _build_git_command(repo: Repository, args: list[str]) -> list[str]:
+    cmd = ["git"]
+    username = os.environ.get("VDS_USERNAME")
+    password = os.environ.get("VDS_PASSWORD")
+    if username and password:
+        token = b64encode(f"{username}:{password}".encode()).decode()
+        header = f"Authorization: Basic {token}"
+        cmd.extend(["-c", f"http.extraHeader={header}"])
+    cmd.extend(args)
+    return cmd
+
+
+async def _run_git(repo: Repository, args: list[str]) -> tuple[int, str, str]:
+    command = _build_git_command(repo, args)
+    return await _run_process(command, repo.absolute_path)
+
+
+async def _run_process(
+    command: list[str], cwd: os.PathLike[str]
+) -> tuple[int, str, str]:
+    process = await asyncio.create_subprocess_exec(
+        *command,
+        cwd=str(cwd),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await process.communicate()
+    return process.returncode or 0, stdout.decode(), stderr.decode()
+
+
+def _remote_base() -> str:
+    return os.environ.get(
+        "VDS_BITBUCKET_HTTP_BASE", "http://bitbucket.digital.vn/scm"
+    ).rstrip("/")
+
+
+def _remote_url(repo: Repository) -> str:
+    return f"{_remote_base()}/{repo.project}/{repo.name}.git"
+
+
+async def _pull_repo(repo: Repository, dry_run: bool, run_id: str) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "pull", extra={"repo": repo.name, "project": repo.project, "run_id": run_id}
+    )
+    returncode, stdout, stderr = await _run_git(repo, ["pull", "--ff-only"])
+
+    if returncode == 0:
+        updated = any(line for line in stdout.splitlines() if "Fast-forward" in line)
+        return RepoResult(
+            repo=repo, success=True, updated=updated, message=stdout.strip()
+        )
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _fetch_repo(repo: Repository, dry_run: bool, run_id: str) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "fetch", extra={"repo": repo.name, "project": repo.project, "run_id": run_id}
+    )
+    returncode, stdout, stderr = await _run_git(repo, ["fetch", "--all"])
+
+    if returncode == 0:
+        return RepoResult(
+            repo=repo, success=True, updated=False, message=stdout.strip()
+        )
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _status_repo(repo: Repository, run_id: str) -> RepoResult:
+    returncode, stdout, stderr = await _run_git(repo, ["status", "--short"])
+    if returncode == 0:
+        message = stdout.strip() or "clean"
+        updated = bool(stdout.strip())
+        return RepoResult(repo=repo, success=True, updated=updated, message=message)
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _clone_repo(repo: Repository, dry_run: bool, run_id: str) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    if repo.absolute_path.exists():
+        return RepoResult(repo=repo, success=True, updated=False, message="exists")
+
+    repo.absolute_path.parent.mkdir(parents=True, exist_ok=True)
+
+    logger.info(
+        "clone", extra={"repo": repo.name, "project": repo.project, "run_id": run_id}
+    )
+
+    remote = _remote_url(repo)
+    branch_args = ["--branch", repo.branch] if repo.branch else []
+    command = _build_git_command(
+        repo, ["clone", *branch_args, remote, str(repo.absolute_path)]
+    )
+    returncode, stdout, stderr = await _run_process(command, repo.absolute_path.parent)
+
+    if returncode != 0 and branch_args:
+        # Retry without explicit branch to match legacy script behaviour.
+        command = _build_git_command(repo, ["clone", remote, str(repo.absolute_path)])
+        returncode, stdout, stderr = await _run_process(
+            command, repo.absolute_path.parent
+        )
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=True, message="cloned")
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _branch_repo(
+    repo: Repository,
+    action: str,
+    branch_name: str | None,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "branch",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "action": action,
+            "run_id": run_id,
+        },
+    )
+
+    if action == "list":
+        returncode, stdout, stderr = await _run_git(repo, ["branch", "-a"])
+    elif action == "create" and branch_name:
+        returncode, stdout, stderr = await _run_git(repo, ["branch", branch_name])
+    elif action == "delete" and branch_name:
+        returncode, stdout, stderr = await _run_git(repo, ["branch", "-d", branch_name])
+    else:
+        return RepoResult(
+            repo=repo, success=False, message=f"Invalid branch action: {action}"
+        )
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=False, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _checkout_repo(
+    repo: Repository,
+    branch_or_path: str,
+    create: bool,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "checkout",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "target": branch_or_path,
+            "run_id": run_id,
+        },
+    )
+
+    args = ["checkout"]
+    if create:
+        args.append("-b")
+    args.append(branch_or_path)
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=True, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _add_repo(
+    repo: Repository,
+    paths: list[str],
+    all_files: bool,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "add",
+        extra={"repo": repo.name, "project": repo.project, "run_id": run_id},
+    )
+
+    args = ["add"]
+    if all_files:
+        args.append(".")
+    else:
+        args.extend(paths)
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=True, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _commit_repo(
+    repo: Repository,
+    message: str,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "commit",
+        extra={"repo": repo.name, "project": repo.project, "run_id": run_id},
+    )
+
+    returncode, stdout, stderr = await _run_git(repo, ["commit", "-m", message])
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=True, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _push_repo(
+    repo: Repository,
+    branch: str | None,
+    remote: str | None,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "push",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "branch": branch,
+            "run_id": run_id,
+        },
+    )
+
+    args = ["push"]
+    if remote:
+        args.append(remote)
+    if branch:
+        args.append(branch)
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=True, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _log_repo(
+    repo: Repository,
+    limit: int | None,
+    oneline: bool,
+    run_id: str,
+) -> RepoResult:
+    logger.info(
+        "log", extra={"repo": repo.name, "project": repo.project, "run_id": run_id}
+    )
+
+    args = ["log"]
+    if oneline:
+        args.append("--oneline")
+    if limit:
+        args.extend(["-n", str(limit)])
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=False, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _diff_repo(
+    repo: Repository,
+    paths: list[str],
+    staged: bool,
+    run_id: str,
+) -> RepoResult:
+    logger.info(
+        "diff", extra={"repo": repo.name, "project": repo.project, "run_id": run_id}
+    )
+
+    args = ["diff"]
+    if staged:
+        args.append("--staged")
+    args.extend(paths)
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=False, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _stash_repo(
+    repo: Repository,
+    action: str,
+    message: str | None,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "stash",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "action": action,
+            "run_id": run_id,
+        },
+    )
+
+    args = ["stash"]
+    if action == "list":
+        args.append("list")
+    elif action == "save" and message:
+        args.extend(["save", message])
+    elif action == "pop":
+        args.append("pop")
+    elif action == "apply":
+        args.append("apply")
+    elif action == "drop":
+        args.append("drop")
+    else:
+        return RepoResult(
+            repo=repo, success=False, message=f"Invalid stash action: {action}"
+        )
+
+    returncode, stdout, stderr = await _run_git(repo, args)
+
+    if returncode == 0:
+        updated = action in ("pop", "apply", "drop")
+        return RepoResult(
+            repo=repo, success=True, updated=updated, message=stdout.strip()
+        )
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _tag_repo(
+    repo: Repository,
+    action: str,
+    tag_name: str | None,
+    message: str | None,
+    dry_run: bool,
+    run_id: str,
+) -> RepoResult:
+    if dry_run:
+        return RepoResult(repo=repo, success=True, updated=False, message="dry-run")
+
+    logger.info(
+        "tag",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "action": action,
+            "run_id": run_id,
+        },
+    )
+
+    if action == "list":
+        returncode, stdout, stderr = await _run_git(repo, ["tag", "-l"])
+    elif action == "create" and tag_name:
+        args = ["tag"]
+        if message:
+            args.extend(["-a", tag_name, "-m", message])
+        else:
+            args.append(tag_name)
+        returncode, stdout, stderr = await _run_git(repo, args)
+    elif action == "delete" and tag_name:
+        returncode, stdout, stderr = await _run_git(repo, ["tag", "-d", tag_name])
+    else:
+        return RepoResult(
+            repo=repo, success=False, message=f"Invalid tag action: {action}"
+        )
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=False, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _remote_repo(
+    repo: Repository,
+    action: str,
+    name: str | None,
+    url: str | None,
+    run_id: str,
+) -> RepoResult:
+    logger.info(
+        "remote",
+        extra={
+            "repo": repo.name,
+            "project": repo.project,
+            "action": action,
+            "run_id": run_id,
+        },
+    )
+
+    if action == "list":
+        returncode, stdout, stderr = await _run_git(repo, ["remote", "-v"])
+    elif action == "add" and name and url:
+        returncode, stdout, stderr = await _run_git(repo, ["remote", "add", name, url])
+    elif action == "remove" and name:
+        returncode, stdout, stderr = await _run_git(repo, ["remote", "remove", name])
+    else:
+        return RepoResult(
+            repo=repo, success=False, message=f"Invalid remote action: {action}"
+        )
+
+    if returncode == 0:
+        return RepoResult(repo=repo, success=True, updated=False, message=stdout.strip())
+
+    return RepoResult(
+        repo=repo, success=False, message=stderr.strip() or stdout.strip()
+    )
+
+
+async def _operate_repo(
+    repo: Repository,
+    operation: GitOperation,
+    dry_run: bool,
+    run_id: str,
+    **kwargs,
+) -> RepoResult:
+    if operation == GitOperation.PULL:
+        return await _pull_repo(repo, dry_run=dry_run, run_id=run_id)
+    if operation == GitOperation.FETCH:
+        return await _fetch_repo(repo, dry_run=dry_run, run_id=run_id)
+    if operation == GitOperation.STATUS:
+        return await _status_repo(repo, run_id=run_id)
+    if operation == GitOperation.CLONE:
+        return await _clone_repo(repo, dry_run=dry_run, run_id=run_id)
+    if operation == GitOperation.BRANCH:
+        return await _branch_repo(
+            repo,
+            kwargs.get("action", "list"),
+            kwargs.get("branch_name"),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.CHECKOUT:
+        return await _checkout_repo(
+            repo,
+            kwargs.get("branch_or_path", "main"),
+            kwargs.get("create", False),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.ADD:
+        return await _add_repo(
+            repo,
+            kwargs.get("paths", []),
+            kwargs.get("all_files", False),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.COMMIT:
+        return await _commit_repo(
+            repo,
+            kwargs.get("message", ""),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.PUSH:
+        return await _push_repo(
+            repo,
+            kwargs.get("branch"),
+            kwargs.get("remote"),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.LOG:
+        return await _log_repo(
+            repo,
+            kwargs.get("limit"),
+            kwargs.get("oneline", False),
+            run_id=run_id,
+        )
+    if operation == GitOperation.DIFF:
+        return await _diff_repo(
+            repo,
+            kwargs.get("paths", []),
+            kwargs.get("staged", False),
+            run_id=run_id,
+        )
+    if operation == GitOperation.STASH:
+        return await _stash_repo(
+            repo,
+            kwargs.get("action", "list"),
+            kwargs.get("message"),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.TAG:
+        return await _tag_repo(
+            repo,
+            kwargs.get("action", "list"),
+            kwargs.get("tag_name"),
+            kwargs.get("message"),
+            dry_run=dry_run,
+            run_id=run_id,
+        )
+    if operation == GitOperation.REMOTE:
+        return await _remote_repo(
+            repo,
+            kwargs.get("action", "list"),
+            kwargs.get("name"),
+            kwargs.get("url"),
+            run_id=run_id,
+        )
+    raise ValueError(f"Unsupported operation: {operation}")
+
+
+async def orchestrate_repositories(
+    manifest: Manifest,
+    operation: GitOperation,
+    filters: ManifestFilters,
+    concurrency: int,
+    dry_run: bool,
+    run_id: str,
+    retries: int = 2,
+    backoff: float = 1.5,
+    allow_serial_recovery: bool = True,
+    **operation_kwargs,
+) -> RunSummary:
+    repositories = manifest.filter(filters)
+    results: dict[str, RepoResult] = {}
+
+    semaphore = asyncio.Semaphore(concurrency)
+    serial_retries = 0
+
+    async def worker(repo: Repository) -> None:
+        async with semaphore:
+            attempt = 0
+            delay = 1.0
+            result: RepoResult
+            while True:
+                attempt += 1
+                try:
+                    # pre hooks
+                    if not dry_run:
+                        pre_key = {
+                            GitOperation.PULL: "pre_pull",
+                            GitOperation.FETCH: "pre_fetch",
+                            GitOperation.STATUS: "pre_status",
+                            GitOperation.BRANCH: "pre_branch",
+                            GitOperation.CHECKOUT: "pre_checkout",
+                            GitOperation.ADD: "pre_add",
+                            GitOperation.COMMIT: "pre_commit",
+                            GitOperation.PUSH: "pre_push",
+                            GitOperation.STASH: "pre_stash",
+                            GitOperation.TAG: "pre_tag",
+                        }.get(operation)
+                        if pre_key:
+                            for command in repo.hooks.get(pre_key, []):
+                                rc, h_stdout, h_stderr = await _run_hook(repo, command)
+                                if rc != 0:
+                                    results[repo.name] = RepoResult(
+                                        repo=repo,
+                                        success=False,
+                                        message=h_stderr.strip()
+                                        or h_stdout.strip()
+                                        or f"hook failed: {command}",
+                                    )
+                                    return
+                    result = await _operate_repo(
+                        repo,
+                        operation=operation,
+                        dry_run=dry_run,
+                        run_id=run_id,
+                        **operation_kwargs,
+                    )
+                except Exception as exc:  # pragma: no cover - fallback safety
+                    logger.exception(
+                        "operation-failed", extra={"repo": repo.name, "run_id": run_id}
+                    )
+                    result = RepoResult(repo=repo, success=False, message=str(exc))
+
+                if result.success or attempt > retries:
+                    results[repo.name] = result
+                    if result.success and not dry_run:
+                        post_key = {
+                            GitOperation.PULL: "post_pull",
+                            GitOperation.FETCH: "post_fetch",
+                            GitOperation.STATUS: "post_status",
+                            GitOperation.BRANCH: "post_branch",
+                            GitOperation.CHECKOUT: "post_checkout",
+                            GitOperation.ADD: "post_add",
+                            GitOperation.COMMIT: "post_commit",
+                            GitOperation.PUSH: "post_push",
+                            GitOperation.STASH: "post_stash",
+                            GitOperation.TAG: "post_tag",
+                        }.get(operation)
+                        if post_key:
+                            for command in repo.hooks.get(post_key, []):
+                                rc, h_stdout, h_stderr = await _run_hook(repo, command)
+                                if rc != 0:
+                                    logger.warning(
+                                        "hook-failed",
+                                        extra={
+                                            "repo": repo.name,
+                                            "hook": command,
+                                            "run_id": run_id,
+                                        },
+                                    )
+                    break
+
+                logger.warning(
+                    "retrying",
+                    extra={
+                        "repo": repo.name,
+                        "project": repo.project,
+                        "run_id": run_id,
+                        "attempt": attempt,
+                    },
+                )
+                await asyncio.sleep(delay)
+                delay *= backoff
+
+    async with asyncio.TaskGroup() as tg:
+        for repo in repositories:
+            tg.create_task(worker(repo))
+
+    if allow_serial_recovery and not dry_run and concurrency > 1:
+        fallback_repos = [
+            repo
+            for repo in repositories
+            if not results[repo.name].success
+            and _is_network_error(results[repo.name].message or "")
+        ]
+        if fallback_repos:
+            logger.warning(
+                "serial-retry",
+                extra={
+                    "run_id": run_id,
+                    "repos": [r.name for r in fallback_repos],
+                },
+            )
+            serial_retries = len(fallback_repos)
+            fallback_manifest = Manifest(fallback_repos)
+            fallback_summary = await orchestrate_repositories(
+                manifest=fallback_manifest,
+                operation=operation,
+                filters=ManifestFilters(),
+                concurrency=1,
+                dry_run=False,
+                run_id=run_id,
+                retries=retries,
+                backoff=backoff,
+                allow_serial_recovery=False,
+            )
+            results.update(fallback_summary.results)
+
+    return RunSummary.from_results(results, serial_retries=serial_retries)
+
+
+def _is_network_error(message: str) -> bool:
+    lowered = message.lower()
+    patterns = (
+        "failed to connect",
+        "couldn't connect",
+        "could not connect",
+        "timeout",
+        "timed out",
+        "connection refused",
+        "could not resolve host",
+    )
+    return any(pattern in lowered for pattern in patterns)