@ngocsangairvds/vsaf 3.2.14 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/collectors/bitbucket_downloader.py

@@ -1,644 +0,0 @@
-"""Bitbucket material downloader for Phase 14."""
-
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import re
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
-from urllib.parse import urlparse
-
-from structlog import get_logger
-
-if TYPE_CHECKING:
-    from vds_audit_orchestrator.clients.bitbucket_cli_client import BitbucketCliClient
-    from vds_audit_orchestrator.collectors.material_cache import MaterialCache
-
-logger = get_logger()
-
-SyncPolicy = Literal["if-changed", "force-refresh", "offline"]
-
-
-class BitbucketMaterialDownloader:
-    """Downloader for Bitbucket repository content (READMEs)."""
-
-    def __init__(
-        self,
-        client: BitbucketCliClient,
-        *,
-        request_timeout: int = 60,
-        include_last_commit: bool = True,
-    ):
-        self.client = client
-        self.request_timeout = max(1, request_timeout)
-        self.include_last_commit = include_last_commit
-
-    def _operation_timeout_seconds(self, *, multiplier: int = 3, floor: int = 20) -> int:
-        return max(floor, self.request_timeout * multiplier)
-
-    def _normalize_sync_policy(self, sync_policy: SyncPolicy | str | None) -> SyncPolicy:
-        if sync_policy is None:
-            return "if-changed"
-        normalized = str(sync_policy).strip().lower()
-        if normalized in {"if-changed", "force-refresh", "offline"}:
-            return normalized  # type: ignore[return-value]
-        logger.warning("bitbucket_sync_policy_invalid", sync_policy=sync_policy, fallback="if-changed")
-        return "if-changed"
-
-    def _is_local_state_repo_reference(self, url: str, project_key: str) -> bool:
-        """Detect synthetic local repo links emitted by state manifests."""
-        try:
-            host = (urlparse(url).hostname or "").strip().lower()
-        except Exception:
-            host = ""
-        return host == "bitbucket.local" and project_key.strip().lower() == "local"
-
-    async def process_link(
-        self,
-        url: str,
-        output_dir: Path,
-        *,
-        cached: list[dict[str, Any]] | None = None,
-        shared_cache: MaterialCache | None = None,
-        sync_policy: SyncPolicy | str | None = None,
-    ) -> list[dict[str, Any]]:
-        """Process a Bitbucket link (Project or Repo) and download materials."""
-        policy = self._normalize_sync_policy(sync_policy)
-        project_key, repo_slug = self._parse_url(url)
-        if not project_key:
-            logger.warning("bitbucket_url_parse_failed", url=url)
-            return []
-
-        if self._is_local_state_repo_reference(url, project_key):
-            checked_at = self._now_iso()
-            logger.info(
-                "bitbucket_local_repo_reference_skipped",
-                project=project_key,
-                repo=repo_slug,
-                url=url,
-            )
-            if not repo_slug:
-                return []
-            return [
-                {
-                    "project": project_key,
-                    "repo": repo_slug,
-                    "repo_url": url,
-                    "file": "README.md",
-                    "path": None,
-                    "sha256": None,
-                    "default_branch": None,
-                    "default_branch_ref": None,
-                    "last_commit": None,
-                    "status": "skipped",
-                    "error": None,
-                    "sync_policy": policy,
-                    "fetch_reason": "bitbucket_local_repo_reference_skipped",
-                    "last_checked_at": checked_at,
-                }
-            ]
-
-        results: list[dict[str, Any]] = []
-        repos_to_process: list[str] = []
-
-        cached_entries: list[dict[str, Any]] = []
-        if cached and policy != "force-refresh":
-            cached_entries.extend(cached)
-        if shared_cache and policy != "force-refresh":
-            cached_entries.extend(shared_cache.materialize_bitbucket(project_key, output_dir))
-
-        if repo_slug:
-            repos_to_process.append(repo_slug)
-        elif policy == "offline":
-            repos_to_process = []
-        else:
-            # Always list repos in project for if-changed/force-refresh.
-            # Cache entries are reused per repo, but should not suppress project scope discovery.
-            logger.info("listing_project_repos", project=project_key)
-            repos = await asyncio.wait_for(
-                self.client.list_repositories(project_key, timeout=self.request_timeout),
-                timeout=float(self._operation_timeout_seconds(multiplier=4)),
-            )
-            for repo_payload in repos:
-                slug = repo_payload.get("slug") if isinstance(repo_payload, dict) else None
-                normalized_slug = self._normalize_repo_slug(slug) if isinstance(slug, str) else None
-                if normalized_slug and normalized_slug not in {"node_modules"}:
-                    repos_to_process.append(normalized_slug)
-
-        bitbucket_dir = output_dir / "bitbucket" / project_key
-        bitbucket_dir.mkdir(parents=True, exist_ok=True)
-
-        cached_by_repo: dict[tuple[str, str, str], dict[str, Any]] = {}
-        for entry in cached_entries:
-            project = entry.get("project")
-            raw_repo = entry.get("repo")
-            repo = self._normalize_repo_slug(raw_repo) if isinstance(raw_repo, str) else None
-            file_name = entry.get("file")
-            if project and repo and file_name:
-                cached_by_repo[(str(project), str(repo), str(file_name))] = entry
-                if policy == "offline" and not repo_slug and str(project) == str(project_key):
-                    repos_to_process.append(str(repo))
-
-        repos_to_process = list(dict.fromkeys(repos_to_process))
-
-        if policy == "offline" and not repos_to_process:
-            return [
-                {
-                    "project": project_key,
-                    "repo": repo_slug,
-                    "file": "README.md",
-                    "path": None,
-                    "sha256": None,
-                    "default_branch": None,
-                    "default_branch_ref": None,
-                    "last_commit": None,
-                    "status": "error",
-                    "error": f"Offline sync policy requires cached Bitbucket entries for project={project_key}",
-                    "sync_policy": "offline",
-                }
-            ]
-
-        for slug in repos_to_process:
-            cache_key = (str(project_key), str(slug), "README.md")
-            cached_entry = cached_by_repo.get(cache_key)
-            started = datetime.now(UTC)
-            try:
-                result = await asyncio.wait_for(
-                    self._download_readme(project_key, slug, bitbucket_dir, cached=cached_entry, sync_policy=policy),
-                    timeout=float(self._operation_timeout_seconds()),
-                )
-            except TimeoutError:
-                result = {
-                    "project": project_key,
-                    "repo": slug,
-                    "file": "README.md",
-                    "path": None,
-                    "sha256": None,
-                    "default_branch": None,
-                    "default_branch_ref": None,
-                    "last_commit": None,
-                    "status": "error",
-                    "error": (
-                        f"Bitbucket README extraction timed out after "
-                        f"{self._operation_timeout_seconds()}s for {project_key}/{slug}"
-                    ),
-                    "sync_policy": policy,
-                    "fetch_reason": "bitbucket_repo_timeout",
-                    "last_checked_at": started.isoformat(),
-                }
-            results.append(result)
-
-        if shared_cache:
-            shared_cache.store_bitbucket(results)
-
-        return results
-
-    def _file_sha256(self, path: Path) -> str:
-        digest = hashlib.sha256()
-        with path.open("rb") as handle:
-            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
-                digest.update(chunk)
-        return digest.hexdigest()
-
-    def _now_iso(self) -> str:
-        return datetime.now(UTC).isoformat()
-
-    def _coerce_int(self, value: Any) -> int | None:
-        if isinstance(value, int):
-            return value
-        if isinstance(value, str) and value.strip().isdigit():
-            return int(value.strip())
-        return None
-
-    def _next_not_modified_count(self, entry: dict[str, Any] | None) -> int:
-        if not isinstance(entry, dict):
-            return 1
-        raw = entry.get("not_modified_count")
-        value = self._coerce_int(raw)
-        return (value or 0) + 1
-
-    def _extract_commit_id(self, last_commit: dict[str, Any] | None) -> str | None:
-        if not isinstance(last_commit, dict):
-            return None
-        for key in ("id", "hash", "commit"):
-            value = last_commit.get(key)
-            if isinstance(value, str) and value.strip():
-                return value.strip()
-        return None
-
-    def _token_from_cached_entry(self, cached: dict[str, Any] | None) -> str | None:
-        if not isinstance(cached, dict):
-            return None
-        branch_ref = cached.get("default_branch_ref") or cached.get("default_branch")
-        branch_ref = branch_ref.strip() if isinstance(branch_ref, str) else ""
-        commit_id = self._extract_commit_id(cached.get("last_commit"))
-        if not branch_ref and not commit_id:
-            return None
-        return f"{branch_ref}:{commit_id or ''}"
-
-    async def _download_readme(
-        self,
-        project: str,
-        repo: str,
-        output_dir: Path,
-        *,
-        cached: dict[str, Any] | None = None,
-        sync_policy: SyncPolicy | str | None = None,
-    ) -> dict[str, Any]:
-        """Download README.md for a repository."""
-        policy = self._normalize_sync_policy(sync_policy)
-        repo_dir = output_dir / repo
-        repo_dir.mkdir(exist_ok=True)
-
-        readme_path = repo_dir / "README.md"
-        status = "missing"
-        error = None
-        content = ""
-
-        if policy == "force-refresh":
-            cached = None
-
-        # Phase 126 Package D TSK-126.12: Check cache freshness before downloading
-        # Use remote signal probe to skip download if repository unchanged
-        fresh_check_result = None
-        if cached and policy == "if-changed":
-            cached_commit_sha = self._extract_commit_id(cached.get("last_commit"))
-            if cached_commit_sha:
-                fresh_check_result = await self.check_freshness(project, repo, cached_commit_sha)
-                # If unchanged based on commit SHA, skip full download
-                if not fresh_check_result.get("changed") and fresh_check_result.get("current_commit_sha"):
-                    cached_path = cached.get("path")
-                    cached_sha256 = cached.get("sha256")
-                    if cached_path and cached_sha256:
-                        cached_file = Path(cached_path)
-                        if cached_file.exists() and self._file_sha256(cached_file) == cached_sha256:
-                            cached_copy = dict(cached)
-                            cached_copy["cached"] = True
-                            cached_copy["status"] = "not_modified"
-                            cached_copy["fetch_reason"] = "cache_hit_remote_unchanged"
-                            cached_copy["not_modified_count"] = self._next_not_modified_count(cached)
-                            cached_copy["last_checked_at"] = self._now_iso()
-                            cached_copy["freshness"] = fresh_check_result.get("freshness", "unknown")
-                            cached_copy["hours_since_commit"] = fresh_check_result.get("hours_since_commit")
-                            cached_copy["token_before"] = self._token_from_cached_entry(cached)
-                            cached_copy["token_after"] = cached_copy["token_before"]
-                            return cached_copy
-
-        default_branch = None
-        default_branch_ref = None
-        last_commit = None
-        token_before = self._token_from_cached_entry(cached)
-        token_after = token_before
-        checked_at = self._now_iso()
-        default_branch_probe_failed = False
-        commit_probe_failed = False
-
-        try:
-            default_branch_timeout = min(self.request_timeout, 10)
-            default_payload = await self.client.get_default_branch(
-                project,
-                repo,
-                timeout=default_branch_timeout,
-            )
-            if isinstance(default_payload, dict):
-                default_branch_ref = (
-                    default_payload.get("id") or default_payload.get("name") or default_payload.get("value")
-                )
-                default_branch = default_payload.get("displayId") or default_branch_ref
-        except Exception as e:
-            default_branch_probe_failed = True
-            logger.debug("default_branch_fetch_failed", project=project, repo=repo, error=str(e))
-
-        if self.include_last_commit:
-            try:
-                commit_probe_timeout = min(self.request_timeout, 10)
-                commits = await self.client.get_commits(project, repo, limit=1, timeout=commit_probe_timeout)
-                if commits:
-                    commit = commits[0]
-                    last_commit = {
-                        "id": commit.get("id") or commit.get("hash"),
-                        "author": (commit.get("author") or {}).get("displayName"),
-                        "date": commit.get("authorTimestamp") or commit.get("date"),
-                    }
-            except Exception as e:
-                commit_probe_failed = True
-                logger.debug("commit_fetch_failed", project=project, repo=repo, error=str(e))
-
-        probe_failed = default_branch_probe_failed or (self.include_last_commit and commit_probe_failed)
-        if probe_failed and cached and token_before:
-            token_after = token_before
-        else:
-            token_after = f"{default_branch_ref or default_branch or ''}:{self._extract_commit_id(last_commit) or ''}"
-
-        if cached:
-            cached_path = cached.get("path")
-            cached_sha256 = cached.get("sha256")
-            if cached_path and cached_sha256:
-                cached_file = Path(cached_path)
-                if cached_file.exists() and self._file_sha256(cached_file) == cached_sha256:
-                    token_changed = (
-                        policy == "if-changed"
-                        and token_before is not None
-                        and token_after is not None
-                        and token_before != token_after
-                    )
-                    if token_changed:
-                        cached = None
-                    else:
-                        cached_copy = dict(cached)
-                        cached_copy["cached"] = True
-                        cached_copy["default_branch"] = default_branch or cached.get("default_branch")
-                        cached_copy["default_branch_ref"] = default_branch_ref or cached.get("default_branch_ref")
-                        cached_copy["last_commit"] = (
-                            last_commit if last_commit is not None else cached.get("last_commit")
-                        )
-                        cached_copy["token_before"] = token_before
-                        cached_copy["token_after"] = token_after
-                        cached_copy["last_checked_at"] = checked_at
-                        if policy == "if-changed" and token_before and token_after and token_before == token_after:
-                            cached_copy["status"] = "not_modified"
-                            cached_copy["fetch_reason"] = "bitbucket_commit_ref_match"
-                            cached_copy["not_modified_count"] = self._next_not_modified_count(cached)
-                        else:
-                            cached_copy["fetch_reason"] = "cache_reuse_no_probe"
-                        return cached_copy
-
-        if policy == "offline":
-            return {
-                "project": project,
-                "repo": repo,
-                "file": "README.md",
-                "path": None,
-                "sha256": None,
-                "default_branch": default_branch,
-                "default_branch_ref": default_branch_ref,
-                "last_commit": last_commit,
-                "status": "error",
-                "error": f"Offline sync policy requires cached Bitbucket README for {project}/{repo}",
-                "sync_policy": "offline",
-                "token_before": token_before,
-                "token_after": token_after,
-                "last_checked_at": checked_at,
-            }
-
-        try:
-            # Prefer the repo's default branch ref when available. Bitbucket Server commonly
-            # defaults to refs/heads/master which may not exist for newer repos.
-            if default_branch_ref:
-                content = await self.client.get_file_content(
-                    project,
-                    repo,
-                    "README.md",
-                    at=str(default_branch_ref),
-                    timeout=self.request_timeout,
-                )
-            else:
-                content = await self.client.get_file_content(
-                    project,
-                    repo,
-                    "README.md",
-                    timeout=self.request_timeout,
-                )
-            if content:
-                readme_path.write_text(content, encoding="utf-8")
-                status = "found"
-            else:
-                status = "empty"
-        except Exception as e:
-            # It's common for README to be missing
-            error = str(e)
-            lowered = error.lower()
-            if any(
-                marker in lowered
-                for marker in (
-                    "404",
-                    "not found",
-                    "does not exist at revision",
-                    'the path "readme.md" does not exist',
-                )
-            ):
-                logger.debug("readme_missing", project=project, repo=repo, error=error)
-            else:
-                logger.warning("readme_download_error", project=project, repo=repo, error=error)
-
-        return {
-            "project": project,
-            "repo": repo,
-            "file": "README.md",
-            "path": str(readme_path) if status == "found" else None,
-            "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest() if status == "found" else None,
-            "default_branch": default_branch,
-            "default_branch_ref": default_branch_ref,
-            "last_commit": last_commit,
-            "status": status,
-            "error": error,
-            "token_before": token_before,
-            "token_after": token_after,
-            "fetch_reason": self._determine_fetch_reason(
-                policy=policy,
-                token_before=token_before,
-                token_after=token_after,
-                fresh_check_result=fresh_check_result,
-                status=status,
-            ),
-            "last_checked_at": checked_at,
-        }
-
-    def _determine_fetch_reason(
-        self,
-        *,
-        policy: SyncPolicy,
-        token_before: str | None,
-        token_after: str | None,
-        fresh_check_result: dict[str, Any] | None,
-        status: str,
-    ) -> str:
-        """Determine the fetch reason for the downloaded README.
-
-        Phase 126 Package D TSK-126.12: Enhanced fetch reason tagging with
-        remote signal probe integration.
-
-        Args:
-            policy: Sync policy used
-            token_before: Cached change token
-            token_after: Current change token
-            fresh_check_result: Result from check_freshness() remote probe
-            status: Download status
-
-        Returns:
-            Fetch reason string
-        """
-        # Signal-based refresh: remote probe indicated change
-        if fresh_check_result and fresh_check_result.get("changed"):
-            return "signal_refresh_bitbucket_commit_changed"
-
-        # Traditional token-based change detection
-        if policy == "if-changed" and token_before and token_after and token_before != token_after:
-            return "bitbucket_commit_ref_changed"
-
-        # Fresh download (no cache or forced)
-        if status == "found":
-            if policy == "force-refresh":
-                return "fresh_download_force_refresh"
-            if not token_before:
-                return "fresh_download_no_cache"
-            return "fresh_download"
-
-        # Missed download (README not found)
-        if status == "missing":
-            return "bitbucket_readme_missing"
-
-        return "bitbucket_readme_downloaded"
-
-    def _parse_url(self, url: str) -> tuple[str | None, str | None]:
-        """Extract project key and repo slug from URL.
-
-        Supported formats:
-        - .../projects/PROJ/repos/repo-slug/...
-        - .../projects/PROJ
-        - .../scm/PROJ/repo-slug.git  (git clone URL)
-        - .../scm/PROJ/repo-slug
-        """
-        # Try REST API format first: /projects/PROJ/repos/repo-slug(.git)
-        project_match = re.search(r"/projects/([^/]+)", url)
-        if project_match:
-            project_key = project_match.group(1)
-            repo_match = re.search(r"/repos/([^/]+)", url)
-            repo_slug = self._normalize_repo_slug(repo_match.group(1)) if repo_match else None
-            return project_key, repo_slug
-
-        # Try git clone format: /scm/PROJ/repo-slug.git
-        scm_match = re.search(r"/scm/([^/]+)/([^/]+?)(?:\.git)?$", url)
-        if scm_match:
-            return scm_match.group(1).upper(), scm_match.group(2)
-
-        return None, None
-
-    @staticmethod
-    def _normalize_repo_slug(value: str | None) -> str | None:
-        slug = str(value or "").strip()
-        if not slug:
-            return None
-        if slug.lower().endswith(".git"):
-            slug = slug[:-4]
-        return slug or None
-
-    # -------------------------------------------------------------------------
-    # Phase 126 Package B: Remote Signal Probe Integration (TSK-126.6)
-    # -------------------------------------------------------------------------
-
-    async def check_freshness(
-        self,
-        project_key: str,
-        repo_slug: str,
-        cached_commit_sha: str | None,
-    ) -> dict[str, Any]:
-        """Check repository freshness using BitbucketProbe.
-
-        Uses remote signal probe to determine if the repository has changed
-        since the cached version. This is a bandwidth-efficient check that
-        only queries commit metadata, not full content.
-
-        Args:
-            project_key: Bitbucket project key (e.g., "PAR", "CEP")
-            repo_slug: Repository slug (e.g., "merchant-report")
-            cached_commit_sha: Previously cached commit SHA to compare against
-
-        Returns:
-            Dict with keys:
-            - changed: bool - True if repo changed since cached version
-            - current_commit_sha: str | None - Current commit SHA
-            - freshness: str - "fresh", "aging", "stale", or "unknown"
-            - hours_since_commit: int | None - Hours since last commit
-            - last_commit_timestamp: str | None - ISO timestamp of last commit
-
-        Example:
-            >>> downloader = BitbucketMaterialDownloader(client)
-            >>> result = await downloader.check_freshness("PAR", "merchant-report", "abc123")
-            >>> if result["changed"]:
-            ...     print(f"Repo updated, new commit: {result['current_commit_sha']}")
-        """
-        from vds_audit_orchestrator.collectors.completeness.bitbucket_probe import BitbucketProbe
-
-        probe = BitbucketProbe(self.client)
-
-        # Get metadata for comprehensive freshness info
-        metadata = await probe.get_repo_metadata(project_key, repo_slug)
-
-        if metadata is None:
-            return {
-                "changed": False,
-                "current_commit_sha": None,
-                "freshness": "unknown",
-                "hours_since_commit": None,
-                "last_commit_timestamp": None,
-                "error": "probe_failed",
-            }
-
-        changed = metadata.get("latest_commit_sha") != cached_commit_sha
-
-        return {
-            "changed": changed,
-            "current_commit_sha": metadata.get("latest_commit_sha"),
-            "freshness": metadata.get("freshness", "unknown"),
-            "hours_since_commit": metadata.get("hours_since_commit"),
-            "last_commit_timestamp": metadata.get("last_commit_timestamp"),
-            "default_branch": metadata.get("default_branch"),
-        }
-
-    async def batch_check_freshness(
-        self,
-        repos: list[dict[str, Any]],
-    ) -> list[dict[str, Any]]:
-        """Check freshness for multiple repositories efficiently.
-
-        Performs remote signal probes for a batch of repositories, returning
-        freshness status for each. Useful for cache health assessment.
-
-        Args:
-            repos: List of dicts with keys:
-                - project: Bitbucket project key
-                - repo: Repository slug
-                - sha256 or last_commit: Cached commit SHA to compare
-
-        Returns:
-            List of dicts with keys from check_freshness plus project and repo
-
-        Example:
-            >>> repos = [
-            ...     {"project": "PAR", "repo": "merchant-report", "sha256": "abc123"},
-            ...     {"project": "PAR", "repo": "merchant-billing", "sha256": "def456"},
-            ... ]
-            >>> results = await downloader.batch_check_freshness(repos)
-            >>> for r in results:
-            ...     print(f"{r['repo']}: {r['freshness']}")
-        """
-        results = []
-
-        for repo_info in repos:
-            project = repo_info.get("project")
-            repo = repo_info.get("repo")
-            cached_sha = repo_info.get("sha256") or repo_info.get("last_commit")
-
-            if not project or not repo:
-                results.append(
-                    {
-                        "project": project,
-                        "repo": repo,
-                        "changed": False,
-                        "current_commit_sha": None,
-                        "freshness": "unknown",
-                        "hours_since_commit": None,
-                        "error": "missing_project_or_repo",
-                    }
-                )
-                continue
-
-            result = await self.check_freshness(project, repo, cached_sha)
-            result["project"] = project
-            result["repo"] = repo
-            results.append(result)
-
-        return results