@kernel-sig/console 0.1.0

package/backend/app/integrations/atomgit/client.py

@@ -0,0 +1,505 @@
+"""AtomGit / GitCode REST API 客户端。
+
+实现依据为**实测行为**而非官方文档：
+
+- 认证必须使用 ``private-token`` 请求头。官方文档所载的
+  ``Authorization: token <TOKEN>`` 在 atomgit.com 上返回 400。
+- 分页总数放在响应头 ``total_count`` / ``total_page``，不在响应体。
+- 文档中描述的若干端点（operate-logs、files-json、issues/{n}/pull-requests 等）
+  在 atomgit.com 实例上并不存在，客户端不依赖它们。
+
+设计约束：客户端不依赖任何内部模块（models/services），
+只做 HTTP 与数据解析，以便被 service、worker 与测试独立复用。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import random
+from dataclasses import dataclass, field
+from typing import Any, TypeVar
+
+import httpx
+import structlog
+from pydantic import BaseModel, ValidationError
+
+from app.integrations.atomgit.models import (
+    AtomGitComment,
+    AtomGitCommit,
+    AtomGitFile,
+    AtomGitIssue,
+    AtomGitLabel,
+    AtomGitPullRequest,
+    AtomGitRepository,
+    AtomGitUser,
+)
+
+logger = structlog.get_logger(__name__)
+
+DEFAULT_BASE_URL = "https://api.atomgit.com/api/v5"
+DEFAULT_TIMEOUT = 30.0
+MAX_PER_PAGE = 100
+
+T = TypeVar("T", bound=BaseModel)
+
+# 重试策略：仅对可恢复的错误重试
+RETRYABLE_STATUS = {429, 500, 502, 503, 504}
+MAX_ATTEMPTS = 4
+BASE_BACKOFF_SECONDS = 1.0
+
+
+class AtomGitError(Exception):
+    """AtomGit 客户端异常基类。"""
+
+
+class AtomGitAuthError(AtomGitError):
+    """凭据无效或权限不足。"""
+
+
+class AtomGitNotFoundError(AtomGitError):
+    """资源不存在。"""
+
+
+class AtomGitRateLimitError(AtomGitError):
+    """触发上游限流。"""
+
+
+class AtomGitUpstreamError(AtomGitError):
+    """上游服务错误（5xx 或网络故障，重试后仍失败）。"""
+
+
+@dataclass
+class Page[T: BaseModel]:
+    """一页结果，附带分页元信息。"""
+
+    items: list[T]
+    page: int
+    per_page: int
+    total_count: int | None = None
+    total_page: int | None = None
+
+    @property
+    def has_more(self) -> bool:
+        if self.total_page is not None:
+            return self.page < self.total_page
+        return len(self.items) == self.per_page
+
+
+@dataclass
+class RequestStats:
+    """累计请求统计，用于同步任务的可观测性。"""
+
+    requests: int = 0
+    retries: int = 0
+    errors: int = 0
+    by_endpoint: dict[str, int] = field(default_factory=dict)
+
+
+class AtomGitClient:
+    """AtomGit API 异步客户端。
+
+    通过 ``base_url`` 参数支持切换到 GitCode 等兼容实例
+    （``https://api.gitcode.com/api/v5``）；两者 token 通用。
+    """
+
+    def __init__(
+        self,
+        token: str,
+        base_url: str = DEFAULT_BASE_URL,
+        *,
+        timeout: float = DEFAULT_TIMEOUT,
+        user_agent: str = "kernel-sig-console/0.1",
+        transport: httpx.AsyncBaseTransport | None = None,
+        max_concurrency: int = 5,
+    ) -> None:
+        if not token:
+            raise ValueError("必须提供 AtomGit token")
+        self._base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(
+            base_url=self._base_url,
+            timeout=timeout,
+            headers={
+                "private-token": token,
+                "Accept": "application/json",
+                "User-Agent": user_agent,
+            },
+            transport=transport,
+            follow_redirects=True,
+        )
+        # 并发上限：避免瞬时打爆上游触发限流
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+        self.stats = RequestStats()
+
+    async def __aenter__(self) -> AtomGitClient:
+        return self
+
+    async def __aexit__(self, *exc_info: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    # ------------------------------------------------------------------
+    # 底层请求
+    # ------------------------------------------------------------------
+
+    async def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: dict[str, Any] | None = None,
+        json: dict[str, Any] | None = None,
+    ) -> httpx.Response:
+        endpoint = path.split("?")[0]
+        self.stats.by_endpoint[endpoint] = self.stats.by_endpoint.get(endpoint, 0) + 1
+
+        # 保留最后一次失败的具体错误类型。限流（429）也属于可重试状态，
+        # 但重试耗尽后应仍然告诉调用方"这是限流"，而不是丢失语义变成通用上游错误。
+        last_error: AtomGitError | None = None
+
+        async with self._semaphore:
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                self.stats.requests += 1
+                try:
+                    response = await self._client.request(method, path, params=params, json=json)
+                except httpx.HTTPError as exc:
+                    last_error = AtomGitUpstreamError(f"网络请求失败：{exc}")
+                    if attempt == MAX_ATTEMPTS:
+                        break
+                    await self._sleep_backoff(attempt)
+                    continue
+
+                if response.status_code in RETRYABLE_STATUS:
+                    detail = self._error_detail(response)
+                    if response.status_code == 429:
+                        last_error = AtomGitRateLimitError(f"触发 AtomGit 限流：{detail}")
+                    else:
+                        last_error = AtomGitUpstreamError(
+                            f"{method} {path} 返回 {response.status_code}：{detail}"
+                        )
+                    if attempt == MAX_ATTEMPTS:
+                        break
+                    self.stats.retries += 1
+                    await self._sleep_backoff(attempt, response=response)
+                    continue
+
+                self._raise_for_status(response, method, path)
+                return response
+
+        self.stats.errors += 1
+        assert last_error is not None  # 循环必然赋值后才可能走到这里
+        # 保持原异常类型，只在文案上补充重试次数，便于排查上游持续性故障
+        raise type(last_error)(f"{last_error}（已重试 {MAX_ATTEMPTS} 次）") from last_error
+
+    @staticmethod
+    def _raise_for_status(response: httpx.Response, method: str, path: str) -> None:
+        code = response.status_code
+        if code < 400:
+            return
+
+        detail = AtomGitClient._error_detail(response)
+
+        if code == 401:
+            raise AtomGitAuthError("AtomGit 凭据无效或已过期，请在「设置 → 凭据」中更新 token")
+        if code == 403:
+            # AtomGit 用 403 同时表达"权限不足"与"触发限流"
+            if "rate limit" in detail.lower() or "too many" in detail.lower():
+                raise AtomGitRateLimitError(f"触发 AtomGit 限流：{detail}")
+            raise AtomGitAuthError(f"当前 token 无权访问 {method} {path}：{detail}")
+        if code == 404:
+            raise AtomGitNotFoundError(f"资源不存在：{method} {path}")
+        if code == 429:
+            raise AtomGitRateLimitError(f"触发 AtomGit 限流：{detail}")
+
+        raise AtomGitUpstreamError(f"{method} {path} 返回 {code}：{detail}")
+
+    @staticmethod
+    def _error_detail(response: httpx.Response) -> str:
+        try:
+            payload = response.json()
+        except ValueError:
+            return response.text[:200]
+        if isinstance(payload, dict):
+            for key in ("message", "error_message", "error", "detail"):
+                value = payload.get(key)
+                if isinstance(value, str) and value:
+                    return value
+        return response.text[:200]
+
+    @staticmethod
+    async def _sleep_backoff(attempt: int, response: httpx.Response | None = None) -> None:
+        """指数退避 + 抖动。若上游给出 Retry-After 则优先遵守。"""
+        delay = BASE_BACKOFF_SECONDS * (2 ** (attempt - 1))
+        if response is not None:
+            retry_after = response.headers.get("Retry-After")
+            if retry_after and retry_after.isdigit():
+                delay = max(delay, float(retry_after))
+        delay += random.uniform(0, delay * 0.25)
+        await asyncio.sleep(min(delay, 30.0))
+
+    async def _get_page(
+        self,
+        path: str,
+        model: type[T],
+        *,
+        params: dict[str, Any] | None = None,
+        page: int = 1,
+        per_page: int = 20,
+    ) -> Page[T]:
+        query = dict(params or {})
+        query.update({"page": page, "per_page": min(per_page, MAX_PER_PAGE)})
+        response = await self._request("GET", path, params=query)
+
+        raw = response.json()
+        if not isinstance(raw, list):
+            raise AtomGitUpstreamError(f"{path} 期望返回列表，实际为 {type(raw).__name__}")
+
+        items = [self._parse(model, entry, path) for entry in raw]
+
+        return Page(
+            items=items,
+            page=page,
+            per_page=query["per_page"],
+            total_count=self._header_int(response, "total_count"),
+            total_page=self._header_int(response, "total_page"),
+        )
+
+    @staticmethod
+    def _header_int(response: httpx.Response, name: str) -> int | None:
+        value = response.headers.get(name) or response.headers.get(name.replace("_", "-"))
+        if value is None:
+            return None
+        try:
+            return int(value)
+        except ValueError:
+            return None
+
+    @staticmethod
+    def _parse(model: type[T], payload: Any, path: str) -> T:
+        try:
+            return model.model_validate(payload)
+        except ValidationError as exc:
+            # 上游字段变化时，明确报出是哪个模型、哪个字段，而不是静默丢弃
+            raise AtomGitUpstreamError(
+                f"解析 {path} 的 {model.__name__} 失败：{exc.error_count()} 处字段不匹配。"
+                f"首个错误：{exc.errors()[0].get('loc')} {exc.errors()[0].get('msg')}"
+            ) from exc
+
+    @staticmethod
+    def _parse_list(model: type[T], payload: Any, path: str) -> list[T]:
+        if not isinstance(payload, list):
+            raise AtomGitUpstreamError(f"{path} 期望返回列表，实际为 {type(payload).__name__}")
+        return [AtomGitClient._parse(model, entry, path) for entry in payload]
+
+    # ------------------------------------------------------------------
+    # 账号与仓库
+    # ------------------------------------------------------------------
+
+    async def get_authenticated_user(self) -> AtomGitUser:
+        """用于在配置凭据时验证 token 有效性并回显账号名。"""
+        response = await self._request("GET", "/user")
+        return self._parse(AtomGitUser, response.json(), "/user")
+
+    async def get_repository(self, owner: str, repo: str) -> AtomGitRepository:
+        response = await self._request("GET", f"/repos/{owner}/{repo}")
+        return self._parse(AtomGitRepository, response.json(), "/repos")
+
+    async def get_file_content(
+        self, owner: str, repo: str, path: str, *, ref: str = "master"
+    ) -> str | None:
+        """读取仓库里的一个文本文件。文件不存在时返回 None。
+
+        SIG 名单（``openeuler/community`` 下的 ``sig/Kernel/*.md``）只能这样取：
+        网页端会重定向到前端 SPA，匿名 raw 链接拿回来的是 HTML 外壳，
+        带 token 的 API 是唯一可靠路径。
+
+        不存在的文件返回 None 而不是抛错：名单文件的位置与名字由上游决定，
+        某天改了文件名不该让整轮同步失败。
+        """
+        try:
+            response = await self._request(
+                "GET", f"/repos/{owner}/{repo}/contents/{path}", params={"ref": ref}
+            )
+        except AtomGitNotFoundError:
+            return None
+
+        payload = response.json()
+        content = payload.get("content")
+        if not isinstance(content, str):
+            return None
+        # 上游返回 base64；encoding 字段实测可能是 "base64" 也可能缺失
+        return base64.b64decode(content).decode("utf-8", "replace")
+
+    # ------------------------------------------------------------------
+    # Pull Request
+    # ------------------------------------------------------------------
+
+    async def list_pulls(
+        self,
+        owner: str,
+        repo: str,
+        *,
+        state: str = "open",
+        page: int = 1,
+        per_page: int = 50,
+        sort: str | None = None,
+        direction: str | None = None,
+    ) -> Page[AtomGitPullRequest]:
+        params: dict[str, Any] = {"state": state}
+        if sort:
+            params["sort"] = sort
+        if direction:
+            params["direction"] = direction
+        return await self._get_page(
+            f"/repos/{owner}/{repo}/pulls",
+            AtomGitPullRequest,
+            params=params,
+            page=page,
+            per_page=per_page,
+        )
+
+    async def get_pull(self, owner: str, repo: str, number: int) -> AtomGitPullRequest:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}")
+        return self._parse(AtomGitPullRequest, response.json(), "/pulls/{n}")
+
+    async def get_pull_files(self, owner: str, repo: str, number: int) -> list[AtomGitFile]:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}/files")
+        return self._parse_list(AtomGitFile, response.json(), "/pulls/{n}/files")
+
+    async def get_pull_commits(self, owner: str, repo: str, number: int) -> list[AtomGitCommit]:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}/commits")
+        return self._parse_list(AtomGitCommit, response.json(), "/pulls/{n}/commits")
+
+    async def get_pull_comments(self, owner: str, repo: str, number: int) -> list[AtomGitComment]:
+        """PR 评论。评审意见的主要来源。"""
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}/comments")
+        return self._parse_list(AtomGitComment, response.json(), "/pulls/{n}/comments")
+
+    async def get_pull_labels(self, owner: str, repo: str, number: int) -> list[AtomGitLabel]:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}/labels")
+        return self._parse_list(AtomGitLabel, response.json(), "/pulls/{n}/labels")
+
+    async def get_pull_linked_issues(
+        self, owner: str, repo: str, number: int
+    ) -> list[AtomGitIssue]:
+        """PR 显式关联的 Issue。优于从正文正则提取。"""
+        response = await self._request("GET", f"/repos/{owner}/{repo}/pulls/{number}/issues")
+        return self._parse_list(AtomGitIssue, response.json(), "/pulls/{n}/issues")
+
+    # ------------------------------------------------------------------
+    # Issue
+    # ------------------------------------------------------------------
+
+    async def list_issues(
+        self,
+        owner: str,
+        repo: str,
+        *,
+        state: str = "open",
+        page: int = 1,
+        per_page: int = 50,
+        labels: str | None = None,
+        since: str | None = None,
+    ) -> Page[AtomGitIssue]:
+        params: dict[str, Any] = {"state": state}
+        if labels:
+            params["labels"] = labels
+        if since:
+            params["since"] = since
+        return await self._get_page(
+            f"/repos/{owner}/{repo}/issues",
+            AtomGitIssue,
+            params=params,
+            page=page,
+            per_page=per_page,
+        )
+
+    async def get_issue(self, owner: str, repo: str, number: int | str) -> AtomGitIssue:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/issues/{number}")
+        return self._parse(AtomGitIssue, response.json(), "/issues/{n}")
+
+    async def get_issue_comments(
+        self, owner: str, repo: str, number: int | str
+    ) -> list[AtomGitComment]:
+        response = await self._request("GET", f"/repos/{owner}/{repo}/issues/{number}/comments")
+        return self._parse_list(AtomGitComment, response.json(), "/issues/{n}/comments")
+
+    async def create_issue_comment(
+        self, owner: str, repo: str, number: int | str, body: str
+    ) -> AtomGitComment:
+        """发表评论。
+
+        ``/issues/{n}/comments`` 同时适用于 PR 与 Issue ——
+        在 AtomGit 的数据模型中两者共用编号空间。
+        """
+        response = await self._request(
+            "POST",
+            f"/repos/{owner}/{repo}/issues/{number}/comments",
+            json={"body": body},
+        )
+        return self._parse(AtomGitComment, response.json(), "/issues/{n}/comments")
+
+    # ------------------------------------------------------------------
+    # 标签与 Webhook
+    # ------------------------------------------------------------------
+
+    async def list_labels(
+        self, owner: str, repo: str, *, page: int = 1, per_page: int = 100
+    ) -> Page[AtomGitLabel]:
+        return await self._get_page(
+            f"/repos/{owner}/{repo}/labels",
+            AtomGitLabel,
+            page=page,
+            per_page=per_page,
+        )
+
+    async def list_webhooks(self, owner: str, repo: str) -> list[dict[str, Any]]:
+        """读取已注册的 webhook。用于诊断事件投递配置。"""
+        response = await self._request("GET", f"/repos/{owner}/{repo}/hooks")
+        payload = response.json()
+        return payload if isinstance(payload, list) else []
+
+    # ------------------------------------------------------------------
+    # 便捷方法
+    # ------------------------------------------------------------------
+
+    async def iter_pulls(
+        self,
+        owner: str,
+        repo: str,
+        *,
+        state: str = "open",
+        per_page: int = 100,
+        max_pages: int = 100,
+    ):
+        """按页迭代 PR，避免一次性把 1000+ 条读进内存。"""
+        page = 1
+        while page <= max_pages:
+            result = await self.list_pulls(owner, repo, state=state, page=page, per_page=per_page)
+            if not result.items:
+                return
+            yield result
+            if not result.has_more:
+                return
+            page += 1
+
+    async def iter_issues(
+        self,
+        owner: str,
+        repo: str,
+        *,
+        state: str = "open",
+        per_page: int = 100,
+        max_pages: int = 100,
+    ):
+        page = 1
+        while page <= max_pages:
+            result = await self.list_issues(owner, repo, state=state, page=page, per_page=per_page)
+            if not result.items:
+                return
+            yield result
+            if not result.has_more:
+                return
+            page += 1