@kernel-sig/console 0.1.0

package/backend/app/services/release_service.py

@@ -0,0 +1,446 @@
+"""版本与分支的统计。
+
+两件事：
+
+1. **把同名分支合并统计。** 库里的 ``target_branch`` 存的是上游原始名，
+   而同一交付有 ``openEuler-24.03-LTS_SP1`` 与 ``-SP1`` 两种写法。
+   统计前必须归一，否则同一个版本会裂成两行，每行的合入数都偏小。
+2. **按双周/月汇总合入情况。** 例会本身就是双周的，所以双周是天然的统计
+   周期；月汇总用于看趋势。两者都从 PR 的 ``merged_at`` 现算，
+   不落库 —— 落库就要考虑失效，而这类聚合每天都会变。
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import UTC, date, datetime, timedelta
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain.release import BranchKind, classify_branch
+from app.domain.review import is_bot
+from app.models.classification import Classification, PRKind, SubjectType
+from app.models.meeting import Meeting, ReleaseReport
+from app.models.pull_request import PullRequest
+from app.models.sig import ReleaseBranch
+
+# 一天。双周与月汇总的边界计算都以它为粒度。
+DAY = timedelta(days=1)
+
+
+@dataclass
+class BranchStats:
+    """一个分支的概览。"""
+
+    name: str
+    """规范化后的名字。"""
+
+    raw_names: list[str]
+    """库中实际出现过的原始写法。查询该分支的 PR 要用这些值。"""
+
+    kind: str
+    series: str | None = None
+    sp: int | None = None
+    upstream_base: str | None = None
+    state: str = "active"
+    note: str | None = None
+    keepers: list[str] = field(default_factory=list)
+    """分支负责人账号，来自 sig-info.yaml。"""
+
+    open_count: int = 0
+    merged_count: int = 0
+    merged_recent: int = 0
+    """近 30 天合入数。"""
+
+    last_merged_at: datetime | None = None
+    latest_tag: str | None = None
+    """最近一次例会上报告的分支 tag。"""
+
+    last_report_on: date | None = None
+
+
+async def _raw_branch_map(session: AsyncSession, repository_id: uuid.UUID) -> dict[str, list[str]]:
+    """规范化分支名 → 库中实际出现过的原始名列表。"""
+    stmt = (
+        select(PullRequest.target_branch)
+        .where(
+            PullRequest.repository_id == repository_id,
+            PullRequest.target_branch.is_not(None),
+        )
+        .group_by(PullRequest.target_branch)
+    )
+    mapping: dict[str, list[str]] = defaultdict(list)
+    for raw in (await session.scalars(stmt)).all():
+        mapping[classify_branch(raw).name].append(raw)
+    return dict(mapping)
+
+
+async def list_branches(
+    session: AsyncSession, repository_id: uuid.UUID, *, recent_days: int = 30
+) -> list[BranchStats]:
+    """全部分支及其合入统计。
+
+    一次查完所有分支的计数再在内存里归并，而不是按分支逐个查 ——
+    分支数在十位量级，但逐个查会变成十几次往返。
+    """
+    raw_map = await _raw_branch_map(session, repository_id)
+    if not raw_map:
+        return []
+
+    canonical = {raw: name for name, raws in raw_map.items() for raw in raws}
+
+    # 一次性取回全部 PR 的分支与状态，在内存里归并。分支数是十位量级，
+    # 按分支逐个查会变成十几次往返，而全量取回的列很窄。
+    stmt = select(PullRequest.target_branch, PullRequest.state, PullRequest.merged_at).where(
+        PullRequest.repository_id == repository_id
+    )
+
+    cutoff = datetime.now(UTC) - timedelta(days=recent_days)
+    stats: dict[str, BranchStats] = {}
+
+    for raw, state, merged_at in (await session.execute(stmt)).all():
+        name = canonical.get(raw)
+        if name is None:
+            continue
+        spec = classify_branch(raw)
+        item = stats.setdefault(
+            name,
+            BranchStats(
+                name=name,
+                raw_names=sorted(raw_map.get(name, [raw])),
+                kind=spec.kind.value,
+                series=spec.series,
+                sp=spec.sp,
+                upstream_base=spec.upstream_base,
+            ),
+        )
+        if state == "open":
+            item.open_count += 1
+        elif state == "merged":
+            item.merged_count += 1
+            if merged_at is not None:
+                if merged_at > cutoff:
+                    item.merged_recent += 1
+                if item.last_merged_at is None or merged_at > item.last_merged_at:
+                    item.last_merged_at = merged_at
+
+    # 登记表里的补充事实（生命周期、维护说明、别名）覆盖推导结果
+    #
+    # 只取本仓库的分支：这张表里每条都带着所属仓库（来自 sig-info.yaml 的
+    # repo_branch），不按它过滤的话，另一个仓库的分支会被当成"登记了但还没有
+    # PR 的分支"凭空出现 —— OLK-6.6 会出现在 src-openeuler/kernel 的分支页上。
+    registry = {
+        row.name: row
+        for row in (
+            await session.scalars(
+                select(ReleaseBranch).where(ReleaseBranch.repository_id == repository_id)
+            )
+        ).all()
+    }
+    for name, row in registry.items():
+        item = stats.get(name)
+        if item is None:
+            # 登记了但还没有 PR 的分支（如刚宣布、尚未启用）也要出现
+            spec = classify_branch(name)
+            item = stats[name] = BranchStats(
+                name=name,
+                raw_names=sorted(set(raw_map.get(name, []) + list(row.aliases or []))) or [name],
+                kind=row.kind or spec.kind.value,
+                series=row.series or spec.series,
+                sp=row.sp if row.sp is not None else spec.sp,
+                upstream_base=row.upstream_base or spec.upstream_base,
+            )
+        item.state = row.state
+        item.note = row.note
+        item.keepers = list(row.keepers or [])
+        if row.aliases:
+            item.raw_names = sorted(set(item.raw_names) | set(row.aliases))
+
+    # 最近一次例会上报告的 tag
+    latest = (
+        select(ReleaseReport.branch, ReleaseReport.to_tag, Meeting.held_on)
+        .join(Meeting, Meeting.id == ReleaseReport.meeting_id)
+        .order_by(Meeting.held_on.desc())
+    )
+    for branch, to_tag, held_on in (await session.execute(latest)).all():
+        item = stats.get(branch)
+        if item is not None and item.latest_tag is None:
+            item.latest_tag = to_tag
+            item.last_report_on = held_on
+
+    return sorted(stats.values(), key=_branch_sort_key)
+
+
+_KIND_ORDER = {
+    BranchKind.OLK.value: 0,
+    BranchKind.LTS.value: 1,
+    BranchKind.INNOVATION.value: 2,
+    BranchKind.OTHER.value: 3,
+}
+
+
+def _branch_sort_key(item: BranchStats) -> tuple:
+    """产品线优先，同线内按 SP 倒序（新版本在前），再按名字。
+
+    OLK 与 LTS 的版本号方向相反：OLK-6.6 比 OLK-5.10 新，而 24.03-LTS 比
+    22.03-LTS 新，但 SP 越大越新。用 SP 倒序并把版本串倒序排，
+    两条线都能得到"新的在前"。
+    """
+    return (
+        _KIND_ORDER.get(item.kind, 9),
+        -(item.sp or 0),
+        _negate_version(item.series or item.name),
+        item.name,
+    )
+
+
+def _negate_version(text: str) -> tuple:
+    """把版本号转成可倒序排序的键。非数字段按 0 处理。"""
+    parts = []
+    for chunk in text.replace("-", ".").replace("_", ".").split("."):
+        parts.append(-int(chunk) if chunk.isdigit() else 0)
+    return tuple(parts)
+
+
+# ---------------------------------------------------------------------------
+# 合入趋势与分期汇总
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MergeBucket:
+    """一个统计周期内的合入情况。"""
+
+    label: str
+    start: date
+    end: date
+    total: int = 0
+    by_kind: dict[str, int] = field(default_factory=dict)
+
+
+async def merge_timeline(
+    session: AsyncSession,
+    repository_id: uuid.UUID,
+    *,
+    raw_branches: list[str],
+    buckets: int = 12,
+    granularity: str = "month",
+) -> list[MergeBucket]:
+    """按周期统计合入量。
+
+    周期边界从**今天**往回推，而不是从第一条数据开始 —— 从数据开始推会让
+    最近一期只有半天，看起来像合入量骤降。
+    """
+    if not raw_branches:
+        return []
+
+    stmt = (
+        select(PullRequest.merged_at, Classification.kind)
+        .outerjoin(
+            Classification,
+            (Classification.subject_id == PullRequest.id)
+            & (Classification.subject_type == SubjectType.PULL_REQUEST),
+        )
+        .where(
+            PullRequest.repository_id == repository_id,
+            PullRequest.target_branch.in_(raw_branches),
+            PullRequest.merged_at.is_not(None),
+        )
+    )
+    rows = (await session.execute(stmt)).all()
+    if not rows:
+        return []
+
+    windows = _bucket_windows(date.today(), buckets, granularity)
+    results = [MergeBucket(label=label, start=start, end=end) for label, start, end in windows]
+
+    for merged_at, kind in rows:
+        merged_on = merged_at.date()
+        for bucket in results:
+            if bucket.start <= merged_on < bucket.end:
+                bucket.total += 1
+                key = kind.value if kind is not None else "unknown"
+                bucket.by_kind[key] = bucket.by_kind.get(key, 0) + 1
+                break
+
+    return results
+
+
+def recent_windows(
+    count: int, granularity: str, today: date | None = None
+) -> list[tuple[str, date, date]]:
+    """最近 N 个统计周期的边界，按时间正序。"""
+    return _bucket_windows(today or date.today(), count, granularity)
+
+
+def _bucket_windows(today: date, count: int, granularity: str) -> list[tuple[str, date, date]]:
+    """从今天往回生成 N 个周期的边界。"""
+    windows: list[tuple[str, date, date]] = []
+    if granularity == "month":
+        cursor = today.replace(day=1)
+        for _ in range(count):
+            nxt = _add_month(cursor)
+            windows.append((cursor.strftime("%Y-%m"), cursor, nxt))
+            cursor = _add_month(cursor, -1)
+    else:
+        # 双周：以 ISO 周的周一对齐，每次回退 14 天
+        cursor = today - timedelta(days=today.weekday())
+        for _ in range(count):
+            windows.append(
+                (
+                    f"{cursor.isoformat()}~{(cursor + 13 * DAY).isoformat()}",
+                    cursor,
+                    cursor + 14 * DAY,
+                )
+            )
+            cursor -= 14 * DAY
+    windows.reverse()
+    return windows
+
+
+def _add_month(day: date, delta: int = 1) -> date:
+    month = day.month - 1 + delta
+    year = day.year + month // 12
+    month = month % 12 + 1
+    return date(year, month, 1)
+
+
+@dataclass
+class PeriodSummary:
+    """一期（双周或月）的合入总结。"""
+
+    label: str
+    start: date
+    end: date
+    total: int = 0
+    by_branch: dict[str, int] = field(default_factory=dict)
+    by_kind: dict[str, int] = field(default_factory=dict)
+    cve_count: int = 0
+    top_contributors: list[tuple[str, int]] = field(default_factory=list)
+
+
+async def period_summary(
+    session: AsyncSession,
+    repository_id: uuid.UUID,
+    *,
+    start: date,
+    end: date,
+    label: str,
+    raw_branches: list[str] | None = None,
+) -> PeriodSummary:
+    """一期内按分支、类别、作者的合入汇总。
+
+    ``raw_branches`` 传入时只统计这些分支 —— 版本页要的是"这个版本这一期
+    合入了什么"，而不是全仓合入了什么。不传则汇总全部。
+    """
+    stmt = (
+        select(PullRequest.target_branch, PullRequest.author_login, Classification.kind)
+        .outerjoin(
+            Classification,
+            (Classification.subject_id == PullRequest.id)
+            & (Classification.subject_type == SubjectType.PULL_REQUEST),
+        )
+        .where(
+            PullRequest.repository_id == repository_id,
+            PullRequest.merged_at.is_not(None),
+            PullRequest.merged_at >= datetime.combine(start, datetime.min.time(), tzinfo=UTC),
+            PullRequest.merged_at < datetime.combine(end, datetime.min.time(), tzinfo=UTC),
+        )
+    )
+    if raw_branches:
+        stmt = stmt.where(PullRequest.target_branch.in_(raw_branches))
+    summary = PeriodSummary(label=label, start=start, end=end)
+    authors: dict[str, int] = defaultdict(int)
+
+    for raw_branch, author, kind in (await session.execute(stmt)).all():
+        summary.total += 1
+        branch = classify_branch(raw_branch).name
+        summary.by_branch[branch] = summary.by_branch.get(branch, 0) + 1
+        key = kind.value if kind is not None else "unknown"
+        summary.by_kind[key] = summary.by_kind.get(key, 0) + 1
+        if key == PRKind.CVE.value:
+            summary.cve_count += 1
+        # 机器人要排除在贡献者之外，否则榜单前几名永远是 CI 与门禁机器人。
+        # 实测某月合入 362 个 PR，其中机器人提了 115 个。
+        if author and not is_bot(author):
+            authors[author] += 1
+
+    summary.top_contributors = sorted(authors.items(), key=lambda item: -item[1])[:10]
+    return summary
+
+
+async def period_summaries(
+    session: AsyncSession,
+    repository_id: uuid.UUID,
+    *,
+    windows: list[tuple[str, date, date]],
+    raw_branches: list[str] | None = None,
+) -> list[PeriodSummary]:
+    """多个连续周期的合入汇总，一次查询完。
+
+    逐期调用 period_summary 会退化成 N 次扫描，而版本页要的正是
+    "最近半年的双周总结"这种连续多期 —— 那样的实现读起来直观，
+    代价是在几万行的表上跑 N 遍。
+    """
+    if not windows:
+        return []
+
+    stmt = (
+        select(
+            PullRequest.target_branch,
+            PullRequest.author_login,
+            Classification.kind,
+            PullRequest.merged_at,
+        )
+        .outerjoin(
+            Classification,
+            (Classification.subject_id == PullRequest.id)
+            & (Classification.subject_type == SubjectType.PULL_REQUEST),
+        )
+        .where(
+            PullRequest.repository_id == repository_id,
+            PullRequest.merged_at.is_not(None),
+            PullRequest.merged_at
+            >= datetime.combine(windows[0][1], datetime.min.time(), tzinfo=UTC),
+            PullRequest.merged_at
+            < datetime.combine(windows[-1][2], datetime.min.time(), tzinfo=UTC),
+        )
+    )
+    if raw_branches:
+        stmt = stmt.where(PullRequest.target_branch.in_(raw_branches))
+
+    summaries = [PeriodSummary(label=label, start=start, end=end) for label, start, end in windows]
+    bounds = [
+        (
+            datetime.combine(start, datetime.min.time(), tzinfo=UTC),
+            datetime.combine(end, datetime.min.time(), tzinfo=UTC),
+        )
+        for _, start, end in windows
+    ]
+    authors: list[dict[str, int]] = [defaultdict(int) for _ in windows]
+
+    for raw_branch, author, kind, merged_at in (await session.execute(stmt)).all():
+        index = next(
+            (i for i, (lo, hi) in enumerate(bounds) if lo <= merged_at < hi),
+            None,
+        )
+        if index is None:
+            continue
+        summary = summaries[index]
+        summary.total += 1
+        branch = classify_branch(raw_branch).name
+        summary.by_branch[branch] = summary.by_branch.get(branch, 0) + 1
+        key = kind.value if kind is not None else "unknown"
+        summary.by_kind[key] = summary.by_kind.get(key, 0) + 1
+        if key == PRKind.CVE.value:
+            summary.cve_count += 1
+        # 机器人要排除在贡献者之外，否则榜单前几名永远是 CI 与门禁机器人
+        if author and not is_bot(author):
+            authors[index][author] += 1
+
+    for summary, counts in zip(summaries, authors, strict=True):
+        summary.top_contributors = sorted(counts.items(), key=lambda item: -item[1])[:10]
+    return summaries

package/backend/app/services/repository_service.py

@@ -0,0 +1,132 @@
+"""仓库纳管服务。"""
+
+import secrets
+import uuid
+from datetime import UTC, datetime
+
+import structlog
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import Settings
+from app.core.exceptions import ConflictError, ExternalServiceError, NotFoundError
+from app.integrations.atomgit.client import (
+    AtomGitAuthError,
+    AtomGitClient,
+    AtomGitError,
+)
+from app.models.repository import DEFAULT_API_BASE_URL, Repository
+from app.services import credential_service
+
+logger = structlog.get_logger(__name__)
+
+WEBHOOK_SECRET_BYTES = 24
+
+
+async def list_repositories(session: AsyncSession) -> list[Repository]:
+    stmt = select(Repository).order_by(Repository.owner, Repository.name)
+    return list((await session.scalars(stmt)).all())
+
+
+async def get_repository(session: AsyncSession, repository_id: uuid.UUID) -> Repository | None:
+    return await session.get(Repository, repository_id)
+
+
+async def require_repository(session: AsyncSession, repository_id: uuid.UUID) -> Repository:
+    repository = await get_repository(session, repository_id)
+    if repository is None:
+        raise NotFoundError("仓库不存在", resource="repository", identifier=str(repository_id))
+    return repository
+
+
+async def get_by_full_name(session: AsyncSession, owner: str, name: str) -> Repository | None:
+    stmt = select(Repository).where(Repository.owner == owner, Repository.name == name)
+    return await session.scalar(stmt)
+
+
+def build_client(settings: Settings, repository: Repository, credential) -> AtomGitClient:
+    """按仓库配置构造客户端。base_url 取自仓库，支持 AtomGit/GitCode 切换。"""
+    token = credential_service.reveal(settings, credential)
+    return AtomGitClient(token, base_url=repository.api_base_url or DEFAULT_API_BASE_URL)
+
+
+async def create_repository(
+    session: AsyncSession,
+    settings: Settings,
+    *,
+    owner: str,
+    name: str,
+    credential_id: uuid.UUID,
+    display_name: str | None = None,
+    api_base_url: str = DEFAULT_API_BASE_URL,
+    tracked_branches: list[str] | None = None,
+    backfill_days: int = 180,
+) -> Repository:
+    """纳管一个仓库。
+
+    **先验证凭据再落库**：提交前用 /user 确认 token 有效，
+    避免把不可用的配置写进数据库、直到第一次同步才暴露问题。
+    """
+    if await get_by_full_name(session, owner, name) is not None:
+        raise ConflictError(f"仓库 {owner}/{name} 已被纳管")
+
+    credential = await credential_service.require_credential(session, credential_id)
+
+    client = AtomGitClient(
+        credential_service.reveal(settings, credential),
+        base_url=api_base_url,
+    )
+    try:
+        async with client:
+            repo_info = await client.get_repository(owner, name)
+    except AtomGitAuthError as exc:
+        credential_service.mark_verified(credential, error=str(exc))
+        await session.flush()
+        raise ExternalServiceError(f"凭据校验失败：{exc}") from exc
+    except AtomGitError as exc:
+        raise ExternalServiceError(f"无法访问 {owner}/{name}：{exc}") from exc
+
+    credential_service.mark_verified(credential)
+
+    repository = Repository(
+        owner=owner,
+        name=name,
+        display_name=display_name or repo_info.human_name or f"{owner}/{name}",
+        description=repo_info.description,
+        api_base_url=api_base_url,
+        credential_id=credential_id,
+        default_branch=repo_info.default_branch,
+        tracked_branches=tracked_branches
+        or ([repo_info.default_branch] if repo_info.default_branch else None),
+        webhook_secret=secrets.token_urlsafe(WEBHOOK_SECRET_BYTES),
+        backfill_days=backfill_days,
+    )
+    session.add(repository)
+    await session.flush()
+    logger.info("repository_created", full_name=repository.full_name)
+    return repository
+
+
+async def update_repository(session: AsyncSession, repository: Repository, **fields) -> Repository:
+    for key, value in fields.items():
+        if value is not None and hasattr(repository, key):
+            setattr(repository, key, value)
+    await session.flush()
+    return repository
+
+
+async def rotate_webhook_secret(session: AsyncSession, repository: Repository) -> str:
+    """轮换 webhook 路径密钥。旧密钥立即失效。"""
+    repository.webhook_secret = secrets.token_urlsafe(WEBHOOK_SECRET_BYTES)
+    await session.flush()
+    return repository.webhook_secret
+
+
+async def get_by_webhook_secret(session: AsyncSession, secret: str) -> Repository | None:
+    stmt = select(Repository).where(Repository.webhook_secret == secret)
+    return await session.scalar(stmt)
+
+
+def mark_synced(repository: Repository, *, error: str | None = None) -> None:
+    repository.last_sync_at = datetime.now(UTC)
+    repository.last_sync_error = error