@kernel-sig/console 0.1.0

package/backend/app/services/analytics_service.py

@@ -0,0 +1,390 @@
+"""统计分析服务。
+
+把已同步的数据转成维护者可决策的指标。
+
+设计约束：所有聚合在数据库内完成。PR 表已有 1100+ 行且会持续增长，
+把行拉到应用层再用 Python 聚合会在数据量上来后迅速劣化。
+"""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain.review import GateStage, is_bot
+from app.models.issue import Issue
+from app.models.pull_request import PRFile, PullRequest, ReviewEvent, ReviewEventType
+from app.models.repository import Repository
+
+
+async def overview(session: AsyncSession, repository_id: uuid.UUID) -> dict:
+    """总览指标。"""
+    pr_counts = dict(
+        (
+            await session.execute(
+                select(PullRequest.state, func.count())
+                .where(PullRequest.repository_id == repository_id)
+                .group_by(PullRequest.state)
+            )
+        ).all()
+    )
+    issue_counts = dict(
+        (
+            await session.execute(
+                select(Issue.state, func.count())
+                .where(Issue.repository_id == repository_id)
+                .group_by(Issue.state)
+            )
+        ).all()
+    )
+
+    now = datetime.now(UTC)
+
+    async def pr_count_since(delta: timedelta) -> int:
+        return (
+            await session.scalar(
+                select(func.count())
+                .select_from(PullRequest)
+                .where(
+                    PullRequest.repository_id == repository_id,
+                    PullRequest.atomgit_created_at >= now - delta,
+                )
+            )
+            or 0
+        )
+
+    merged_recently = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.merged_at.is_not(None),
+                PullRequest.merged_at >= now - timedelta(days=7),
+            )
+        )
+        or 0
+    )
+
+    ready = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.gate_stage == GateStage.READY_TO_MERGE.value,
+            )
+        )
+        or 0
+    )
+    blocked = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.gate_stage.in_(
+                    [
+                        GateStage.CI_FAILED.value,
+                        GateStage.CLA_DENIED.value,
+                        GateStage.REJECTED.value,
+                    ]
+                ),
+            )
+        )
+        or 0
+    )
+
+    return {
+        "pulls": {
+            "open": pr_counts.get("open", 0),
+            "merged": pr_counts.get("merged", 0),
+            "closed": pr_counts.get("closed", 0),
+            "created_last_7d": await pr_count_since(timedelta(days=7)),
+            "created_last_30d": await pr_count_since(timedelta(days=30)),
+            "merged_last_7d": merged_recently,
+            "ready_to_merge": ready,
+            "blocked": blocked,
+        },
+        "issues": {
+            "open": issue_counts.get("open", 0),
+            "closed": issue_counts.get("closed", 0),
+        },
+    }
+
+
+async def waiting_time_stats(session: AsyncSession, repository_id: uuid.UUID) -> dict:
+    """等待时长统计。
+
+    只看仍未评审的 PR：把已评审的混进来会把平均值拉平，
+    掩盖"确实有一批 PR 长期没人看"这个事实。
+    """
+    now = datetime.now(UTC)
+    age_expr = func.extract("epoch", now - PullRequest.atomgit_created_at) / 86400.0
+
+    row = (
+        await session.execute(
+            select(
+                func.count(),
+                func.avg(age_expr),
+                func.percentile_cont(0.5).within_group(age_expr),
+                func.max(age_expr),
+            ).where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.gate_stage == GateStage.REVIEW_PENDING.value,
+            )
+        )
+    ).one()
+
+    over_30 = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.gate_stage == GateStage.REVIEW_PENDING.value,
+                age_expr > 30,
+            )
+        )
+        or 0
+    )
+
+    return {
+        "pending_review_count": row[0] or 0,
+        "avg_waiting_days": round(float(row[1]), 1) if row[1] is not None else 0.0,
+        "median_waiting_days": round(float(row[2]), 1) if row[2] is not None else 0.0,
+        "max_waiting_days": round(float(row[3]), 1) if row[3] is not None else 0.0,
+        "over_30_days": over_30,
+    }
+
+
+async def reviewer_workload(
+    session: AsyncSession, repository_id: uuid.UUID, *, days: int = 90, limit: int = 20
+) -> list[dict]:
+    """评审人工作量。
+
+    数据来自 comment 来源的评审事件 —— 评审人身份只存在于评论里
+    （标签是汇总结果，不含具体人）。这也是为什么必须解析评论。
+    """
+    since = datetime.now(UTC) - timedelta(days=days)
+
+    rows = (
+        await session.execute(
+            select(
+                ReviewEvent.actor_login,
+                func.count().filter(ReviewEvent.event_type == ReviewEventType.LGTM.value),
+                func.count().filter(ReviewEvent.event_type == ReviewEventType.APPROVE.value),
+                func.count().filter(ReviewEvent.event_type == ReviewEventType.ACK.value),
+                func.count(),
+            )
+            .where(
+                ReviewEvent.occurred_at >= since,
+                ReviewEvent.source == "comment",
+                ReviewEvent.actor_login != "",
+                ReviewEvent.pull_request_id.in_(
+                    select(PullRequest.id).where(PullRequest.repository_id == repository_id)
+                ),
+            )
+            .group_by(ReviewEvent.actor_login)
+            .order_by(func.count().desc())
+            .limit(limit)
+        )
+    ).all()
+
+    return [
+        {
+            "actor_login": row[0],
+            "lgtm_count": row[1],
+            "approve_count": row[2],
+            "ack_count": row[3],
+            "total": row[4],
+        }
+        for row in rows
+    ]
+
+
+async def stage_distribution(session: AsyncSession, repository_id: uuid.UUID) -> list[dict]:
+    """按门禁阶段的 PR 分布。这是"卡在哪一环"的直接答案。"""
+    rows = (
+        await session.execute(
+            select(PullRequest.gate_stage, func.count())
+            .where(PullRequest.repository_id == repository_id, PullRequest.state == "open")
+            .group_by(PullRequest.gate_stage)
+            .order_by(func.count().desc())
+        )
+    ).all()
+    return [{"stage": row[0], "count": row[1]} for row in rows]
+
+
+async def branch_distribution(session: AsyncSession, repository_id: uuid.UUID) -> list[dict]:
+    rows = (
+        await session.execute(
+            select(PullRequest.target_branch, func.count())
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.target_branch.is_not(None),
+            )
+            .group_by(PullRequest.target_branch)
+            .order_by(func.count().desc())
+            .limit(15)
+        )
+    ).all()
+    return [{"branch": row[0], "count": row[1]} for row in rows]
+
+
+async def author_distribution(
+    session: AsyncSession, repository_id: uuid.UUID, *, limit: int = 15
+) -> list[dict]:
+    """人类作者的提交量。
+
+    **必须排除机器人**：实测作者榜前列被 openeuler-infra-bot、devstation-robot、
+    ci-robot1 占据（合计 333 个 PR），不排除则这份榜单毫无参考价值。
+    机器人判定复用评审解析器中的同一套规则，避免两处判定不一致。
+    """
+    rows = (
+        await session.execute(
+            select(PullRequest.author_login, func.count())
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.author_login.is_not(None),
+            )
+            .group_by(PullRequest.author_login)
+            .order_by(func.count().desc())
+        )
+    ).all()
+
+    humans = [{"author": row[0], "count": row[1]} for row in rows if not is_bot(row[0])]
+    bot_count = sum(row[1] for row in rows if is_bot(row[0]))
+
+    result = humans[:limit]
+    if bot_count:
+        # 机器人合计单独列出而非丢弃：自动化贡献本身是有信息量的
+        result.append({"author": "（机器人合计）", "count": bot_count, "is_bot": True})
+    return result
+
+
+async def interval_stats(session: AsyncSession, repository_id: uuid.UUID) -> dict:
+    """各阶段的平均滞留时长。
+
+    从阶段进入时间算起并不直接可得（需要状态变更历史），
+    因此这里使用可观测的近似：以 PR 创建时间到合入/当前时间的区间，
+    配合阶段分布来定位瓶颈。比不提供任何时长信息更有用，
+    但不宣称是精确的阶段停留时间。
+    """
+    merged_duration = (
+        func.extract("epoch", PullRequest.merged_at - PullRequest.atomgit_created_at) / 86400.0
+    )
+
+    row = (
+        await session.execute(
+            select(
+                func.avg(merged_duration),
+                func.percentile_cont(0.5).within_group(merged_duration),
+                func.count(),
+            ).where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.merged_at.is_not(None),
+                PullRequest.atomgit_created_at.is_not(None),
+            )
+        )
+    ).one()
+
+    recent_merged = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.merged_at >= datetime.now(UTC) - timedelta(days=30),
+            )
+        )
+        or 0
+    )
+
+    return {
+        "merged_count": row[2] or 0,
+        "avg_merge_days": round(float(row[0]), 1) if row[0] is not None else 0.0,
+        "median_merge_days": round(float(row[1]), 1) if row[1] is not None else 0.0,
+        "merged_last_30d": recent_merged,
+    }
+
+
+async def quality_signals(session: AsyncSession, repository_id: uuid.UUID) -> dict:
+    """数据质量信号。
+
+    这些是自动可查、且直接影响评审成本的事实，比主观判断可靠。
+    """
+    binary_files = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PRFile)
+            .join(PullRequest, PullRequest.id == PRFile.pull_request_id)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PRFile.is_binary.is_(True),
+            )
+        )
+        or 0
+    )
+    conflicts = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.merge_conflict.is_(True),
+            )
+        )
+        or 0
+    )
+    drafts = (
+        await session.scalar(
+            select(func.count())
+            .select_from(PullRequest)
+            .where(
+                PullRequest.repository_id == repository_id,
+                PullRequest.state == "open",
+                PullRequest.draft.is_(True),
+            )
+        )
+        or 0
+    )
+    return {
+        "binary_files": binary_files,
+        "merge_conflicts": conflicts,
+        "drafts": drafts,
+    }
+
+
+async def repository_summary(session: AsyncSession, repository: Repository) -> dict:
+    """一次性返回仪表盘所需的全部指标。"""
+    return {
+        "overview": await overview(session, repository.id),
+        "waiting": await waiting_time_stats(session, repository.id),
+        "stages": await stage_distribution(session, repository.id),
+        "branches": await branch_distribution(session, repository.id),
+        "authors": await author_distribution(session, repository.id),
+        "reviewers": await reviewer_workload(session, repository.id),
+        "intervals": await interval_stats(session, repository.id),
+        "quality": await quality_signals(session, repository.id),
+    }
+
+
+__all__ = [
+    "author_distribution",
+    "branch_distribution",
+    "interval_stats",
+    "overview",
+    "quality_signals",
+    "repository_summary",
+    "reviewer_workload",
+    "stage_distribution",
+    "waiting_time_stats",
+]