@kernel-sig/console 0.1.0

package/backend/app/services/sig_service.py

@@ -0,0 +1,385 @@
+"""SIG 例会、成员与模块归属的采集。
+
+两个外部数据源，都是"整份文档同步"而非增量：
+
+- **Etherpad 纪要**（``etherpad.openeuler.org/p/Kernel-meetings``）：一个持续追加的
+  活文档，历次会议倒序累积。同步方式是整页拉取、逐场解析、按会议日期 upsert。
+  会议纪要会被事后修改（补链接、改主持人），所以 upsert 而不是只插新的。
+- **community 仓库的 SIG 文档**：``sig/Kernel/README.md``（Maintainer）
+  与 ``sig/Kernel/committers.md``（模块归属）。低频变化，每次全量替换。
+
+成员与模块归属采用全量替换而非增量合并：它们是人维护的权威名单，
+有人被移除时必须消失。增量合并会让"已退出的人"永久留在库里。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from datetime import UTC, datetime
+
+import httpx
+import structlog
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain import meeting as meeting_domain
+from app.domain import sig as sig_domain
+from app.domain import sig_info as sig_info_domain
+from app.domain.release import classify_branch, normalize_branch_name
+from app.models.meeting import (
+    Meeting,
+    MeetingAgenda,
+    MeetingStatus,
+    ReleaseReport,
+    ReleaseReportPatch,
+)
+from app.models.repository import Repository
+from app.models.sig import MemberSource, ReleaseBranch, SIGMember, SubsystemOwner
+
+logger = structlog.get_logger(__name__)
+
+ETHER_PAD_URL = "https://etherpad.openeuler.org/p/Kernel-meetings"
+ETHER_PAD_EXPORT = f"{ETHER_PAD_URL}/export/txt"
+SIG_REPO_OWNER = "openeuler"
+SIG_REPO_NAME = "community"
+SIG_README_PATH = "sig/Kernel/README.md"
+SIG_COMMITTERS_PATH = "sig/Kernel/committers.md"
+SIG_SIG_INFO_PATH = "sig/Kernel/sig-info.yaml"
+
+# 会议的"已结束"判定：会议日期早于今天即为已开过。
+# 不用纪要里有没有内容判断 —— 会前一天主持人就会把模板填好。
+
+
+async def fetch_etherpad(client: httpx.AsyncClient | None = None) -> str:
+    """拉取纪要全文。
+
+    单独抽出来是为了能在测试里喂本地样本 —— 解析逻辑才是需要覆盖的部分，
+    而每次跑测试都去打上游会让测试依赖网络。
+    """
+    owns_client = client is None
+    client = client or httpx.AsyncClient(timeout=30.0, follow_redirects=True)
+    try:
+        response = await client.get(ETHER_PAD_EXPORT)
+        response.raise_for_status()
+        return response.text
+    finally:
+        if owns_client:
+            await client.aclose()
+
+
+def _section_hash(block: dict) -> str:
+    """一节纪要的内容指纹。
+
+    Etherpad 整页同步，但只有少数几节会变。按节算指纹，未变的跳过重写 ——
+    重写会级联删除并重建议题与报告，让它们的 id 每次都变，而前端可能已经
+    在引用这些 id。
+    """
+    payload = repr(sorted(block.items(), key=lambda item: item[0]))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
+
+
+async def sync_meetings(session: AsyncSession, text: str) -> dict[str, int]:
+    """把一份纪要全文同步进库。返回统计。
+
+    整页同步是**幂等**的：重复执行结果相同。按会议日期 upsert，
+    内容有变化时重建该场会议的议题与报告。
+    """
+    drafts = meeting_domain.parse_pad(text)
+    if not drafts:
+        logger.warning("meeting_parse_empty", url=ETHER_PAD_URL)
+        return {"meetings": 0, "created": 0, "updated": 0, "unchanged": 0}
+
+    dates = [draft.date for draft in drafts]
+    existing = {
+        row.held_on: row
+        for row in (await session.scalars(select(Meeting).where(Meeting.held_on.in_(dates)))).all()
+    }
+
+    today = datetime.now(UTC).date()
+    created = updated = unchanged = 0
+
+    for draft in drafts:
+        # 会议日期早于今天即已开过。不用"纪要里有没有内容"判断：
+        # 会前一天主持人就会把模板填好，那会显示成已结束。
+        status = MeetingStatus.DONE.value if draft.date < today else MeetingStatus.SCHEDULED.value
+        row = existing.get(draft.date)
+
+        if row is not None and row.source_hash == _section_hash(_draft_fingerprint(draft)):
+            unchanged += 1
+            continue
+
+        if row is None:
+            row = Meeting(held_on=draft.date, source_url=ETHER_PAD_URL)
+            session.add(row)
+            created += 1
+        else:
+            updated += 1
+
+        row.title = draft.title
+        row.start_time = draft.time
+        row.host = draft.host
+        row.next_host = draft.next_host
+        row.host_order = draft.host_order or None
+        row.meeting_url = draft.meeting_url
+        row.minutes_url = draft.minutes_url
+        row.legacy = draft.legacy or None
+        row.status = status
+        row.source_hash = _section_hash(_draft_fingerprint(draft))
+        row.last_synced_at = datetime.now(UTC)
+        await session.flush()
+
+        # 议题与报告整场重建：议题编号会变（有人插了一条），
+        # 逐条 diff 得不偿失，而一场会议的数据量很小。
+        await session.execute(delete(MeetingAgenda).where(MeetingAgenda.meeting_id == row.id))
+        await session.execute(delete(ReleaseReport).where(ReleaseReport.meeting_id == row.id))
+
+        for index, agenda in enumerate(draft.agendas):
+            session.add(
+                MeetingAgenda(
+                    meeting_id=row.id,
+                    sequence=index,
+                    num=agenda.num,
+                    title=agenda.title,
+                    owner=agenda.owner,
+                    body=agenda.body or None,
+                    pull_numbers=sorted(set(agenda.pull_numbers)) or None,
+                    issue_numbers=sorted(set(agenda.issue_numbers)) or None,
+                )
+            )
+
+        for progress in _collect_progress(draft):
+            report = ReleaseReport(
+                meeting_id=row.id,
+                branch=progress.branch,
+                from_tag=progress.from_tag,
+                to_tag=progress.to_tag,
+                patch_count=progress.patch_count,
+                listed_count=progress.listed_patch_count,
+                iso_urls=progress.iso_urls or None,
+                cve_ids=sorted(set(progress.cve_ids)) or None,
+                groups=[{"name": group.name, "count": group.count} for group in progress.groups]
+                or None,
+            )
+            session.add(report)
+            await session.flush()
+
+            sequence = 0
+            for group in progress.groups:
+                for item in group.items:
+                    session.add(
+                        ReleaseReportPatch(
+                            report_id=report.id,
+                            sequence=sequence,
+                            group_name=group.name,
+                            title=item.title[:2000],
+                            pull_number=item.pull_number,
+                            url=item.url,
+                        )
+                    )
+                    sequence += 1
+
+    await session.flush()
+    logger.info(
+        "meetings_synced",
+        parsed=len(drafts),
+        created=created,
+        updated=updated,
+        unchanged=unchanged,
+    )
+    return {
+        "meetings": len(drafts),
+        "created": created,
+        "updated": updated,
+        "unchanged": unchanged,
+    }
+
+
+def _draft_fingerprint(draft: meeting_domain.MeetingDraft) -> dict:
+    """转成可哈希的字典。dataclass 里含 date 与嵌套列表，不能直接 repr 排序。"""
+    return {
+        "date": draft.date.isoformat(),
+        "time": draft.time,
+        "host": draft.host,
+        "next_host": draft.next_host,
+        "meeting_url": draft.meeting_url,
+        "title": draft.title,
+        "legacy": draft.legacy,
+        "agendas": [
+            (agenda.num, agenda.title, agenda.owner, len(agenda.body)) for agenda in draft.agendas
+        ],
+        "reports": [
+            (p.branch, p.from_tag, p.to_tag, p.patch_count, p.listed_patch_count)
+            for p in _collect_progress(draft)
+        ],
+    }
+
+
+def _collect_progress(draft: meeting_domain.MeetingDraft) -> list[meeting_domain.BranchProgress]:
+    """一场会议里所有分支的进展报告。
+
+    同一分支在同一场会议里可能出现两次（不同议题各报告一次），
+    以先出现的为准 —— 后一份通常是补记，字段更少。
+    """
+    merged: dict[str, meeting_domain.BranchProgress] = {}
+    for agenda in draft.agendas:
+        for progress in agenda.progress:
+            merged.setdefault(progress.branch, progress)
+    return list(merged.values())
+
+
+# ---------------------------------------------------------------------------
+# SIG 名单
+# ---------------------------------------------------------------------------
+
+
+async def sync_roster(
+    session: AsyncSession,
+    *,
+    sig_info_yaml: str | None,
+    committers_md: str | None = None,
+    source_ref: str = "master",
+) -> dict[str, int]:
+    """同步成员名单、分支负责人与模块归属。
+
+    成员名单以 ``sig-info.yaml`` 为准 —— 它是 openEuler 的机器可读 SIG 元数据，
+    字段完整（姓名、邮箱、组织、atomgit 与 gitee 账号）。此前退而用 README 的
+    维护者列表加 committers.md 的模块表，只有 101 人且缺邮箱与组织账号。
+
+    全量替换而非增量合并：这是一份权威名单，有人退出时必须消失。
+    也刻意不用某列做冲突键 —— 名单里唯一可用的键是平台账号，而账号是可空的
+    （133 人中 16 人只有 gitee），以它建唯一索引会让这些人无法入库。
+    """
+    if sig_info_yaml is None:
+        logger.warning("sig_info_unavailable")
+        return {"members": 0, "branches": 0, "subsystems": 0, "skipped": 1}
+
+    info = sig_info_domain.parse_sig_info(sig_info_yaml)
+    if info is None or not info.roster:
+        logger.warning("sig_info_parse_failed")
+        return {"members": 0, "branches": 0, "subsystems": 0, "skipped": 1}
+
+    now = datetime.now(UTC)
+    source_url = (
+        f"https://atomgit.com/{SIG_REPO_OWNER}/{SIG_REPO_NAME}"
+        f"/blob/{source_ref}/{SIG_SIG_INFO_PATH}"
+    )
+
+    await session.execute(delete(SIGMember))
+    for entry in info.roster:
+        session.add(
+            SIGMember(
+                name=entry.name[:128],
+                role=entry.role,
+                roles=entry.roles or None,
+                atomgit_id=entry.atomgit_id,
+                gitee_id=entry.gitee_id,
+                email=entry.email,
+                org=entry.organization,
+                repositories=entry.repositories or None,
+                admin_repositories=entry.admin_repositories or None,
+                kept_branches=entry.kept_branches or None,
+                is_active=True,
+                source=MemberSource.SIG_INFO.value,
+                source_url=source_url,
+                last_synced_at=now,
+            )
+        )
+
+    # 分支负责人写进登记表：分支页据此显示"该分支谁负责"。
+    # 只在登记表里补 keepers，不覆盖人工填的生命周期与说明。
+    #
+    # 键是 (仓库, 分支名) 而不是分支名：上游文档里每条 repo_branch 都写着
+    # 这个分支属于哪个仓库，而 master、openEuler-24.03-LTS-SP4 这类名字
+    # 两个仓库都有 —— 只按名字存，两个仓库就会互相看到对方的分支。
+    repositories = {
+        f"{row.owner}/{row.name}": row.id
+        for row in (await session.scalars(select(Repository))).all()
+    }
+    keepers_by_branch: dict[tuple[str, str], list[str]] = {}
+    for branch in info.branch_keepers:
+        key = (branch.repo, normalize_branch_name(branch.branch))
+        keepers_by_branch.setdefault(key, [])
+        for login in branch.keepers:
+            if login not in keepers_by_branch[key]:
+                keepers_by_branch[key].append(login)
+
+    for (repo, name), keepers in keepers_by_branch.items():
+        repository_id = repositories.get(repo)
+        row = await session.scalar(
+            select(ReleaseBranch).where(
+                ReleaseBranch.name == name,
+                ReleaseBranch.repository_id == repository_id,
+            )
+        )
+        spec = classify_branch(name)
+        if row is None:
+            row = ReleaseBranch(
+                repository_id=repository_id,
+                name=name,
+                kind=spec.kind.value,
+                series=spec.series,
+                sp=spec.sp,
+                upstream_base=spec.upstream_base,
+            )
+            session.add(row)
+        row.keepers = keepers or None
+    await session.flush()
+
+    subsystems = 0
+    if committers_md is not None:
+        rows = sig_domain.parse_committers(committers_md)
+        await session.execute(delete(SubsystemOwner))
+        for index, row in enumerate(rows):
+            session.add(
+                SubsystemOwner(
+                    module=row.module[:64],
+                    section=row.section or None,
+                    paths=row.paths or None,
+                    committer_logins=sorted({a.account_id for a in row.committers}) or None,
+                    sequence=index,
+                    source_url=(
+                        f"https://atomgit.com/{SIG_REPO_OWNER}/{SIG_REPO_NAME}"
+                        f"/blob/{source_ref}/{SIG_COMMITTERS_PATH}"
+                    ),
+                    last_synced_at=now,
+                )
+            )
+            subsystems += 1
+
+    await session.flush()
+    logger.info(
+        "sig_roster_synced",
+        members=len(info.roster),
+        branches=len(keepers_by_branch),
+        subsystems=subsystems,
+    )
+    return {
+        "members": len(info.roster),
+        "branches": len(keepers_by_branch),
+        "subsystems": subsystems,
+        "skipped": 0,
+    }
+
+
+async def fetch_sig_documents(client) -> tuple[str | None, str | None, str | None]:
+    """从 AtomGit 取 SIG 的三份文档：sig-info.yaml、committers.md、README.md。
+
+    取不到的那份返回 None —— 其中某一份改路径或临时不可读，不该让整轮同步失败。
+    """
+    results = await asyncio.gather(
+        client.get_file_content(SIG_REPO_OWNER, SIG_REPO_NAME, SIG_SIG_INFO_PATH),
+        client.get_file_content(SIG_REPO_OWNER, SIG_REPO_NAME, SIG_COMMITTERS_PATH),
+        client.get_file_content(SIG_REPO_OWNER, SIG_REPO_NAME, SIG_README_PATH),
+        return_exceptions=True,
+    )
+    return tuple(item if isinstance(item, str) else None for item in results)  # type: ignore[return-value]
+
+
+__all__ = [
+    "ETHER_PAD_URL",
+    "fetch_etherpad",
+    "fetch_sig_documents",
+    "sync_meetings",
+    "sync_roster",
+]