@kernel-sig/console 0.1.0

package/scripts/verify-analysis-pipeline.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+"""分类与关注项流水线的端到端校验。
+
+跑在真实数据库上，验证两件靠单测覆盖不到的事：
+1. 全量数据上不抛异常、结果落在枚举取值域内
+2. **重算是幂等的** —— 关注项的 first_detected_at 不被刷新、
+   解决状态不被重复置位、分类不因重复执行而漂移
+
+用法（在 backend 目录下）：
+    KSC_SECRET_KEY=... python ../scripts/verify-analysis-pipeline.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+from sqlalchemy import func, select
+
+from app.core.config import get_settings
+from app.core.db import dispose_engine, init_engine, session_scope
+from app.domain.classification import SUBSYSTEM_LABELS
+from app.models.attention import AttentionItem, AttentionRule, AttentionSeverity
+from app.models.classification import Classification, PRKind
+from app.models.issue import Issue
+from app.models.pull_request import PullRequest
+from app.services import (
+    attention_service,
+    classification_service,
+    repository_service,
+)
+
+failures: list[str] = []
+
+
+def check(condition: bool, message: str) -> None:
+    if condition:
+        print(f"  ok   {message}")
+    else:
+        print(f"  FAIL {message}")
+        failures.append(message)
+
+
+async def main() -> int:
+    settings = get_settings()
+    init_engine(settings)
+
+    async with session_scope() as session:
+        repositories = await repository_service.list_repositories(session)
+        check(bool(repositories), f"纳管仓库数 = {len(repositories)}")
+        if not repositories:
+            await dispose_engine()
+            return 1
+
+        for repository in repositories:
+            print(f"\n=== {repository.full_name} ===")
+
+            pulls = list(
+                (
+                    await session.scalars(
+                        select(PullRequest).where(
+                            PullRequest.repository_id == repository.id,
+                            PullRequest.state == "open",
+                        )
+                    )
+                ).all()
+            )
+            issues = list(
+                (
+                    await session.scalars(
+                        select(Issue).where(
+                            Issue.repository_id == repository.id, Issue.state == "open"
+                        )
+                    )
+                ).all()
+            )
+            print(f"  open PR = {len(pulls)}, open Issue = {len(issues)}")
+
+            # --- 分类 ---
+            await classification_service.classify_pulls(session, repository, pulls)
+            await classification_service.classify_issues(session, repository, issues)
+
+            kinds = dict(
+                (
+                    await session.execute(
+                        select(Classification.kind, func.count())
+                        .where(Classification.repository_id == repository.id)
+                        .group_by(Classification.kind)
+                    )
+                ).all()
+            )
+            total = sum(kinds.values())
+            print(f"  分类结果：{total} 条")
+            for kind, count in sorted(kinds.items(), key=lambda kv: -kv[1]):
+                share = count / total * 100 if total else 0
+                print(f"    {kind.value:<10} {count:>5}  {share:5.1f}%")
+
+            # 断言"每个 open 对象都被分类"，而不是"分类条数等于 open 对象数"：
+            # 分类覆盖全部状态（月度汇总要统计已合入的 PR 属于哪一类），
+            # 总数本来就大于 open 的数量，用等号衡量的是统计口径而非覆盖率。
+            classified_ids = set(
+                (await session.scalars(
+                    select(Classification.subject_id).where(
+                        Classification.repository_id == repository.id
+                    )
+                )).all()
+            )
+            uncovered = [row.number for row in pulls if row.id not in classified_ids]
+            uncovered += [row.number for row in issues if row.id not in classified_ids]
+            print(f"  未分类的 open 对象：{len(uncovered)} 个")
+            check(not uncovered, "每个 open PR/Issue 都有分类结果")
+            check(
+                all(isinstance(k, PRKind) for k in kinds),
+                "全部类型落在 PRKind 枚举内",
+            )
+
+            # 覆盖率不做通过/失败判定 —— 内核补丁标题描述的是"改了什么"而非
+            # "属于哪一类"，未命中是规则的固有上限，由 AI 兜底接住。
+            # 这里只报告数字，让覆盖率的退化能被看见。
+            unknown_share = kinds.get(PRKind.UNKNOWN, 0) / total if total else 0
+            print(f"  → 规则未命中率 {unknown_share:.1%}（这部分由 AI 补判）")
+
+            # 真正要守住的是**精确率**，不是覆盖率
+            rows = (
+                await session.execute(
+                    select(Classification).where(Classification.repository_id == repository.id)
+                )
+            ).scalars()
+            cve_rows = []
+            rule_rows = []
+            for row in rows:
+                if row.kind is PRKind.CVE:
+                    cve_rows.append(row)
+                if row.source.value == "rule":
+                    rule_rows.append(row)
+
+            hit_rows = [row for row in rule_rows if row.kind is not PRKind.UNKNOWN]
+
+            # 自动判定的 CVE 必须带得出发现在正文里的编号：规则层靠匹配编号
+            # 才判 cve，所以天然满足；AI 层则一度凭"这是安全修复"的印象判 cve
+            # 而给不出编号，让对外报的「CVE 数」混进未编号的加固（已由
+            # normalize_ai_kind 堵住，存量由迁移 e5b27c9d3a41 修正）。
+            #
+            # 人工指定不在此列：那是维护者的显式判断，他可能知道编号之外的
+            # 实情，平台不该替他改，也不该因此判他错。
+            auto_cve_rows = [row for row in cve_rows if row.source.value != "manual"]
+            manual_cve = len(cve_rows) - len(auto_cve_rows)
+            check(
+                all(row.cve_ids for row in auto_cve_rows),
+                f"自动判定的 CVE 都带编号（{len(auto_cve_rows)} 条，"
+                f"另有 {manual_cve} 条人工指定不计）",
+            )
+            check(
+                all(row.rule_name for row in hit_rows),
+                f"规则命中项都记录命中规则名（{len(hit_rows)} 条，可解释性）",
+            )
+            check(
+                all(row.confidence > 0 for row in rule_rows if row.kind is not PRKind.UNKNOWN),
+                "规则命中项的置信度大于 0",
+            )
+            check(
+                all(
+                    row.kind is PRKind.UNKNOWN and row.confidence == 0
+                    for row in rule_rows
+                    if row.kind is PRKind.UNKNOWN
+                ),
+                "规则未命中的条目标为 unknown 且置信度为 0（AI 补判的识别信号）",
+            )
+
+            subsystems = [
+                row[0]
+                for row in (
+                    await session.execute(
+                        select(Classification.subsystem)
+                        .where(
+                            Classification.repository_id == repository.id,
+                            Classification.subsystem.is_not(None),
+                        )
+                        .distinct()
+                    )
+                ).all()
+            ]
+            unknown_subsystems = [s for s in subsystems if s not in SUBSYSTEM_LABELS]
+            check(
+                not unknown_subsystems,
+                f"子系统取值全部在已知集合内（异常项：{unknown_subsystems[:5]}）",
+            )
+
+            cve_pulls = await session.scalar(
+                select(func.count())
+                .select_from(Classification)
+                .where(
+                    Classification.repository_id == repository.id,
+                    Classification.kind == PRKind.CVE,
+                )
+            )
+            print(f"  判定为 CVE 的条目：{cve_pulls}")
+
+            # --- 关注项，第一轮 ---
+            stats_before = await attention_service.compute_for_repository(session, repository)
+            print(f"  关注项：{stats_before}")
+            first_pass = {
+                (row.subject_type, row.subject_id, row.rule): row.first_detected_at
+                for row in (
+                    await session.scalars(
+                        select(AttentionItem).where(AttentionItem.repository_id == repository.id)
+                    )
+                ).all()
+            }
+
+            # --- 关注项，第二轮（幂等性验证）---
+            await attention_service.compute_for_repository(session, repository)
+            second_pass = {
+                (row.subject_type, row.subject_id, row.rule): row.first_detected_at
+                for row in (
+                    await session.scalars(
+                        select(AttentionItem).where(AttentionItem.repository_id == repository.id)
+                    )
+                ).all()
+            }
+            check(
+                first_pass == second_pass,
+                "重算不改变任何关注项的 key 或 first_detected_at",
+            )
+
+            unresolved = await session.scalar(
+                select(func.count())
+                .select_from(AttentionItem)
+                .where(
+                    AttentionItem.repository_id == repository.id,
+                    AttentionItem.resolved_at.is_(None),
+                )
+            )
+            check(
+                unresolved == stats_before["open_items"],
+                f"重算后未解决数不变（{unresolved}）",
+            )
+
+            by_severity = await attention_service.severity_counts(session, repository.id)
+            print(f"  按严重程度：{by_severity}")
+            by_rule = await attention_service.rule_counts(session, repository.id)
+            active_rules = {k: v for k, v in by_rule.items() if v}
+            print(f"  命中的规则：{active_rules}")
+
+            check(
+                all(k in {s.value for s in AttentionSeverity} for k in by_severity),
+                "严重程度取值合法",
+            )
+            check(
+                all(k in {r.value for r in AttentionRule} for k in by_rule),
+                "规则取值合法",
+            )
+            # 规则与领域层的一致性：两边的枚举必须完全对应。
+            # 多一个值意味着库里能存下规则层永远不会产生的取值
+            from app.domain.attention import Rule as DomainRule
+
+            model_rules = {r.value for r in AttentionRule}
+            domain_rules = {r.value for r in DomainRule}
+            check(
+                model_rules == domain_rules,
+                f"模型枚举与领域规则一致（模型独有：{model_rules - domain_rules}；"
+                f"领域独有：{domain_rules - model_rules}）",
+            )
+
+            # 分层抽样：列出 blocker / critical 的前几条，人工核对是否合理
+            top = list(
+                (
+                    await session.scalars(
+                        select(AttentionItem)
+                        .where(
+                            AttentionItem.repository_id == repository.id,
+                            AttentionItem.resolved_at.is_(None),
+                            AttentionItem.severity.in_(
+                                [AttentionSeverity.BLOCKER.value, AttentionSeverity.CRITICAL.value]
+                            ),
+                        )
+                        .limit(8)
+                    )
+                ).all()
+            )
+            print("  最高优先级的几条：")
+            for item in top:
+                print(f"    [{item.severity.value}] #{item.subject_number} {item.title}")
+
+            check(
+                all(item.subject_id is not None for item in top),
+                "关注项都关联到具体对象",
+            )
+
+    await dispose_engine()
+
+    print()
+    if failures:
+        print(f"失败 {len(failures)} 项：")
+        for message in failures:
+            print(f"  - {message}")
+        return 1
+    print("全部通过")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))