@kernel-sig/console 0.1.0

package/scripts/verify-ai-pipeline.py

@@ -0,0 +1,616 @@
+#!/usr/bin/env python
+"""AI 分析链路的端到端校验。
+
+用一个本地桩服务冒充 OpenAI 兼容端点，验证真实代码路径：
+路由选模型 → 解密凭据 → 发请求 → 解析 JSON → 落库 → 幂等。
+
+没有桩服务就只能测到 mock 层，而这一链路上最容易出错的地方
+（请求头、payload 结构、响应解析、状态机）恰恰都在真实 HTTP 之后。
+
+用法（在 backend 目录下）：
+    KSC_SECRET_KEY=... python ../scripts/verify-ai-pipeline.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+import threading
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from sqlalchemy import delete, func, inspect as sa_inspect, select
+
+from app.core.config import get_settings
+from app.core.db import dispose_engine, init_engine, session_scope
+from app.core.exceptions import ValidationError
+from app.models.ai import AIAnalysis, AITask, AnalysisStatus, LLMProvider
+from app.models.classification import Classification, ClassificationSource, PRKind, SubjectType
+from app.models.pull_request import PullRequest
+from app.models.credential import Credential, CredentialKind
+from app.services import ai_service, classification_service, credential_service, repository_service
+
+STUB_PORT = 18099
+STUB_URL = f"http://127.0.0.1:{STUB_PORT}/v1"
+PROVIDER_NAME = "verify-stub"
+
+failures: list[str] = []
+
+
+def check(condition: bool, message: str) -> None:
+    print(f"  {'ok  ' if condition else 'FAIL'} {message}")
+    if not condition:
+        failures.append(message)
+
+
+# ---------------------------------------------------------------------------
+# 桩服务
+# ---------------------------------------------------------------------------
+
+
+class _Stub:
+    """记录收到的请求，并按模式返回响应。
+
+    模式：ok（正常 JSON）/ garbage（散文包着 JSON）/ auth_error（401）
+    """
+
+    def __init__(self) -> None:
+        self.requests: list[dict] = []
+        self.mode = "ok"
+
+    def payload(self) -> dict:
+        return {
+            "model": "stub-model",
+            "choices": [
+                {
+                    "message": {
+                        "content": json.dumps(
+                            {
+                                "kind": "bugfix",
+                                "subsystem": "net",
+                                "confidence": 0.82,
+                                "reason": "标题描述了并发相关的缺陷修复",
+                            },
+                            ensure_ascii=False,
+                        )
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+            "usage": {"prompt_tokens": 120, "completion_tokens": 30},
+        }
+
+
+STUB = _Stub()
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def do_POST(self) -> None:
+        length = int(self.headers.get("Content-Length", 0))
+        body = json.loads(self.rfile.read(length) or b"{}")
+        STUB.requests.append({"path": self.path, "body": body, "headers": dict(self.headers)})
+
+        if STUB.mode == "auth_error":
+            self._send(401, {"error": {"message": "invalid api key"}})
+            return
+        if STUB.mode == "bad_kind":
+            # 模型偶尔会自造一个不在枚举里的取值 —— 必须被显式拒绝
+            content = '{"kind": "perf_optimization", "subsystem": "net", "confidence": 0.9}'
+            self._send(200, {"choices": [{"message": {"content": content}}], "model": "stub-model"})
+            return
+        if STUB.mode == "garbage":
+            # 模型不顾"只输出 JSON"的指示，包了围栏又加了前言：
+            # 这是常态而非异常，必须能被 extract_json 兜住
+            content = '好的，我的判断如下：\n```json\n{"kind": "feature", "subsystem": "mm", '
+            content += '"confidence": 0.7, "reason": "新增了内存管理接口"}\n```\n希望有帮助。'
+            self._send(200, {"choices": [{"message": {"content": content}}], "model": "stub-model"})
+            return
+        self._send(200, STUB.payload())
+
+    def _send(self, status: int, payload: dict) -> None:
+        data = json.dumps(payload).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def log_message(self, *args: object) -> None:
+        return  # 静默，避免污染校验输出
+
+
+def start_stub() -> ThreadingHTTPServer:
+    server = ThreadingHTTPServer(("127.0.0.1", STUB_PORT), _Handler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server
+
+
+# ---------------------------------------------------------------------------
+
+
+async def main() -> int:
+    settings = get_settings()
+    init_engine(settings)
+    server = start_stub()
+    touched_subjects: list = []
+
+    try:
+        async with session_scope() as session:
+            repository = (await repository_service.list_repositories(session))[0]
+            print(f"仓库：{repository.full_name}")
+
+            # --- 准备：桩凭据 + 桩模型 ---
+            credential = await session.scalar(
+                select(Credential).where(Credential.name == "stub-llm-key")
+            )
+            if credential is None:
+                credential = await credential_service.create_credential(
+                    session,
+                    settings,
+                    name="stub-llm-key",
+                    kind=CredentialKind.LLM_API_KEY,
+                    plaintext="sk-stub-verify",
+                )
+
+            provider = await session.scalar(
+                select(LLMProvider).where(LLMProvider.name == PROVIDER_NAME)
+            )
+            if provider is None:
+                provider = LLMProvider(
+                    name=PROVIDER_NAME,
+                    base_url=STUB_URL,
+                    credential_id=credential.id,
+                    model="stub-model",
+                    is_default=True,
+                )
+                session.add(provider)
+                await session.flush()
+            else:
+                provider.enabled = True
+                provider.is_default = True
+                provider.credential_id = credential.id
+
+            # --- 选两个 PR 作为分析对象 ---
+            #
+            # 对象必须是**规则判不出来的** PR，这不是随便挑的：后面要验证
+            # "AI 判定扛得住规则重算"，而 `reclassify` 的结论是"规则优先、AI
+            # 兜底" —— 规则判得出来时它当然写回规则答案，只有判不出来时才回落
+            # 到模型此前的结论。拿规则判得出来的 PR 去测这一条，测的是别的东西。
+            #
+            # 也不能直接问"有哪些待补判的 PR"：平台收敛，定时任务每半小时补判
+            # 一批，跑久了池子就空了，那句断言会以"找到 0 个"失败 —— 校验结果
+            # 取决于任务跑到哪一刻，那是环境，不是被测的代码。
+            #
+            # 所以自己算：把 open PR 全取出来，现算规则结论，挑出 unknown 的那些。
+            # 算完再把它们的分类暂存成 unknown 造出待补判状态，收尾原样还回去。
+            # 中途删掉的 AI 分析记录由 touched_subjects 清理。
+            #
+            # **不能只扫最近更新的那批**：实测最近 200 个恰好全部规则可判
+            # （新同步进来的 PR 一旦补齐提交信息，97.7% 都能靠自声明标注判出来），
+            # 而判不出来的 181 个散在更早的 PR 里。按更新时间取窗口会时灵时不灵，
+            # 又是"结果取决于环境"。全量读 open PR 与兄弟脚本 verify-analysis 一致。
+            candidates = list(
+                (
+                    await session.scalars(
+                        select(PullRequest).where(
+                            PullRequest.repository_id == repository.id,
+                            PullRequest.state == "open",
+                        )
+                    )
+                ).all()
+            )
+            files = await classification_service.files_for_pulls(
+                session, [pull.id for pull in candidates]
+            )
+            commits = await classification_service.commits_for_pulls(
+                session, [pull.id for pull in candidates]
+            )
+            targets = [
+                pull
+                for pull in candidates
+                if PRKind(
+                    classification_service.build_result(
+                        title=pull.title,
+                        body=pull.body,
+                        filenames=files.get(pull.id, ()),
+                        commit_messages=commits.get(pull.id, ()),
+                    ).kind.value
+                )
+                is PRKind.UNKNOWN
+            ][:2]
+            check(
+                len(targets) == 2,
+                f"在 {len(candidates)} 个 open PR 里找到 {len(targets)} 个规则判不出来的",
+            )
+            if len(targets) < 2:
+                return 1
+
+            saved: list[tuple[uuid.UUID, dict]] = []
+            stale: list[Classification] = []
+            for pull in targets:
+                touched_subjects.append(pull.id)
+                row = await classification_service.get_classification(
+                    session, SubjectType.PULL_REQUEST, pull.id
+                )
+                if row is None:
+                    continue
+                saved.append(
+                    (
+                        row.id,
+                        {c.key: getattr(row, c.key) for c in sa_inspect(row).mapper.column_attrs},
+                    )
+                )
+                row.kind = PRKind.UNKNOWN
+                row.source = ClassificationSource.RULE
+                row.confidence = 0
+                stale.append(row)
+            await session.flush()
+
+            # flush 之后再把对象从会话里摘掉（收尾时按 id 取回）。顺序不能反：
+            # 摘掉一个脏对象会连它待提交的修改一起丢，那两行就根本没变成 unknown。
+            #
+            # 摘掉是为了避免身份映射挡住后面读到的真实值：`apply_ai_result` 走的是
+            # Core 的 pg_insert(...on_conflict_do_update)，绕开 ORM，库里改了而
+            # 会话里这个对象还是旧值，get_classification 会把 unknown 递回来。
+            for row in stale:
+                session.expunge(row)
+
+            missing_now = {
+                pull.id
+                for pull in await classification_service.pulls_missing_classification(
+                    session, repository.id, limit=200
+                )
+            }
+            check(
+                {pull.id for pull in targets} <= missing_now,
+                "两个分析对象现在都处于「待补判」状态",
+            )
+
+            target = targets[0]
+
+            # --- 正常路径 ---
+            #
+            # 必须 force：平台按内容哈希去重，同一对象内容没变就不再调用模型
+            # （这是省钱的机制）。而这一段的断言全都是关于**请求本身**的 ——
+            # 路径、鉴权头、模型名、消息条数 —— 命中缓存就一条请求都不会发，
+            # 断言随即变成对着空列表取下标而崩，且崩的地方指向的是
+            # "请求路径不对"这件根本不存在的事。补判对象有没有被分析过
+            # 取决于 worker 跑到哪一刻，那是环境，不是这里要测的东西。
+            STUB.requests.clear()
+            STUB.mode = "ok"
+            analysis = await ai_service.classify_subject(
+                session, settings, repository, pull=target, triggered_by=None, force=True
+            )
+
+            check(
+                analysis.status is AnalysisStatus.SUCCEEDED,
+                f"分析成功（status={analysis.status.value}, error={analysis.error}）",
+            )
+            check(len(STUB.requests) == 1, f"恰好发出 1 次请求（实际 {len(STUB.requests)}）")
+
+            sent = STUB.requests[0]
+            check(sent["path"] == "/v1/chat/completions", f"请求路径 {sent['path']}")
+            check(
+                sent["headers"].get("Authorization") == "Bearer sk-stub-verify",
+                "使用解密后的凭据作为 Bearer token",
+            )
+            check(sent["body"]["model"] == "stub-model", "模型名来自 provider 配置")
+            check(sent["body"]["stream"] is False, "非流式调用")
+            check(
+                sent["body"]["response_format"] == {"type": "json_object"},
+                "分类任务要求上游返回 JSON",
+            )
+            check(len(sent["body"]["messages"]) == 2, "system + user 两条消息")
+            # 分类是轻任务，Prompt 刻意不带 KABI/补丁格式那套领域规则：
+            # 判定"属于哪一类"用不上它们，带上只是把每次调用的成本抬高。
+            # 真正必须有的是取值清单 —— 否则模型会自造类型。
+            system_prompt = sent["body"]["messages"][0]["content"]
+            check(
+                all(kind in system_prompt for kind in ("cve", "backport", "bugfix", "feature")),
+                "system prompt 列出全部可选类型（约束模型不得自造）",
+            )
+            print(
+                f"  → prompt_tokens={analysis.prompt_tokens} "
+                f"completion_tokens={analysis.completion_tokens}"
+            )
+
+            # --- 结果写回分类表 ---
+            row = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, target.id
+            )
+            check(row is not None, "分类记录存在")
+            check(
+                row.source is ClassificationSource.AI,
+                f"分类来源标记为 ai（实际 {row.source.value}）",
+            )
+            check(row.kind is PRKind.BUGFIX, f"类型来自模型输出（{row.kind.value}）")
+            check(row.subsystem == "net", f"子系统已归一化（{row.subsystem}）")
+            check(0 < row.confidence < 1, f"置信度记录为模型的 0.82（{row.confidence}）")
+
+            # --- 幂等：内容未变不重复调用模型 ---
+            STUB.requests.clear()
+            again = await ai_service.classify_subject(session, settings, repository, pull=target)
+            check(
+                len(STUB.requests) == 0,
+                f"输入未变时不重复调用模型（实际发出 {len(STUB.requests)} 次）",
+            )
+            check(again.id == analysis.id, "返回同一条分析记录")
+
+            # --- 强制重跑必须真的重跑 ---
+            STUB.requests.clear()
+            await ai_service.classify_subject(
+                session, settings, repository, pull=target, force=True
+            )
+            check(len(STUB.requests) == 1, "force=True 时绕过幂等检查")
+
+            # --- 人工覆盖优先：自动任务不得改写 ---
+            await classification_service.override(
+                session,
+                row,
+                kind=PRKind.FEATURE,
+                subsystem="mm",
+                user_id=(await session.scalar(select(Credential.created_by))) or None,
+            )
+            await session.flush()
+            await classification_service.classify_pulls(session, repository, [target])
+            refreshed = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, target.id
+            )
+            check(
+                refreshed.kind is PRKind.FEATURE and refreshed.source.value == "manual",
+                f"人工覆盖不被规则重算改写（当前 {refreshed.kind.value}/{refreshed.source.value}）",
+            )
+            STUB.requests.clear()
+            await ai_service.classify_subject(session, settings, repository, pull=target)
+            refreshed = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, target.id
+            )
+            check(
+                refreshed.kind is PRKind.FEATURE,
+                "人工覆盖同样不被 AI 结果改写",
+            )
+            await classification_service.clear_override(session, refreshed)
+
+            # --- AI 判定必须扛得住规则重算 ---
+            #
+            # 规则每 10 分钟全量重算一次。若允许它把 AI 判出的结果覆盖回
+            # unknown，那条记录下一轮又会被挑去"补判" —— 模型会被反复
+            # 调用同一个对象，既不收敛也持续计费。
+            await classification_service.classify_pulls(session, repository, targets)
+            after_rule_pass = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, target.id
+            )
+            check(
+                after_rule_pass.source is ClassificationSource.AI
+                and after_rule_pass.kind is PRKind.BUGFIX,
+                f"规则重算不打回 AI 判定（当前 {after_rule_pass.source.value}/"
+                f"{after_rule_pass.kind.value}）",
+            )
+
+            still_missing = await classification_service.pulls_missing_classification(
+                session, repository.id, limit=200
+            )
+            check(
+                target.id not in {pull.id for pull in still_missing},
+                "AI 判定过的条目不再出现在待补判列表里（不会重复计费）",
+            )
+
+            # --- 畸形输出：模型加围栏 + 前言 ---
+            STUB.mode = "garbage"
+            STUB.requests.clear()
+            second = targets[1] if len(targets) > 1 else target
+            garbage = await ai_service.classify_subject(
+                session, settings, repository, pull=second, force=True
+            )
+            check(
+                garbage.status is AnalysisStatus.SUCCEEDED,
+                f"散文包裹的 JSON 能被解析（status={garbage.status.value}）",
+            )
+            check(
+                garbage.result.get("kind") == "feature", f"解析出 kind={garbage.result.get('kind')}"
+            )
+
+            # --- 上游拒绝：失败要落状态，且不向上抛 ---
+            STUB.mode = "auth_error"
+            STUB.requests.clear()
+            failed = await ai_service.classify_subject(
+                session, settings, repository, pull=second, force=True
+            )
+            check(
+                failed.status is AnalysisStatus.FAILED,
+                f"401 使分析标记为 failed（实际 {failed.status.value}）",
+            )
+            check(bool(failed.error), f"失败原因已记录：{(failed.error or '')[:60]}")
+            check(
+                failed.finished_at is not None and failed.started_at is not None,
+                "失败也记录起止时间（用于统计耗时）",
+            )
+            # 401 是确定性失败，重试只会把错误的凭据再送两遍
+            check(
+                len(STUB.requests) == 1,
+                f"401 不重试（实际请求 {len(STUB.requests)} 次）",
+            )
+
+            token_sum = sum(
+                a.total_tokens
+                for a in (
+                    await session.scalars(
+                        select(AIAnalysis).where(AIAnalysis.repository_id == repository.id)
+                    )
+                ).all()
+            )
+            print(f"  → 累计 token：{token_sum}")
+
+            # --- 模型自造枚举值：显式拒绝，且不写坏分类表 ---
+            STUB.mode = "bad_kind"
+            STUB.requests.clear()
+            before = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, second.id
+            )
+            before_kind = before.kind if before else None
+            bogus = await ai_service.classify_subject(
+                session, settings, repository, pull=second, force=True
+            )
+            after = await classification_service.get_classification(
+                session, SubjectType.PULL_REQUEST, second.id
+            )
+            check(
+                bogus.status is AnalysisStatus.SUCCEEDED,
+                "模型返回的是合法 JSON，分析本身算成功",
+            )
+            check(
+                bogus.result.get("kind") == "perf_optimization",
+                "原始响应如实保存，便于事后追查模型说了什么",
+            )
+            check(
+                (after.kind if after else None) == before_kind,
+                f"非法类型不写入分类表（仍为 {before_kind.value if before_kind else None}）",
+            )
+
+            # --- 未配置模型时的行为 ---
+            #
+            # 必须**抛错**而不是落一条 FAILED：未配置是部署问题，不是分析失败。
+            # 若落库，定时任务每轮都会写几条"失败"，用量面板上看着像模型故障，
+            # 实际只是还没配；这些空记录还会一直累积。
+            # 桩 provider 未必是库里唯一启用的一个 —— 仓库纳管后可能已经配了
+            # 真实端点。只关桩的那个，"无可用模型"这个前提就不成立，检查结果
+            # 于是取决于本机环境而非代码。这里暂时关掉全部，断言后原样恢复：
+            # 校验脚本不该把本机的端点配置改掉。
+            enabled_before = list(
+                (await session.scalars(select(LLMProvider).where(LLMProvider.enabled.is_(True)))).all()
+            )
+            for row in enabled_before:
+                row.enabled = False
+            await session.flush()
+            STUB.mode = "ok"
+            STUB.requests.clear()
+
+            rows_before = await session.scalar(
+                select(func.count())
+                .select_from(AIAnalysis)
+                .where(AIAnalysis.repository_id == repository.id)
+            )
+            raised = ""
+            try:
+                await ai_service.classify_subject(
+                    session, settings, repository, pull=second, force=True
+                )
+            except ValidationError as exc:
+                raised = str(exc)
+
+            check("尚未配置" in raised, f"无可用模型时抛出明确提示：{raised[:40]}")
+            check(len(STUB.requests) == 0, "无可用模型时不发请求")
+
+            # 恢复本机原有的端点配置，并让桩 provider 重新可用
+            for row in enabled_before:
+                row.enabled = True
+            provider.enabled = True
+            await session.flush()
+
+            rows_after = await session.scalar(
+                select(func.count())
+                .select_from(AIAnalysis)
+                .where(AIAnalysis.repository_id == repository.id)
+            )
+            check(
+                rows_after == rows_before,
+                f"无可用模型时不留下失败记录（{rows_before} → {rows_after}）",
+            )
+
+            # --- 并发：定时任务与人工重跑撞上同一个对象 ---
+            #
+            # 唯一约束 (subject_type, subject_id, task) 规定一个对象只有一行，
+            # 而 execute() 是"先查再插"。定时任务和用户点「重新分析」可能同时
+            # 选中同一个对象：两边的 SELECT 都看不见对方还没提交的行，都去插入，
+            # 后到的被约束拦下。生产上就是这么炸的 —— 实测到 worker 用真实端点
+            # （v100-llm-local）写的那一行，与脚本自己那次插入撞在同一个 key 上。
+            #
+            # 竞态没法在脚本里稳定复现，但它的**本质条件**可以：让 SELECT 明明
+            # 有一次看不见那行已经存在的记录。这里用一次性的 monkeypatch 造出
+            # 这个条件，比拉两个进程去撞时间窗可靠。
+            real_get = ai_service.get_analysis
+            hidden = {"done": False}
+
+            async def _hide_once(session_, subject_type, subject_id, task):  # noqa: ANN001
+                found = await real_get(session_, subject_type, subject_id, task)
+                if found is not None and not hidden["done"]:
+                    hidden["done"] = True
+                    return None
+                return found
+
+            ai_service.get_analysis = _hide_once
+            STUB.mode = "ok"
+            STUB.requests.clear()
+            crashed = ""
+            try:
+                raced = await ai_service.classify_subject(
+                    session, settings, repository, pull=second, force=True
+                )
+            except Exception as exc:  # noqa: BLE001
+                crashed = f"{type(exc).__name__}: {exc}"
+                raced = None
+            finally:
+                ai_service.get_analysis = real_get
+
+            check(hidden["done"], "竞态条件已造出（有一次 SELECT 看不见已存在的行）")
+            check(not crashed, f"并发插入被接住而不是抛出去：{crashed[:60]}")
+            check(
+                raced is not None and raced.status is AnalysisStatus.SUCCEEDED,
+                f"撞上后仍完成分析（status={raced.status.value if raced else None}）",
+            )
+            dup = await session.scalar(
+                select(func.count())
+                .select_from(AIAnalysis)
+                .where(
+                    AIAnalysis.subject_id == second.id,
+                    AIAnalysis.task == AITask.CLASSIFY,
+                )
+            )
+            check(dup == 1, f"仍然只有一行分析记录（{dup}）")
+
+            # 会话必须还能用：worker 一个会话要连着分析几十个对象，
+            # 一次冲突把会话作废就等于整批白跑。
+            usable = await session.scalar(select(func.count()).select_from(AIAnalysis))
+            check(usable is not None, "冲突之后会话仍可继续使用")
+
+            # --- 把分类还原 ---
+            #
+            # 开头为了造出"待补判"而暂存的那两行，逐字段写回去。不还原的话
+            # 这台机器上会平白多出两条 unknown，规则重算虽然能救回来一部分，
+            # 但 AI 判出的类别（比如 perf）会被打回未知并重新计费。
+            for classification_id, values in saved:
+                row = await session.get(Classification, classification_id)
+                if row is None:
+                    continue
+                for key, value in values.items():
+                    setattr(row, key, value)
+            await session.flush()
+            print(f"  ok   已还原 {len(saved)} 条分类记录")
+
+        # --- 清理 ---
+        #
+        # 按 subject_id 删而不是按 provider_name：未配置模型时那条失败记录
+        # 的 provider_name 是 NULL，按 provider_name 过滤会把它留在库里，
+        # 界面上就会凭空多出几条"失败"的用量统计。
+        async with session_scope() as session:
+            for subject_id in touched_subjects:
+                await session.execute(delete(AIAnalysis).where(AIAnalysis.subject_id == subject_id))
+            await session.execute(delete(LLMProvider).where(LLMProvider.name == PROVIDER_NAME))
+        print("  ok   已清理桩数据")
+    finally:
+        server.shutdown()
+        await dispose_engine()
+
+    print()
+    if failures:
+        print(f"失败 {len(failures)} 项：")
+        for message in failures:
+            print(f"  - {message}")
+        return 1
+    print("全部通过")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))