@kernel-sig/console 0.1.0

package/backend/app/integrations/llm/provider.py

@@ -0,0 +1,326 @@
+"""LLM 提供方抽象。
+
+统一走 OpenAI 兼容的 /chat/completions 协议，即可对接 DeepSeek、通义千问、
+智谱、Kimi、OpenAI 以及本地 vLLM / Ollama —— 换供应商只需改 base_url 与模型名。
+
+不依赖任何内部模块，便于独立测试与复用。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import random
+import re
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+import httpx
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+RETRYABLE_STATUS = {408, 429, 500, 502, 503, 504}
+MAX_ATTEMPTS = 3
+BASE_BACKOFF_SECONDS = 1.5
+
+
+class LLMError(Exception):
+    """LLM 调用异常基类。"""
+
+
+class LLMAuthError(LLMError):
+    """凭据无效或权限不足。"""
+
+
+class LLMRateLimitError(LLMError):
+    """触发上游限流。"""
+
+
+class LLMUpstreamError(LLMError):
+    """上游服务错误（5xx 或网络故障）。"""
+
+
+class LLMResponseError(LLMError):
+    """响应内容不符合预期（如要求 JSON 却返回了非 JSON）。"""
+
+
+@dataclass
+class ChatMessage:
+    role: str  # system | user | assistant
+    content: str
+
+
+@dataclass
+class CompletionRequest:
+    messages: list[ChatMessage]
+    model: str
+    temperature: float = 0.1
+    max_tokens: int = 4096
+    # 要求上游返回 JSON。并非所有供应商都支持，不支持时由 prompt 兜底。
+    json_mode: bool = False
+    timeout: float = 180.0
+
+
+@dataclass
+class CompletionResult:
+    content: str
+    model: str
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    finish_reason: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def total_tokens(self) -> int:
+        return self.prompt_tokens + self.completion_tokens
+
+
+class LLMProvider(Protocol):
+    """任意模型提供方需实现的接口。"""
+
+    name: str
+
+    async def complete(self, request: CompletionRequest) -> CompletionResult: ...
+
+    def stream(self, request: CompletionRequest) -> AsyncIterator[str]: ...
+
+    async def aclose(self) -> None: ...
+
+
+# ---------------------------------------------------------------------------
+# OpenAI 兼容实现
+# ---------------------------------------------------------------------------
+
+
+class OpenAICompatibleProvider:
+    """OpenAI 兼容协议的客户端。
+
+    这是目前唯一需要的实现 —— DeepSeek、Qwen、GLM、Kimi、OpenAI、
+    vLLM、Ollama 均提供该协议。若将来遇到不兼容的供应商，
+    只需新增一个实现 ``LLMProvider`` 的类，上层无需改动。
+    """
+
+    def __init__(
+        self,
+        name: str,
+        base_url: str,
+        api_key: str,
+        *,
+        default_model: str = "",
+        timeout: float = 180.0,
+        transport: httpx.AsyncBaseTransport | None = None,
+        max_concurrency: int = 3,
+    ) -> None:
+        if not base_url:
+            raise ValueError("必须提供 base_url")
+        self.name = name
+        self._default_model = default_model
+        self._client = httpx.AsyncClient(
+            base_url=base_url.rstrip("/"),
+            timeout=timeout,
+            headers={
+                # 各家兼容实现的鉴权头基本一致；未配置 key 时省略该头以支持本地服务
+                **({"Authorization": f"Bearer {api_key}"} if api_key else {}),
+                "Content-Type": "application/json",
+            },
+            transport=transport,
+            follow_redirects=True,
+        )
+        # 并发上限：多数供应商按并发限流，串行化比触发 429 再退避更划算
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+        self.requests = 0
+        self.retries = 0
+        self.errors = 0
+
+    async def __aenter__(self) -> OpenAICompatibleProvider:
+        return self
+
+    async def __aexit__(self, *exc_info: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    # ------------------------------------------------------------------
+
+    def _payload(self, request: CompletionRequest, *, stream: bool) -> dict[str, Any]:
+        payload: dict[str, Any] = {
+            "model": request.model or self._default_model,
+            "messages": [
+                {"role": message.role, "content": message.content} for message in request.messages
+            ],
+            "temperature": request.temperature,
+            "max_tokens": request.max_tokens,
+            "stream": stream,
+        }
+        if request.json_mode:
+            payload["response_format"] = {"type": "json_object"}
+        return payload
+
+    async def complete(self, request: CompletionRequest) -> CompletionResult:
+        payload = self._payload(request, stream=False)
+        response = await self._post("/chat/completions", payload, request.timeout)
+
+        try:
+            body = response.json()
+            choice = body["choices"][0]
+            usage = body.get("usage") or {}
+        except (ValueError, KeyError, IndexError) as exc:
+            raise LLMResponseError(f"响应结构不符合预期：{exc}") from exc
+
+        return CompletionResult(
+            content=(choice.get("message") or {}).get("content") or "",
+            model=body.get("model") or payload["model"],
+            prompt_tokens=usage.get("prompt_tokens", 0),
+            completion_tokens=usage.get("completion_tokens", 0),
+            finish_reason=choice.get("finish_reason"),
+            raw=body,
+        )
+
+    async def stream(self, request: CompletionRequest) -> AsyncIterator[str]:
+        """流式输出。用于前端 SSE 实时展示分析过程。"""
+        payload = self._payload(request, stream=True)
+        async with (
+            self._semaphore,
+            self._client.stream(
+                "POST", "/chat/completions", json=payload, timeout=request.timeout
+            ) as response,
+        ):
+            self.requests += 1
+            if response.status_code >= 400:
+                await response.aread()
+                self._raise_for_status(response)
+            async for line in response.aiter_lines():
+                if not line.startswith("data:"):
+                    continue
+                data = line[5:].strip()
+                if data == "[DONE]":
+                    return
+                try:
+                    chunk = json.loads(data)
+                    delta = chunk["choices"][0].get("delta") or {}
+                    if content := delta.get("content"):
+                        yield content
+                except (ValueError, KeyError, IndexError):
+                    # 个别供应商会插入心跳或注释行，忽略即可
+                    continue
+
+    async def _post(self, path: str, payload: dict[str, Any], timeout: float) -> httpx.Response:
+        last_error: LLMError | None = None
+
+        async with self._semaphore:
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                self.requests += 1
+                try:
+                    response = await self._client.post(path, json=payload, timeout=timeout)
+                except httpx.HTTPError as exc:
+                    last_error = LLMUpstreamError(f"网络请求失败：{exc}")
+                    if attempt == MAX_ATTEMPTS:
+                        break
+                    await self._backoff(attempt)
+                    continue
+
+                if response.status_code in RETRYABLE_STATUS:
+                    detail = _error_detail(response)
+                    last_error = (
+                        LLMRateLimitError(f"触发限流：{detail}")
+                        if response.status_code == 429
+                        else LLMUpstreamError(f"上游返回 {response.status_code}：{detail}")
+                    )
+                    if attempt == MAX_ATTEMPTS:
+                        break
+                    self.retries += 1
+                    await self._backoff(attempt, response)
+                    continue
+
+                self._raise_for_status(response)
+                return response
+
+        self.errors += 1
+        assert last_error is not None
+        raise last_error
+
+    @staticmethod
+    def _raise_for_status(response: httpx.Response) -> None:
+        code = response.status_code
+        if code < 400:
+            return
+        detail = _error_detail(response)
+        if code in (401, 403):
+            raise LLMAuthError(f"凭据无效或无权访问：{detail}")
+        if code == 429:
+            raise LLMRateLimitError(f"触发限流：{detail}")
+        raise LLMUpstreamError(f"上游返回 {code}：{detail}")
+
+    @staticmethod
+    async def _backoff(attempt: int, response: httpx.Response | None = None) -> None:
+        delay = BASE_BACKOFF_SECONDS * (2 ** (attempt - 1))
+        if response is not None:
+            retry_after = response.headers.get("Retry-After")
+            if retry_after and retry_after.isdigit():
+                delay = max(delay, float(retry_after))
+        await asyncio.sleep(min(delay + random.uniform(0, delay * 0.25), 30.0))
+
+
+def _error_detail(response: httpx.Response) -> str:
+    try:
+        payload = response.json()
+    except ValueError:
+        return response.text[:200]
+    if isinstance(payload, dict):
+        error = payload.get("error")
+        if isinstance(error, dict) and isinstance(error.get("message"), str):
+            return error["message"][:300]
+        if isinstance(error, str):
+            return error[:300]
+        for key in ("message", "detail"):
+            if isinstance(payload.get(key), str):
+                return payload[key][:300]
+    return response.text[:200]
+
+
+# ---------------------------------------------------------------------------
+# 结构化输出解析
+# ---------------------------------------------------------------------------
+
+_FENCE_PATTERN = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL)
+
+
+def extract_json(text: str) -> dict[str, Any]:
+    """从模型输出中稳健地取出 JSON 对象。
+
+    模型经常不顾"只输出 JSON"的指示，包上 ```json 围栏或加一段前言。
+    直接 json.loads 会失败，因此这里逐层退让：
+    原样解析 → 剥离代码围栏 → 截取首个 { 到末个 }。
+
+    全部失败时抛 LLMResponseError —— 上层据此重试或标记失败，
+    而不是把一段散文塞进结构化字段。
+    """
+    candidate = (text or "").strip()
+    if not candidate:
+        raise LLMResponseError("模型返回了空内容")
+
+    for attempt in (candidate, _strip_fence(candidate), _slice_braces(candidate)):
+        if not attempt:
+            continue
+        try:
+            parsed = json.loads(attempt)
+        except ValueError:
+            continue
+        if isinstance(parsed, dict):
+            return parsed
+        raise LLMResponseError(f"期望 JSON 对象，实际为 {type(parsed).__name__}")
+
+    raise LLMResponseError(f"无法从模型输出中解析出 JSON：{candidate[:200]}")
+
+
+def _strip_fence(text: str) -> str:
+    match = _FENCE_PATTERN.search(text)
+    return match.group(1) if match else ""
+
+
+def _slice_braces(text: str) -> str:
+    start, end = text.find("{"), text.rfind("}")
+    return text[start : end + 1] if start != -1 and end > start else ""

package/backend/app/main.py

@@ -0,0 +1,242 @@
+"""FastAPI 应用装配。"""
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+import structlog
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+from starlette.middleware.cors import CORSMiddleware
+
+from app.api.v1.router import router as v1_router
+from app.core.config import Settings, get_settings
+from app.core.db import dispose_engine, init_engine, session_scope
+from app.core.exceptions import PlatformError
+from app.core.logging import configure_logging
+from app.core.permissions import Role
+from app.middleware.audit import AuditMiddleware
+from app.middleware.request_context import RequestContextMiddleware
+from app.models.credential import Credential
+
+logger = structlog.get_logger(__name__)
+
+
+async def _bootstrap_admin(settings: Settings) -> None:
+    """按环境变量创建首个管理员（仅当系统无用户时）。
+
+    这是容器化部署的便利手段 —— 无需手工打开浏览器完成初始化。
+    已初始化则静默跳过。
+    """
+    if not settings.bootstrap_admin_password:
+        return
+
+    from app.services import user_service
+
+    async with session_scope() as session:
+        if await user_service.count_users(session) > 0:
+            return
+        await user_service.create_user(
+            session,
+            username=settings.bootstrap_admin_username,
+            display_name=settings.bootstrap_admin_username,
+            email=settings.bootstrap_admin_email,
+            password=settings.bootstrap_admin_password,
+            role=Role.ADMIN,
+        )
+        logger.info("bootstrap_admin_created", username=settings.bootstrap_admin_username)
+
+
+# 首次启动导入的凭据在界面上的名字，让人一眼看出它是哪来的
+BOOTSTRAP_CREDENTIAL_NAME = "AtomGit（首次启动导入）"
+
+
+async def _bootstrap_credential(
+    session: AsyncSession, settings: Settings, token: str
+) -> Credential:
+    """取或建引导用的 AtomGit 凭据。
+
+    已存在就复用：重启容器时 token 一般还在，不该因此再建一条重名凭据
+    （凭据名唯一，会撞约束）。
+    """
+    from sqlalchemy import select
+
+    from app.models.credential import CredentialKind
+    from app.services import credential_service
+
+    credential = await session.scalar(
+        select(Credential).where(Credential.name == BOOTSTRAP_CREDENTIAL_NAME)
+    )
+    if credential is None:
+        credential = await credential_service.create_credential(
+            session,
+            settings,
+            name=BOOTSTRAP_CREDENTIAL_NAME,
+            kind=CredentialKind.ATOMGIT_TOKEN,
+            plaintext=token,
+        )
+    return credential
+
+
+async def _bootstrap_repository(settings: Settings) -> None:
+    """首次启动纳管默认仓库。
+
+    使用者打开界面时不该是一片空白，所以**没配 token 也照样建这条记录** ——
+    此时同步不跑（同步层会跳过无凭据的仓库），界面上标明未配置凭据、
+    指引去补 token。
+
+    配了 token 就一并导入成凭据并关联上，数据随即开始拉取。令牌在库里是
+    AES-256-GCM 加密的，明文只存在于环境变量。
+
+    已纳管则静默跳过，所以重启容器不会重复建。
+    """
+    target = settings.bootstrap_repository
+    if not target:
+        return
+
+    owner, _, name = target.partition("/")
+    if not owner or not name:
+        logger.warning("bootstrap_repository_malformed", value=target)
+        return
+
+    from app.models.repository import Repository
+    from app.services import repository_service
+
+    token = settings.atomgit_token
+
+    async with session_scope() as session:
+        existing = await repository_service.get_by_full_name(session, owner, name)
+
+        if existing is not None:
+            # 已纳管。但如果它是上次**没带 token**时建的，凭据还是空的 ——
+            # 这次配上了就补上去。否则"先启动、后给 token"这条路走不通：
+            # 使用者得自己去界面上再关联一次，而他已经把 token 给了命令行。
+            if existing.credential_id is None and token:
+                credential = await _bootstrap_credential(session, settings, token)
+                existing.credential_id = credential.id
+                logger.info("bootstrap_repository_credential_linked", repository=target)
+            return
+
+        credential_id = None
+        if token:
+            credential_id = (await _bootstrap_credential(session, settings, token)).id
+
+        session.add(
+            Repository(
+                owner=owner,
+                name=name,
+                display_name=target,
+                credential_id=credential_id,
+            )
+        )
+        logger.info(
+            "bootstrap_repository_created",
+            repository=target,
+            has_credential=credential_id is not None,
+        )
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    settings = get_settings()
+    configure_logging(level=settings.log_level, json_output=settings.is_production)
+    init_engine(settings)
+
+    try:
+        await _bootstrap_admin(settings)
+    except Exception:
+        logger.exception("bootstrap_admin_failed")
+
+    try:
+        await _bootstrap_repository(settings)
+    except Exception:
+        # 引导失败不该让服务起不来：仓库可以之后在界面上手动纳管
+        logger.exception("bootstrap_repository_failed")
+
+    logger.info(
+        "application_started",
+        environment=settings.environment,
+        app_name=settings.app_name,
+    )
+    try:
+        yield
+    finally:
+        await dispose_engine()
+        logger.info("application_stopped")
+
+
+def _problem_response(
+    status_code: int,
+    title: str,
+    detail: str,
+    instance: str | None,
+    code: str,
+    context: dict | None = None,
+) -> JSONResponse:
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "type": f"https://kernel-sig.local/errors/{code}",
+            "title": title,
+            "status": status_code,
+            "detail": detail,
+            "instance": instance,
+            "code": code,
+            "context": context or None,
+        },
+    )
+
+
+def create_app() -> FastAPI:
+    settings = get_settings()
+
+    app = FastAPI(
+        title=settings.app_name,
+        version="0.1.0",
+        docs_url=None if settings.is_production else "/api/docs",
+        redoc_url=None,
+        openapi_url="/api/openapi.json",
+        lifespan=lifespan,
+    )
+
+    # 中间件注册顺序：后添加的先执行。
+    # 实际执行序：RequestContext → Audit → CORS → 路由
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:15173", "http://localhost:18080"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+    app.add_middleware(AuditMiddleware)
+    app.add_middleware(RequestContextMiddleware)
+
+    @app.exception_handler(PlatformError)
+    async def _handle_platform_error(request: Request, exc: PlatformError) -> JSONResponse:
+        return _problem_response(
+            exc.status_code, exc.title, exc.detail, request.url.path, exc.code, exc.context
+        )
+
+    @app.exception_handler(RequestValidationError)
+    async def _handle_validation_error(
+        request: Request, exc: RequestValidationError
+    ) -> JSONResponse:
+        errors = [
+            {"field": ".".join(str(p) for p in err["loc"][1:]), "message": err["msg"]}
+            for err in exc.errors()
+        ]
+        return _problem_response(
+            422,
+            "Validation Failed",
+            "请求参数校验失败",
+            request.url.path,
+            "validation_error",
+            {"errors": errors},
+        )
+
+    app.include_router(v1_router, prefix=settings.api_prefix)
+    return app
+
+
+app = create_app()

package/backend/app/middleware/audit.py

@@ -0,0 +1,129 @@
+"""自动审计中间件：记录所有写操作。
+
+GET/HEAD/OPTIONS 不记录（读操作量大且审计价值低）。
+敏感字段在写入前脱敏 —— 审计日志本身不能成为凭据泄露点。
+"""
+
+import json
+from typing import Any
+
+import structlog
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+from app.core.db import get_session_factory
+from app.models.audit import AuditLog, AuditResult
+
+AUDITED_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
+REDACTED_KEYS = {
+    "password",
+    "current_password",
+    "new_password",
+    "token",
+    "secret",
+    "api_key",
+    "private_token",
+    "refresh_token",
+    "access_token",
+}
+# 登录/刷新/初始化的载荷本身是凭据，整体不记录。
+# 但登录**尝试**必须可归属，否则无法从审计日志发现暴力破解，
+# 因此单独提取 username（绝非 password）作为行为归属。
+CREDENTIAL_PATHS = {
+    "/api/v1/auth/login",
+    "/api/v1/auth/refresh",
+    "/api/v1/auth/setup",
+}
+
+
+def _redact(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {k: ("***" if k.lower() in REDACTED_KEYS else _redact(v)) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_redact(item) for item in value]
+    return value
+
+
+def _username_from(body: Any) -> str | None:
+    if isinstance(body, dict):
+        username = body.get("username")
+        if isinstance(username, str) and username:
+            return username
+    return None
+
+
+class AuditMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
+        if request.method not in AUDITED_METHODS:
+            return await call_next(request)
+
+        raw_body = await self._read_json(request)
+        payload = self._extract_payload(raw_body, request.url.path)
+        claimed_actor = _username_from(raw_body) if request.url.path in CREDENTIAL_PATHS else None
+
+        try:
+            response = await call_next(request)
+        except Exception as exc:
+            await self._write_log(request, payload, AuditResult.FAILURE, claimed_actor, str(exc))
+            raise
+
+        result = AuditResult.SUCCESS if response.status_code < 400 else AuditResult.FAILURE
+        await self._write_log(request, payload, result, claimed_actor)
+        return response
+
+    @staticmethod
+    async def _read_json(request: Request) -> Any:
+        try:
+            body = await request.body()
+        except Exception:
+            return None
+        if not body:
+            return None
+        try:
+            return json.loads(body)
+        except (ValueError, UnicodeDecodeError):
+            return None
+
+    @staticmethod
+    def _extract_payload(body: Any, path: str) -> dict[str, Any] | None:
+        if path in CREDENTIAL_PATHS:
+            username = _username_from(body)
+            # 记录尝试登录的账号名（非机密），不记录密码
+            return {"username": username, "_note": "凭据类请求，密码不记录"}
+        if body is None:
+            return None
+        return _redact(body) if isinstance(body, dict) else {"body": _redact(body)}
+
+    async def _write_log(
+        self,
+        request: Request,
+        payload: dict[str, Any] | None,
+        result: AuditResult,
+        claimed_actor: str | None = None,
+        error: str | None = None,
+    ) -> None:
+        # 只读取原始值：此处数据库会话已关闭，触碰 ORM 实例会触发
+        # DetachedInstanceError。认证依赖已把归属信息存为标量。
+        actor_user_id = getattr(request.state, "actor_user_id", None)
+        actor_username = getattr(request.state, "actor_username", None) or claimed_actor
+
+        try:
+            factory = get_session_factory()
+            async with factory() as session:
+                session.add(
+                    AuditLog(
+                        actor_user_id=actor_user_id,
+                        actor_username=actor_username,
+                        actor_ip=request.client.host if request.client else None,
+                        action=f"{request.method} {request.url.path}",
+                        after=payload,
+                        result=result,
+                        error=error,
+                        request_id=getattr(request.state, "request_id", None),
+                    )
+                )
+                await session.commit()
+        except Exception:
+            # 审计写入失败绝不能影响主请求，但必须留痕便于排查
+            structlog.get_logger(__name__).exception("audit_log_write_failed")