create-windy 0.3.2 → 0.3.5

package/template/apps/document-server/src/southwind_document_server/adapter.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import tempfile
+from pathlib import Path
+
+import httpx
+
+from .config import DocumentRuntimeConfig
+from .contracts import ArtifactInput, JsonValue, StartDocumentRunRequest
+
+
+class SafeDocumentError(Exception):
+    def __init__(self, code: str, message: str, retryable: bool) -> None:
+        super().__init__(message)
+        self.code = code
+        self.safe_message = message
+        self.retryable = retryable
+
+
+class DocumentAdapter:
+    def __init__(self, config: DocumentRuntimeConfig) -> None:
+        self.config = config
+
+    @property
+    def available(self) -> bool:
+        return self.config.adapter_url is not None
+
+    async def execute(self, request: StartDocumentRunRequest) -> JsonValue:
+        if not self.config.adapter_url:
+            raise SafeDocumentError(
+                "CAPABILITY_UNAVAILABLE",
+                "Document Adapter 未配置",
+                True,
+            )
+        missing = set(request.required_capabilities) - set(self.config.capabilities)
+        if missing:
+            raise SafeDocumentError(
+                "CAPABILITY_UNAVAILABLE",
+                "Document Adapter 缺少所需能力",
+                True,
+            )
+        self._authorize_deployment(request)
+        if request.limits.max_units > self.config.max_units:
+            raise SafeDocumentError(
+                "BUDGET_EXCEEDED",
+                "Document 页数上限超过 Runtime 配置",
+                False,
+            )
+        paths: list[Path] = []
+        try:
+            for source in request.sources:
+                paths.append(await self._download(request, source))
+            return await self._invoke(request, paths)
+        finally:
+            for path in paths:
+                await remove(path)
+
+    async def _download(
+        self, request: StartDocumentRunRequest, source: ArtifactInput
+    ) -> Path:
+        digest = hashlib.sha256()
+        total = 0
+        with tempfile.NamedTemporaryFile(delete=False) as target:
+            path = Path(target.name)
+            try:
+                async with httpx.AsyncClient(
+                    timeout=httpx.Timeout(
+                        self.config.request_timeout_seconds,
+                        connect=self.config.connect_timeout_seconds,
+                    )
+                ) as client:
+                    async with client.stream(
+                        "GET",
+                        self.config.artifact_url,
+                        headers={
+                            "authorization": f"Bearer {self.config.internal_token}",
+                            "x-ai-artifact-ref": source.value,
+                        },
+                    ) as response:
+                        response.raise_for_status()
+                        async for chunk in response.aiter_bytes():
+                            total += len(chunk)
+                            if total > min(
+                                source.byte_size,
+                                request.limits.max_input_bytes,
+                            ):
+                                raise SafeDocumentError(
+                                    "BUDGET_EXCEEDED",
+                                    "文档输入超过声明上限",
+                                    False,
+                                )
+                            digest.update(chunk)
+                            target.write(chunk)
+            except SafeDocumentError:
+                await remove(path)
+                raise
+            except Exception as error:
+                await remove(path)
+                raise SafeDocumentError(
+                    "ARTIFACT_UNAVAILABLE",
+                    "Document Runtime 无法读取 Artifact",
+                    True,
+                ) from error
+        if total != source.byte_size or digest.hexdigest() != source.sha256:
+            await remove(path)
+            raise SafeDocumentError(
+                "ARTIFACT_UNAVAILABLE",
+                "Artifact 完整性校验失败",
+                False,
+            )
+        return path
+
+    async def _invoke(
+        self, request: StartDocumentRunRequest, paths: list[Path]
+    ) -> JsonValue:
+        adapter_url = self.config.adapter_url
+        if not adapter_url:
+            raise SafeDocumentError(
+                "CAPABILITY_UNAVAILABLE", "Document Adapter 未配置", True
+            )
+        headers = (
+            {"authorization": f"Bearer {self.config.adapter_token}"}
+            if self.config.adapter_token
+            else {}
+        )
+        try:
+            async with httpx.AsyncClient(
+                timeout=httpx.Timeout(
+                    self.config.request_timeout_seconds,
+                    connect=self.config.connect_timeout_seconds,
+                )
+            ) as client:
+                opened = [path.open("rb") for path in paths]
+                try:
+                    response = await client.post(
+                        f"{adapter_url.rstrip('/')}/v1/document/process",
+                        headers=headers,
+                        files=[
+                            (
+                                "file" if len(opened) == 1 else "files",
+                                (
+                                    f"source-{index + 1}",
+                                    source,
+                                    request.sources[index].media_type,
+                                ),
+                            )
+                            for index, source in enumerate(opened)
+                        ],
+                        data={
+                            "operationKey": request.operation_key,
+                            "dataClassification": request.data_classification,
+                            "options": json.dumps(
+                                request.options,
+                                ensure_ascii=False,
+                                separators=(",", ":"),
+                            ),
+                            "providerEgress": request.provider_egress.model_dump_json(
+                                by_alias=True
+                            ),
+                            "providerDataPolicy": (
+                                request.provider_data_policy.model_dump_json(
+                                    by_alias=True
+                                )
+                                if request.provider_data_policy
+                                else "null"
+                            ),
+                            "limits": request.limits.model_dump_json(by_alias=True),
+                        },
+                    )
+                finally:
+                    for source in opened:
+                        source.close()
+            response.raise_for_status()
+            if len(response.content) > request.limits.max_output_bytes:
+                raise SafeDocumentError(
+                    "BUDGET_EXCEEDED",
+                    "Document 输出超过上限",
+                    False,
+                )
+            value: JsonValue = response.json()
+            validate_output(value, request.limits.max_units)
+            return value
+        except SafeDocumentError:
+            raise
+        except Exception as error:
+            raise SafeDocumentError(
+                "RUNTIME_UNAVAILABLE",
+                "Document Adapter 执行失败",
+                True,
+            ) from error
+
+    def _authorize_deployment(self, request: StartDocumentRunRequest) -> None:
+        if request.provider_egress.mode == "forbidden":
+            return
+        if not request.provider_data_policy:
+            raise SafeDocumentError(
+                "FORBIDDEN",
+                "Document Operation 缺少 Provider 字段治理策略",
+                False,
+            )
+        if (
+            not self.config.deployment_key
+            or request.provider_egress.deployment_key != self.config.deployment_key
+        ):
+            raise SafeDocumentError(
+                "FORBIDDEN",
+                "Document Operation 未授权当前 Deployment",
+                False,
+            )
+
+
+def validate_output(value: JsonValue, max_units: int) -> None:
+    if not isinstance(value, dict):
+        raise SafeDocumentError(
+            "OUTPUT_VALIDATION_FAILED", "Document 输出结构无效", False
+        )
+    pages = value.get("pages")
+    if not isinstance(pages, list):
+        raise SafeDocumentError(
+            "OUTPUT_VALIDATION_FAILED", "Document 输出缺少页级结果", False
+        )
+    if len(pages) > max_units:
+        raise invalid_output("Document 输出页数超过上限")
+    page_numbers: set[int] = set()
+    for page in pages:
+        if not isinstance(page, dict):
+            raise invalid_output("Document 页级结果无效")
+        page_number = page.get("pageNumber")
+        if (
+            not isinstance(page_number, int)
+            or isinstance(page_number, bool)
+            or page_number < 1
+            or page_number > max_units
+            or page_number in page_numbers
+            or not isinstance(page.get("text"), str)
+            or not isinstance(page.get("blocks"), list)
+        ):
+            raise invalid_output("Document 页码、文本或 Block 无效")
+        page_numbers.add(page_number)
+        blocks = page.get("blocks")
+        if not isinstance(blocks, list):
+            raise invalid_output("Document Block 无效")
+        for block in blocks:
+            validate_block(block)
+        layout = page.get("layout")
+        if layout is not None:
+            if not isinstance(layout, list):
+                raise invalid_output("Document 版面树无效")
+            for node in layout:
+                validate_layout(node, 0)
+    tables = value.get("tables", [])
+    if not isinstance(tables, list):
+        raise invalid_output("Document 表格结果无效")
+    for table in tables:
+        rows = table.get("rows") if isinstance(table, dict) else None
+        if (
+            not isinstance(table, dict)
+            or table.get("pageNumber") not in page_numbers
+            or not isinstance(rows, list)
+            or any(
+                not isinstance(row, list)
+                or any(not isinstance(cell, str) for cell in row)
+                for row in rows
+            )
+        ):
+            raise invalid_output("Document 表格结果无效")
+        validate_bounds(table.get("bounds"))
+    if value.get("artifacts", []) != []:
+        raise invalid_output("Document Adapter 不得直接签发 Artifact")
+    metadata = value.get("metadata", {})
+    if not isinstance(metadata, dict) or any(
+        not isinstance(item, str | int | float | bool) for item in metadata.values()
+    ):
+        raise invalid_output("Document 元数据无效")
+
+
+def validate_block(value: object) -> None:
+    if (
+        not isinstance(value, dict)
+        or not isinstance(value.get("id"), str)
+        or not isinstance(value.get("text"), str)
+    ):
+        raise invalid_output("Document 文本 Block 无效")
+    confidence = value.get("confidence")
+    if confidence is not None and (
+        not isinstance(confidence, int | float)
+        or isinstance(confidence, bool)
+        or confidence < 0
+        or confidence > 1
+    ):
+        raise invalid_output("Document Block 置信度无效")
+    validate_bounds(value.get("bounds"))
+
+
+def validate_layout(value: object, depth: int) -> None:
+    if depth > 32 or not isinstance(value, dict):
+        raise invalid_output("Document 版面树无效")
+    if not isinstance(value.get("id"), str) or not isinstance(value.get("type"), str):
+        raise invalid_output("Document 版面节点无效")
+    text = value.get("text")
+    if text is not None and not isinstance(text, str):
+        raise invalid_output("Document 版面节点文本无效")
+    validate_bounds(value.get("bounds"))
+    children = value.get("children", [])
+    if not isinstance(children, list):
+        raise invalid_output("Document 版面子节点无效")
+    for child in children:
+        validate_layout(child, depth + 1)
+
+
+def validate_bounds(value: object) -> None:
+    if value is None:
+        return
+    if not isinstance(value, dict) or value.get("unit") not in {
+        "pixel",
+        "normalized",
+    }:
+        raise invalid_output("Document 坐标无效")
+    for field in ("x", "y", "width", "height"):
+        coordinate = value.get(field)
+        if (
+            not isinstance(coordinate, int | float)
+            or isinstance(coordinate, bool)
+            or coordinate < 0
+        ):
+            raise invalid_output("Document 坐标无效")
+
+
+def invalid_output(message: str) -> SafeDocumentError:
+    return SafeDocumentError("OUTPUT_VALIDATION_FAILED", message, False)
+
+
+async def remove(path: Path) -> None:
+    await asyncio.to_thread(path.unlink, missing_ok=True)

package/template/apps/document-server/src/southwind_document_server/app.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import asyncio
+import hmac
+import os
+import uuid
+from collections.abc import AsyncIterator
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse, StreamingResponse
+
+from .adapter import DocumentAdapter, SafeDocumentError
+from .config import DocumentRuntimeConfig
+from .contracts import (
+    ErrorResponse,
+    StartDocumentRunRequest,
+    StartRunResponse,
+)
+from .service import DocumentRunService
+from .store import DocumentRunStore, RunConflictError
+
+
+def create_app(config: DocumentRuntimeConfig) -> FastAPI:
+    app = FastAPI(title="Southwind Document Runtime", docs_url=None, redoc_url=None)
+    store = DocumentRunStore(config.store_path)
+    service = DocumentRunService(DocumentAdapter(config), store)
+
+    def authorize(authorization: str | None = Header(default=None)) -> None:
+        provided = (authorization or "").removeprefix("Bearer ")
+        if not provided or not hmac.compare_digest(provided, config.internal_token):
+            raise HTTPException(status_code=403, detail="FORBIDDEN")
+
+    @app.get("/health/ready")
+    async def ready(_: None = Depends(authorize)) -> JSONResponse:
+        if not service.adapter.available:
+            return JSONResponse(
+                {"status": "unavailable", "capabilities": []},
+                status_code=503,
+            )
+        return JSONResponse(
+            {
+                "status": "ready",
+                "capabilities": list(config.capabilities),
+                "limits": {"maxUnits": config.max_units},
+            }
+        )
+
+    @app.post("/api/document/runs", status_code=202)
+    async def start(
+        request: StartDocumentRunRequest,
+        idempotency_key: str = Header(alias="idempotency-key"),
+        _: None = Depends(authorize),
+    ) -> StartRunResponse:
+        if not idempotency_key.strip() or len(idempotency_key) > 160:
+            raise HTTPException(status_code=400, detail="INVALID_INPUT")
+        try:
+            record = await service.start(request, idempotency_key)
+        except RunConflictError:
+            raise HTTPException(status_code=409, detail="RUN_CONFLICT") from None
+        return StartRunResponse(
+            run_id=record.run_id,
+            status=record.status,
+            events_url=f"/api/document/runs/{record.run_id}/events",
+        )
+
+    @app.get("/api/document/runs/{run_id}/events")
+    async def events(
+        run_id: str,
+        after: int = Query(default=0, ge=0),
+        _: None = Depends(authorize),
+    ) -> StreamingResponse:
+        service.require(run_id)
+
+        async def stream() -> AsyncIterator[str]:
+            cursor = after
+            for _attempt in range(30):
+                current = service.require(run_id)
+                events = store.events_after(run_id, cursor)
+                for event in events:
+                    sequence = event.get("sequence")
+                    if not isinstance(sequence, int):
+                        raise RuntimeError("Document Event sequence 无效")
+                    cursor = sequence
+                    yield f"data: {encode_event(event)}\n\n"
+                if current.terminal:
+                    return
+                if not events:
+                    yield ": heartbeat\n\n"
+                await asyncio.sleep(1)
+
+        return StreamingResponse(stream(), media_type="text/event-stream")
+
+    @app.get("/api/document/runs/{run_id}/result")
+    async def result(run_id: str, _: None = Depends(authorize)) -> JSONResponse:
+        record = service.require(run_id)
+        if record.status != "completed":
+            return JSONResponse({"code": "RUN_NOT_READY"}, status_code=409)
+        return JSONResponse(
+            {"runId": run_id, "status": "completed", "output": record.output}
+        )
+
+    @app.post("/api/document/runs/{run_id}/cancel")
+    async def cancel(run_id: str, _: None = Depends(authorize)) -> JSONResponse:
+        record = await service.cancel(run_id)
+        return JSONResponse({"runId": run_id, "status": record.status})
+
+    @app.exception_handler(KeyError)
+    async def missing(_request: object, _error: KeyError) -> JSONResponse:
+        return safe_error("RUN_NOT_FOUND", "Document Run 不存在", 404, False)
+
+    @app.exception_handler(HTTPException)
+    async def http_error(_request: object, error: HTTPException) -> JSONResponse:
+        code = str(error.detail)
+        return safe_error(code, "Document 请求被拒绝", error.status_code, False)
+
+    @app.exception_handler(RequestValidationError)
+    async def validation_error(
+        _request: object, _error: RequestValidationError
+    ) -> JSONResponse:
+        return safe_error("INVALID_INPUT", "Document 请求格式无效", 400, False)
+
+    @app.exception_handler(SafeDocumentError)
+    async def safe_document_error(
+        _request: object, error: SafeDocumentError
+    ) -> JSONResponse:
+        status = 403 if error.code == "FORBIDDEN" else 400
+        return safe_error(error.code, error.safe_message, status, error.retryable)
+
+    return app
+
+
+def safe_error(code: str, message: str, status: int, retryable: bool) -> JSONResponse:
+    return JSONResponse(
+        ErrorResponse(
+            code=code,
+            message=message,
+            request_id=f"document_{uuid.uuid4().hex}",
+            retryable=retryable,
+        ).model_dump(by_alias=True),
+        status_code=status,
+    )
+
+
+def encode_event(event: dict[str, object]) -> str:
+    import json
+
+    return json.dumps(event, ensure_ascii=False, separators=(",", ":"))
+
+
+app = create_app(DocumentRuntimeConfig.from_environment(dict(os.environ)))

package/template/apps/document-server/src/southwind_document_server/config.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import ipaddress
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class DocumentRuntimeConfig:
+    internal_token: str
+    store_path: str
+    artifact_url: str
+    adapter_url: str | None
+    adapter_token: str | None
+    deployment_key: str | None
+    capabilities: tuple[str, ...]
+    max_units: int = 500
+    connect_timeout_seconds: float = 5
+    request_timeout_seconds: float = 300
+
+    @classmethod
+    def from_environment(cls, environment: dict[str, str]) -> DocumentRuntimeConfig:
+        internal_token = required(environment, "DOCUMENT_SERVER_INTERNAL_TOKEN")
+        store_path = environment.get(
+            "DOCUMENT_RUN_STORE_PATH", "/app/data/document/runs.sqlite3"
+        ).strip()
+        artifact_url = environment.get(
+            "DOCUMENT_ARTIFACT_URL",
+            "http://server:9747/api/internal/ai/artifacts",
+        ).strip()
+        validate_intranet_url(artifact_url)
+        adapter_url = environment.get("DOCUMENT_ADAPTER_URL", "").strip() or None
+        if adapter_url:
+            validate_intranet_url(adapter_url)
+        capabilities = tuple(
+            sorted(
+                {
+                    item.strip()
+                    for item in environment.get(
+                        "DOCUMENT_ADAPTER_CAPABILITIES",
+                        "document.ocr,document.layout,document.table",
+                    ).split(",")
+                    if item.strip()
+                }
+            )
+        )
+        return cls(
+            internal_token=internal_token,
+            store_path=store_path,
+            artifact_url=artifact_url,
+            adapter_url=adapter_url,
+            adapter_token=environment.get("DOCUMENT_ADAPTER_TOKEN", "").strip() or None,
+            deployment_key=environment.get("DOCUMENT_DEPLOYMENT_KEY", "").strip()
+            or None,
+            capabilities=capabilities,
+            max_units=positive_int(environment, "DOCUMENT_MAX_UNITS", 500),
+        )
+
+
+def required(environment: dict[str, str], key: str) -> str:
+    value = environment.get(key, "").strip()
+    if not value:
+        raise ValueError(f"缺少 {key}")
+    return value
+
+
+def positive_int(environment: dict[str, str], key: str, default: int) -> int:
+    raw = environment.get(key, str(default)).strip()
+    try:
+        value = int(raw)
+    except ValueError:
+        raise ValueError(f"{key} 必须是正整数") from None
+    if value <= 0:
+        raise ValueError(f"{key} 必须是正整数")
+    return value
+
+
+def validate_intranet_url(raw: str) -> None:
+    parsed = urlparse(raw)
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+        raise ValueError("Document Adapter URL 无效")
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ValueError("Document Adapter URL 不允许凭据、查询串或片段")
+    host = parsed.hostname.lower()
+    if host == "localhost" or host.endswith(".internal") or "." not in host:
+        return
+    try:
+        address = ipaddress.ip_address(host)
+    except ValueError:
+        raise ValueError(
+            "Document Adapter 必须使用 localhost、内部服务名、私网 IP 或 .internal"
+        ) from None
+    if not (address.is_private or address.is_loopback):
+        raise ValueError("Document Adapter 禁止使用公网地址")

package/template/apps/document-server/src/southwind_document_server/contracts.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from typing import Literal, Self
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+type JsonValue = (
+    str | int | float | bool | None | list[JsonValue] | dict[str, JsonValue]
+)
+type RunStatus = Literal["accepted", "running", "completed", "failed", "cancelled"]
+
+
+def to_camel(value: str) -> str:
+    first, *rest = value.split("_")
+    return first + "".join(part.capitalize() for part in rest)
+
+
+class CanonicalModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+        extra="forbid",
+    )
+
+
+class ArtifactInput(CanonicalModel):
+    value: str = Field(min_length=1, max_length=4096)
+    media_type: str = Field(min_length=1, max_length=160)
+    byte_size: int = Field(ge=0, le=512 * 1024 * 1024)
+    sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+
+
+class RuntimeLimits(CanonicalModel):
+    max_duration_seconds: int = Field(ge=1, le=3600)
+    max_input_bytes: int = Field(ge=1, le=512 * 1024 * 1024)
+    max_output_bytes: int = Field(ge=1024, le=64 * 1024 * 1024)
+    max_units: int = Field(ge=1, le=10_000)
+    max_artifacts: int = Field(ge=1, le=100)
+
+
+class ProviderEgress(CanonicalModel):
+    mode: Literal["forbidden", "configured-endpoint"]
+    deployment_key: str | None = None
+
+    @model_validator(mode="after")
+    def validate_deployment(self) -> Self:
+        if self.mode == "configured-endpoint":
+            if not self.deployment_key or not self.deployment_key.strip():
+                raise ValueError("Provider Deployment Key 缺失")
+        elif self.deployment_key is not None:
+            raise ValueError("禁止出网时不能指定 Provider Deployment")
+        return self
+
+
+class ProviderFieldPolicy(CanonicalModel):
+    field: Literal["artifact-content", "provider-output"]
+    classification: Literal["public", "internal", "sensitive"]
+    dlp_action: Literal["reject", "redact"]
+
+
+class ProviderInputFieldPolicy(CanonicalModel):
+    json_pointer: str = Field(pattern=r"^/[^/]+(?:/[^/]+)*$")
+    classification: Literal["public", "internal", "sensitive"]
+    dlp_action: Literal["reject", "redact"]
+
+
+class ProviderDataPolicy(CanonicalModel):
+    input_fields: list[ProviderInputFieldPolicy] = Field(min_length=1)
+    envelope_fields: list[ProviderFieldPolicy] = Field(min_length=2)
+
+    @model_validator(mode="after")
+    def validate_fields(self) -> Self:
+        fields = [rule.field for rule in self.envelope_fields]
+        if set(fields) != {"artifact-content", "provider-output"}:
+            raise ValueError("Document Provider 字段治理策略不完整")
+        if len(fields) != len(set(fields)):
+            raise ValueError("Document Provider 字段治理策略重复")
+        return self
+
+
+class StartDocumentRunRequest(CanonicalModel):
+    kind: Literal["document"]
+    operation_key: str = Field(min_length=1, max_length=160)
+    options: JsonValue
+    sources: list[ArtifactInput] = Field(min_length=1, max_length=100)
+    required_capabilities: list[str] = Field(min_length=1, max_length=32)
+    data_classification: Literal["public", "internal", "sensitive"]
+    limits: RuntimeLimits
+    provider_egress: ProviderEgress
+    provider_data_policy: ProviderDataPolicy | None = None
+
+
+class StartRunResponse(CanonicalModel):
+    run_id: str
+    status: RunStatus
+    events_url: str
+
+
+class ErrorResponse(CanonicalModel):
+    code: str
+    message: str
+    request_id: str
+    retryable: bool