create-windy 0.5.2 → 0.6.1

package/template/apps/document-server/src/southwind_document_server/adapter.py

@@ -4,42 +4,60 @@ import asyncio
 import hashlib
 import json
 import tempfile
+import time
+from collections.abc import Callable
+from datetime import UTC, datetime
 from pathlib import Path
 
 import httpx
 
+from .adapter_errors import SafeDocumentError
+from .artifacts import AdapterExecution
 from .config import DocumentRuntimeConfig
-from .contracts import ArtifactInput, JsonValue, StartDocumentRunRequest
-
-
-class SafeDocumentError(Exception):
-    def __init__(self, code: str, message: str, retryable: bool) -> None:
-        super().__init__(message)
-        self.code = code
-        self.safe_message = message
-        self.retryable = retryable
+from .contracts import ArtifactInput, StartDocumentRunRequest
+from .multipart_output import parse_adapter_response
+from .output_validation import validate_output
+from .readiness import DocumentReadinessProbe
 
 
 class DocumentAdapter:
-    def __init__(self, config: DocumentRuntimeConfig) -> None:
+    def __init__(
+        self,
+        config: DocumentRuntimeConfig,
+        transport: httpx.AsyncBaseTransport | None = None,
+        sink_readiness_transport: httpx.AsyncBaseTransport | None = None,
+        wall_clock: Callable[[], datetime] = lambda: datetime.now(UTC),
+        monotonic_clock: Callable[[], float] = time.monotonic,
+    ) -> None:
         self.config = config
+        self._transport = transport
+        self.readiness = DocumentReadinessProbe(
+            config,
+            transport,
+            sink_readiness_transport,
+            wall_clock,
+            monotonic_clock,
+        )
 
     @property
     def available(self) -> bool:
-        return self.config.adapter_url is not None
+        return self.config.readiness_configured
+
+    async def readiness_snapshot(self) -> dict[str, object]:
+        return (await self.readiness.inspect()).snapshot
 
-    async def execute(self, request: StartDocumentRunRequest) -> JsonValue:
-        if not self.config.adapter_url:
+    async def execute(self, request: StartDocumentRunRequest) -> AdapterExecution:
+        inspection = await self.readiness.inspect()
+        if not inspection.ready:
             raise SafeDocumentError(
                 "CAPABILITY_UNAVAILABLE",
-                "Document Adapter 未配置",
+                "Document Adapter 尚未通过 readiness 探测",
                 True,
             )
-        missing = set(request.required_capabilities) - set(self.config.capabilities)
-        if missing:
+        if not self.readiness.supports(inspection, request):
             raise SafeDocumentError(
                 "CAPABILITY_UNAVAILABLE",
-                "Document Adapter 缺少所需能力",
+                "Document Adapter 缺少原子 Operation 能力",
                 True,
             )
         self._authorize_deployment(request)
@@ -59,29 +77,27 @@ class DocumentAdapter:
                 await remove(path)
 
     async def _download(
-        self, request: StartDocumentRunRequest, source: ArtifactInput
+        self,
+        request: StartDocumentRunRequest,
+        source: ArtifactInput,
     ) -> Path:
         digest = hashlib.sha256()
         total = 0
-        with tempfile.NamedTemporaryFile(delete=False) as target:
-            path = Path(target.name)
-            try:
-                async with httpx.AsyncClient(
-                    timeout=httpx.Timeout(
-                        self.config.request_timeout_seconds,
-                        connect=self.config.connect_timeout_seconds,
-                    )
-                ) as client:
+        path: Path | None = None
+        try:
+            with tempfile.NamedTemporaryFile(delete=False) as target:
+                path = Path(target.name)
+                async with self._client() as client:
                     async with client.stream(
                         "GET",
                         self.config.artifact_url,
                         headers={
-                            "authorization": f"Bearer {self.config.internal_token}",
+                            "authorization": (f"Bearer {self.config.internal_token}"),
                             "x-ai-artifact-ref": source.value,
                         },
                     ) as response:
                         response.raise_for_status()
-                        async for chunk in response.aiter_bytes():
+                        async for chunk in response.aiter_bytes(chunk_size=64 * 1024):
                             total += len(chunk)
                             if total > min(
                                 source.byte_size,
@@ -94,32 +110,41 @@ class DocumentAdapter:
                                 )
                             digest.update(chunk)
                             target.write(chunk)
-            except SafeDocumentError:
-                await remove(path)
-                raise
-            except Exception as error:
-                await remove(path)
+            if total != source.byte_size or digest.hexdigest() != source.sha256:
                 raise SafeDocumentError(
                     "ARTIFACT_UNAVAILABLE",
-                    "Document Runtime 无法读取 Artifact",
-                    True,
-                ) from error
-        if total != source.byte_size or digest.hexdigest() != source.sha256:
-            await remove(path)
+                    "Artifact 完整性校验失败",
+                    False,
+                )
+            return path
+        except SafeDocumentError:
+            if path is not None:
+                path.unlink(missing_ok=True)
+            raise
+        except Exception as error:
+            if path is not None:
+                path.unlink(missing_ok=True)
             raise SafeDocumentError(
                 "ARTIFACT_UNAVAILABLE",
-                "Artifact 完整性校验失败",
-                False,
-            )
-        return path
+                "Document Runtime 无法读取 Artifact",
+                True,
+            ) from error
+        except BaseException:
+            if path is not None:
+                path.unlink(missing_ok=True)
+            raise
 
     async def _invoke(
-        self, request: StartDocumentRunRequest, paths: list[Path]
-    ) -> JsonValue:
+        self,
+        request: StartDocumentRunRequest,
+        paths: list[Path],
+    ) -> AdapterExecution:
         adapter_url = self.config.adapter_url
         if not adapter_url:
             raise SafeDocumentError(
-                "CAPABILITY_UNAVAILABLE", "Document Adapter 未配置", True
+                "CAPABILITY_UNAVAILABLE",
+                "Document Adapter 未配置",
+                True,
             )
         headers = (
             {"authorization": f"Bearer {self.config.adapter_token}"}
@@ -127,15 +152,11 @@ class DocumentAdapter:
             else {}
         )
         try:
-            async with httpx.AsyncClient(
-                timeout=httpx.Timeout(
-                    self.config.request_timeout_seconds,
-                    connect=self.config.connect_timeout_seconds,
-                )
-            ) as client:
+            async with self._client() as client:
                 opened = [path.open("rb") for path in paths]
                 try:
-                    response = await client.post(
+                    async with client.stream(
+                        "POST",
                         f"{adapter_url.rstrip('/')}/v1/document/process",
                         headers=headers,
                         files=[
@@ -149,40 +170,18 @@ class DocumentAdapter:
                             )
                             for index, source in enumerate(opened)
                         ],
-                        data={
-                            "operationKey": request.operation_key,
-                            "dataClassification": request.data_classification,
-                            "options": json.dumps(
-                                request.options,
-                                ensure_ascii=False,
-                                separators=(",", ":"),
-                            ),
-                            "providerEgress": request.provider_egress.model_dump_json(
-                                by_alias=True
-                            ),
-                            "providerDataPolicy": (
-                                request.provider_data_policy.model_dump_json(
-                                    by_alias=True
-                                )
-                                if request.provider_data_policy
-                                else "null"
-                            ),
-                            "limits": request.limits.model_dump_json(by_alias=True),
-                        },
-                    )
+                        data=self._adapter_fields(request),
+                    ) as response:
+                        response.raise_for_status()
+                        return await parse_adapter_response(
+                            response,
+                            request.limits.max_output_bytes,
+                            request.limits.max_units,
+                            request.output_artifact_sink,
+                        )
                 finally:
                     for source in opened:
                         source.close()
-            response.raise_for_status()
-            if len(response.content) > request.limits.max_output_bytes:
-                raise SafeDocumentError(
-                    "BUDGET_EXCEEDED",
-                    "Document 输出超过上限",
-                    False,
-                )
-            value: JsonValue = response.json()
-            validate_output(value, request.limits.max_units)
-            return value
         except SafeDocumentError:
             raise
         except Exception as error:
@@ -192,6 +191,37 @@ class DocumentAdapter:
                 True,
             ) from error
 
+    def _adapter_fields(
+        self,
+        request: StartDocumentRunRequest,
+    ) -> dict[str, str]:
+        return {
+            "operationKey": request.operation_key,
+            "dataClassification": request.data_classification,
+            "options": json.dumps(
+                request.options,
+                ensure_ascii=False,
+                separators=(",", ":"),
+            ),
+            "providerEgress": request.provider_egress.model_dump_json(by_alias=True),
+            "providerDataPolicy": (
+                request.provider_data_policy.model_dump_json(by_alias=True)
+                if request.provider_data_policy
+                else "null"
+            ),
+            "limits": request.limits.model_dump_json(by_alias=True),
+        }
+
+    def _client(self) -> httpx.AsyncClient:
+        return httpx.AsyncClient(
+            timeout=httpx.Timeout(
+                self.config.request_timeout_seconds,
+                connect=self.config.connect_timeout_seconds,
+            ),
+            transport=self._transport,
+            follow_redirects=False,
+        )
+
     def _authorize_deployment(self, request: StartDocumentRunRequest) -> None:
         if request.provider_egress.mode == "forbidden":
             return
@@ -212,126 +242,8 @@ class DocumentAdapter:
             )
 
 
-def validate_output(value: JsonValue, max_units: int) -> None:
-    if not isinstance(value, dict):
-        raise SafeDocumentError(
-            "OUTPUT_VALIDATION_FAILED", "Document 输出结构无效", False
-        )
-    pages = value.get("pages")
-    if not isinstance(pages, list):
-        raise SafeDocumentError(
-            "OUTPUT_VALIDATION_FAILED", "Document 输出缺少页级结果", False
-        )
-    if len(pages) > max_units:
-        raise invalid_output("Document 输出页数超过上限")
-    page_numbers: set[int] = set()
-    for page in pages:
-        if not isinstance(page, dict):
-            raise invalid_output("Document 页级结果无效")
-        page_number = page.get("pageNumber")
-        if (
-            not isinstance(page_number, int)
-            or isinstance(page_number, bool)
-            or page_number < 1
-            or page_number > max_units
-            or page_number in page_numbers
-            or not isinstance(page.get("text"), str)
-            or not isinstance(page.get("blocks"), list)
-        ):
-            raise invalid_output("Document 页码、文本或 Block 无效")
-        page_numbers.add(page_number)
-        blocks = page.get("blocks")
-        if not isinstance(blocks, list):
-            raise invalid_output("Document Block 无效")
-        for block in blocks:
-            validate_block(block)
-        layout = page.get("layout")
-        if layout is not None:
-            if not isinstance(layout, list):
-                raise invalid_output("Document 版面树无效")
-            for node in layout:
-                validate_layout(node, 0)
-    tables = value.get("tables", [])
-    if not isinstance(tables, list):
-        raise invalid_output("Document 表格结果无效")
-    for table in tables:
-        rows = table.get("rows") if isinstance(table, dict) else None
-        if (
-            not isinstance(table, dict)
-            or table.get("pageNumber") not in page_numbers
-            or not isinstance(rows, list)
-            or any(
-                not isinstance(row, list)
-                or any(not isinstance(cell, str) for cell in row)
-                for row in rows
-            )
-        ):
-            raise invalid_output("Document 表格结果无效")
-        validate_bounds(table.get("bounds"))
-    if value.get("artifacts", []) != []:
-        raise invalid_output("Document Adapter 不得直接签发 Artifact")
-    metadata = value.get("metadata", {})
-    if not isinstance(metadata, dict) or any(
-        not isinstance(item, str | int | float | bool) for item in metadata.values()
-    ):
-        raise invalid_output("Document 元数据无效")
-
-
-def validate_block(value: object) -> None:
-    if (
-        not isinstance(value, dict)
-        or not isinstance(value.get("id"), str)
-        or not isinstance(value.get("text"), str)
-    ):
-        raise invalid_output("Document 文本 Block 无效")
-    confidence = value.get("confidence")
-    if confidence is not None and (
-        not isinstance(confidence, int | float)
-        or isinstance(confidence, bool)
-        or confidence < 0
-        or confidence > 1
-    ):
-        raise invalid_output("Document Block 置信度无效")
-    validate_bounds(value.get("bounds"))
-
-
-def validate_layout(value: object, depth: int) -> None:
-    if depth > 32 or not isinstance(value, dict):
-        raise invalid_output("Document 版面树无效")
-    if not isinstance(value.get("id"), str) or not isinstance(value.get("type"), str):
-        raise invalid_output("Document 版面节点无效")
-    text = value.get("text")
-    if text is not None and not isinstance(text, str):
-        raise invalid_output("Document 版面节点文本无效")
-    validate_bounds(value.get("bounds"))
-    children = value.get("children", [])
-    if not isinstance(children, list):
-        raise invalid_output("Document 版面子节点无效")
-    for child in children:
-        validate_layout(child, depth + 1)
-
-
-def validate_bounds(value: object) -> None:
-    if value is None:
-        return
-    if not isinstance(value, dict) or value.get("unit") not in {
-        "pixel",
-        "normalized",
-    }:
-        raise invalid_output("Document 坐标无效")
-    for field in ("x", "y", "width", "height"):
-        coordinate = value.get(field)
-        if (
-            not isinstance(coordinate, int | float)
-            or isinstance(coordinate, bool)
-            or coordinate < 0
-        ):
-            raise invalid_output("Document 坐标无效")
-
-
-def invalid_output(message: str) -> SafeDocumentError:
-    return SafeDocumentError("OUTPUT_VALIDATION_FAILED", message, False)
-
-
 async def remove(path: Path) -> None:
     await asyncio.to_thread(path.unlink, missing_ok=True)
+
+
+__all__ = ["DocumentAdapter", "SafeDocumentError", "validate_output"]

package/template/apps/document-server/src/southwind_document_server/adapter_errors.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class SafeDocumentError(Exception):
+    def __init__(self, code: str, message: str, retryable: bool) -> None:
+        super().__init__(message)
+        self.code = code
+        self.safe_message = message
+        self.retryable = retryable
+
+
+def invalid_output(message: str) -> SafeDocumentError:
+    return SafeDocumentError("OUTPUT_VALIDATION_FAILED", message, False)

package/template/apps/document-server/src/southwind_document_server/app.py

@@ -5,7 +5,9 @@ import hmac
 import os
 import uuid
 from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 
+import httpx
 from fastapi import Depends, FastAPI, Header, HTTPException, Query
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse, StreamingResponse
@@ -17,14 +19,44 @@ from .contracts import (
     StartDocumentRunRequest,
     StartRunResponse,
 )
+from .grants import OutputGrantCell
 from .service import DocumentRunService
+from .sink import OutputArtifactSinkClient
 from .store import DocumentRunStore, RunConflictError
 
 
-def create_app(config: DocumentRuntimeConfig) -> FastAPI:
-    app = FastAPI(title="Southwind Document Runtime", docs_url=None, redoc_url=None)
+def create_app(
+    config: DocumentRuntimeConfig,
+    *,
+    adapter_transport: httpx.AsyncBaseTransport | None = None,
+    sink_transport: httpx.AsyncBaseTransport | None = None,
+) -> FastAPI:
     store = DocumentRunStore(config.store_path)
-    service = DocumentRunService(DocumentAdapter(config), store)
+    grants = OutputGrantCell()
+    sink = OutputArtifactSinkClient(config, grants, sink_transport)
+    adapter = DocumentAdapter(config, adapter_transport, sink_transport)
+    service = DocumentRunService(
+        adapter,
+        store,
+        sink,
+        grants,
+    )
+
+    @asynccontextmanager
+    async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
+        service.start_reconciler()
+        try:
+            yield
+        finally:
+            await service.close()
+
+    app = FastAPI(
+        title="Southwind Document Runtime",
+        docs_url=None,
+        redoc_url=None,
+        lifespan=lifespan,
+    )
+    app.state.document_service = service
 
     def authorize(authorization: str | None = Header(default=None)) -> None:
         provided = (authorization or "").removeprefix("Bearer ")
@@ -33,29 +65,31 @@ def create_app(config: DocumentRuntimeConfig) -> FastAPI:
 
     @app.get("/health/ready")
     async def ready(_: None = Depends(authorize)) -> JSONResponse:
-        if not service.adapter.available:
-            return JSONResponse(
-                {"status": "unavailable", "capabilities": []},
-                status_code=503,
-            )
+        snapshot = await service.adapter.readiness_snapshot()
         return JSONResponse(
-            {
-                "status": "ready",
-                "capabilities": list(config.capabilities),
-                "limits": {"maxUnits": config.max_units},
-            }
+            snapshot,
+            status_code=200 if snapshot.get("status") == "ready" else 503,
+            headers={"cache-control": "private, no-store"},
         )
 
     @app.post("/api/document/runs", status_code=202)
     async def start(
         request: StartDocumentRunRequest,
         idempotency_key: str = Header(alias="idempotency-key"),
+        output_artifact_grant: str | None = Header(
+            default=None,
+            alias="x-ai-output-artifact-grant",
+        ),
         _: None = Depends(authorize),
     ) -> StartRunResponse:
         if not idempotency_key.strip() or len(idempotency_key) > 160:
             raise HTTPException(status_code=400, detail="INVALID_INPUT")
         try:
-            record = await service.start(request, idempotency_key)
+            record = await service.start(
+                request,
+                idempotency_key,
+                output_artifact_grant,
+            )
         except RunConflictError:
             raise HTTPException(status_code=409, detail="RUN_CONFLICT") from None
         return StartRunResponse(
@@ -125,8 +159,22 @@ def create_app(config: DocumentRuntimeConfig) -> FastAPI:
         _request: object, error: SafeDocumentError
     ) -> JSONResponse:
         status = 403 if error.code == "FORBIDDEN" else 400
+        if error.code == "RUNTIME_UNAVAILABLE":
+            status = 503
         return safe_error(error.code, error.safe_message, status, error.retryable)
 
+    @app.exception_handler(Exception)
+    async def unexpected_error(
+        _request: object,
+        _error: Exception,
+    ) -> JSONResponse:
+        return safe_error(
+            "RUNTIME_UNAVAILABLE",
+            "Document Runtime 暂时不可用",
+            503,
+            True,
+        )
+
     return app
 
 

package/template/apps/document-server/src/southwind_document_server/artifacts.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+
+from .contracts import JsonValue, OutputArtifactDeclaration
+
+
+@dataclass(frozen=True)
+class ProducedArtifact:
+    declaration: OutputArtifactDeclaration
+    path: Path
+
+
+@dataclass(frozen=True)
+class AdapterExecution:
+    output: JsonValue
+    artifacts: tuple[ProducedArtifact, ...] = ()
+
+    async def cleanup(self) -> None:
+        await asyncio.gather(
+            *(
+                asyncio.to_thread(artifact.path.unlink, missing_ok=True)
+                for artifact in self.artifacts
+            )
+        )