create-windy 0.5.1 → 0.6.0

package/template/apps/document-server/src/southwind_document_server/readiness_support.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import time
+from datetime import UTC, datetime
+
+from .contracts import StartDocumentRunRequest
+from .readiness_contracts import DOCUMENT_OUTPUT_ARTIFACT_CAPABILITY_V1
+
+type Snapshot = dict[str, object]
+
+
+def supports_offer(
+    offer: dict[str, object],
+    capabilities: set[str],
+    inputs: set[str],
+    outputs: set[str],
+    request: StartDocumentRunRequest,
+) -> bool:
+    available = set(string_list(offer["capabilities"]))
+    input_modalities = set(string_list(offer["inputModalities"]))
+    output_modalities = set(string_list(offer["outputModalities"]))
+    limits = offer["limits"]
+    return bool(
+        isinstance(limits, dict)
+        and capabilities.issubset(available)
+        and inputs.issubset(input_modalities)
+        and outputs.issubset(output_modalities)
+        and number(limits, "maxDurationSeconds") >= request.limits.max_duration_seconds
+        and number(limits, "maxInputBytes") >= request.limits.max_input_bytes
+        and number(limits, "maxOutputBytes") >= request.limits.max_output_bytes
+        and number(limits, "maxUnits") >= request.limits.max_units
+        and number(limits, "maxArtifacts") >= request.limits.max_artifacts
+    )
+
+
+def sink_matches_snapshot(sink: object, snapshot: Snapshot) -> bool:
+    deployment = snapshot.get("deployment")
+    return bool(
+        isinstance(deployment, dict)
+        and getattr(sink, "deployment_identity", None) == deployment.get("identity")
+        and getattr(sink, "release_digest", None) == deployment.get("releaseDigest")
+        and getattr(sink, "offers_digest", None) == deployment.get("offersDigest")
+    )
+
+
+def not_configured(observed_at: str) -> Snapshot:
+    return {
+        "schemaVersion": 1,
+        "status": "unavailable",
+        "configured": False,
+        "runtimeKind": "document",
+        "observedAt": observed_at,
+        "downstreamProbe": {"status": "not-run", "reason": "NOT_CONFIGURED"},
+        "deployment": None,
+        "offers": [],
+        "unavailableReasons": [{"code": "RUNTIME_NOT_CONFIGURED", "retryable": True}],
+    }
+
+
+def stale_snapshot(
+    *,
+    checked_at: datetime,
+    observed_at: datetime,
+    valid_until: datetime,
+    latency: int,
+    deployment: dict[str, str],
+) -> Snapshot:
+    return {
+        "schemaVersion": 1,
+        "status": "unavailable",
+        "configured": True,
+        "runtimeKind": "document",
+        "observedAt": canonical_time(observed_at),
+        "downstreamProbe": {
+            "status": "passed",
+            "checkedAt": canonical_time(checked_at),
+            "validUntil": canonical_time(valid_until),
+            "latencyMs": latency,
+            "deploymentIdentity": deployment["identity"],
+            "releaseDigest": deployment["releaseDigest"],
+            "offersDigest": deployment["offersDigest"],
+        },
+        "deployment": deployment,
+        "offers": [],
+        "unavailableReasons": [{"code": "PROBE_STALE", "retryable": True}],
+    }
+
+
+def canonical_time(value: datetime) -> str:
+    return (
+        value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+    )
+
+
+def latency_ms(started: float, monotonic: float | None = None) -> int:
+    finished = time.monotonic() if monotonic is None else monotonic
+    return max(0, round((finished - started) * 1000))
+
+
+def required(value: str | None) -> str:
+    if not value:
+        raise ValueError("Document readiness 配置缺失")
+    return value
+
+
+def string_list(value: object) -> list[str]:
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise ValueError("Document readiness list 无效")
+    return value
+
+
+def number(value: dict[str, object], key: str) -> int:
+    item = value.get(key)
+    return item if isinstance(item, int) and not isinstance(item, bool) else -1
+
+
+def output_capability_requested(request: StartDocumentRunRequest) -> bool:
+    return request.output_artifact_sink is not None or (
+        DOCUMENT_OUTPUT_ARTIFACT_CAPABILITY_V1 in request.required_capabilities
+    )

package/template/apps/document-server/src/southwind_document_server/request_validation.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from .adapter_errors import SafeDocumentError
+from .contracts import StartDocumentRunRequest
+
+
+def validate_request_budget(request: StartDocumentRunRequest) -> None:
+    if len(request.sources) > request.limits.max_artifacts:
+        raise SafeDocumentError(
+            "BUDGET_EXCEEDED",
+            "Document Artifact 数量超过输入上限",
+            False,
+        )
+    total_bytes = sum(source.byte_size for source in request.sources)
+    if total_bytes > request.limits.max_input_bytes:
+        raise SafeDocumentError(
+            "BUDGET_EXCEEDED",
+            "Document Artifact 超过输入上限",
+            False,
+        )
+    options = request.options
+    if not isinstance(options, dict):
+        return
+    page_range = options.get("pageRange")
+    if page_range is None:
+        return
+    if not isinstance(page_range, dict):
+        raise SafeDocumentError(
+            "INVALID_INPUT",
+            "Document 页码范围无效",
+            False,
+        )
+    start = page_range.get("from")
+    end = page_range.get("to")
+    if (
+        not isinstance(start, int)
+        or isinstance(start, bool)
+        or not isinstance(end, int)
+        or isinstance(end, bool)
+        or start < 1
+        or end < start
+        or end - start + 1 > request.limits.max_units
+    ):
+        raise SafeDocumentError(
+            "INVALID_INPUT",
+            "Document 页码范围无效",
+            False,
+        )

package/template/apps/document-server/src/southwind_document_server/service.py

@@ -1,57 +1,236 @@
 from __future__ import annotations
 
 import asyncio
-import hashlib
-import json
+import time
 import uuid
+from collections.abc import Callable
 
-from .adapter import DocumentAdapter, SafeDocumentError
-from .contracts import ErrorResponse, StartDocumentRunRequest
-from .store import DocumentRunStore, RunRecord
+from .adapter import DocumentAdapter
+from .adapter_errors import SafeDocumentError
+from .artifacts import AdapterExecution
+from .contracts import (
+    ErrorResponse,
+    OutputArtifactSinkDescriptor,
+    StartDocumentRunRequest,
+)
+from .fingerprint import document_request_fingerprint
+from .grants import GrantStateError, OutputGrantCell
+from .output_validation import replace_artifacts
+from .progress import record_page_progress
+from .request_validation import validate_request_budget
+from .service_recovery import RECOVERY_ERROR, RunRecoveryCoordinator
+from .sink import OutputArtifactSinkClient, SinkClientError
+from .store import DocumentRunStore, RunConflictError, RunRecord
 
 
 class DocumentRunService:
-    def __init__(self, adapter: DocumentAdapter, store: DocumentRunStore) -> None:
+    def __init__(
+        self,
+        adapter: DocumentAdapter,
+        store: DocumentRunStore,
+        sink: OutputArtifactSinkClient | None = None,
+        grants: OutputGrantCell | None = None,
+        *,
+        clock: Callable[[], float] = time.time,
+        recovery_timeout_seconds: float = 60.0,
+        reconciliation_interval_seconds: float = 1.0,
+    ) -> None:
         self.adapter = adapter
         self.store = store
+        self.grants = grants or OutputGrantCell()
+        self.sink = sink or OutputArtifactSinkClient(
+            adapter.config,
+            self.grants,
+        )
         self.tasks: dict[str, asyncio.Task[None]] = {}
+        self._requests: dict[str, StartDocumentRunRequest] = {}
+        self._recovery = RunRecoveryCoordinator(
+            self.store,
+            self.sink,
+            self.grants,
+            self.tasks,
+            self._requests,
+            clock=clock,
+            timeout_seconds=recovery_timeout_seconds,
+            interval_seconds=reconciliation_interval_seconds,
+        )
 
     async def start(
-        self, request: StartDocumentRunRequest, idempotency_key: str
+        self,
+        request: StartDocumentRunRequest,
+        idempotency_key: str,
+        raw_grant: str | None = None,
     ) -> RunRecord:
         validate_request_budget(request)
-        fingerprint = hashlib.sha256(
-            json.dumps(
-                {
-                    "operationKey": request.operation_key,
-                    "sourceSha256": [source.sha256 for source in request.sources],
-                    "options": request.options,
-                    "capabilities": sorted(request.required_capabilities),
-                },
-                ensure_ascii=False,
-                separators=(",", ":"),
-                sort_keys=True,
-            ).encode()
-        ).hexdigest()
-        record, created = self.store.create(
-            f"doc_{uuid.uuid4().hex}",
+        await self._recovery.reconcile_once()
+        existing = self.store.get_by_idempotency(idempotency_key)
+        run_id = request.run_id or (
+            existing.run_id if existing else f"doc_{uuid.uuid4().hex}"
+        )
+        try:
+            fingerprint = document_request_fingerprint(request, run_id)
+        except ValueError:
+            raise SafeDocumentError(
+                "INVALID_INPUT",
+                "Document 请求包含不可规范化 JSON",
+                False,
+            ) from None
+        sink = request.output_artifact_sink
+        if sink and existing is None and raw_grant is None:
+            raise RunConflictError
+        if sink is None and raw_grant is not None:
+            raise SafeDocumentError(
+                "INVALID_INPUT",
+                "Document 请求包含未绑定的 Output Grant",
+                False,
+            )
+        record, _created = self.store.create(
+            run_id,
             request.operation_key,
             idempotency_key,
             fingerprint,
+            sink.scope if sink else None,
+            self._recovery.new_deadline() if sink else None,
         )
-        if created:
-            task = asyncio.create_task(self._execute(record.run_id, request))
-            self.tasks[record.run_id] = task
-            task.add_done_callback(lambda _task: self.tasks.pop(record.run_id, None))
+        if sink and record.terminal and record.status != "completed":
+            raise RunConflictError
+        if sink and not record.terminal:
+            if raw_grant is None:
+                raise RunConflictError
+            recovering = record.recovery_required
+            allow_same_generation = False
+            if recovering:
+                allow_same_generation = (
+                    await self.grants.generation(sink.scope) == sink.grant_generation
+                )
+                record = self.store.require_recovery_redispatch(
+                    run_id,
+                    sink.grant_generation,
+                    self._recovery.clock(),
+                    allow_same_generation=allow_same_generation,
+                )
+            try:
+                await self.grants.install(
+                    sink.scope,
+                    sink.grant_generation,
+                    raw_grant,
+                    record.grant_generation,
+                )
+                try:
+                    if recovering:
+                        record = self.store.commit_recovery_redispatch(
+                            run_id,
+                            sink.grant_generation,
+                            self._recovery.clock(),
+                            allow_same_generation=allow_same_generation,
+                        )
+                    else:
+                        record = self.store.refresh_grant_generation(
+                            run_id,
+                            sink.grant_generation,
+                        )
+                except RunConflictError:
+                    await self.grants.discard_if_generation(
+                        sink.scope,
+                        sink.grant_generation,
+                    )
+                    raise
+                except Exception:
+                    await self.grants.discard_if_generation(
+                        sink.scope,
+                        sink.grant_generation,
+                    )
+                    raise SafeDocumentError(
+                        "RUNTIME_UNAVAILABLE",
+                        "Document Run 状态暂时不可用",
+                        True,
+                    ) from None
+            except GrantStateError:
+                raise RunConflictError from None
+            self._requests[run_id] = request
+            try:
+                await self.sink.accept(sink)
+                if recovering:
+                    self._schedule(record.run_id, request)
+                    try:
+                        record = self.store.clear_recovery(
+                            run_id,
+                            sink.grant_generation,
+                            self._recovery.clock(),
+                        )
+                    except RunConflictError:
+                        current = self.store.get(run_id)
+                        if (
+                            current is not None
+                            and current.status in {"accepted", "running", "completed"}
+                            and not current.recovery_required
+                            and current.grant_generation == sink.grant_generation
+                        ):
+                            record = current
+                        else:
+                            task = self.tasks.get(run_id)
+                            if task:
+                                task.cancel()
+                            raise
+            except asyncio.CancelledError:
+                await self._recovery.preserve_cancelled_ack(
+                    run_id,
+                    sink.scope,
+                    sink.grant_generation,
+                )
+                raise
+            except RunConflictError:
+                result = await self._recovery.cleanup_run(run_id)
+                self.store.fail(run_id, RECOVERY_ERROR)
+                await self._recovery.finalize_cleanup(run_id, result)
+                self._requests.pop(run_id, None)
+                raise
+            except SinkClientError as error:
+                if error.ambiguous:
+                    self.store.begin_recovery(
+                        run_id,
+                        self._recovery.clock(),
+                        self._recovery.timeout_seconds,
+                    )
+                else:
+                    result = await self._recovery.cleanup_run(run_id)
+                    self.store.fail(run_id, RECOVERY_ERROR)
+                    await self._recovery.finalize_cleanup(run_id, result)
+                    if result:
+                        await self.grants.discard_if_generation(
+                            sink.scope,
+                            result[1],
+                        )
+                    self._requests.pop(run_id, None)
+                raise SafeDocumentError(
+                    "RUNTIME_UNAVAILABLE",
+                    "Document Output Artifact ACK 失败",
+                    True,
+                ) from None
+        if not record.terminal:
+            self._requests[run_id] = request
+            self._schedule(record.run_id, request)
         return record
 
+    def start_reconciler(self) -> None:
+        self._recovery.start()
+
+    async def close(self) -> None:
+        await self._recovery.close()
+
+    @property
+    def reconciler_task(self) -> asyncio.Task[None] | None:
+        return self._recovery.task
+
     async def cancel(self, run_id: str) -> RunRecord:
         record = self.require(run_id)
-        if not record.terminal:
-            self.store.append(run_id, "run.cancelled")
-            task = self.tasks.get(run_id)
-            if task:
-                task.cancel()
+        record, transitioned = self.store.cancel(run_id)
+        if not transitioned:
+            return record
+        task = self.tasks.get(run_id)
+        if task:
+            task.cancel()
+        await self._recovery.cleanup_terminal(run_id)
         return self.require(run_id)
 
     def require(self, run_id: str) -> RunRecord:
@@ -60,90 +239,112 @@ class DocumentRunService:
             raise KeyError(run_id)
         return record
 
-    async def _execute(self, run_id: str, request: StartDocumentRunRequest) -> None:
-        self.store.append(run_id, "run.started")
+    def _schedule(
+        self,
+        run_id: str,
+        request: StartDocumentRunRequest,
+    ) -> None:
+        if run_id in self.tasks:
+            return
+        task = asyncio.create_task(self._execute(run_id, request))
+        self.tasks[run_id] = task
+        task.add_done_callback(lambda _task: self._forget(run_id))
+
+    def _forget(self, run_id: str) -> None:
+        self.tasks.pop(run_id, None)
         try:
+            if self.require(run_id).terminal:
+                self._requests.pop(run_id, None)
+        except Exception:
+            pass
+
+    async def _execute(
+        self,
+        run_id: str,
+        request: StartDocumentRunRequest,
+    ) -> None:
+        execution: AdapterExecution | None = None
+        sink = request.output_artifact_sink
+        try:
+            if self.store.mark_running(run_id).terminal:
+                return
             async with asyncio.timeout(request.limits.max_duration_seconds):
-                output = await self.adapter.execute(request)
-                if self.require(run_id).status == "cancelled":
+                execution = await self.adapter.execute(request)
+                if self.require(run_id).terminal:
+                    await self._recovery.cleanup_terminal(run_id)
                     return
-                pages = output.get("pages") if isinstance(output, dict) else None
-                if isinstance(pages, list):
-                    total = len(pages)
-                    for index in range(total):
-                        self.store.append(
-                            run_id,
-                            "document.page.completed",
-                            {
-                                "pageNumber": index + 1,
-                                "completedPages": index + 1,
-                                "totalPages": total,
-                            },
+                record_page_progress(self.store, run_id, execution)
+                output = execution.output
+                if sink:
+                    references = await self.sink.publish(
+                        sink,
+                        execution.artifacts,
+                    )
+                    output = replace_artifacts(output, references)
+                completed, transitioned = self.store.complete(run_id, output)
+                if sink:
+                    if transitioned and completed.status == "completed":
+                        await self.grants.discard_if_generation(
+                            sink.scope,
+                            completed.grant_generation,
                         )
-                self.store.complete(run_id, output)
+                    else:
+                        await self._recovery.cleanup_terminal(run_id)
         except TimeoutError:
-            self._fail(run_id, "RUN_TIMEOUT", "Document 任务超时", True)
+            await self._fail(
+                run_id,
+                sink,
+                "RUN_TIMEOUT",
+                "Document 任务超时",
+                True,
+            )
+        except asyncio.CancelledError:
+            result = await self._recovery.cleanup_run(run_id)
+            try:
+                self.store.cancel(run_id)
+                await self._recovery.finalize_cleanup(run_id, result)
+            except Exception:
+                pass
         except SafeDocumentError as error:
-            self._fail(
+            await self._fail(
                 run_id,
+                sink,
                 error.code,
                 error.safe_message,
                 error.retryable,
             )
         except Exception:
-            self._fail(
+            await self._fail(
                 run_id,
+                sink,
                 "RUNTIME_UNAVAILABLE",
                 "Document Runtime 执行失败",
                 True,
             )
+        finally:
+            if execution:
+                await execution.cleanup()
 
-    def _fail(self, run_id: str, code: str, message: str, retryable: bool) -> None:
-        self.store.append(
-            run_id,
-            "run.failed",
-            {
-                "error": ErrorResponse(
+    async def _fail(
+        self,
+        run_id: str,
+        sink: OutputArtifactSinkDescriptor | None,
+        code: str,
+        message: str,
+        retryable: bool,
+    ) -> None:
+        result = await self._recovery.cleanup_run(run_id)
+        try:
+            record, _transitioned = self.store.fail(
+                run_id,
+                ErrorResponse(
                     code=code,
                     message=message,
                     request_id=f"document_{uuid.uuid4().hex}",
                     retryable=retryable,
-                ).model_dump(by_alias=True)
-            },
-        )
-
-
-def validate_request_budget(request: StartDocumentRunRequest) -> None:
-    if len(request.sources) > request.limits.max_artifacts:
-        raise SafeDocumentError(
-            "BUDGET_EXCEEDED",
-            "Document Artifact 数量超过输入上限",
-            False,
-        )
-    total_bytes = sum(source.byte_size for source in request.sources)
-    if total_bytes > request.limits.max_input_bytes:
-        raise SafeDocumentError(
-            "BUDGET_EXCEEDED",
-            "Document Artifact 超过输入上限",
-            False,
-        )
-    options = request.options
-    if not isinstance(options, dict):
-        return
-    page_range = options.get("pageRange")
-    if page_range is None:
-        return
-    if not isinstance(page_range, dict):
-        raise SafeDocumentError("INVALID_INPUT", "Document 页码范围无效", False)
-    start = page_range.get("from")
-    end = page_range.get("to")
-    if (
-        not isinstance(start, int)
-        or isinstance(start, bool)
-        or not isinstance(end, int)
-        or isinstance(end, bool)
-        or start < 1
-        or end < start
-        or end - start + 1 > request.limits.max_units
-    ):
-        raise SafeDocumentError("INVALID_INPUT", "Document 页码范围无效", False)
+                ).model_dump(by_alias=True),
+            )
+            if sink and record.terminal:
+                await self._recovery.finalize_cleanup(run_id, result)
+        except Exception:
+            pass