create-windy 0.3.2 → 0.3.5

package/template/apps/document-server/src/southwind_document_server/service.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import uuid
+
+from .adapter import DocumentAdapter, SafeDocumentError
+from .contracts import ErrorResponse, StartDocumentRunRequest
+from .store import DocumentRunStore, RunRecord
+
+
+class DocumentRunService:
+    def __init__(self, adapter: DocumentAdapter, store: DocumentRunStore) -> None:
+        self.adapter = adapter
+        self.store = store
+        self.tasks: dict[str, asyncio.Task[None]] = {}
+
+    async def start(
+        self, request: StartDocumentRunRequest, idempotency_key: str
+    ) -> RunRecord:
+        validate_request_budget(request)
+        fingerprint = hashlib.sha256(
+            json.dumps(
+                {
+                    "operationKey": request.operation_key,
+                    "sourceSha256": [source.sha256 for source in request.sources],
+                    "options": request.options,
+                    "capabilities": sorted(request.required_capabilities),
+                },
+                ensure_ascii=False,
+                separators=(",", ":"),
+                sort_keys=True,
+            ).encode()
+        ).hexdigest()
+        record, created = self.store.create(
+            f"doc_{uuid.uuid4().hex}",
+            request.operation_key,
+            idempotency_key,
+            fingerprint,
+        )
+        if created:
+            task = asyncio.create_task(self._execute(record.run_id, request))
+            self.tasks[record.run_id] = task
+            task.add_done_callback(lambda _task: self.tasks.pop(record.run_id, None))
+        return record
+
+    async def cancel(self, run_id: str) -> RunRecord:
+        record = self.require(run_id)
+        if not record.terminal:
+            self.store.append(run_id, "run.cancelled")
+            task = self.tasks.get(run_id)
+            if task:
+                task.cancel()
+        return self.require(run_id)
+
+    def require(self, run_id: str) -> RunRecord:
+        record = self.store.get(run_id)
+        if not record:
+            raise KeyError(run_id)
+        return record
+
+    async def _execute(self, run_id: str, request: StartDocumentRunRequest) -> None:
+        self.store.append(run_id, "run.started")
+        try:
+            async with asyncio.timeout(request.limits.max_duration_seconds):
+                output = await self.adapter.execute(request)
+                if self.require(run_id).status == "cancelled":
+                    return
+                pages = output.get("pages") if isinstance(output, dict) else None
+                if isinstance(pages, list):
+                    total = len(pages)
+                    for index in range(total):
+                        self.store.append(
+                            run_id,
+                            "document.page.completed",
+                            {
+                                "pageNumber": index + 1,
+                                "completedPages": index + 1,
+                                "totalPages": total,
+                            },
+                        )
+                self.store.complete(run_id, output)
+        except TimeoutError:
+            self._fail(run_id, "RUN_TIMEOUT", "Document 任务超时", True)
+        except SafeDocumentError as error:
+            self._fail(
+                run_id,
+                error.code,
+                error.safe_message,
+                error.retryable,
+            )
+        except Exception:
+            self._fail(
+                run_id,
+                "RUNTIME_UNAVAILABLE",
+                "Document Runtime 执行失败",
+                True,
+            )
+
+    def _fail(self, run_id: str, code: str, message: str, retryable: bool) -> None:
+        self.store.append(
+            run_id,
+            "run.failed",
+            {
+                "error": ErrorResponse(
+                    code=code,
+                    message=message,
+                    request_id=f"document_{uuid.uuid4().hex}",
+                    retryable=retryable,
+                ).model_dump(by_alias=True)
+            },
+        )
+
+
+def validate_request_budget(request: StartDocumentRunRequest) -> None:
+    if len(request.sources) > request.limits.max_artifacts:
+        raise SafeDocumentError(
+            "BUDGET_EXCEEDED",
+            "Document Artifact 数量超过输入上限",
+            False,
+        )
+    total_bytes = sum(source.byte_size for source in request.sources)
+    if total_bytes > request.limits.max_input_bytes:
+        raise SafeDocumentError(
+            "BUDGET_EXCEEDED",
+            "Document Artifact 超过输入上限",
+            False,
+        )
+    options = request.options
+    if not isinstance(options, dict):
+        return
+    page_range = options.get("pageRange")
+    if page_range is None:
+        return
+    if not isinstance(page_range, dict):
+        raise SafeDocumentError("INVALID_INPUT", "Document 页码范围无效", False)
+    start = page_range.get("from")
+    end = page_range.get("to")
+    if (
+        not isinstance(start, int)
+        or isinstance(start, bool)
+        or not isinstance(end, int)
+        or isinstance(end, bool)
+        or start < 1
+        or end < start
+        or end - start + 1 > request.limits.max_units
+    ):
+        raise SafeDocumentError("INVALID_INPUT", "Document 页码范围无效", False)

package/template/apps/document-server/src/southwind_document_server/store.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from .contracts import JsonValue, RunStatus
+
+
+@dataclass(frozen=True)
+class RunRecord:
+    run_id: str
+    operation_key: str
+    idempotency_key: str
+    fingerprint: str
+    status: RunStatus
+    output: JsonValue = None
+
+    @property
+    def terminal(self) -> bool:
+        return self.status in {"completed", "failed", "cancelled"}
+
+
+class RunConflictError(Exception):
+    pass
+
+
+class DocumentRunStore:
+    def __init__(self, path: str) -> None:
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        self.database = sqlite3.connect(path, check_same_thread=False)
+        self.database.row_factory = sqlite3.Row
+        self._initialize()
+        self._recover()
+
+    def create(
+        self,
+        run_id: str,
+        operation_key: str,
+        idempotency_key: str,
+        fingerprint: str,
+    ) -> tuple[RunRecord, bool]:
+        existing = self.database.execute(
+            "select run_id from document_runs where idempotency_key = ?",
+            (idempotency_key,),
+        ).fetchone()
+        if existing:
+            record = self.get(str(existing["run_id"]))
+            if not record or record.fingerprint != fingerprint:
+                raise RunConflictError
+            return record, False
+        now = time.time()
+        self.database.execute(
+            """
+            insert into document_runs values (?, ?, ?, ?, 'accepted', 'null', ?, ?)
+            """,
+            (run_id, operation_key, idempotency_key, fingerprint, now, now),
+        )
+        self.database.commit()
+        return self.get(run_id) or _missing(), True
+
+    def get(self, run_id: str) -> RunRecord | None:
+        row = self.database.execute(
+            "select * from document_runs where run_id = ?", (run_id,)
+        ).fetchone()
+        if not row:
+            return None
+        return RunRecord(
+            run_id=str(row["run_id"]),
+            operation_key=str(row["operation_key"]),
+            idempotency_key=str(row["idempotency_key"]),
+            fingerprint=str(row["fingerprint"]),
+            status=cast(RunStatus, str(row["status"])),
+            output=json.loads(str(row["output_json"])),
+        )
+
+    def append(
+        self, run_id: str, event_type: str, fields: dict[str, object] | None = None
+    ) -> None:
+        record = self.get(run_id)
+        if not record or record.terminal:
+            return
+        sequence = (
+            int(
+                self.database.execute(
+                    "select count(*) from document_events where run_id = ?", (run_id,)
+                ).fetchone()[0]
+            )
+            + 1
+        )
+        event = {
+            "type": event_type,
+            "runId": run_id,
+            "sequence": sequence,
+            **(fields or {}),
+        }
+        status = {
+            "run.started": "running",
+            "run.completed": "completed",
+            "run.failed": "failed",
+            "run.cancelled": "cancelled",
+        }.get(event_type, record.status)
+        self.database.execute(
+            "insert into document_events values (?, ?, ?)",
+            (run_id, sequence, encode(event)),
+        )
+        self.database.execute(
+            "update document_runs set status = ?, updated_at = ? where run_id = ?",
+            (status, time.time(), run_id),
+        )
+        self.database.commit()
+
+    def complete(self, run_id: str, output: JsonValue) -> None:
+        self.database.execute(
+            "update document_runs set output_json = ? where run_id = ?",
+            (encode(output), run_id),
+        )
+        self.database.commit()
+        self.append(run_id, "run.completed")
+
+    def events_after(self, run_id: str, after: int) -> list[dict[str, object]]:
+        return [
+            json.loads(str(row["event_json"]))
+            for row in self.database.execute(
+                """
+                select event_json from document_events
+                where run_id = ? and sequence > ? order by sequence
+                """,
+                (run_id, after),
+            )
+        ]
+
+    def _initialize(self) -> None:
+        self.database.executescript(
+            """
+            create table if not exists document_runs (
+              run_id text primary key,
+              operation_key text not null,
+              idempotency_key text not null unique,
+              fingerprint text not null,
+              status text not null,
+              output_json text not null,
+              created_at real not null,
+              updated_at real not null
+            );
+            create table if not exists document_events (
+              run_id text not null references document_runs(run_id) on delete cascade,
+              sequence integer not null,
+              event_json text not null,
+              primary key (run_id, sequence)
+            );
+            """
+        )
+        self.database.commit()
+
+    def _recover(self) -> None:
+        rows = self.database.execute(
+            "select run_id from document_runs where status in ('accepted', 'running')"
+        ).fetchall()
+        for row in rows:
+            self.append(
+                str(row["run_id"]),
+                "run.failed",
+                {
+                    "error": {
+                        "code": "RUNTIME_UNAVAILABLE",
+                        "message": (
+                            "Document Runtime 重启中断了任务，请使用新幂等键重试"
+                        ),
+                        "requestId": "document-recovery",
+                        "retryable": True,
+                    }
+                },
+            )
+
+
+def encode(value: object) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def _missing() -> RunRecord:
+    raise RuntimeError("Document Run 创建后不可读")

package/template/apps/document-server/tests/__init__.py

@@ -0,0 +1 @@
+"""Document Runtime tests."""

package/template/apps/document-server/tests/test_config.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import pytest
+
+from southwind_document_server.config import DocumentRuntimeConfig
+
+
+def test_adapter_must_be_local_or_intranet() -> None:
+    with pytest.raises(ValueError, match="禁止使用公网地址"):
+        DocumentRuntimeConfig.from_environment(
+            {
+                "DOCUMENT_SERVER_INTERNAL_TOKEN": "internal-token",
+                "DOCUMENT_ADAPTER_URL": "https://8.8.8.8",
+            }
+        )
+
+
+def test_artifact_endpoint_must_be_local_or_intranet() -> None:
+    with pytest.raises(ValueError, match="禁止使用公网地址"):
+        DocumentRuntimeConfig.from_environment(
+            {
+                "DOCUMENT_SERVER_INTERNAL_TOKEN": "internal-token",
+                "DOCUMENT_ARTIFACT_URL": "https://8.8.8.8/artifacts",
+            }
+        )
+
+
+def test_adapter_rejects_public_dns_names() -> None:
+    with pytest.raises(ValueError, match="内部服务名"):
+        DocumentRuntimeConfig.from_environment(
+            {
+                "DOCUMENT_SERVER_INTERNAL_TOKEN": "internal-token",
+                "DOCUMENT_ADAPTER_URL": "https://public.example.com",
+            }
+        )
+
+
+def test_unconfigured_adapter_keeps_runtime_bootable() -> None:
+    config = DocumentRuntimeConfig.from_environment(
+        {"DOCUMENT_SERVER_INTERNAL_TOKEN": "internal-token"}
+    )
+    assert config.adapter_url is None

package/template/apps/document-server/tests/test_service.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+
+from southwind_document_server.adapter import (
+    DocumentAdapter,
+    SafeDocumentError,
+    validate_output,
+)
+from southwind_document_server.config import DocumentRuntimeConfig
+from southwind_document_server.contracts import (
+    ArtifactInput,
+    JsonValue,
+    ProviderEgress,
+    RuntimeLimits,
+    StartDocumentRunRequest,
+)
+from southwind_document_server.service import DocumentRunService
+from southwind_document_server.store import DocumentRunStore, RunConflictError
+
+
+class FakeAdapter(DocumentAdapter):
+    def __init__(self) -> None:
+        super().__init__(
+            DocumentRuntimeConfig(
+                internal_token="internal",
+                store_path=":memory:",
+                artifact_url="http://server.internal/api/internal/ai/artifacts",
+                adapter_url="http://127.0.0.1:9000",
+                adapter_token=None,
+                deployment_key=None,
+                capabilities=("document.ocr",),
+            )
+        )
+
+    async def execute(self, request: StartDocumentRunRequest) -> JsonValue:
+        return {
+            "pages": [
+                {
+                    "pageNumber": 1,
+                    "text": "第一页",
+                    "blocks": [],
+                }
+            ],
+            "tables": [],
+            "artifacts": [],
+            "metadata": {"engine": "fake"},
+        }
+
+
+async def test_idempotent_run_emits_page_progress_and_terminal_result() -> None:
+    store = DocumentRunStore(":memory:")
+    service = DocumentRunService(FakeAdapter(), store)
+    request = document_request()
+
+    created = await service.start(request, "document-idempotent")
+    replayed = await service.start(request, "document-idempotent")
+    assert replayed.run_id == created.run_id
+    await asyncio.gather(*service.tasks.values())
+
+    completed = service.require(created.run_id)
+    assert completed.status == "completed"
+    assert completed.output == {
+        "pages": [{"pageNumber": 1, "text": "第一页", "blocks": []}],
+        "tables": [],
+        "artifacts": [],
+        "metadata": {"engine": "fake"},
+    }
+    assert [event["type"] for event in store.events_after(created.run_id, 0)] == [
+        "run.started",
+        "document.page.completed",
+        "run.completed",
+    ]
+
+
+async def test_same_key_with_changed_source_is_rejected() -> None:
+    service = DocumentRunService(FakeAdapter(), DocumentRunStore(":memory:"))
+    request = document_request()
+    await service.start(request, "document-conflict")
+    changed = request.model_copy(
+        update={"sources": [request.sources[0].model_copy(update={"sha256": "b" * 64})]}
+    )
+    try:
+        await service.start(changed, "document-conflict")
+    except RunConflictError:
+        pass
+    else:
+        raise AssertionError("相同幂等键的语义漂移必须拒绝")
+
+
+def test_output_rejects_page_overrun_and_provider_artifact() -> None:
+    output: JsonValue = {
+        "pages": [
+            {"pageNumber": 1, "text": "第一页", "blocks": []},
+            {"pageNumber": 2, "text": "第二页", "blocks": []},
+        ]
+    }
+    try:
+        validate_output(output, 1)
+    except SafeDocumentError as error:
+        assert error.code == "OUTPUT_VALIDATION_FAILED"
+    else:
+        raise AssertionError("超过页数上限的输出必须拒绝")
+
+    output = {
+        "pages": [{"pageNumber": 1, "text": "第一页", "blocks": []}],
+        "artifacts": [{"url": "https://provider.example/output.pdf"}],
+    }
+    try:
+        validate_output(output, 1)
+    except SafeDocumentError as error:
+        assert error.code == "OUTPUT_VALIDATION_FAILED"
+    else:
+        raise AssertionError("Adapter 不能直接签发 Artifact")
+
+
+def document_request() -> StartDocumentRunRequest:
+    return StartDocumentRunRequest(
+        kind="document",
+        operation_key="analysis.transcript.ocr",
+        options={"outputs": ["pages"]},
+        sources=[
+            ArtifactInput(
+                value="opaque-artifact-ref",
+                media_type="application/pdf",
+                byte_size=128,
+                sha256="a" * 64,
+            )
+        ],
+        required_capabilities=["document.ocr"],
+        data_classification="sensitive",
+        limits=RuntimeLimits(
+            max_duration_seconds=60,
+            max_input_bytes=1_024,
+            max_output_bytes=1_024,
+            max_units=20,
+            max_artifacts=1,
+        ),
+        provider_egress=ProviderEgress(mode="forbidden"),
+    )