create-windy 0.5.1 → 0.6.0

package/template/apps/document-server/tests/test_store.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from southwind_document_server.store import DocumentRunStore, RunConflictError
+
+from .support import RAW_GRANT, RUN_ID
+
+SECOND_RUN_ID = f"doc_{'b' * 64}"
+RECOVERY_ERROR = {
+    "code": "RUNTIME_UNAVAILABLE",
+    "message": "Document Runtime 重启恢复期限已到",
+    "requestId": "document_recovery",
+    "retryable": True,
+}
+
+
+def test_store_replays_identical_binding_and_rejects_drift() -> None:
+    store = DocumentRunStore(":memory:")
+    scope = f"outscope_{'2' * 32}"
+    created, is_new = store.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "idem",
+        "a" * 64,
+        scope,
+    )
+    replayed, replay_new = store.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "idem",
+        "a" * 64,
+        scope,
+    )
+    assert is_new is True
+    assert replay_new is False
+    assert replayed == created
+
+    for fingerprint, changed_scope in (
+        ("b" * 64, scope),
+        ("a" * 64, f"outscope_{'3' * 32}"),
+    ):
+        try:
+            store.create(
+                RUN_ID,
+                "evidence.document.ocr",
+                "idem",
+                fingerprint,
+                changed_scope,
+            )
+        except RunConflictError:
+            pass
+        else:
+            raise AssertionError("持久化语义或 scope 漂移必须冲突")
+
+
+def test_terminal_result_and_events_commit_monotonically() -> None:
+    store = DocumentRunStore(":memory:")
+    store.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "terminal",
+        "a" * 64,
+        None,
+    )
+    assert store.mark_running(RUN_ID).status == "running"
+    cancelled, transitioned = store.cancel(RUN_ID)
+    assert transitioned is True
+    assert cancelled.status == "cancelled"
+
+    after, completed = store.complete(RUN_ID, {"artifacts": [{"value": "ref"}]})
+    assert completed is False
+    assert after.status == "cancelled"
+    assert after.output is None
+    events = store.events_after(RUN_ID, 0)
+    assert [event["type"] for event in events] == [
+        "run.started",
+        "run.cancelled",
+    ]
+    assert [event["sequence"] for event in events] == [1, 2]
+
+
+def test_restart_preserves_nonterminal_generation_without_raw_secret(
+    tmp_path: Path,
+) -> None:
+    database = tmp_path / "runs.sqlite3"
+    scope = f"outscope_{'2' * 32}"
+    first = DocumentRunStore(str(database))
+    first.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "restart",
+        "a" * 64,
+        scope,
+    )
+    first.refresh_grant_generation(RUN_ID, 3)
+    first.mark_running(RUN_ID)
+    first.close()
+
+    recovered = DocumentRunStore(str(database))
+    record = recovered.get(RUN_ID)
+    assert record is not None
+    assert record.status == "running"
+    assert record.grant_generation == 3
+    columns = {
+        str(row["name"])
+        for row in recovered.database.execute("pragma table_info(document_runs)")
+    }
+    assert not {"grant", "grant_token", "grant_hash"} & columns
+    recovered.close()
+    assert RAW_GRANT.encode() not in database.read_bytes()
+
+
+def test_startup_reconcile_fails_legacy_and_never_slides_sink_deadline() -> None:
+    store = DocumentRunStore(":memory:")
+    store.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "legacy-restart",
+        "a" * 64,
+        None,
+    )
+    store.mark_running(RUN_ID)
+    scope = f"outscope_{'2' * 32}"
+    store.create(
+        SECOND_RUN_ID,
+        "evidence.document.ocr",
+        "sink-restart",
+        "b" * 64,
+        scope,
+    )
+    store.refresh_grant_generation(SECOND_RUN_ID, 1)
+    store.mark_running(SECOND_RUN_ID)
+
+    store.reconcile_startup(100.0, 30.0, RECOVERY_ERROR)
+    legacy = store.get(RUN_ID)
+    recovering = store.get(SECOND_RUN_ID)
+    assert legacy is not None
+    assert legacy.status == "failed"
+    assert recovering is not None
+    assert recovering.status == "running"
+    assert recovering.recovery_required is True
+    assert recovering.recovery_deadline == 130.0
+
+    store.reconcile_startup(120.0, 30.0, RECOVERY_ERROR)
+    unchanged = store.get(SECOND_RUN_ID)
+    assert unchanged is not None
+    assert unchanged.recovery_deadline == 130.0
+    assert [event["type"] for event in store.events_after(RUN_ID, 0)] == [
+        "run.started",
+        "run.failed",
+    ]
+
+
+def test_expired_recovery_commits_one_terminal_event_across_instances(
+    tmp_path: Path,
+) -> None:
+    database = tmp_path / "atomic-recovery.sqlite3"
+    first = DocumentRunStore(str(database))
+    first.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "atomic-expiry",
+        "a" * 64,
+        f"outscope_{'2' * 32}",
+    )
+    first.refresh_grant_generation(RUN_ID, 1)
+    first.mark_running(RUN_ID)
+    first.reconcile_startup(10.0, 5.0, RECOVERY_ERROR)
+    second = DocumentRunStore(str(database))
+
+    assert len(first.expire_recoveries(16.0, RECOVERY_ERROR)) == 1
+    assert second.expire_recoveries(16.0, RECOVERY_ERROR) == []
+    record = second.get(RUN_ID)
+    assert record is not None
+    assert record.status == "failed"
+    assert record.cleanup_status == "pending-host-ttl"
+    assert [event["type"] for event in second.events_after(RUN_ID, 0)] == [
+        "run.started",
+        "run.failed",
+    ]
+
+
+def test_deleted_cleanup_fact_is_absorbing() -> None:
+    store = DocumentRunStore(":memory:")
+    store.create(
+        RUN_ID,
+        "evidence.document.ocr",
+        "absorbing-cleanup",
+        "a" * 64,
+        f"outscope_{'2' * 32}",
+    )
+    store.refresh_grant_generation(RUN_ID, 1)
+    store.fail(RUN_ID, RECOVERY_ERROR)
+    store.mark_cleanup(RUN_ID, 1, "deleted")
+
+    record = store.mark_cleanup(RUN_ID, 1, "pending-host-ttl")
+
+    assert record.cleanup_status == "deleted"

package/template/apps/embedding-server/pyproject.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 description = "Southwind AI governed text and image Embedding Runtime"
 requires-python = ">=3.12,<3.13"
 dependencies = [
+  "cryptography>=50.0.0",
   "fastapi==0.139.2",
   "httpx==0.28.1",
   "pillow==12.3.0",
@@ -36,6 +37,10 @@ python_version = "3.12"
 strict = true
 files = ["src", "tests"]
 
+[[tool.mypy.overrides]]
+module = ["cryptography.*"]
+ignore_missing_imports = true
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

package/template/apps/embedding-server/src/southwind_embedding_server/adapter.py

@@ -27,7 +27,40 @@ class EmbeddingAdapter:
     def available(self) -> bool:
         return self.config.adapter_url is not None
 
-    async def embed(self, request: EmbeddingRequest) -> EmbeddingResponse:
+    async def readiness(self) -> bool:
+        """A configured URL is not readiness.
+
+        The real Adapter must answer a bounded probe.
+        """
+        if not self.config.adapter_url:
+            return False
+        headers = (
+            {"authorization": f"Bearer {self.config.adapter_token}"}
+            if self.config.adapter_token
+            else {}
+        )
+        try:
+            async with httpx.AsyncClient(
+                timeout=httpx.Timeout(5, connect=self.config.connect_timeout_seconds)
+            ) as client:
+                response = await client.get(
+                    f"{self.config.adapter_url.rstrip('/')}/health/ready",
+                    headers=headers,
+                )
+            if response.status_code != 200:
+                return False
+            body = response.json()
+            return (
+                isinstance(body, dict)
+                and body.get("model") == self.config.model
+                and body.get("dimensions") == self.config.dimensions
+            )
+        except Exception:
+            return False
+
+    async def embed(
+        self, request: EmbeddingRequest, run_id: str | None = None
+    ) -> EmbeddingResponse:
         self._authorize(request)
         if not self.config.adapter_url:
             raise SafeEmbeddingError(
@@ -44,7 +77,7 @@ class EmbeddingAdapter:
                 "Embedding Deployment 缺少 Operation 所需能力",
                 True,
             )
-        prepared, input_hashes = await self._prepare(request)
+        prepared, input_hashes = await self._prepare(request, run_id)
         raw = await self._invoke(request, prepared)
         vectors = validate_vectors(
             raw,
@@ -68,6 +101,7 @@ class EmbeddingAdapter:
     async def _prepare(
         self,
         request: EmbeddingRequest,
+        run_id: str | None,
     ) -> tuple[list[dict[str, JsonValue]], list[str]]:
         artifacts = sum(item.source is not None for item in request.items)
         if artifacts > (request.limits.max_artifacts or 0):
@@ -93,6 +127,8 @@ class EmbeddingAdapter:
                 self.config,
                 remaining,
                 request.limits.max_pixels_per_artifact,
+                run_id,
+                item.id,
             )
             remaining -= item.source.byte_size
             hashes.append(digest)

package/template/apps/embedding-server/src/southwind_embedding_server/app.py

@@ -4,15 +4,20 @@ import asyncio
 import hmac
 import os
 import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 
-from fastapi import Depends, FastAPI, Header, HTTPException
+from fastapi import Depends, FastAPI, Header, HTTPException, Query
 from fastapi.exceptions import RequestValidationError
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, StreamingResponse
 
 from .adapter import EmbeddingAdapter
 from .config import EmbeddingConfig
 from .contracts import EmbeddingRequest, EmbeddingResponse, ErrorResponse
 from .error import SafeEmbeddingError
+from .run_cipher import EmbeddingRunCipher
+from .run_service import EmbeddingRunService
+from .run_store import EmbeddingRunStore, RunConflictError
 
 
 def create_app(
@@ -21,6 +26,19 @@ def create_app(
 ) -> FastAPI:
     app = FastAPI(title="Southwind Embedding Runtime", docs_url=None, redoc_url=None)
     runtime = adapter or EmbeddingAdapter(config)
+    store = EmbeddingRunStore(
+        config.run_store_path,
+        EmbeddingRunCipher(config.run_encryption_secret),
+        config.terminal_ttl_seconds,
+    )
+    service = EmbeddingRunService(runtime, store, config.max_concurrent_runs)
+
+    @asynccontextmanager
+    async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
+        await service.recover()
+        yield
+
+    app.router.lifespan_context = lifespan
 
     def authorize(authorization: str | None = Header(default=None)) -> None:
         provided = (authorization or "").removeprefix("Bearer ")
@@ -29,7 +47,7 @@ def create_app(
 
     @app.get("/health/ready")
     async def ready(_: None = Depends(authorize)) -> JSONResponse:
-        if not runtime.available:
+        if not await runtime.readiness():
             return JSONResponse(
                 {"status": "unavailable", "capabilities": []},
                 status_code=503,
@@ -67,11 +85,86 @@ def create_app(
                 True,
             ) from error
 
+    @app.post("/api/embeddings/runs", status_code=202)
+    async def start_run(
+        request: EmbeddingRequest,
+        idempotency_key: str = Header(alias="idempotency-key"),
+        run_id: str = Header(alias="x-embedding-run-id"),
+        _: None = Depends(authorize),
+    ) -> JSONResponse:
+        if (
+            not idempotency_key.strip()
+            or len(idempotency_key) > 160
+            or not valid_run_id(run_id)
+        ):
+            raise HTTPException(status_code=400, detail="INVALID_INPUT")
+        try:
+            record = await service.start(request, idempotency_key, run_id)
+        except RunConflictError:
+            raise HTTPException(status_code=409, detail="RUN_CONFLICT") from None
+        return JSONResponse(
+            {
+                "runId": record.run_id,
+                "status": record.status,
+                "eventsUrl": f"/api/embeddings/runs/{record.run_id}/events",
+            },
+            status_code=202,
+        )
+
+    @app.get("/api/embeddings/runs/{run_id}/events")
+    async def events(
+        run_id: str,
+        after: int = Query(default=0, ge=0),
+        _: None = Depends(authorize),
+    ) -> StreamingResponse:
+        await service.require(run_id)
+
+        async def stream() -> AsyncIterator[str]:
+            cursor = after
+            for _attempt in range(30):
+                record = await service.require(run_id)
+                for event in await store.events_after(run_id, cursor):
+                    sequence = event.get("sequence")
+                    if not isinstance(sequence, int):
+                        raise RuntimeError("Embedding Event sequence 无效")
+                    cursor = sequence
+                    yield f"data: {encode_event(event)}\\n\\n"
+                if record.terminal:
+                    return
+                yield ": heartbeat\\n\\n"
+                await asyncio.sleep(1)
+
+        return StreamingResponse(stream(), media_type="text/event-stream")
+
+    @app.get("/api/embeddings/runs/{run_id}/result")
+    async def result(run_id: str, _: None = Depends(authorize)) -> JSONResponse:
+        record = await service.require(run_id)
+        if await store.tombstone_if_expired(run_id):
+            return JSONResponse({"code": "RESULT_EXPIRED"}, status_code=410)
+        if record.status != "completed" or record.output is None:
+            return JSONResponse({"code": "RUN_NOT_READY"}, status_code=409)
+        return JSONResponse(
+            {
+                "runId": run_id,
+                "status": "completed",
+                "output": record.output.model_dump(by_alias=True),
+            }
+        )
+
+    @app.post("/api/embeddings/runs/{run_id}/cancel")
+    async def cancel(run_id: str, _: None = Depends(authorize)) -> JSONResponse:
+        record = await service.cancel(run_id)
+        return JSONResponse({"runId": run_id, "status": record.status})
+
     @app.exception_handler(HTTPException)
     async def http_error(_request: object, error: HTTPException) -> JSONResponse:
         code = str(error.detail)
         return safe_error(code, "Embedding 请求被拒绝", error.status_code, False)
 
+    @app.exception_handler(KeyError)
+    async def missing(_request: object, _error: KeyError) -> JSONResponse:
+        return safe_error("RUN_NOT_FOUND", "Embedding Run 不存在", 404, False)
+
     @app.exception_handler(RequestValidationError)
     async def validation_error(
         _request: object,
@@ -94,6 +187,12 @@ def create_app(
     return app
 
 
+def encode_event(event: dict[str, object]) -> str:
+    import json
+
+    return json.dumps(event, ensure_ascii=False, separators=(",", ":"))
+
+
 def error_status(code: str) -> int:
     if code == "FORBIDDEN":
         return 403
@@ -104,6 +203,15 @@ def error_status(code: str) -> int:
     return 400
 
 
+def valid_run_id(value: str) -> bool:
+    return (
+        bool(value)
+        and len(value) <= 80
+        and value.startswith("embedding_")
+        and value.replace("_", "").replace("-", "").isalnum()
+    )
+
+
 def safe_error(code: str, message: str, status: int, retryable: bool) -> JSONResponse:
     return JSONResponse(
         ErrorResponse(

package/template/apps/embedding-server/src/southwind_embedding_server/config.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 class EmbeddingConfig:
     internal_token: str
     artifact_url: str
+    run_artifact_url: str
     adapter_url: str | None
     adapter_token: str | None
     deployment_key: str | None
@@ -21,6 +22,10 @@ class EmbeddingConfig:
     max_units: int = 256
     connect_timeout_seconds: float = 5
     request_timeout_seconds: float = 120
+    run_store_path: str = "/app/data/embedding/runs.sqlite3"
+    run_encryption_secret: str = ""
+    max_concurrent_runs: int = 1
+    terminal_ttl_seconds: int = 7 * 24 * 60 * 60
 
     @classmethod
     def from_environment(cls, environment: dict[str, str]) -> EmbeddingConfig:
@@ -29,12 +34,22 @@ class EmbeddingConfig:
             "http://server:9747/api/internal/ai/artifacts",
         ).strip()
         validate_intranet_url(artifact_url)
+        run_artifact_url = (
+            environment.get(
+                "EMBEDDING_RUN_ARTIFACT_URL",
+                "http://server:9747/api/internal/embeddings/runs",
+            )
+            .strip()
+            .rstrip("/")
+        )
+        validate_intranet_url(run_artifact_url)
         adapter_url = environment.get("EMBEDDING_ADAPTER_URL", "").strip() or None
         if adapter_url:
             validate_intranet_url(adapter_url)
         return cls(
             internal_token=required(environment, "EMBEDDING_SERVER_INTERNAL_TOKEN"),
             artifact_url=artifact_url,
+            run_artifact_url=run_artifact_url,
             adapter_url=adapter_url,
             adapter_token=environment.get("EMBEDDING_ADAPTER_TOKEN", "").strip()
             or None,
@@ -60,6 +75,18 @@ class EmbeddingConfig:
                 )
             ),
             max_units=positive_int(environment, "EMBEDDING_MAX_UNITS", 256),
+            run_store_path=environment.get(
+                "EMBEDDING_RUN_STORE_PATH", "/app/data/embedding/runs.sqlite3"
+            ).strip(),
+            run_encryption_secret=required(
+                environment, "EMBEDDING_RUN_ENCRYPTION_SECRET"
+            ),
+            max_concurrent_runs=positive_int(
+                environment, "EMBEDDING_MAX_CONCURRENT_RUNS", 1
+            ),
+            terminal_ttl_seconds=positive_int(
+                environment, "EMBEDDING_RUN_TERMINAL_TTL_SECONDS", 7 * 24 * 60 * 60
+            ),
         )
 
 

package/template/apps/embedding-server/src/southwind_embedding_server/image_input.py

@@ -4,6 +4,7 @@ import asyncio
 import base64
 import hashlib
 from io import BytesIO
+from urllib.parse import quote
 
 import httpx
 from PIL import Image, UnidentifiedImageError
@@ -20,6 +21,8 @@ async def load_image(
     config: EmbeddingConfig,
     max_bytes: int,
     max_pixels: int,
+    run_id: str | None = None,
+    item_id: str | None = None,
 ) -> tuple[str, str]:
     if source.media_type.lower() not in _IMAGE_TYPES:
         raise invalid("Embedding 图片格式不受支持")
@@ -32,13 +35,21 @@ async def load_image(
                 connect=config.connect_timeout_seconds,
             )
         ) as client:
+            url = config.artifact_url
+            headers = {
+                "authorization": f"Bearer {config.internal_token}",
+                "x-ai-artifact-ref": source.value,
+            }
+            if run_id and item_id:
+                url = (
+                    f"{config.run_artifact_url}/{quote(run_id, safe='')}"
+                    f"/artifacts/{quote(item_id, safe='')}"
+                )
+                headers.pop("x-ai-artifact-ref")
             async with client.stream(
                 "GET",
-                config.artifact_url,
-                headers={
-                    "authorization": f"Bearer {config.internal_token}",
-                    "x-ai-artifact-ref": source.value,
-                },
+                url,
+                headers=headers,
             ) as response:
                 response.raise_for_status()
                 async for chunk in response.aiter_bytes():

package/template/apps/embedding-server/src/southwind_embedding_server/run_cipher.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from .contracts import JsonValue
+
+
+class EmbeddingRunCipher:
+    def __init__(self, secret: str) -> None:
+        if len(secret.encode()) < 32:
+            raise ValueError("EMBEDDING_RUN_ENCRYPTION_SECRET 至少需要 32 字节")
+        self._key = hashlib.sha256(secret.encode()).digest()
+
+    def seal(self, value: JsonValue) -> str:
+        nonce = os.urandom(12)
+        body = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode()
+        encrypted = AESGCM(self._key).encrypt(nonce, body, b"embedding-run-v1")
+        return base64.urlsafe_b64encode(nonce + encrypted).decode()
+
+    def open(self, value: str) -> JsonValue:
+        try:
+            raw = base64.urlsafe_b64decode(value.encode())
+            body = AESGCM(self._key).decrypt(raw[:12], raw[12:], b"embedding-run-v1")
+            return json.loads(body)  # type: ignore[no-any-return]
+        except Exception as error:
+            raise ValueError("Embedding Run 加密 invocation 无法解密") from error