create-windy 0.5.2 → 0.6.0

package/template/apps/document-server/src/southwind_document_server/store_recovery.py

@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from collections.abc import Callable
+from contextlib import AbstractContextManager
+
+from .contracts import JsonValue, RunStatus
+from .store_models import (
+    CleanupStatus,
+    RunConflictError,
+    RunRecord,
+    row_to_record,
+)
+
+type Transaction = Callable[[], AbstractContextManager[None]]
+type RequireRow = Callable[[str], RunRecord]
+type TerminalLocked = Callable[
+    [
+        str,
+        RunStatus,
+        str,
+        dict[str, object] | None,
+        JsonValue,
+        CleanupStatus | None,
+    ],
+    tuple[RunRecord, bool],
+]
+
+
+class RecoveryRunStore:
+    def __init__(
+        self,
+        database: sqlite3.Connection,
+        transaction: Transaction,
+        require_row: RequireRow,
+        terminal_locked: TerminalLocked,
+    ) -> None:
+        self.database = database
+        self.transaction = transaction
+        self.require_row = require_row
+        self.terminal_locked = terminal_locked
+
+    def reconcile_startup(
+        self,
+        now: float,
+        timeout_seconds: float,
+        error: dict[str, object],
+    ) -> None:
+        with self.transaction():
+            rows = self.database.execute(
+                """
+                select run_id from document_runs
+                where status in ('accepted', 'running')
+                order by created_at, run_id
+                """
+            ).fetchall()
+            for row in rows:
+                run_id = str(row["run_id"])
+                record = self.require_row(run_id)
+                if not record.has_output_sink:
+                    self.terminal_locked(
+                        run_id,
+                        "failed",
+                        "run.failed",
+                        {"error": error},
+                        None,
+                        None,
+                    )
+                else:
+                    self.database.execute(
+                        """
+                        update document_runs
+                        set recovery_required = 1,
+                            recovery_deadline = coalesce(recovery_deadline, ?),
+                            updated_at = ?
+                        where run_id = ? and status in ('accepted', 'running')
+                        """,
+                        (now + timeout_seconds, now, run_id),
+                    )
+
+    def begin(
+        self,
+        run_id: str,
+        now: float,
+        timeout_seconds: float,
+    ) -> RunRecord:
+        with self.transaction():
+            record = self.require_row(run_id)
+            if record.terminal or not record.has_output_sink:
+                raise RunConflictError
+            self.database.execute(
+                """
+                update document_runs
+                set recovery_required = 1,
+                    recovery_deadline = coalesce(recovery_deadline, ?),
+                    updated_at = ?
+                where run_id = ?
+                """,
+                (now + timeout_seconds, now, run_id),
+            )
+            return self.require_row(run_id)
+
+    def require_redispatch(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+        *,
+        allow_same_generation: bool,
+    ) -> RunRecord:
+        with self.transaction():
+            record = self.require_row(run_id)
+            if (
+                record.terminal
+                or not record.has_output_sink
+                or not record.recovery_required
+                or record.recovery_deadline is None
+                or record.recovery_deadline <= now
+                or generation < record.grant_generation
+                or (generation == record.grant_generation and not allow_same_generation)
+            ):
+                raise RunConflictError
+            return record
+
+    def commit_redispatch(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+        *,
+        allow_same_generation: bool,
+    ) -> RunRecord:
+        with self.transaction():
+            record = self.require_row(run_id)
+            if (
+                record.terminal
+                or not record.recovery_required
+                or record.recovery_deadline is None
+                or record.recovery_deadline <= now
+                or generation < record.grant_generation
+                or (generation == record.grant_generation and not allow_same_generation)
+            ):
+                raise RunConflictError
+            if generation > record.grant_generation:
+                self.database.execute(
+                    """
+                    update document_runs
+                    set grant_generation = ?, updated_at = ?
+                    where run_id = ?
+                    """,
+                    (generation, now, run_id),
+                )
+            return self.require_row(run_id)
+
+    def clear(self, run_id: str, generation: int, now: float) -> RunRecord:
+        with self.transaction():
+            record = self.require_row(run_id)
+            if (
+                record.terminal
+                or not record.recovery_required
+                or record.grant_generation != generation
+                or record.recovery_deadline is None
+                or record.recovery_deadline <= now
+            ):
+                raise RunConflictError
+            self.database.execute(
+                """
+                update document_runs
+                set recovery_required = 0, recovery_deadline = null,
+                    updated_at = ?
+                where run_id = ?
+                """,
+                (now, run_id),
+            )
+            return self.require_row(run_id)
+
+    def expire(
+        self,
+        now: float,
+        error: dict[str, object],
+    ) -> list[RunRecord]:
+        expired: list[RunRecord] = []
+        with self.transaction():
+            rows = self.database.execute(
+                """
+                select run_id from document_runs
+                where status in ('accepted', 'running')
+                  and recovery_required = 1
+                  and recovery_deadline <= ?
+                order by recovery_deadline, run_id
+                """,
+                (now,),
+            ).fetchall()
+            for row in rows:
+                record, transitioned = self.terminal_locked(
+                    str(row["run_id"]),
+                    "failed",
+                    "run.failed",
+                    {"error": error},
+                    None,
+                    "pending-host-ttl",
+                )
+                if transitioned:
+                    expired.append(record)
+        return expired
+
+    def pending_cleanups(self) -> list[RunRecord]:
+        with self.transaction():
+            return [
+                row_to_record(row)
+                for row in self.database.execute(
+                    """
+                    select * from document_runs
+                    where status in ('failed', 'cancelled')
+                      and has_output_sink = 1
+                      and cleanup_status = 'pending-host-ttl'
+                    order by updated_at, run_id
+                    """
+                )
+            ]
+
+    def mark_cleanup(
+        self,
+        run_id: str,
+        generation: int,
+        status: CleanupStatus,
+    ) -> RunRecord:
+        with self.transaction():
+            record = self.require_row(run_id)
+            if (
+                not record.terminal
+                or not record.has_output_sink
+                or record.grant_generation != generation
+            ):
+                raise RunConflictError
+            if record.cleanup_status == "deleted":
+                return record
+            self.database.execute(
+                """
+                update document_runs
+                set cleanup_status = ?, updated_at = ?
+                where run_id = ?
+                """,
+                (status, time.time(), run_id),
+            )
+            return self.require_row(run_id)

package/template/apps/document-server/src/southwind_document_server/store_schema.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import sqlite3
+
+
+def initialize_store(database: sqlite3.Connection) -> None:
+    database.executescript(
+        """
+        create table if not exists document_runs (
+          run_id text primary key,
+          operation_key text not null,
+          idempotency_key text not null unique,
+          fingerprint text not null,
+          status text not null,
+          output_json text not null,
+          has_output_sink integer not null default 0,
+          output_scope text,
+          grant_generation integer not null default 0,
+          recovery_required integer not null default 0,
+          recovery_deadline real,
+          cleanup_status text not null default 'none',
+          created_at real not null,
+          updated_at real not null
+        );
+        create table if not exists document_events (
+          run_id text not null
+            references document_runs(run_id) on delete cascade,
+          sequence integer not null,
+          event_json text not null,
+          primary key (run_id, sequence)
+        );
+        """
+    )
+    columns = {
+        str(row["name"]) for row in database.execute("pragma table_info(document_runs)")
+    }
+    additions = {
+        "has_output_sink": "integer not null default 0",
+        "output_scope": "text",
+        "grant_generation": "integer not null default 0",
+        "recovery_required": "integer not null default 0",
+        "recovery_deadline": "real",
+        "cleanup_status": "text not null default 'none'",
+    }
+    for column, definition in additions.items():
+        if column not in columns:
+            database.execute(
+                f"alter table document_runs add column {column} {definition}"
+            )
+    database.commit()

package/template/apps/document-server/tests/recovery_support.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Sequence
+
+from southwind_document_server.adapter import DocumentAdapter
+from southwind_document_server.adapter_errors import SafeDocumentError
+from southwind_document_server.artifacts import AdapterExecution, ProducedArtifact
+from southwind_document_server.contracts import (
+    OutputArtifactReference,
+    OutputArtifactSinkDescriptor,
+    StartDocumentRunRequest,
+)
+from southwind_document_server.fingerprint import document_request_fingerprint
+from southwind_document_server.grants import OutputGrantCell
+from southwind_document_server.sink import OutputArtifactSinkClient, SinkClientError
+from southwind_document_server.store import CleanupStatus, DocumentRunStore, RunRecord
+
+from .support import RUN_ID, document_output, runtime_config
+
+
+class FakeClock:
+    def __init__(self, now: float) -> None:
+        self.now = now
+
+    def __call__(self) -> float:
+        return self.now
+
+
+class RecoveryAdapter(DocumentAdapter):
+    def __init__(self) -> None:
+        super().__init__(runtime_config())
+        self.executions = 0
+
+    async def execute(self, request: StartDocumentRunRequest) -> AdapterExecution:
+        self.executions += 1
+        return AdapterExecution(document_output())
+
+
+class LateFailAdapter(RecoveryAdapter):
+    def __init__(self) -> None:
+        super().__init__()
+        self.started = asyncio.Event()
+        self.release = asyncio.Event()
+
+    async def execute(self, request: StartDocumentRunRequest) -> AdapterExecution:
+        self.executions += 1
+        self.started.set()
+        await self.release.wait()
+        raise SafeDocumentError("RUNTIME_UNAVAILABLE", "late failure", True)
+
+
+class ImmediateFailAdapter(RecoveryAdapter):
+    async def execute(self, request: StartDocumentRunRequest) -> AdapterExecution:
+        self.executions += 1
+        raise SafeDocumentError("RUNTIME_UNAVAILABLE", "immediate failure", True)
+
+
+class RecoverySink(OutputArtifactSinkClient):
+    def __init__(
+        self,
+        grants: OutputGrantCell,
+        *,
+        ambiguous_accept: bool = False,
+    ) -> None:
+        super().__init__(runtime_config(), grants)
+        self.grants = grants
+        self.ambiguous_accept = ambiguous_accept
+        self.accepted: list[int] = []
+        self.aborted = 0
+        self.aborted_generations: list[int] = []
+
+    async def accept(self, sink: OutputArtifactSinkDescriptor) -> None:
+        await self.grants.token_for(sink.scope, sink.grant_generation)
+        if self.ambiguous_accept:
+            raise SinkClientError(ambiguous=True)
+        self.accepted.append(sink.grant_generation)
+
+    async def publish(
+        self,
+        sink: OutputArtifactSinkDescriptor,
+        artifacts: Sequence[ProducedArtifact],
+    ) -> list[OutputArtifactReference]:
+        await self.grants.token(sink.scope)
+        return []
+
+    async def abort(self, sink: OutputArtifactSinkDescriptor) -> bool:
+        return await self.abort_scope(sink.scope, sink.grant_generation)
+
+    async def abort_scope(self, scope: str, generation: int) -> bool:
+        await self.grants.token_for(scope, generation)
+        self.aborted += 1
+        self.aborted_generations.append(generation)
+        return True
+
+
+class BlockingAckSink(RecoverySink):
+    def __init__(self, grants: OutputGrantCell) -> None:
+        super().__init__(grants)
+        self.ack_started = asyncio.Event()
+
+    async def accept(self, sink: OutputArtifactSinkDescriptor) -> None:
+        await self.grants.token_for(sink.scope, sink.grant_generation)
+        self.ack_started.set()
+        await asyncio.Event().wait()
+
+
+class DeadlineAdvancingSink(RecoverySink):
+    def __init__(self, grants: OutputGrantCell, clock: FakeClock) -> None:
+        super().__init__(grants)
+        self.clock = clock
+
+    async def accept(self, sink: OutputArtifactSinkDescriptor) -> None:
+        await super().accept(sink)
+        self.clock.now = 106.0
+
+
+class ConcurrentAckSink(RecoverySink):
+    def __init__(self, grants: OutputGrantCell) -> None:
+        super().__init__(grants, ambiguous_accept=True)
+        self.concurrent_mode = False
+        self.arrivals = 0
+        self.both_arrived = asyncio.Event()
+
+    async def accept(self, sink: OutputArtifactSinkDescriptor) -> None:
+        if not self.concurrent_mode:
+            await super().accept(sink)
+            return
+        await self.grants.token_for(sink.scope, sink.grant_generation)
+        self.arrivals += 1
+        if self.arrivals == 2:
+            self.both_arrived.set()
+        await self.both_arrived.wait()
+
+
+class StaggeredAckSink(RecoverySink):
+    def __init__(self, grants: OutputGrantCell) -> None:
+        super().__init__(grants, ambiguous_accept=True)
+        self.concurrent_mode = False
+        self.arrivals = 0
+        self.both_arrived = asyncio.Event()
+        self.release_second = asyncio.Event()
+
+    async def accept(self, sink: OutputArtifactSinkDescriptor) -> None:
+        if not self.concurrent_mode:
+            await super().accept(sink)
+            return
+        await self.grants.token_for(sink.scope, sink.grant_generation)
+        self.arrivals += 1
+        if self.arrivals == 1:
+            await self.both_arrived.wait()
+            return
+        self.both_arrived.set()
+        await self.release_second.wait()
+
+
+class CleanupMarkFailingStore(DocumentRunStore):
+    def __init__(self) -> None:
+        super().__init__(":memory:")
+        self.fail_cleanup_mark = True
+
+    def mark_cleanup(
+        self,
+        run_id: str,
+        generation: int,
+        status: CleanupStatus,
+    ) -> RunRecord:
+        if self.fail_cleanup_mark:
+            self.fail_cleanup_mark = False
+            raise RuntimeError("cleanup persistence unavailable")
+        return super().mark_cleanup(run_id, generation, status)
+
+
+class CommitRecoveryFailingStore(DocumentRunStore):
+    def __init__(self) -> None:
+        super().__init__(":memory:")
+        self.fail_commit = True
+
+    def commit_recovery_redispatch(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+        *,
+        allow_same_generation: bool,
+    ) -> RunRecord:
+        if self.fail_commit:
+            self.fail_commit = False
+            raise RuntimeError("generation persistence unavailable")
+        return super().commit_recovery_redispatch(
+            run_id,
+            generation,
+            now,
+            allow_same_generation=allow_same_generation,
+        )
+
+
+class BeginRecoveryFailingStore(DocumentRunStore):
+    def __init__(self) -> None:
+        super().__init__(":memory:")
+        self.fail_begin_recovery = True
+
+    def begin_recovery(
+        self,
+        run_id: str,
+        now: float,
+        recovery_timeout_seconds: float,
+    ) -> RunRecord:
+        if self.fail_begin_recovery:
+            self.fail_begin_recovery = False
+            raise RuntimeError("recovery persistence unavailable")
+        return super().begin_recovery(run_id, now, recovery_timeout_seconds)
+
+
+def seed_active_run(
+    store: DocumentRunStore,
+    request: StartDocumentRunRequest,
+    idempotency_key: str,
+) -> None:
+    sink = request.output_artifact_sink
+    store.create(
+        RUN_ID,
+        request.operation_key,
+        idempotency_key,
+        document_request_fingerprint(request, RUN_ID),
+        sink.scope if sink else None,
+    )
+    if sink:
+        store.refresh_grant_generation(RUN_ID, sink.grant_generation)
+    store.mark_running(RUN_ID)