create-windy 0.5.1 → 0.6.0

package/template/apps/document-server/src/southwind_document_server/store.py

@@ -2,30 +2,26 @@ from __future__ import annotations
 
 import json
 import sqlite3
+import threading
 import time
-from dataclasses import dataclass
+from collections.abc import Iterator
+from contextlib import contextmanager
 from pathlib import Path
 from typing import cast
 
 from .contracts import JsonValue, RunStatus
+from .store_events import insert_event
+from .store_models import (
+    CleanupStatus,
+    RunConflictError,
+    RunRecord,
+    encode,
+    row_to_record,
+)
+from .store_recovery import RecoveryRunStore
+from .store_schema import initialize_store
 
-
-@dataclass(frozen=True)
-class RunRecord:
-    run_id: str
-    operation_key: str
-    idempotency_key: str
-    fingerprint: str
-    status: RunStatus
-    output: JsonValue = None
-
-    @property
-    def terminal(self) -> bool:
-        return self.status in {"completed", "failed", "cancelled"}
-
-
-class RunConflictError(Exception):
-    pass
+__all__ = ["CleanupStatus", "DocumentRunStore", "RunConflictError", "RunRecord"]
 
 
 class DocumentRunStore:
@@ -33,8 +29,15 @@ class DocumentRunStore:
         Path(path).parent.mkdir(parents=True, exist_ok=True)
         self.database = sqlite3.connect(path, check_same_thread=False)
         self.database.row_factory = sqlite3.Row
-        self._initialize()
-        self._recover()
+        self.database.execute("pragma foreign_keys = on")
+        self._lock = threading.RLock()
+        initialize_store(self.database)
+        self._recovery = RecoveryRunStore(
+            self.database,
+            self._transaction,
+            self._require_row,
+            self._terminal_locked,
+        )
 
     def create(
         self,
@@ -42,144 +45,301 @@ class DocumentRunStore:
         operation_key: str,
         idempotency_key: str,
         fingerprint: str,
+        output_scope: str | None,
+        recovery_deadline: float | None = None,
     ) -> tuple[RunRecord, bool]:
-        existing = self.database.execute(
-            "select run_id from document_runs where idempotency_key = ?",
-            (idempotency_key,),
-        ).fetchone()
-        if existing:
-            record = self.get(str(existing["run_id"]))
-            if not record or record.fingerprint != fingerprint:
-                raise RunConflictError
-            return record, False
-        now = time.time()
-        self.database.execute(
-            """
-            insert into document_runs values (?, ?, ?, ?, 'accepted', 'null', ?, ?)
-            """,
-            (run_id, operation_key, idempotency_key, fingerprint, now, now),
-        )
-        self.database.commit()
-        return self.get(run_id) or _missing(), True
+        with self._transaction():
+            row = self._row_by_idempotency(idempotency_key)
+            if row:
+                record = row_to_record(row)
+                if (
+                    record.run_id != run_id
+                    or record.operation_key != operation_key
+                    or record.fingerprint != fingerprint
+                    or record.output_scope != output_scope
+                    or record.has_output_sink != (output_scope is not None)
+                ):
+                    raise RunConflictError
+                return record, False
+            now = time.time()
+            self.database.execute(
+                """
+                insert into document_runs (
+                  run_id, operation_key, idempotency_key, fingerprint, status,
+                  output_json, has_output_sink, output_scope, grant_generation,
+                  recovery_required, recovery_deadline, created_at, updated_at
+                ) values (?, ?, ?, ?, 'accepted', 'null', ?, ?, 0, ?, ?, ?, ?)
+                """,
+                (
+                    run_id,
+                    operation_key,
+                    idempotency_key,
+                    fingerprint,
+                    int(output_scope is not None),
+                    output_scope,
+                    int(output_scope is not None and recovery_deadline is not None),
+                    recovery_deadline if output_scope is not None else None,
+                    now,
+                    now,
+                ),
+            )
+            return self._require_row(run_id), True
 
     def get(self, run_id: str) -> RunRecord | None:
-        row = self.database.execute(
-            "select * from document_runs where run_id = ?", (run_id,)
-        ).fetchone()
-        if not row:
-            return None
-        return RunRecord(
-            run_id=str(row["run_id"]),
-            operation_key=str(row["operation_key"]),
-            idempotency_key=str(row["idempotency_key"]),
-            fingerprint=str(row["fingerprint"]),
-            status=cast(RunStatus, str(row["status"])),
-            output=json.loads(str(row["output_json"])),
-        )
+        with self._lock:
+            row = self.database.execute(
+                "select * from document_runs where run_id = ?",
+                (run_id,),
+            ).fetchone()
+            return row_to_record(row) if row else None
 
-    def append(
-        self, run_id: str, event_type: str, fields: dict[str, object] | None = None
-    ) -> None:
-        record = self.get(run_id)
-        if not record or record.terminal:
-            return
-        sequence = (
-            int(
+    def get_by_idempotency(self, idempotency_key: str) -> RunRecord | None:
+        with self._lock:
+            row = self._row_by_idempotency(idempotency_key)
+            return row_to_record(row) if row else None
+
+    def refresh_grant_generation(self, run_id: str, generation: int) -> RunRecord:
+        with self._transaction():
+            record = self._require_row(run_id)
+            if (
+                not record.has_output_sink
+                or record.terminal
+                or generation < record.grant_generation
+            ):
+                raise RunConflictError
+            if generation > record.grant_generation:
                 self.database.execute(
-                    "select count(*) from document_events where run_id = ?", (run_id,)
-                ).fetchone()[0]
-            )
-            + 1
-        )
-        event = {
-            "type": event_type,
-            "runId": run_id,
-            "sequence": sequence,
-            **(fields or {}),
-        }
-        status = {
-            "run.started": "running",
-            "run.completed": "completed",
-            "run.failed": "failed",
-            "run.cancelled": "cancelled",
-        }.get(event_type, record.status)
-        self.database.execute(
-            "insert into document_events values (?, ?, ?)",
-            (run_id, sequence, encode(event)),
+                    """
+                    update document_runs
+                    set grant_generation = ?, updated_at = ?
+                    where run_id = ?
+                    """,
+                    (generation, time.time(), run_id),
+                )
+            return self._require_row(run_id)
+
+    def reconcile_startup(
+        self,
+        now: float,
+        recovery_timeout_seconds: float,
+        error: dict[str, object],
+    ) -> None:
+        self._recovery.reconcile_startup(
+            now,
+            recovery_timeout_seconds,
+            error,
         )
-        self.database.execute(
-            "update document_runs set status = ?, updated_at = ? where run_id = ?",
-            (status, time.time(), run_id),
+
+    def begin_recovery(
+        self,
+        run_id: str,
+        now: float,
+        recovery_timeout_seconds: float,
+    ) -> RunRecord:
+        return self._recovery.begin(run_id, now, recovery_timeout_seconds)
+
+    def require_recovery_redispatch(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+        *,
+        allow_same_generation: bool,
+    ) -> RunRecord:
+        return self._recovery.require_redispatch(
+            run_id,
+            generation,
+            now,
+            allow_same_generation=allow_same_generation,
         )
-        self.database.commit()
 
-    def complete(self, run_id: str, output: JsonValue) -> None:
-        self.database.execute(
-            "update document_runs set output_json = ? where run_id = ?",
-            (encode(output), run_id),
+    def clear_recovery(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+    ) -> RunRecord:
+        return self._recovery.clear(run_id, generation, now)
+
+    def commit_recovery_redispatch(
+        self,
+        run_id: str,
+        generation: int,
+        now: float,
+        *,
+        allow_same_generation: bool,
+    ) -> RunRecord:
+        return self._recovery.commit_redispatch(
+            run_id,
+            generation,
+            now,
+            allow_same_generation=allow_same_generation,
         )
-        self.database.commit()
-        self.append(run_id, "run.completed")
 
-    def events_after(self, run_id: str, after: int) -> list[dict[str, object]]:
-        return [
-            json.loads(str(row["event_json"]))
-            for row in self.database.execute(
+    def expire_recoveries(
+        self,
+        now: float,
+        error: dict[str, object],
+    ) -> list[RunRecord]:
+        return self._recovery.expire(now, error)
+
+    def pending_cleanups(self) -> list[RunRecord]:
+        return self._recovery.pending_cleanups()
+
+    def mark_cleanup(
+        self,
+        run_id: str,
+        generation: int,
+        status: CleanupStatus,
+    ) -> RunRecord:
+        return self._recovery.mark_cleanup(run_id, generation, status)
+
+    def mark_running(self, run_id: str) -> RunRecord:
+        with self._transaction():
+            record = self._require_row(run_id)
+            if record.status != "accepted":
+                return record
+            insert_event(self.database, run_id, "run.started")
+            self.database.execute(
                 """
-                select event_json from document_events
-                where run_id = ? and sequence > ? order by sequence
+                update document_runs
+                set status = 'running', updated_at = ?
+                where run_id = ? and status = 'accepted'
                 """,
-                (run_id, after),
+                (time.time(), run_id),
             )
-        ]
+            return self._require_row(run_id)
 
-    def _initialize(self) -> None:
-        self.database.executescript(
-            """
-            create table if not exists document_runs (
-              run_id text primary key,
-              operation_key text not null,
-              idempotency_key text not null unique,
-              fingerprint text not null,
-              status text not null,
-              output_json text not null,
-              created_at real not null,
-              updated_at real not null
-            );
-            create table if not exists document_events (
-              run_id text not null references document_runs(run_id) on delete cascade,
-              sequence integer not null,
-              event_json text not null,
-              primary key (run_id, sequence)
-            );
-            """
+    def append_progress(
+        self,
+        run_id: str,
+        event_type: str,
+        fields: dict[str, object],
+    ) -> bool:
+        with self._transaction():
+            record = self._require_row(run_id)
+            if record.terminal:
+                return False
+            insert_event(self.database, run_id, event_type, fields)
+            self.database.execute(
+                "update document_runs set updated_at = ? where run_id = ?",
+                (time.time(), run_id),
+            )
+            return True
+
+    def complete(
+        self,
+        run_id: str,
+        output: JsonValue,
+    ) -> tuple[RunRecord, bool]:
+        return self._terminal(run_id, "completed", "run.completed", None, output)
+
+    def fail(
+        self,
+        run_id: str,
+        error: dict[str, object],
+    ) -> tuple[RunRecord, bool]:
+        return self._terminal(
+            run_id,
+            "failed",
+            "run.failed",
+            {"error": error},
+            None,
         )
-        self.database.commit()
-
-    def _recover(self) -> None:
-        rows = self.database.execute(
-            "select run_id from document_runs where status in ('accepted', 'running')"
-        ).fetchall()
-        for row in rows:
-            self.append(
-                str(row["run_id"]),
-                "run.failed",
-                {
-                    "error": {
-                        "code": "RUNTIME_UNAVAILABLE",
-                        "message": (
-                            "Document Runtime 重启中断了任务，请使用新幂等键重试"
-                        ),
-                        "requestId": "document-recovery",
-                        "retryable": True,
-                    }
-                },
+
+    def cancel(self, run_id: str) -> tuple[RunRecord, bool]:
+        return self._terminal(run_id, "cancelled", "run.cancelled", None, None)
+
+    def events_after(self, run_id: str, after: int) -> list[dict[str, object]]:
+        with self._lock:
+            return [
+                json.loads(str(row["event_json"]))
+                for row in self.database.execute(
+                    """
+                    select event_json from document_events
+                    where run_id = ? and sequence > ? order by sequence
+                    """,
+                    (run_id, after),
+                )
+            ]
+
+    def close(self) -> None:
+        with self._lock:
+            self.database.close()
+
+    def _terminal(
+        self,
+        run_id: str,
+        status: RunStatus,
+        event_type: str,
+        fields: dict[str, object] | None,
+        output: JsonValue,
+    ) -> tuple[RunRecord, bool]:
+        with self._transaction():
+            return self._terminal_locked(
+                run_id,
+                status,
+                event_type,
+                fields,
+                output,
             )
 
+    def _terminal_locked(
+        self,
+        run_id: str,
+        status: RunStatus,
+        event_type: str,
+        fields: dict[str, object] | None,
+        output: JsonValue,
+        cleanup_status: CleanupStatus | None = None,
+    ) -> tuple[RunRecord, bool]:
+        record = self._require_row(run_id)
+        if record.terminal:
+            return record, False
+        insert_event(self.database, run_id, event_type, fields)
+        self.database.execute(
+            """
+            update document_runs
+            set status = ?, output_json = ?, recovery_required = 0,
+                cleanup_status = coalesce(?, cleanup_status), updated_at = ?
+            where run_id = ? and status in ('accepted', 'running')
+            """,
+            (
+                status,
+                encode(output),
+                cleanup_status,
+                time.time(),
+                run_id,
+            ),
+        )
+        return self._require_row(run_id), True
 
-def encode(value: object) -> str:
-    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+    def _row_by_idempotency(self, idempotency_key: str) -> sqlite3.Row | None:
+        return cast(
+            sqlite3.Row | None,
+            self.database.execute(
+                "select * from document_runs where idempotency_key = ?",
+                (idempotency_key,),
+            ).fetchone(),
+        )
 
+    def _require_row(self, run_id: str) -> RunRecord:
+        row = self.database.execute(
+            "select * from document_runs where run_id = ?",
+            (run_id,),
+        ).fetchone()
+        if not row:
+            raise RuntimeError("Document Run 不存在")
+        return row_to_record(row)
 
-def _missing() -> RunRecord:
-    raise RuntimeError("Document Run 创建后不可读")
+    @contextmanager
+    def _transaction(self) -> Iterator[None]:
+        with self._lock:
+            self.database.execute("begin immediate")
+            try:
+                yield
+            except Exception:
+                self.database.rollback()
+                raise
+            else:
+                self.database.commit()

package/template/apps/document-server/src/southwind_document_server/store_events.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import sqlite3
+
+from .store_models import encode
+
+
+def insert_event(
+    database: sqlite3.Connection,
+    run_id: str,
+    event_type: str,
+    fields: dict[str, object] | None = None,
+) -> None:
+    sequence = int(
+        database.execute(
+            """
+            select coalesce(max(sequence), 0) + 1
+            from document_events where run_id = ?
+            """,
+            (run_id,),
+        ).fetchone()[0]
+    )
+    database.execute(
+        """
+        insert into document_events (run_id, sequence, event_json)
+        values (?, ?, ?)
+        """,
+        (
+            run_id,
+            sequence,
+            encode(
+                {
+                    "type": event_type,
+                    "runId": run_id,
+                    "sequence": sequence,
+                    **(fields or {}),
+                }
+            ),
+        ),
+    )

package/template/apps/document-server/src/southwind_document_server/store_models.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from dataclasses import dataclass
+from typing import Literal, cast
+
+from .contracts import JsonValue, RunStatus
+
+TERMINAL_STATUSES = {"completed", "failed", "cancelled"}
+type CleanupStatus = Literal["none", "pending-host-ttl", "deleted"]
+
+
+@dataclass(frozen=True)
+class RunRecord:
+    run_id: str
+    operation_key: str
+    idempotency_key: str
+    fingerprint: str
+    status: RunStatus
+    output: JsonValue = None
+    has_output_sink: bool = False
+    output_scope: str | None = None
+    grant_generation: int = 0
+    recovery_required: bool = False
+    recovery_deadline: float | None = None
+    cleanup_status: CleanupStatus = "none"
+
+    @property
+    def terminal(self) -> bool:
+        return self.status in TERMINAL_STATUSES
+
+
+class RunConflictError(Exception):
+    pass
+
+
+def row_to_record(row: sqlite3.Row) -> RunRecord:
+    return RunRecord(
+        run_id=str(row["run_id"]),
+        operation_key=str(row["operation_key"]),
+        idempotency_key=str(row["idempotency_key"]),
+        fingerprint=str(row["fingerprint"]),
+        status=cast(RunStatus, str(row["status"])),
+        output=json.loads(str(row["output_json"])),
+        has_output_sink=bool(row["has_output_sink"]),
+        output_scope=(
+            str(row["output_scope"]) if row["output_scope"] is not None else None
+        ),
+        grant_generation=int(row["grant_generation"]),
+        recovery_required=bool(row["recovery_required"]),
+        recovery_deadline=(
+            float(row["recovery_deadline"])
+            if row["recovery_deadline"] is not None
+            else None
+        ),
+        cleanup_status=cast(CleanupStatus, str(row["cleanup_status"])),
+    )
+
+
+def encode(value: object) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))