create-windy 0.5.1 → 0.6.0

package/template/apps/document-server/tests/test_recovery.py

@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import pytest
+
+from southwind_document_server.adapter_errors import SafeDocumentError
+from southwind_document_server.grants import OutputGrantCell
+from southwind_document_server.service import DocumentRunService
+from southwind_document_server.store import DocumentRunStore, RunConflictError
+
+from .recovery_support import FakeClock, RecoveryAdapter, RecoverySink, seed_active_run
+from .support import (
+    RAW_GRANT,
+    RUN_ID,
+    document_request,
+    sink_descriptor,
+)
+
+
+def test_legacy_restart_fails_once_with_stable_runtime_error() -> None:
+    store = DocumentRunStore(":memory:")
+    request = document_request()
+    seed_active_run(store, request, "legacy-restart")
+
+    DocumentRunService(RecoveryAdapter(), store, clock=FakeClock(100.0))
+    DocumentRunService(RecoveryAdapter(), store, clock=FakeClock(110.0))
+
+    record = store.get(RUN_ID)
+    assert record is not None
+    assert record.status == "failed"
+    events = store.events_after(RUN_ID, 0)
+    assert [event["type"] for event in events] == ["run.started", "run.failed"]
+    error = events[-1]["error"]
+    assert isinstance(error, dict)
+    assert error["code"] == "RUNTIME_UNAVAILABLE"
+    assert error["requestId"] == "document_recovery"
+
+
+async def test_restart_redispatch_requires_higher_generation_and_fixed_deadline(
+    tmp_path: Path,
+) -> None:
+    database = tmp_path / "redispatch.sqlite3"
+    request = document_request(sink=sink_descriptor())
+    seeded = DocumentRunStore(str(database))
+    seed_active_run(seeded, request, "redispatch")
+    seeded.close()
+    clock = FakeClock(100.0)
+
+    first_grants = OutputGrantCell()
+    first = DocumentRunService(
+        RecoveryAdapter(),
+        DocumentRunStore(str(database)),
+        RecoverySink(first_grants),
+        first_grants,
+        clock=clock,
+        recovery_timeout_seconds=30.0,
+    )
+    recovering = first.require(RUN_ID)
+    assert recovering.recovery_required is True
+    assert recovering.recovery_deadline == 130.0
+    with pytest.raises(RunConflictError):
+        await first.start(request, "redispatch", RAW_GRANT)
+    first.store.refresh_grant_generation(RUN_ID, 2)
+    lower = request.model_copy(
+        update={"output_artifact_sink": sink_descriptor(generation=1)}
+    )
+    with pytest.raises(RunConflictError):
+        await first.start(lower, "redispatch", RAW_GRANT)
+    same = request.model_copy(
+        update={"output_artifact_sink": sink_descriptor(generation=2)}
+    )
+    with pytest.raises(RunConflictError):
+        await first.start(same, "redispatch", f"outgrant_v1_{'B' * 43}")
+
+    changed = request.model_copy(
+        update={
+            "options": {"outputs": ["tables"]},
+            "output_artifact_sink": sink_descriptor(generation=3),
+        }
+    )
+    with pytest.raises(RunConflictError):
+        await first.start(changed, "redispatch", f"outgrant_v1_{'B' * 43}")
+
+    clock.now = 110.0
+    next_grants = OutputGrantCell()
+    resumed = DocumentRunService(
+        RecoveryAdapter(),
+        DocumentRunStore(str(database)),
+        RecoverySink(next_grants),
+        next_grants,
+        clock=clock,
+        recovery_timeout_seconds=30.0,
+    )
+    unchanged = resumed.require(RUN_ID)
+    assert unchanged.recovery_deadline == 130.0
+    next_request = request.model_copy(
+        update={"output_artifact_sink": sink_descriptor(generation=3)}
+    )
+    record = await resumed.start(
+        next_request,
+        "redispatch",
+        f"outgrant_v1_{'B' * 43}",
+    )
+    assert record.recovery_required is False
+    await asyncio.gather(*resumed.tasks.values())
+    assert resumed.require(RUN_ID).status == "completed"
+
+
+async def test_background_expiry_without_credential_records_host_ttl(
+    tmp_path: Path,
+) -> None:
+    database = tmp_path / "ttl-cleanup.sqlite3"
+    request = document_request(sink=sink_descriptor())
+    seeded = DocumentRunStore(str(database))
+    seed_active_run(seeded, request, "ttl-cleanup")
+    seeded.close()
+    clock = FakeClock(10.0)
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants)
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        DocumentRunStore(str(database)),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=5.0,
+        reconciliation_interval_seconds=0.001,
+    )
+
+    service.start_reconciler()
+    clock.now = 16.0
+    for _attempt in range(20):
+        if service.require(RUN_ID).terminal:
+            break
+        await asyncio.sleep(0.005)
+    await service.close()
+
+    record = service.require(RUN_ID)
+    assert record.status == "failed"
+    assert record.cleanup_status == "pending-host-ttl"
+    assert sink.aborted == 0
+    assert [event["type"] for event in service.store.events_after(RUN_ID, 0)] == [
+        "run.started",
+        "run.failed",
+    ]
+
+
+async def test_background_expiry_with_current_credential_deletes_scope() -> None:
+    clock = FakeClock(20.0)
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants, ambiguous_accept=True)
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=5.0,
+        reconciliation_interval_seconds=0.001,
+    )
+    request = document_request(sink=sink_descriptor())
+
+    with pytest.raises(SafeDocumentError):
+        await service.start(request, "ambiguous-ack", RAW_GRANT)
+    assert service.tasks == {}
+    recovering = service.require(RUN_ID)
+    assert recovering.recovery_deadline == 25.0
+
+    service.start_reconciler()
+    clock.now = 26.0
+    for _attempt in range(20):
+        if service.require(RUN_ID).terminal:
+            break
+        await asyncio.sleep(0.005)
+    await service.close()
+
+    record = service.require(RUN_ID)
+    assert record.status == "failed"
+    assert record.cleanup_status == "deleted"
+    assert sink.aborted == 1
+    wire_events = json.dumps(
+        service.store.events_after(RUN_ID, 0),
+        ensure_ascii=False,
+    )
+    assert "recoveryRequired" not in wire_events
+    assert "recoveryDeadline" not in wire_events
+    assert "cleanupStatus" not in wire_events
+
+
+async def test_transport_ambiguity_allows_only_exact_same_process_retry() -> None:
+    clock = FakeClock(40.0)
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants, ambiguous_accept=True)
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=10.0,
+    )
+    request = document_request(sink=sink_descriptor())
+
+    with pytest.raises(SafeDocumentError):
+        await service.start(request, "same-process", RAW_GRANT)
+    deadline = service.require(RUN_ID).recovery_deadline
+    with pytest.raises(RunConflictError):
+        await service.start(
+            request,
+            "same-process",
+            f"outgrant_v1_{'B' * 43}",
+        )
+    assert service.require(RUN_ID).recovery_deadline == deadline
+
+    sink.ambiguous_accept = False
+    resumed = await service.start(request, "same-process", RAW_GRANT)
+    assert resumed.recovery_required is False
+    await asyncio.gather(*service.tasks.values())
+    assert service.require(RUN_ID).status == "completed"

package/template/apps/document-server/tests/test_recovery_faults.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from southwind_document_server.adapter_errors import SafeDocumentError
+from southwind_document_server.grants import OutputGrantCell
+from southwind_document_server.service import DocumentRunService
+
+from .recovery_support import (
+    BeginRecoveryFailingStore,
+    BlockingAckSink,
+    CommitRecoveryFailingStore,
+    FakeClock,
+    RecoveryAdapter,
+    RecoverySink,
+)
+from .support import RAW_GRANT, RUN_ID, document_request, sink_descriptor
+
+
+async def test_generation_persistence_failure_enters_bounded_recovery() -> None:
+    clock = FakeClock(100.0)
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants)
+    store = CommitRecoveryFailingStore()
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        store,
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=10.0,
+        reconciliation_interval_seconds=0.001,
+    )
+    request = document_request(sink=sink_descriptor())
+
+    with pytest.raises(SafeDocumentError) as raised:
+        await service.start(request, "generation-failure", RAW_GRANT)
+
+    assert raised.value.code == "RUNTIME_UNAVAILABLE"
+    recovering = service.require(RUN_ID)
+    assert recovering.status == "accepted"
+    assert recovering.recovery_required is True
+    assert recovering.recovery_deadline == 110.0
+    assert await grants.generation(sink_descriptor().scope) is None
+    assert sink.accepted == []
+
+    clock.now = 111.0
+    service.start_reconciler()
+    for _attempt in range(20):
+        if service.require(RUN_ID).terminal:
+            break
+        await asyncio.sleep(0.005)
+    await service.close()
+    failed = service.require(RUN_ID)
+    assert failed.status == "failed"
+    assert failed.cleanup_status == "pending-host-ttl"
+
+
+async def test_ack_cancellation_preserves_cancel_when_recovery_write_fails() -> None:
+    clock = FakeClock(200.0)
+    grants = OutputGrantCell()
+    sink = BlockingAckSink(grants)
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        BeginRecoveryFailingStore(),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=10.0,
+        reconciliation_interval_seconds=0.001,
+    )
+    task = asyncio.create_task(
+        service.start(
+            document_request(sink=sink_descriptor()),
+            "cancel-recovery-failure",
+            RAW_GRANT,
+        )
+    )
+    await sink.ack_started.wait()
+    task.cancel()
+
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    recovering = service.require(RUN_ID)
+    assert recovering.status == "accepted"
+    assert recovering.recovery_required is True
+    assert recovering.recovery_deadline == 210.0
+    assert await grants.generation(sink_descriptor().scope) is None
+
+    clock.now = 211.0
+    service.start_reconciler()
+    for _attempt in range(20):
+        if service.require(RUN_ID).terminal:
+            break
+        await asyncio.sleep(0.005)
+    await service.close()
+    failed = service.require(RUN_ID)
+    assert failed.status == "failed"
+    assert failed.cleanup_status == "pending-host-ttl"

package/template/apps/document-server/tests/test_recovery_races.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+import pytest
+
+from southwind_document_server.adapter_errors import SafeDocumentError
+from southwind_document_server.grants import OutputGrantCell
+from southwind_document_server.service import DocumentRunService
+from southwind_document_server.store import DocumentRunStore, RunConflictError
+
+from .recovery_support import (
+    BlockingAckSink,
+    CleanupMarkFailingStore,
+    ConcurrentAckSink,
+    DeadlineAdvancingSink,
+    FakeClock,
+    ImmediateFailAdapter,
+    LateFailAdapter,
+    RecoveryAdapter,
+    RecoverySink,
+    StaggeredAckSink,
+    seed_active_run,
+)
+from .support import RAW_GRANT, RUN_ID, document_request, sink_descriptor
+
+
+async def test_concurrent_exact_recovery_retries_share_success() -> None:
+    grants = OutputGrantCell()
+    sink = ConcurrentAckSink(grants)
+    adapter = RecoveryAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+    )
+    request = document_request(sink=sink_descriptor())
+    with pytest.raises(SafeDocumentError):
+        await service.start(request, "concurrent-retry", RAW_GRANT)
+    sink.ambiguous_accept = False
+    sink.concurrent_mode = True
+
+    results = await asyncio.gather(
+        service.start(request, "concurrent-retry", RAW_GRANT),
+        service.start(request, "concurrent-retry", RAW_GRANT),
+        return_exceptions=True,
+    )
+
+    assert all(not isinstance(result, BaseException) for result in results)
+    tasks = list(service.tasks.values())
+    if tasks:
+        await asyncio.gather(*tasks)
+    assert service.require(RUN_ID).status == "completed"
+    assert adapter.executions == 1
+    assert sink.aborted_generations == []
+
+
+async def test_late_exact_recovery_ack_shares_completed_success() -> None:
+    grants = OutputGrantCell()
+    sink = StaggeredAckSink(grants)
+    adapter = RecoveryAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+    )
+    request = document_request(sink=sink_descriptor())
+    with pytest.raises(SafeDocumentError):
+        await service.start(request, "late-concurrent-retry", RAW_GRANT)
+    sink.ambiguous_accept = False
+    sink.concurrent_mode = True
+    retries = {
+        asyncio.create_task(service.start(request, "late-concurrent-retry", RAW_GRANT)),
+        asyncio.create_task(service.start(request, "late-concurrent-retry", RAW_GRANT)),
+    }
+
+    await sink.both_arrived.wait()
+    done, pending = await asyncio.wait(
+        retries,
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+    assert len(done) == 1
+    assert len(pending) == 1
+    first = done.pop().result()
+    assert first.recovery_required is False
+    tasks = list(service.tasks.values())
+    if tasks:
+        await asyncio.gather(*tasks)
+    assert service.require(RUN_ID).status == "completed"
+
+    sink.release_second.set()
+    results = await asyncio.gather(*pending, return_exceptions=True)
+    assert all(not isinstance(result, BaseException) for result in results)
+    record = service.require(RUN_ID)
+    assert record.status == "completed"
+    assert record.cleanup_status == "none"
+    assert adapter.executions == 1
+    assert sink.aborted_generations == []
+
+
+async def test_late_exact_recovery_ack_preserves_deleted_failure_cleanup() -> None:
+    grants = OutputGrantCell()
+    sink = StaggeredAckSink(grants)
+    adapter = ImmediateFailAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+    )
+    request = document_request(sink=sink_descriptor())
+    with pytest.raises(SafeDocumentError):
+        await service.start(request, "late-failed-retry", RAW_GRANT)
+    sink.ambiguous_accept = False
+    sink.concurrent_mode = True
+    retries = {
+        asyncio.create_task(service.start(request, "late-failed-retry", RAW_GRANT)),
+        asyncio.create_task(service.start(request, "late-failed-retry", RAW_GRANT)),
+    }
+
+    await sink.both_arrived.wait()
+    done, pending = await asyncio.wait(
+        retries,
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+    assert len(done) == 1
+    done.pop().result()
+    tasks = list(service.tasks.values())
+    if tasks:
+        await asyncio.gather(*tasks)
+    before = service.require(RUN_ID)
+    assert before.status == "failed"
+    assert before.cleanup_status == "deleted"
+
+    sink.release_second.set()
+    results = await asyncio.gather(*pending, return_exceptions=True)
+    assert len(results) == 1
+    assert isinstance(results[0], RunConflictError)
+    after = service.require(RUN_ID)
+    assert after.status == "failed"
+    assert after.cleanup_status == "deleted"
+    assert sink.aborted_generations == [1]
+
+
+async def test_running_generation_rotation_cleans_current_generation() -> None:
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants)
+    adapter = LateFailAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+    )
+    request = document_request(sink=sink_descriptor())
+    await service.start(request, "rotate-running", RAW_GRANT)
+    await adapter.started.wait()
+
+    rotated = request.model_copy(
+        update={"output_artifact_sink": sink_descriptor(generation=2)}
+    )
+    await service.start(
+        rotated,
+        "rotate-running",
+        f"outgrant_v1_{'B' * 43}",
+    )
+    adapter.release.set()
+    await asyncio.gather(*service.tasks.values())
+
+    record = service.require(RUN_ID)
+    assert record.status == "failed"
+    assert record.grant_generation == 2
+    assert record.cleanup_status == "deleted"
+    assert sink.aborted_generations == [2]
+    assert await grants.generation(sink_descriptor().scope) is None
+
+
+async def test_ack_deadline_race_cancels_task_and_records_deleted_cleanup(
+    tmp_path: Path,
+) -> None:
+    database = tmp_path / "ack-deadline.sqlite3"
+    request = document_request(sink=sink_descriptor())
+    seeded = DocumentRunStore(str(database))
+    seed_active_run(seeded, request, "ack-deadline")
+    seeded.close()
+    clock = FakeClock(100.0)
+    grants = OutputGrantCell()
+    sink = DeadlineAdvancingSink(grants, clock)
+    adapter = RecoveryAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(str(database)),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=5.0,
+    )
+    resumed = request.model_copy(
+        update={"output_artifact_sink": sink_descriptor(generation=2)}
+    )
+
+    with pytest.raises(RunConflictError):
+        await service.start(
+            resumed,
+            "ack-deadline",
+            f"outgrant_v1_{'B' * 43}",
+        )
+    await asyncio.sleep(0)
+
+    record = service.require(RUN_ID)
+    assert record.status == "failed"
+    assert record.cleanup_status == "deleted"
+    assert sink.aborted_generations == [2]
+    assert adapter.executions == 0
+
+
+async def test_ack_cancellation_enters_recovery_and_preserves_cancellation() -> None:
+    clock = FakeClock(200.0)
+    grants = OutputGrantCell()
+    sink = BlockingAckSink(grants)
+    adapter = RecoveryAdapter()
+    service = DocumentRunService(
+        adapter,
+        DocumentRunStore(":memory:"),
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=10.0,
+    )
+    task = asyncio.create_task(
+        service.start(
+            document_request(sink=sink_descriptor()),
+            "cancelled-ack",
+            RAW_GRANT,
+        )
+    )
+    await sink.ack_started.wait()
+    task.cancel()
+
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    record = service.require(RUN_ID)
+    assert record.status == "accepted"
+    assert record.recovery_required is True
+    assert record.recovery_deadline == 210.0
+    assert service.tasks == {}
+    assert adapter.executions == 0
+
+
+async def test_cleanup_mark_failure_keeps_credential_for_reconcile_retry() -> None:
+    clock = FakeClock(300.0)
+    grants = OutputGrantCell()
+    sink = RecoverySink(grants, ambiguous_accept=True)
+    store = CleanupMarkFailingStore()
+    service = DocumentRunService(
+        RecoveryAdapter(),
+        store,
+        sink,
+        grants,
+        clock=clock,
+        recovery_timeout_seconds=5.0,
+    )
+    with pytest.raises(SafeDocumentError):
+        await service.start(
+            document_request(sink=sink_descriptor()),
+            "cleanup-retry",
+            RAW_GRANT,
+        )
+    clock.now = 306.0
+
+    await service._recovery.reconcile_once()
+    first = service.require(RUN_ID)
+    assert first.cleanup_status == "pending-host-ttl"
+    assert await grants.generation(sink_descriptor().scope) == 1
+
+    await service._recovery.reconcile_once()
+    second = service.require(RUN_ID)
+    assert second.cleanup_status == "deleted"
+    assert await grants.generation(sink_descriptor().scope) is None
+    assert sink.aborted_generations == [1, 1]

package/template/apps/document-server/tests/test_service.py

@@ -7,6 +7,7 @@ from southwind_document_server.adapter import (
     SafeDocumentError,
     validate_output,
 )
+from southwind_document_server.artifacts import AdapterExecution
 from southwind_document_server.config import DocumentRuntimeConfig
 from southwind_document_server.contracts import (
     ArtifactInput,
@@ -33,19 +34,21 @@ class FakeAdapter(DocumentAdapter):
             )
         )
 
-    async def execute(self, request: StartDocumentRunRequest) -> JsonValue:
-        return {
-            "pages": [
-                {
-                    "pageNumber": 1,
-                    "text": "第一页",
-                    "blocks": [],
-                }
-            ],
-            "tables": [],
-            "artifacts": [],
-            "metadata": {"engine": "fake"},
-        }
+    async def execute(self, request: StartDocumentRunRequest) -> AdapterExecution:
+        return AdapterExecution(
+            {
+                "pages": [
+                    {
+                        "pageNumber": 1,
+                        "text": "第一页",
+                        "blocks": [],
+                    }
+                ],
+                "tables": [],
+                "artifacts": [],
+                "metadata": {"engine": "fake"},
+            }
+        )
 
 
 async def test_idempotent_run_emits_page_progress_and_terminal_result() -> None: