create-windy 0.4.1 → 0.5.0

package/template/apps/agent-server/tests/test_report_resilience.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import json
+from typing import Literal, cast
+
+import httpx
+import pytest
+
+from southwind_agent_server.app import create_app
+from southwind_agent_server.contracts import StartRunRequest
+from southwind_agent_server.deterministic import DeterministicEngineAdapter
+from southwind_agent_server.engine import EngineCompleted
+from southwind_agent_server.openai_engine import OpenAiCompatibleEngineAdapter
+from southwind_agent_server.report_store import InMemoryAgentRuntimeReportStore
+from southwind_agent_server.report_types import (
+    AttemptReport,
+    AttemptTerminalOutcome,
+    UsageFacts,
+)
+from southwind_agent_server.tool_host import ToolHostClient
+from tests.openai_provider_test_support import (
+    create_provider,
+    governance,
+    runtime_request,
+    sse_response,
+)
+
+
+async def test_report_store_outage_does_not_rewrite_canonical_terminal() -> None:
+    report_store = UnavailableRunReportStore()
+    application = create_app(
+        DeterministicEngineAdapter(),
+        heartbeat_seconds=0.01,
+        report_store=report_store,
+    )
+    async with httpx.AsyncClient(
+        transport=httpx.ASGITransport(app=application),
+        base_url="http://agent.test",
+    ) as client:
+        response = await client.post(
+            "/api/agent/runs",
+            headers={"Idempotency-Key": "report-store-outage"},
+            json={"operationKey": "test.echo", "input": {"text": "completed"}},
+        )
+        run_id = str(cast(dict[str, object], response.json())["runId"])
+        event_response = await client.get(f"/api/agent/runs/{run_id}/events")
+
+    emitted = events(event_response)
+    assert emitted[-1]["type"] == "run.completed"
+    assert "run.failed" not in {event["type"] for event in emitted}
+    rendered = report_store.metrics.render()
+    for stage in ("run_ensure", "run_start", "run_terminal"):
+        assert (
+            f'southwind_agent_runtime_reporting_failures_total{{stage="{stage}"}} 1'
+            in rendered
+        )
+
+
+@pytest.mark.parametrize("failure_stage", ["attempt_start", "attempt_terminal"])
+async def test_report_store_outage_does_not_block_model_request(
+    failure_stage: Literal["attempt_start", "attempt_terminal"],
+) -> None:
+    provider_calls = 0
+
+    async def provider_handler(_request: httpx.Request) -> httpx.Response:
+        nonlocal provider_calls
+        provider_calls += 1
+        return sse_response([{"choices": [{"delta": {"content": "完成"}}]}])
+
+    report_store = FailingAttemptReportStore(failure_stage)
+    engine = OpenAiCompatibleEngineAdapter(
+        create_provider(httpx.MockTransport(provider_handler)),
+        ToolHostClient("internal-token"),
+        governance(),
+        report_store,
+    )
+
+    signals = [
+        signal
+        async for signal in engine.execute(runtime_request("business.records.list"))
+    ]
+
+    assert provider_calls == 1
+    assert isinstance(signals[-1], EngineCompleted)
+    assert (
+        "southwind_agent_runtime_reporting_failures_total"
+        f'{{stage="{failure_stage}"}} 1' in report_store.metrics.render()
+    )
+
+
+class UnavailableRunReportStore(InMemoryAgentRuntimeReportStore):
+    async def ensure_run(
+        self,
+        run_id: str,
+        request: StartRunRequest,
+        created_at: float,
+    ) -> None:
+        raise RuntimeError("report store unavailable")
+
+    async def start_run(self, run_id: str, started_at: float) -> None:
+        raise RuntimeError("report store unavailable")
+
+    async def set_run_status(
+        self,
+        run_id: str,
+        status: Literal["running", "awaiting-approval"],
+        updated_at: float,
+    ) -> None:
+        raise RuntimeError("report store unavailable")
+
+    async def finish_run(
+        self,
+        run_id: str,
+        status: Literal["completed", "failed", "cancelled"],
+        finished_at: float,
+        safe_error_code: str | None = None,
+    ) -> None:
+        raise RuntimeError("report store unavailable")
+
+
+class FailingAttemptReportStore(InMemoryAgentRuntimeReportStore):
+    def __init__(
+        self,
+        failure_stage: Literal["attempt_start", "attempt_terminal"],
+    ) -> None:
+        super().__init__()
+        self.failure_stage = failure_stage
+
+    async def start_attempt(
+        self,
+        run_id: str,
+        tool_round: int,
+        deployment_key: str,
+        requested_model: str,
+        started_at: float,
+    ) -> AttemptReport:
+        if self.failure_stage == "attempt_start":
+            raise RuntimeError("report store unavailable")
+        return await super().start_attempt(
+            run_id,
+            tool_round,
+            deployment_key,
+            requested_model,
+            started_at,
+        )
+
+    async def finish_attempt(
+        self,
+        attempt_id: str,
+        outcome: AttemptTerminalOutcome,
+        usage: UsageFacts,
+        finished_at: float,
+        reported_model: str | None = None,
+        safe_error_code: str | None = None,
+    ) -> None:
+        if self.failure_stage == "attempt_terminal":
+            raise RuntimeError("report store unavailable")
+        await super().finish_attempt(
+            attempt_id,
+            outcome,
+            usage,
+            finished_at,
+            reported_model,
+            safe_error_code,
+        )
+
+
+def events(response: httpx.Response) -> list[dict[str, object]]:
+    result: list[dict[str, object]] = []
+    for frame in response.text.split("\n\n"):
+        data = next(
+            (
+                line.removeprefix("data: ")
+                for line in frame.splitlines()
+                if line.startswith("data: ")
+            ),
+            None,
+        )
+        if data is not None:
+            result.append(json.loads(data))
+    return result

package/template/apps/agent-server/tests/test_report_store.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+from typing import Literal
+
+import pytest
+
+from southwind_agent_server.contracts import (
+    RunReportingContext,
+    StartRunRequest,
+)
+from southwind_agent_server.postgres_report_insert import (
+    RUN_INSERT_SQL,
+    run_insert_values,
+)
+from southwind_agent_server.report_lifecycle import sync_status
+from southwind_agent_server.report_store import InMemoryAgentRuntimeReportStore
+from southwind_agent_server.report_types import ReportFilter, UsageFacts
+
+
+def request(actor_id: str) -> StartRunRequest:
+    return StartRunRequest(
+        operation_key="business.answer",
+        input={"secret": "must-not-be-reported"},
+        reporting=RunReportingContext(
+            actor_id=actor_id,
+            actor_type="user",
+            department_id="department_1",
+        ),
+    )
+
+
+def test_postgres_run_insert_binds_every_persisted_timestamp() -> None:
+    values = run_insert_values("run_1", request("actor_1"), 100.0, 90)
+
+    assert RUN_INSERT_SQL.count("%s") == len(values)
+    assert values[-3:] == (100.0, 100.0, 7_776_100.0)
+
+
+async def test_summary_rates_exclude_cancelled_and_unknown_cache() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    values: tuple[tuple[str, str, Literal["completed", "failed", "cancelled"]], ...] = (
+        ("run_completed", "actor_1", "completed"),
+        ("run_failed", "actor_2", "failed"),
+        ("run_cancelled", "actor_3", "cancelled"),
+    )
+    for run_id, actor_id, status in values:
+        await store.ensure_run(run_id, request(actor_id), 100.0)
+        await store.start_run(run_id, 101.0)
+        await store.finish_run(run_id, status, 102.0)
+
+    attempts = (
+        ("run_completed", UsageFacts(cache_status="hit")),
+        ("run_failed", UsageFacts(cache_status="miss")),
+        ("run_cancelled", UsageFacts()),
+    )
+    for run_id, usage in attempts:
+        attempt = await store.start_attempt(
+            run_id,
+            1,
+            "ai.openai-compatible",
+            "requested-model",
+            101.0,
+        )
+        await store.finish_attempt(
+            attempt.attempt_id,
+            "completed",
+            usage,
+            102.0,
+        )
+
+    summary = await store.summary(ReportFilter(0, 500))
+
+    assert summary["runs"] == {
+        "total": 3,
+        "completed": 1,
+        "failed": 1,
+        "cancelled": 1,
+        "inProgress": 0,
+        "successRate": 0.5,
+    }
+    assert summary["cache"] == {
+        "hit": 1,
+        "miss": 1,
+        "unknown": 1,
+        "hitRate": 0.5,
+        "tokenRate": None,
+        "coverageRate": 2 / 3,
+    }
+
+
+async def test_approval_status_transitions_and_terminal_never_regresses() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    await store.ensure_run("run_1", request("actor_1"), 100.0)
+    await store.start_run("run_1", 101.0)
+
+    await sync_status(store, "run_1", "approval.required")
+    waiting = await store.get_run("run_1")
+    assert waiting is not None
+    assert waiting["status"] == "awaiting-approval"
+
+    await sync_status(store, "run_1", "approval.resolved")
+    resumed = await store.get_run("run_1")
+    assert resumed is not None
+    assert resumed["status"] == "running"
+
+    await store.finish_run("run_1", "completed", 104.0)
+    await sync_status(store, "run_1", "approval.required")
+    terminal = await store.get_run("run_1")
+    assert terminal is not None
+    assert terminal["status"] == "completed"
+
+
+async def test_unknown_usage_stays_null_and_empty_history_has_no_origin() -> None:
+    empty = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    empty_summary = await empty.summary(ReportFilter(0, 500))
+    assert empty_summary["reportingAvailableFrom"] is None
+
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    await store.ensure_run("run_1", request("actor_secret"), 100.0)
+    attempt = await store.start_attempt(
+        "run_1",
+        1,
+        "ai.openai-compatible",
+        "requested-model",
+        101.0,
+    )
+    await store.finish_attempt(
+        attempt.attempt_id,
+        "completed",
+        UsageFacts(),
+        102.0,
+    )
+
+    detail = await store.get_run("run_1")
+
+    assert detail is not None
+    assert detail["usage"] == {
+        "inputTokens": None,
+        "outputTokens": None,
+        "cachedInputTokens": None,
+        "reasoningTokens": None,
+        "totalTokens": None,
+    }
+    rendered_metrics = store.metrics.render()
+    assert "actor_secret" not in rendered_metrics
+    assert "requested-model" not in rendered_metrics
+
+
+async def test_reporting_origin_is_limited_to_filter_and_scope() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    await store.ensure_run("run_other", request("actor_other"), 100.0)
+    await store.ensure_run("run_visible", request("actor_visible"), 200.0)
+
+    selected = await store.summary(
+        ReportFilter(0, 500, selected_actor_id="actor_visible")
+    )
+    scoped = await store.summary(
+        ReportFilter(
+            0,
+            500,
+            scope_restricted=True,
+            actor_ids=("actor_visible",),
+        )
+    )
+    empty = await store.summary(
+        ReportFilter(
+            0,
+            500,
+            scope_restricted=True,
+            actor_ids=("actor_missing",),
+        )
+    )
+
+    assert selected["reportingAvailableFrom"] == "1970-01-01T00:03:20Z"
+    assert scoped["reportingAvailableFrom"] == "1970-01-01T00:03:20Z"
+    assert empty["reportingAvailableFrom"] is None
+
+
+async def test_attempt_terminal_replay_rejects_changed_facts() -> None:
+    store = InMemoryAgentRuntimeReportStore()
+    attempt = await store.start_attempt(
+        "run_1",
+        1,
+        "ai.openai-compatible",
+        "requested-model",
+        100.0,
+    )
+    usage = UsageFacts(
+        source="provider-reported",
+        input_tokens=10,
+        total_tokens=10,
+    )
+    await store.finish_attempt(
+        attempt.attempt_id,
+        "completed",
+        usage,
+        101.0,
+        "reported-model",
+    )
+    await store.finish_attempt(
+        attempt.attempt_id,
+        "completed",
+        usage,
+        102.0,
+        "reported-model",
+    )
+
+    with pytest.raises(ValueError, match="Attempt 终态冲突"):
+        await store.finish_attempt(
+            attempt.attempt_id,
+            "completed",
+            usage,
+            103.0,
+            "different-model",
+        )
+
+
+async def test_token_rates_only_use_attempts_with_matching_known_facts() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    await store.ensure_run("run_1", request("actor_1"), 100.0)
+    for usage in (
+        UsageFacts(
+            source="provider-reported",
+            input_tokens=10,
+            cached_input_tokens=None,
+            total_tokens=None,
+            cache_status="unknown",
+        ),
+        UsageFacts(
+            source="provider-reported",
+            input_tokens=10,
+            cached_input_tokens=5,
+            total_tokens=15,
+            cache_status="hit",
+        ),
+    ):
+        attempt = await store.start_attempt(
+            "run_1",
+            1,
+            "ai.openai-compatible",
+            "requested-model",
+            101.0,
+        )
+        await store.finish_attempt(
+            attempt.attempt_id,
+            "completed",
+            usage,
+            102.0,
+        )
+
+    summary = await store.summary(ReportFilter(0, 500))
+
+    assert summary["cache"]["tokenRate"] == 0.5  # type: ignore[index]
+    assert summary["tokens"]["coverageRate"] == 0.5  # type: ignore[index]
+    assert summary["activity"][0]["totalTokens"] is None  # type: ignore[index]
+
+
+async def test_completely_unknown_aggregate_tokens_stay_null() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 400.0)
+    await store.ensure_run("run_1", request("actor_1"), 100.0)
+    attempt = await store.start_attempt(
+        "run_1",
+        1,
+        "ai.openai-compatible",
+        "requested-model",
+        101.0,
+    )
+    await store.finish_attempt(
+        attempt.attempt_id,
+        "completed",
+        UsageFacts(),
+        102.0,
+    )
+
+    summary = await store.summary(ReportFilter(0, 500))
+
+    assert summary["tokens"] == {
+        "input": None,
+        "output": None,
+        "cachedInput": None,
+        "reasoning": None,
+        "total": None,
+        "coverageRate": 0,
+    }
+    assert summary["byModel"][0]["totalTokens"] is None  # type: ignore[index]
+    assert summary["activity"][0]["totalTokens"] is None  # type: ignore[index]

package/template/apps/agent-server/tests/test_report_summary_parity.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from southwind_agent_server.postgres_report_summary import present_summary
+from southwind_agent_server.report_presenter import aggregate_summary
+from southwind_agent_server.report_types import (
+    AttemptReport,
+    RunReport,
+    UsageFacts,
+)
+
+
+def test_postgres_and_in_memory_summary_present_same_dual_timeline() -> None:
+    run = RunReport(
+        run_id="run_1",
+        operation_key="business.answer",
+        release_id="release_1",
+        trace_id="trace_1",
+        actor_id="actor_1",
+        actor_type="user",
+        department_id=None,
+        status="completed",
+        safe_error_code=None,
+        started_at=1,
+        finished_at=2,
+        duration_ms=1_000,
+        created_at=0,
+        updated_at=2,
+        expires_at=1_000,
+    )
+    usage = UsageFacts(
+        source="provider-reported",
+        input_tokens=10,
+        output_tokens=2,
+        cached_input_tokens=5,
+        total_tokens=12,
+        cache_status="hit",
+    )
+    attempt = AttemptReport(
+        attempt_id="attempt_1",
+        run_id="run_1",
+        sequence=1,
+        tool_round=1,
+        deployment_key="ai.openai-compatible",
+        requested_model="requested",
+        reported_model="reported",
+        outcome="completed",
+        usage=usage,
+        duration_ms=1_000,
+        safe_error_code=None,
+        started_at=172_800,
+        finished_at=172_801,
+    )
+
+    memory = aggregate_summary([run], [attempt], 90, 0, 400)
+    postgres = present_summary(
+        {
+            "total": 1,
+            "completed": 1,
+            "failed": 0,
+            "cancelled": 0,
+            "available_from": 0,
+        },
+        {
+            "attempts": 1,
+            "available_from": 172_800,
+            "cache_hit": 1,
+            "cache_miss": 0,
+            "cache_unknown": 0,
+            "known_total_tokens": 1,
+            "input_tokens": 10,
+            "output_tokens": 2,
+            "cached_input_tokens": 5,
+            "reasoning_tokens": None,
+            "total_tokens": 12,
+            "paired_input_tokens": 10,
+            "paired_cached_tokens": 5,
+        },
+        [
+            {
+                "deployment_key": "ai.openai-compatible",
+                "model": "reported",
+                "attempts": 1,
+                "input_tokens": 10,
+                "output_tokens": 2,
+                "cached_input_tokens": 5,
+                "reasoning_tokens": None,
+                "total_tokens": 12,
+                "unknown_usage_count": 0,
+            }
+        ],
+        [
+            {
+                "date": "1970-01-01",
+                "runs": 1,
+                "completed": 1,
+                "failed": 0,
+                "attempts": 0,
+                "known_token_attempts": 0,
+                "total_tokens": None,
+            },
+            {
+                "date": "1970-01-03",
+                "runs": 0,
+                "completed": 0,
+                "failed": 0,
+                "attempts": 1,
+                "known_token_attempts": 1,
+                "total_tokens": 12,
+            },
+        ],
+        90,
+        400,
+    )
+
+    assert postgres == memory