create-windy 0.4.0 → 0.5.0

package/template/apps/agent-server/tests/test_postgres_report_summary.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+from typing import Any, cast
+
+from southwind_agent_server.postgres_report_summary import (
+    present_summary,
+    summary,
+)
+from southwind_agent_server.report_types import ReportFilter
+
+
+async def test_summary_uses_bounded_sql_aggregates_and_projects_result() -> None:
+    connection = FakeConnection()
+    result = await summary(
+        cast(Any, FakePool(connection)),
+        ReportFilter(
+            0,
+            500,
+            selected_actor_id="actor_visible",
+            scope_restricted=True,
+            actor_ids=("actor_visible",),
+            department_ids=("department_visible",),
+        ),
+        retention_days=90,
+        generated_at=400,
+    )
+
+    assert len(connection.queries) == 4
+    rendered_sql = "\n".join(query for query, _ in connection.queries)
+    assert "select *" not in rendered_sql.lower()
+    assert "run_id = any" not in rendered_sql.lower()
+    assert "group by a.deployment_key" in rendered_sql
+    assert "group by 1" in rendered_sql
+    assert "at time zone 'UTC'" in rendered_sql
+    unbounded_attempt_rollup = (
+        "from agent_runtime_report_attempts\n              group by run_id"
+    )
+    assert unbounded_attempt_rollup not in rendered_sql
+    assert "with run_activity as" in rendered_sql
+    assert "attempt_activity as" in rendered_sql
+    for index, (query, parameters) in enumerate(connection.queries):
+        assert "r.actor_id = %s" in query
+        assert "r.department_id = any(%s)" in query
+        expected = (
+            0,
+            500,
+            400,
+            "actor_visible",
+            ["actor_visible"],
+            ["department_visible"],
+        )
+        assert parameters == (expected * 2 if index == 3 else expected)
+        if index > 0:
+            assert "a.started_at >= %s" in query
+            assert "a.started_at <= %s" in query
+            assert "a.finished_at is not null" in query
+
+    assert result == {
+        "generatedAt": "1970-01-01T00:06:40Z",
+        "reportingAvailableFrom": "1970-01-01T00:01:40Z",
+        "retentionDays": 90,
+        "runs": {
+            "total": 3,
+            "completed": 1,
+            "failed": 1,
+            "cancelled": 1,
+            "inProgress": 0,
+            "successRate": 0.5,
+        },
+        "cache": {
+            "hit": 1,
+            "miss": 1,
+            "unknown": 1,
+            "hitRate": 0.5,
+            "tokenRate": 0.5,
+            "coverageRate": 2 / 3,
+        },
+        "tokens": {
+            "input": 30,
+            "output": 10,
+            "cachedInput": 5,
+            "reasoning": 2,
+            "total": 40,
+            "coverageRate": 2 / 3,
+        },
+        "byModel": [
+            {
+                "deploymentKey": "ai.openai-compatible",
+                "model": "qwen",
+                "attempts": 3,
+                "inputTokens": 30,
+                "outputTokens": 10,
+                "cachedInputTokens": 5,
+                "reasoningTokens": 2,
+                "totalTokens": 40,
+                "unknownUsageCount": 1,
+            }
+        ],
+        "activity": [
+            {
+                "date": "2026-07-30",
+                "runs": 3,
+                "completed": 1,
+                "failed": 1,
+                "totalTokens": None,
+            },
+            {
+                "date": "2026-07-31",
+                "runs": 0,
+                "completed": 0,
+                "failed": 0,
+                "totalTokens": None,
+            },
+        ],
+    }
+
+
+def test_summary_keeps_completely_unknown_token_facts_null() -> None:
+    result = present_summary(
+        {
+            "total": 1,
+            "completed": 1,
+            "failed": 0,
+            "cancelled": 0,
+            "available_from": 100,
+        },
+        {
+            "attempts": 1,
+            "available_from": 100,
+            "cache_hit": 0,
+            "cache_miss": 0,
+            "cache_unknown": 1,
+            "known_total_tokens": 0,
+            "input_tokens": None,
+            "output_tokens": None,
+            "cached_input_tokens": None,
+            "reasoning_tokens": None,
+            "total_tokens": None,
+            "paired_input_tokens": 0,
+            "paired_cached_tokens": 0,
+        },
+        [
+            {
+                "deployment_key": "ai.openai-compatible",
+                "model": "qwen",
+                "attempts": 1,
+                "input_tokens": None,
+                "output_tokens": None,
+                "cached_input_tokens": None,
+                "reasoning_tokens": None,
+                "total_tokens": None,
+                "unknown_usage_count": 1,
+            }
+        ],
+        [
+            {
+                "date": "2026-07-31",
+                "runs": 1,
+                "completed": 1,
+                "failed": 0,
+                "attempts": 1,
+                "known_token_attempts": 0,
+                "total_tokens": None,
+            }
+        ],
+        90,
+        400,
+    )
+
+    assert result["tokens"] == {
+        "input": None,
+        "output": None,
+        "cachedInput": None,
+        "reasoning": None,
+        "total": None,
+        "coverageRate": 0,
+    }
+    assert result["byModel"][0]["totalTokens"] is None  # type: ignore[index]
+    assert result["activity"][0]["totalTokens"] is None  # type: ignore[index]
+
+
+class FakePool:
+    def __init__(self, connection: FakeConnection) -> None:
+        self.value = connection
+
+    def connection(self) -> FakeLease:
+        return FakeLease(self.value)
+
+
+class FakeLease:
+    def __init__(self, connection: FakeConnection) -> None:
+        self.value = connection
+
+    async def __aenter__(self) -> FakeConnection:
+        return self.value
+
+    async def __aexit__(
+        self,
+        exception_type: object,
+        exception: object,
+        traceback: object,
+    ) -> None:
+        return None
+
+
+class FakeConnection:
+    def __init__(self) -> None:
+        self.queries: list[tuple[str, tuple[object, ...]]] = []
+
+    async def execute(
+        self,
+        query: str,
+        parameters: tuple[object, ...],
+    ) -> FakeCursor:
+        self.queries.append((query, parameters))
+        if "min(r.created_at) as available_from" in query:
+            return FakeCursor(
+                [
+                    {
+                        "total": 3,
+                        "completed": 1,
+                        "failed": 1,
+                        "cancelled": 1,
+                        "available_from": 100,
+                    }
+                ]
+            )
+        if "as paired_input_tokens" in query:
+            return FakeCursor(
+                [
+                    {
+                        "attempts": 3,
+                        "available_from": 200,
+                        "cache_hit": 1,
+                        "cache_miss": 1,
+                        "cache_unknown": 1,
+                        "known_total_tokens": 2,
+                        "input_tokens": 30,
+                        "output_tokens": 10,
+                        "cached_input_tokens": 5,
+                        "reasoning_tokens": 2,
+                        "total_tokens": 40,
+                        "paired_input_tokens": 10,
+                        "paired_cached_tokens": 5,
+                    }
+                ]
+            )
+        if "group by a.deployment_key" in query:
+            return FakeCursor(
+                [
+                    {
+                        "deployment_key": "ai.openai-compatible",
+                        "model": "qwen",
+                        "attempts": 3,
+                        "input_tokens": 30,
+                        "output_tokens": 10,
+                        "cached_input_tokens": 5,
+                        "reasoning_tokens": 2,
+                        "total_tokens": 40,
+                        "unknown_usage_count": 1,
+                    }
+                ]
+            )
+        if "with run_activity as" in query:
+            return FakeCursor(
+                [
+                    {
+                        "date": "2026-07-30",
+                        "runs": 3,
+                        "completed": 1,
+                        "failed": 1,
+                        "attempts": 0,
+                        "known_token_attempts": 0,
+                        "total_tokens": None,
+                    },
+                    {
+                        "date": "2026-07-31",
+                        "runs": 0,
+                        "completed": 0,
+                        "failed": 0,
+                        "attempts": 3,
+                        "known_token_attempts": 2,
+                        "total_tokens": 40,
+                    },
+                ]
+            )
+        raise AssertionError("unexpected SQL")
+
+
+class FakeCursor:
+    def __init__(self, rows: list[dict[str, Any]]) -> None:
+        self.rows = rows
+
+    async def fetchone(self) -> dict[str, Any] | None:
+        return self.rows[0] if self.rows else None
+
+    async def fetchall(self) -> list[dict[str, Any]]:
+        return self.rows

package/template/apps/agent-server/tests/test_provider_attempt_reporting.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from southwind_agent_server.adk_adapter import AdkEngineAdapter
+from southwind_agent_server.contracts import StartRunRequest
+from southwind_agent_server.engine import AgentEngineAdapter, SafeEngineError
+from southwind_agent_server.openai_engine import (
+    OpenAiCompatibleEngineAdapter,
+)
+from southwind_agent_server.report_store import (
+    InMemoryAgentRuntimeReportStore,
+)
+from southwind_agent_server.tool_host import ToolHostClient
+from tests.openai_provider_test_support import (
+    create_provider,
+    governance,
+    runtime_request,
+    sse_response,
+)
+
+
+@pytest.mark.parametrize("adapter_kind", ["direct", "adk"])
+async def test_blocked_response_preserves_provider_usage(
+    adapter_kind: str,
+) -> None:
+    async def provider_handler(_request: httpx.Request) -> httpx.Response:
+        return sse_response(
+            [
+                {
+                    "model": "provider-model-v2",
+                    "choices": [
+                        {
+                            "delta": {
+                                "tool_calls": [
+                                    {
+                                        "index": 0,
+                                        "id": "call_bad",
+                                        "function": {
+                                            "name": "business.admin.delete",
+                                            "arguments": "{}",
+                                        },
+                                    }
+                                ]
+                            }
+                        }
+                    ],
+                    "usage": {
+                        "prompt_tokens": 10,
+                        "completion_tokens": 2,
+                        "total_tokens": 12,
+                        "prompt_tokens_details": {"cached_tokens": 5},
+                    },
+                }
+            ]
+        )
+
+    report = InMemoryAgentRuntimeReportStore(now=lambda: 200.0)
+    await report.ensure_run(
+        "run_1",
+        StartRunRequest(operation_key="business.answer", input=None),
+        100.0,
+    )
+    await report.start_run("run_1", 101.0)
+    provider = create_provider(httpx.MockTransport(provider_handler))
+    adapter: AgentEngineAdapter
+    if adapter_kind == "direct":
+        adapter = OpenAiCompatibleEngineAdapter(
+            provider,
+            ToolHostClient("internal-token"),
+            governance(),
+            report,
+        )
+    else:
+        adapter = AdkEngineAdapter(
+            provider,
+            ToolHostClient("internal-token"),
+            governance(),
+            report,
+        )
+
+    with pytest.raises(SafeEngineError) as captured:
+        async for _ in adapter.execute(
+            runtime_request("business.records.list"),
+        ):
+            pass
+
+    detail = await report.get_run("run_1")
+    assert detail is not None
+    attempt = detail["attempts"][0]  # type: ignore[index]
+
+    assert captured.value.code == "TOOL_DENIED"
+    assert attempt["outcome"] == "blocked"
+    assert attempt["usageSource"] == "provider-reported"
+    assert attempt["inputTokens"] == 10
+    assert attempt["outputTokens"] == 2
+    assert attempt["cachedInputTokens"] == 5
+    assert attempt["totalTokens"] == 12
+    assert attempt["cacheStatus"] == "hit"
+    assert attempt["firstTokenLatencyMs"] is not None
+    assert attempt["model"] == "provider-model-v2"
+
+
+async def test_adk_cancel_after_response_preserves_provider_usage() -> None:
+    adapter: AdkEngineAdapter
+
+    async def provider_handler(_request: httpx.Request) -> httpx.Response:
+        await adapter.cancel("run_1")
+        return sse_response(
+            [
+                {
+                    "model": "provider-model-v2",
+                    "choices": [{"delta": {"content": "late"}}],
+                    "usage": {
+                        "prompt_tokens": 3,
+                        "completion_tokens": 1,
+                        "total_tokens": 4,
+                    },
+                }
+            ]
+        )
+
+    report = InMemoryAgentRuntimeReportStore(now=lambda: 200.0)
+    await report.ensure_run(
+        "run_1",
+        StartRunRequest(operation_key="business.answer", input=None),
+        100.0,
+    )
+    await report.start_run("run_1", 101.0)
+    adapter = AdkEngineAdapter(
+        create_provider(httpx.MockTransport(provider_handler)),
+        ToolHostClient("internal-token"),
+        governance(),
+        report,
+    )
+
+    with pytest.raises(SafeEngineError) as captured:
+        async for _ in adapter.execute(
+            runtime_request("business.records.list"),
+        ):
+            pass
+
+    detail = await report.get_run("run_1")
+    assert detail is not None
+    attempt = detail["attempts"][0]  # type: ignore[index]
+    assert captured.value.code == "RUN_CANCELLED"
+    assert attempt["outcome"] == "cancelled"
+    assert attempt["usageSource"] == "provider-reported"
+    assert attempt["totalTokens"] == 4
+    assert attempt["firstTokenLatencyMs"] is not None
+    assert attempt["model"] == "provider-model-v2"

package/template/apps/agent-server/tests/test_report_attempt_lifecycle.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from southwind_agent_server.contracts import (
+    RunReportingContext,
+    StartRunRequest,
+)
+from southwind_agent_server.report_store import (
+    InMemoryAgentRuntimeReportStore,
+)
+from southwind_agent_server.report_types import ReportFilter, UsageFacts
+
+DAY = 24 * 60 * 60
+
+
+def request() -> StartRunRequest:
+    return StartRunRequest(
+        operation_key="business.answer",
+        input=None,
+        reporting=RunReportingContext(
+            actor_id="actor_1",
+            actor_type="user",
+        ),
+    )
+
+
+async def test_running_attempt_is_visible_but_excluded_from_summary() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 4 * DAY)
+    await store.ensure_run("run_1", request(), 0)
+    await store.start_run("run_1", 1)
+    attempt = await store.start_attempt(
+        "run_1",
+        1,
+        "ai.openai-compatible",
+        "requested-model",
+        2 * DAY,
+    )
+
+    detail = await store.get_run("run_1")
+    summary = await store.summary(ReportFilter(0, 3 * DAY))
+
+    assert detail is not None
+    assert detail["attempts"][0]["outcome"] == "running"  # type: ignore[index]
+    assert detail["attempts"][0]["finishedAt"] is None  # type: ignore[index]
+    assert summary["cache"] == {
+        "hit": 0,
+        "miss": 0,
+        "unknown": 0,
+        "hitRate": None,
+        "tokenRate": None,
+        "coverageRate": 0,
+    }
+    assert summary["tokens"]["coverageRate"] == 0  # type: ignore[index]
+    assert summary["byModel"] == []
+
+    await store.start_run("run_1", 2 * DAY + 10)
+    recovered = await store.get_run("run_1")
+
+    assert recovered is not None
+    recovered_attempt = recovered["attempts"][0]  # type: ignore[index]
+    assert recovered_attempt["outcome"] == "abandoned"
+    assert recovered_attempt["finishedAt"] is not None
+    assert attempt.outcome == "running"
+
+
+async def test_attempt_metrics_use_attempt_day_not_run_day() -> None:
+    store = InMemoryAgentRuntimeReportStore(now=lambda: 4 * DAY)
+    await store.ensure_run("run_1", request(), 0)
+    await store.start_run("run_1", 1)
+    attempt = await store.start_attempt(
+        "run_1",
+        1,
+        "ai.openai-compatible",
+        "requested-model",
+        2 * DAY,
+    )
+    await store.finish_attempt(
+        attempt.attempt_id,
+        "completed",
+        UsageFacts(
+            source="provider-reported",
+            input_tokens=10,
+            output_tokens=2,
+            total_tokens=12,
+            cache_status="miss",
+        ),
+        2 * DAY + 1,
+        "reported-model",
+    )
+    await store.finish_run("run_1", "completed", 2 * DAY + 2)
+
+    attempt_window = await store.summary(
+        ReportFilter(DAY, 3 * DAY),
+    )
+    whole_window = await store.summary(
+        ReportFilter(-1, 3 * DAY),
+    )
+
+    assert attempt_window["runs"]["total"] == 0  # type: ignore[index]
+    assert attempt_window["reportingAvailableFrom"] == "1970-01-03T00:00:00Z"
+    assert attempt_window["tokens"]["total"] == 12  # type: ignore[index]
+    assert attempt_window["byModel"][0]["attempts"] == 1  # type: ignore[index]
+    assert attempt_window["activity"] == [
+        {
+            "date": "1970-01-03",
+            "runs": 0,
+            "completed": 0,
+            "failed": 0,
+            "totalTokens": 12,
+        }
+    ]
+    assert whole_window["activity"] == [
+        {
+            "date": "1970-01-01",
+            "runs": 1,
+            "completed": 1,
+            "failed": 0,
+            "totalTokens": None,
+        },
+        {
+            "date": "1970-01-03",
+            "runs": 0,
+            "completed": 0,
+            "failed": 0,
+            "totalTokens": 12,
+        },
+    ]