create-windy 0.2.33 → 0.2.34

package/template/apps/agent-server/src/southwind_agent_server/provider_governance.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import cast
+
+from .contracts import (
+    JsonValue,
+    ProviderDataField,
+    ProviderDataPolicy,
+    RuntimeToolDefinition,
+)
+from .engine import SafeEngineError
+from .provider_audit import ProviderAttemptAudit, ProviderAuditStore
+
+_DLP_PATTERNS = (
+    re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{8,}"),
+    re.compile(r"(?i)\b(?:api[_-]?key|secret|password|token)\s*[:=]\s*\S+"),
+    re.compile(r"\b1[3-9]\d{9}\b"),
+    re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.I),
+    re.compile(r"\b\d{17}[\dXx]\b"),
+    re.compile(r"\b(?:\d[ -]?){16,30}\b"),
+)
+_REDACTED = "[REDACTED]"
+
+
+@dataclass(frozen=True)
+class GovernedAttempt:
+    messages: list[dict[str, JsonValue]]
+    tools: tuple[RuntimeToolDefinition, ...]
+
+
+class ProviderRequestGovernance:
+    def __init__(self, audit: ProviderAuditStore) -> None:
+        self.audit = audit
+
+    def authorize(
+        self,
+        *,
+        deployment_key: str,
+        operation_key: str,
+        run_id: str,
+        attempt: int,
+        messages: list[dict[str, JsonValue]],
+        tools: tuple[RuntimeToolDefinition, ...],
+        policy: ProviderDataPolicy,
+        input_value: JsonValue = None,
+        measure: Callable[
+            [list[dict[str, JsonValue]], tuple[RuntimeToolDefinition, ...]], int
+        ]
+        | None = None,
+    ) -> GovernedAttempt:
+        rules = {rule.field: rule for rule in policy.envelope_fields}
+        classified: set[str] = set()
+        classified.update(input_classifications(input_value, policy))
+        redacted = False
+        governed_messages: list[dict[str, JsonValue]] = []
+        try:
+            for message in messages:
+                field = field_for_message(message)
+                rule = rules.get(field)
+                if rule is None:
+                    raise GovernanceDenied(f"Provider 未授权发送字段：{field}")
+                classified.add(rule.classification)
+                value, changed = govern_value(message, rule.dlp_action)
+                redacted = redacted or changed
+                governed_messages.append(cast(dict[str, JsonValue], value))
+            governed_tools = tools
+            if tools:
+                rule = rules.get("tool-definitions")
+                if rule is None:
+                    raise GovernanceDenied("Provider 未授权发送字段：tool-definitions")
+                classified.add(rule.classification)
+                raw_tools = [tool.model_dump(by_alias=True) for tool in tools]
+                governed, changed = govern_value(raw_tools, rule.dlp_action)
+                redacted = redacted or changed
+                governed_tools = tuple(
+                    RuntimeToolDefinition.model_validate(item)
+                    for item in cast(list[object], governed)
+                )
+            sent_bytes = (measure or payload_size)(
+                governed_messages,
+                governed_tools,
+            )
+            decision = "redact" if redacted else "allow"
+            self._record(
+                deployment_key,
+                operation_key,
+                run_id,
+                attempt,
+                classified,
+                sent_bytes,
+                decision,
+            )
+            return GovernedAttempt(governed_messages, governed_tools)
+        except GovernanceDenied:
+            self._record(
+                deployment_key,
+                operation_key,
+                run_id,
+                attempt,
+                classified,
+                0,
+                "reject",
+            )
+            raise SafeEngineError(
+                "FORBIDDEN",
+                "Provider 请求被字段治理策略拒绝",
+                retryable=False,
+            ) from None
+
+    def health(self) -> dict[str, int | float | str | None]:
+        return self.audit.health()
+
+    def _record(
+        self,
+        deployment_key: str,
+        operation_key: str,
+        run_id: str,
+        attempt: int,
+        classified: set[str],
+        sent_bytes: int,
+        decision: str,
+    ) -> None:
+        try:
+            self.audit.append(
+                ProviderAttemptAudit(
+                    deployment_key,
+                    operation_key,
+                    run_id,
+                    attempt,
+                    tuple(sorted(classified)),
+                    sent_bytes,
+                    decision,
+                )
+            )
+        except Exception:
+            raise SafeEngineError(
+                "FORBIDDEN",
+                "Provider 发送审计不可用",
+                retryable=False,
+            ) from None
+
+
+class GovernanceDenied(Exception):
+    pass
+
+
+def field_for_message(message: dict[str, JsonValue]) -> ProviderDataField:
+    role = message.get("role")
+    if role == "system":
+        return "instructions"
+    if role == "user":
+        return "user-prompt"
+    if role == "assistant":
+        return "provider-output"
+    if role == "tool":
+        return "tool-results"
+    raise GovernanceDenied("Provider 消息角色未知")
+
+
+def input_classifications(
+    value: JsonValue,
+    policy: ProviderDataPolicy,
+) -> set[str]:
+    result: set[str] = set()
+    for rule in policy.input_fields:
+        current = value
+        found = True
+        for encoded in rule.json_pointer.removeprefix("/").split("/"):
+            segment = encoded.replace("~1", "/").replace("~0", "~")
+            if not isinstance(current, dict) or segment not in current:
+                found = False
+                break
+            current = current[segment]
+        if found:
+            result.add(rule.classification)
+    return result
+
+
+def govern_value(value: object, action: str) -> tuple[object, bool]:
+    if isinstance(value, str):
+        detected = any(pattern.search(value) for pattern in _DLP_PATTERNS)
+        if detected and action == "reject":
+            raise GovernanceDenied("DLP reject")
+        if detected:
+            result = value
+            for pattern in _DLP_PATTERNS:
+                result = pattern.sub(_REDACTED, result)
+            return result, result != value
+        return value, False
+    if isinstance(value, list):
+        changed = False
+        items: list[object] = []
+        for item in value:
+            governed, item_changed = govern_value(item, action)
+            items.append(governed)
+            changed = changed or item_changed
+        return items, changed
+    if isinstance(value, dict):
+        changed = False
+        governed_record: dict[str, object] = {}
+        for key, item in value.items():
+            governed, item_changed = govern_value(item, action)
+            governed_record[key] = governed
+            changed = changed or item_changed
+        return governed_record, changed
+    return value, False
+
+
+def payload_size(
+    messages: list[dict[str, JsonValue]],
+    tools: tuple[RuntimeToolDefinition, ...],
+) -> int:
+    payload = {
+        "messages": messages,
+        "tools": [tool.model_dump(by_alias=True) for tool in tools],
+    }
+    return len(
+        json.dumps(
+            payload,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        ).encode()
+    )

package/template/apps/agent-server/src/southwind_agent_server/service.py

@@ -86,6 +86,7 @@ class AgentRunService:
                 allowed_tools=tuple(invocation.allowed_tools),
                 limits=invocation.limits,
                 provider_egress=invocation.provider_egress,
+                provider_data_policy=invocation.provider_data_policy,
                 tool_network_policy=invocation.tool_network_policy,
                 execution_grant=invocation.execution_grant,
                 tool_host_url=invocation.tool_host_url,

package/template/apps/agent-server/tests/test_api.py

@@ -9,7 +9,7 @@ import httpx
 import pytest
 from fastapi import FastAPI
 
-from southwind_agent_server.app import create_app
+from southwind_agent_server.app import create_app, provider_governance_snapshot
 from southwind_agent_server.deterministic import DeterministicEngineAdapter
 
 
@@ -233,3 +233,18 @@ async def test_liveness_does_not_claim_provider_readiness(
     assert live.json() == {"status": "live"}
     assert ready.status_code == 503
     assert ready.json()["code"] == "RUNTIME_UNAVAILABLE"
+
+
+def test_provider_governance_health_filters_content_identifiers() -> None:
+    snapshot = provider_governance_snapshot(
+        {
+            "status": "healthy",
+            "retainedRecords": 3,
+            "runId": "run_secret",
+            "prompt": "do-not-expose",
+        }
+    )
+    assert snapshot["status"] == "healthy"
+    assert snapshot["retainedRecords"] == 3
+    assert "runId" not in snapshot
+    assert "prompt" not in snapshot

package/template/apps/agent-server/tests/test_openai_provider.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+import sqlite3
+import tempfile
 
 import httpx
 import pytest
@@ -11,7 +13,10 @@ from southwind_agent_server.config import (
     SecretRef,
 )
 from southwind_agent_server.contracts import (
+    ProviderDataPolicy,
     ProviderEgress,
+    ProviderFieldPolicy,
+    ProviderInputFieldPolicy,
     RuntimeLimits,
     RuntimeToolDefinition,
     ToolNetworkPolicy,
@@ -24,6 +29,8 @@ from southwind_agent_server.engine import (
 )
 from southwind_agent_server.openai_engine import OpenAiCompatibleEngineAdapter
 from southwind_agent_server.openai_provider import OpenAiCompatibleProvider
+from southwind_agent_server.provider_audit import ProviderAuditStore
+from southwind_agent_server.provider_governance import ProviderRequestGovernance
 from southwind_agent_server.tool_host import ToolHostClient
 
 
@@ -76,11 +83,13 @@ async def test_provider_error_never_leaks_body_or_secret() -> None:
 
 async def test_engine_executes_only_allowed_tool_through_host() -> None:
     provider_calls = 0
+    provider_body_bytes: list[int] = []
     tool_requests: list[dict[str, object]] = []
 
     async def provider_handler(request: httpx.Request) -> httpx.Response:
         nonlocal provider_calls
         provider_calls += 1
+        provider_body_bytes.append(len(request.content))
         if provider_calls == 1:
             return sse_response(
                 [
@@ -111,12 +120,14 @@ async def test_engine_executes_only_allowed_tool_through_host() -> None:
         tool_requests.append(json.loads(request.content))
         return httpx.Response(200, json={"value": {"total": 1}})
 
+    audit_path = tempfile.NamedTemporaryFile(suffix=".sqlite3", delete=False).name
     engine = OpenAiCompatibleEngineAdapter(
         create_provider(httpx.MockTransport(provider_handler)),
         ToolHostClient(
             "internal-token",
             transport=httpx.MockTransport(tool_handler),
         ),
+        governance(audit_path),
     )
     signals = [
         signal
@@ -132,6 +143,20 @@ async def test_engine_executes_only_allowed_tool_through_host() -> None:
         "arguments": {"page": 2},
     }
     assert signals[-1].output == {"text": "共有一条记录"}  # type: ignore[union-attr]
+    audits = (
+        sqlite3.connect(audit_path)
+        .execute(
+            """
+        select attempt, sent_bytes, decision
+        from provider_attempt_audit order by attempt
+        """
+        )
+        .fetchall()
+    )
+    assert audits == [
+        (1, provider_body_bytes[0], "allow"),
+        (2, provider_body_bytes[1], "allow"),
+    ]
 
 
 async def test_engine_denies_unknown_model_tool_before_host_callback() -> None:
@@ -172,6 +197,7 @@ async def test_engine_denies_unknown_model_tool_before_host_callback() -> None:
             "internal-token",
             transport=httpx.MockTransport(tool_handler),
         ),
+        governance(),
     )
     with pytest.raises(SafeEngineError) as captured:
         async for _signal in engine.execute(runtime_request("business.records.list")):
@@ -191,6 +217,7 @@ async def test_engine_denies_provider_before_network_without_explicit_egress() -
     engine = OpenAiCompatibleEngineAdapter(
         create_provider(httpx.MockTransport(provider_handler)),
         ToolHostClient("internal-token"),
+        governance(),
     )
     request = runtime_request("business.records.list")
     request = EngineRunRequest(
@@ -211,6 +238,7 @@ async def test_engine_denies_mismatched_provider_deployment() -> None:
     engine = OpenAiCompatibleEngineAdapter(
         create_provider(httpx.MockTransport(lambda _request: sse_response([]))),
         ToolHostClient("internal-token"),
+        governance(),
     )
     request = runtime_request("business.records.list")
     request = EngineRunRequest(
@@ -267,12 +295,63 @@ def runtime_request(tool_key: str) -> EngineRunRequest:
             mode="configured-endpoint",
             deployment_key="ai.openai-compatible",
         ),
+        provider_data_policy=provider_data_policy(),
         tool_network_policy=ToolNetworkPolicy(mode="forbidden"),
         execution_grant="grant_1",
         tool_host_url="http://server:9747",
     )
 
 
+def provider_data_policy() -> ProviderDataPolicy:
+    return ProviderDataPolicy(
+        input_fields=[
+            ProviderInputFieldPolicy(
+                json_pointer="/question",
+                classification="internal",
+                dlp_action="redact",
+            )
+        ],
+        envelope_fields=[
+            ProviderFieldPolicy(
+                field="instructions",
+                classification="internal",
+                dlp_action="reject",
+            ),
+            ProviderFieldPolicy(
+                field="user-prompt",
+                classification="internal",
+                dlp_action="redact",
+            ),
+            ProviderFieldPolicy(
+                field="tool-definitions",
+                classification="internal",
+                dlp_action="reject",
+            ),
+            ProviderFieldPolicy(
+                field="provider-output",
+                classification="internal",
+                dlp_action="redact",
+            ),
+            ProviderFieldPolicy(
+                field="tool-results",
+                classification="sensitive",
+                dlp_action="redact",
+            ),
+        ],
+    )
+
+
+def governance(path: str | None = None) -> ProviderRequestGovernance:
+    path = (
+        path
+        or tempfile.NamedTemporaryFile(
+            suffix=".sqlite3",
+            delete=False,
+        ).name
+    )
+    return ProviderRequestGovernance(ProviderAuditStore(path, retention_days=90))
+
+
 def sse_response(chunks: list[object]) -> httpx.Response:
     body = "".join(
         f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n" for chunk in chunks

package/template/apps/agent-server/tests/test_provider_governance.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Literal, cast
+
+import pytest
+
+from southwind_agent_server.contracts import (
+    ProviderDataPolicy,
+    ProviderFieldPolicy,
+    ProviderInputFieldPolicy,
+)
+from southwind_agent_server.engine import SafeEngineError
+from southwind_agent_server.provider_audit import (
+    ProviderAttemptAudit,
+    ProviderAuditStore,
+)
+from southwind_agent_server.provider_governance import ProviderRequestGovernance
+
+
+def test_dlp_redacts_before_send_and_audits_metadata_only(tmp_path: Path) -> None:
+    path = tmp_path / "provider-audit.sqlite3"
+    governance = ProviderRequestGovernance(
+        ProviderAuditStore(str(path), retention_days=90)
+    )
+
+    result = governance.authorize(
+        deployment_key="ai.openai-compatible",
+        operation_key="business.answer",
+        run_id="run_1",
+        attempt=1,
+        messages=[
+            {"role": "system", "content": "只回答问题"},
+            {"role": "user", "content": "联系 me@example.com"},
+        ],
+        tools=(),
+        policy=policy(user_action="redact"),
+    )
+
+    assert result.messages[1]["content"] == "联系 [REDACTED]"
+    row = (
+        sqlite3.connect(path)
+        .execute(
+            """
+        select deployment_key, operation_key, run_id, attempt,
+          classifications, sent_bytes, decision
+        from provider_attempt_audit
+        """
+        )
+        .fetchone()
+    )
+    assert row == (
+        "ai.openai-compatible",
+        "business.answer",
+        "run_1",
+        1,
+        "internal",
+        row[5],
+        "redact",
+    )
+    assert row[5] > 0
+    assert "me@example.com" not in path.read_bytes().decode(errors="ignore")
+
+
+def test_missing_allowlist_and_dlp_reject_before_network(tmp_path: Path) -> None:
+    path = tmp_path / "provider-audit.sqlite3"
+    governance = ProviderRequestGovernance(
+        ProviderAuditStore(str(path), retention_days=90)
+    )
+
+    with pytest.raises(SafeEngineError) as missing:
+        governance.authorize(
+            deployment_key="ai.openai-compatible",
+            operation_key="business.answer",
+            run_id="run_missing",
+            attempt=1,
+            messages=[{"role": "system", "content": "只回答问题"}],
+            tools=(),
+            policy=ProviderDataPolicy(input_fields=[], envelope_fields=[]),
+        )
+    assert missing.value.code == "FORBIDDEN"
+
+    with pytest.raises(SafeEngineError) as rejected:
+        governance.authorize(
+            deployment_key="ai.openai-compatible",
+            operation_key="business.answer",
+            run_id="run_reject",
+            attempt=1,
+            messages=[
+                {"role": "system", "content": "只回答问题"},
+                {"role": "user", "content": "password=do-not-send"},
+            ],
+            tools=(),
+            policy=policy(user_action="reject"),
+        )
+    assert rejected.value.code == "FORBIDDEN"
+    decisions = (
+        sqlite3.connect(path)
+        .execute(
+            "select decision, sent_bytes from provider_attempt_audit "
+            "order by occurred_at"
+        )
+        .fetchall()
+    )
+    assert decisions == [("reject", 0), ("reject", 0)]
+
+
+def test_attempts_are_unique_append_only_and_ttl_purge_is_bounded(
+    tmp_path: Path,
+) -> None:
+    path = tmp_path / "provider-audit.sqlite3"
+    store = ProviderAuditStore(str(path), retention_days=1)
+    event = ProviderAttemptAudit(
+        "ai.openai-compatible",
+        "business.answer",
+        "run_1",
+        1,
+        ("internal",),
+        42,
+        "allow",
+    )
+    store.append(event)
+    with pytest.raises(sqlite3.IntegrityError):
+        store.append(event)
+
+    database = sqlite3.connect(path)
+    database.execute(
+        "update provider_attempt_audit set occurred_at = ?",
+        (time.time() - 2 * 24 * 60 * 60,),
+    )
+    database.commit()
+    database.close()
+
+    assert store.purge_expired() == 1
+    health = store.health()
+    assert health["retainedRecords"] == 0
+    assert health["lastDeletedRecords"] == 1
+    assert health["totalDeletedRecords"] == 1
+
+
+def test_audit_store_serializes_concurrent_single_replica_writes(
+    tmp_path: Path,
+) -> None:
+    store = ProviderAuditStore(
+        str(tmp_path / "provider-audit.sqlite3"),
+        retention_days=90,
+    )
+
+    def append(attempt: int) -> None:
+        store.append(
+            ProviderAttemptAudit(
+                "ai.openai-compatible",
+                "business.answer",
+                "run_concurrent",
+                attempt,
+                ("internal",),
+                42,
+                "allow",
+            )
+        )
+
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        list(executor.map(append, range(1, 21)))
+
+    assert store.health()["retainedRecords"] == 20
+
+
+def test_audit_failure_denies_provider_attempt() -> None:
+    class BrokenAudit:
+        def append(self, _event: ProviderAttemptAudit) -> None:
+            raise sqlite3.OperationalError("do-not-expose")
+
+    governance = ProviderRequestGovernance(cast(ProviderAuditStore, BrokenAudit()))
+    with pytest.raises(SafeEngineError) as captured:
+        governance.authorize(
+            deployment_key="ai.openai-compatible",
+            operation_key="business.answer",
+            run_id="run_audit_failure",
+            attempt=1,
+            messages=[
+                {"role": "system", "content": "只回答问题"},
+                {"role": "user", "content": "正常内容"},
+            ],
+            tools=(),
+            policy=policy(user_action="redact"),
+        )
+    assert captured.value.code == "FORBIDDEN"
+    assert "do-not-expose" not in str(captured.value)
+
+
+def policy(*, user_action: Literal["reject", "redact"]) -> ProviderDataPolicy:
+    return ProviderDataPolicy(
+        input_fields=[
+            ProviderInputFieldPolicy(
+                json_pointer="/question",
+                classification="internal",
+                dlp_action="redact",
+            )
+        ],
+        envelope_fields=[
+            ProviderFieldPolicy(
+                field="instructions",
+                classification="internal",
+                dlp_action="reject",
+            ),
+            ProviderFieldPolicy(
+                field="user-prompt",
+                classification="internal",
+                dlp_action=user_action,
+            ),
+        ],
+    )