create-windy 0.3.2 → 0.3.5

package/template/apps/agent-server/src/southwind_agent_server/sqlite_catalog.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import asyncio
+import sqlite3
+import time
+import uuid
+
+from .contracts import JsonValue
+from .sqlite_serialization import encode
+from .store import IdempotencyConflictError, ReleaseNotFoundError, RunRecord
+from .store_cipher import RunStoreCipher
+
+
+class SqliteCatalogMixin:
+    _database: sqlite3.Connection
+    _condition: asyncio.Condition
+    _cipher: RunStoreCipher
+
+    async def save_invocation(self, run_id: str, value: JsonValue) -> None:
+        async with self._condition:
+            updated = self._database.execute(
+                """
+                update agent_runs set invocation_cipher = ?, updated_at = ?
+                where run_id = ? and status not in ('completed', 'failed', 'cancelled')
+                """,
+                (self._cipher.seal(value), time.time(), run_id),
+            )
+            if updated.rowcount != 1:
+                raise KeyError(run_id)
+            self._database.commit()
+
+    async def record_evaluation(
+        self,
+        release_id: str,
+        evaluation_set_key: str,
+        evaluation_set_version: str,
+        metrics: dict[str, float],
+    ) -> str:
+        async with self._condition:
+            release = self._database.execute(
+                "select release_id from agent_releases where release_id = ?",
+                (release_id,),
+            ).fetchone()
+            if not release:
+                raise ReleaseNotFoundError
+            existing = self._database.execute(
+                """
+                select evaluation_id, metrics_json
+                from agent_evaluation_results
+                where release_id = ? and evaluation_set_key = ?
+                  and evaluation_set_version = ?
+                """,
+                (release_id, evaluation_set_key, evaluation_set_version),
+            ).fetchone()
+            encoded = encode(metrics)
+            if existing:
+                if str(existing["metrics_json"]) != encoded:
+                    raise IdempotencyConflictError
+                return str(existing["evaluation_id"])
+            evaluation_id = f"eval_{uuid.uuid4().hex}"
+            try:
+                self._database.execute(
+                    """
+                    insert into agent_evaluation_results values (?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        evaluation_id,
+                        release_id,
+                        evaluation_set_key,
+                        evaluation_set_version,
+                        encoded,
+                        time.time(),
+                    ),
+                )
+                self._database.commit()
+            except sqlite3.IntegrityError:
+                self._database.rollback()
+                raise IdempotencyConflictError from None
+            return evaluation_id
+
+    def _ensure_release(self, record: RunRecord) -> None:
+        if not isinstance(record.release, dict):
+            raise IdempotencyConflictError
+        release_id = record.release.get("releaseId")
+        if not isinstance(release_id, str):
+            raise IdempotencyConflictError
+        encoded = encode(record.release)
+        existing = self._database.execute(
+            """
+            select operation_key, snapshot_json from agent_releases
+            where release_id = ?
+            """,
+            (release_id,),
+        ).fetchone()
+        if existing and (
+            str(existing["operation_key"]) != record.operation_key
+            or str(existing["snapshot_json"]) != encoded
+        ):
+            raise IdempotencyConflictError
+        self._database.execute(
+            """
+            insert or ignore into agent_releases (
+              release_id, operation_key, snapshot_json, created_at
+            ) values (?, ?, ?, ?)
+            """,
+            (release_id, record.operation_key, encoded, time.time()),
+        )

package/template/apps/agent-server/src/southwind_agent_server/sqlite_schema.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import sqlite3
+
+
+def initialize_database(database: sqlite3.Connection) -> None:
+    database.execute("pragma foreign_keys = on")
+    database.executescript(
+        """
+        create table if not exists agent_runs (
+          run_id text primary key,
+          operation_key text not null,
+          input_json text not null,
+          idempotency_key text not null unique,
+          fingerprint text not null,
+          status text not null,
+          output_json text not null,
+          created_at real not null,
+          updated_at real not null,
+          invocation_cipher text,
+          release_json text not null default 'null',
+          trace_id text not null default ''
+        );
+        create table if not exists agent_events (
+          run_id text not null references agent_runs(run_id) on delete cascade,
+          sequence integer not null,
+          event_json text not null,
+          primary key (run_id, sequence)
+        );
+        """
+    )
+    columns = {
+        str(row["name"]) for row in database.execute("pragma table_info(agent_runs)")
+    }
+    if "invocation_cipher" not in columns:
+        database.execute("alter table agent_runs add column invocation_cipher text")
+    if "release_json" not in columns:
+        database.execute(
+            "alter table agent_runs add column release_json "
+            "text not null default 'null'"
+        )
+    if "trace_id" not in columns:
+        database.execute(
+            "alter table agent_runs add column trace_id text not null default ''"
+        )
+    database.executescript(
+        """
+        create table if not exists agent_releases (
+          release_id text primary key,
+          operation_key text not null,
+          snapshot_json text not null,
+          created_at real not null
+        );
+        create table if not exists agent_evaluation_results (
+          evaluation_id text primary key,
+          release_id text not null references agent_releases(release_id),
+          evaluation_set_key text not null,
+          evaluation_set_version text not null,
+          metrics_json text not null,
+          created_at real not null,
+          unique(release_id, evaluation_set_key, evaluation_set_version)
+        );
+        """
+    )
+    database.commit()

package/template/apps/agent-server/src/southwind_agent_server/sqlite_serialization.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import json
+
+from .store import CanonicalEvent
+
+
+def encode(value: object) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def sequence_of(event: CanonicalEvent) -> int:
+    value = event.get("sequence")
+    if not isinstance(value, int):
+        raise RuntimeError("Agent Event sequence 无效")
+    return value

package/template/apps/agent-server/src/southwind_agent_server/sqlite_store.py

@@ -8,24 +8,29 @@ from pathlib import Path
 from typing import cast
 
 from .contracts import AgentError, AgentErrorCode, JsonValue, RunStatus
+from .sqlite_catalog import SqliteCatalogMixin
+from .sqlite_schema import initialize_database
+from .sqlite_serialization import encode, sequence_of
 from .store import (
     CanonicalEvent,
     IdempotencyConflictError,
     RunRecord,
     status_for,
 )
+from .store_cipher import RunStoreCipher
 
 
 class RunStoreCapacityError(Exception):
     pass
 
 
-class SqliteRunStore:
+class SqliteRunStore(SqliteCatalogMixin):
     """Single-replica durable Run store with bounded rows, events and output."""
 
     def __init__(
         self,
         path: str,
+        cipher: RunStoreCipher,
         *,
         max_runs: int = 10_000,
         max_events_per_run: int = 2_000,
@@ -40,10 +45,15 @@ class SqliteRunStore:
         self._max_output_bytes = max_output_bytes
         self._terminal_ttl = terminal_ttl_seconds
         self._condition = asyncio.Condition()
-        self._initialize()
+        self._cipher = cipher
+        initialize_database(self._database)
         self._recover_incomplete()
 
-    async def create_or_get(self, record: RunRecord) -> tuple[RunRecord, bool]:
+    async def create_or_get(
+        self,
+        record: RunRecord,
+        invocation: JsonValue = None,
+    ) -> tuple[RunRecord, bool]:
         async with self._condition:
             existing = self._database.execute(
                 "select run_id from agent_runs where idempotency_key = ?",
@@ -61,24 +71,35 @@ class SqliteRunStore:
             if count >= self._max_runs:
                 raise RunStoreCapacityError("Agent Run Store 容量已满")
             now = time.time()
-            self._database.execute(
-                """
-                insert into agent_runs (
-                  run_id, operation_key, input_json, idempotency_key,
-                  fingerprint, status, output_json, created_at, updated_at
-                ) values (?, ?, ?, ?, ?, 'accepted', 'null', ?, ?)
-                """,
-                (
-                    record.run_id,
-                    record.operation_key,
-                    "null",
-                    record.idempotency_key,
-                    record.fingerprint,
-                    now,
-                    now,
-                ),
-            )
-            self._database.commit()
+            try:
+                self._database.execute(
+                    """
+                    insert into agent_runs (
+                      run_id, operation_key, input_json, idempotency_key,
+                      fingerprint, status, output_json, created_at, updated_at,
+                      release_json, trace_id, invocation_cipher
+                    ) values (?, ?, ?, ?, ?, 'accepted', 'null', ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        record.run_id,
+                        record.operation_key,
+                        "null",
+                        record.idempotency_key,
+                        record.fingerprint,
+                        now,
+                        now,
+                        encode(record.release),
+                        record.trace_id,
+                        self._cipher.seal(invocation)
+                        if invocation is not None
+                        else None,
+                    ),
+                )
+                self._ensure_release(record)
+                self._database.commit()
+            except Exception:
+                self._database.rollback()
+                raise
             self._condition.notify_all()
             return record, True
 
@@ -168,38 +189,58 @@ class SqliteRunStore:
                 record.terminal,
             )
 
-    def close(self) -> None:
-        self._database.close()
+    async def recoverable(self) -> list[tuple[RunRecord, JsonValue]]:
+        async with self._condition:
+            rows = self._database.execute(
+                """
+                select run_id, invocation_cipher from agent_runs
+                where status not in ('completed', 'failed', 'cancelled')
+                  and invocation_cipher is not null
+                order by created_at
+                """
+            ).fetchall()
+            result: list[tuple[RunRecord, JsonValue]] = []
+            for row in rows:
+                record = self._read(str(row["run_id"]))
+                if record is not None:
+                    result.append(
+                        (
+                            record,
+                            self._cipher.open(str(row["invocation_cipher"])),
+                        )
+                    )
+            return result
 
-    def _initialize(self) -> None:
-        self._database.executescript(
-            """
-            create table if not exists agent_runs (
-              run_id text primary key,
-              operation_key text not null,
-              input_json text not null,
-              idempotency_key text not null unique,
-              fingerprint text not null,
-              status text not null,
-              output_json text not null,
-              created_at real not null,
-              updated_at real not null
-            );
-            create table if not exists agent_events (
-              run_id text not null references agent_runs(run_id) on delete cascade,
-              sequence integer not null,
-              event_json text not null,
-              primary key (run_id, sequence)
-            );
-            """
-        )
-        self._database.commit()
+    async def clear_invocation(self, run_id: str) -> None:
+        async with self._condition:
+            self._database.execute(
+                "update agent_runs set invocation_cipher = null where run_id = ?",
+                (run_id,),
+            )
+            self._database.commit()
+
+    async def renew_lease(self, run_id: str) -> None:
+        record = await self.get(run_id)
+        if record is None or record.terminal:
+            raise KeyError(run_id)
+
+    async def release_lease(self, run_id: str) -> None:
+        if await self.get(run_id) is None:
+            raise KeyError(run_id)
+
+    async def probe(self) -> None:
+        async with self._condition:
+            self._database.execute("select 1").fetchone()
+
+    async def close(self) -> None:
+        self._database.close()
 
     def _recover_incomplete(self) -> None:
         rows = self._database.execute(
             """
             select run_id from agent_runs
             where status not in ('completed', 'failed', 'cancelled')
+              and invocation_cipher is null
             """
         ).fetchall()
         for row in rows:
@@ -248,6 +289,8 @@ class SqliteRunStore:
             status=cast(RunStatus, str(row["status"])),
             events=events,
             output=json.loads(str(row["output_json"])),
+            release=json.loads(str(row["release_json"])),
+            trace_id=str(row["trace_id"]),
         )
 
     def _insert_event(
@@ -304,14 +347,3 @@ class SqliteRunStore:
             (cutoff,),
         )
         self._database.commit()
-
-
-def encode(value: object) -> str:
-    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
-
-
-def sequence_of(event: CanonicalEvent) -> int:
-    value = event.get("sequence")
-    if not isinstance(value, int):
-        raise RuntimeError("Agent Event sequence 无效")
-    return value

package/template/apps/agent-server/src/southwind_agent_server/store.py

@@ -17,6 +17,8 @@ class RunRecord:
     input: JsonValue
     idempotency_key: str
     fingerprint: str
+    release: JsonValue = None
+    trace_id: str = ""
     status: RunStatus = "accepted"
     events: list[CanonicalEvent] = field(default_factory=list)
     output: JsonValue = None
@@ -30,8 +32,16 @@ class IdempotencyConflictError(Exception):
     pass
 
 
+class ReleaseNotFoundError(Exception):
+    pass
+
+
 class RunStore(Protocol):
-    async def create_or_get(self, record: RunRecord) -> tuple[RunRecord, bool]: ...
+    async def create_or_get(
+        self,
+        record: RunRecord,
+        invocation: JsonValue = None,
+    ) -> tuple[RunRecord, bool]: ...
     async def get(self, run_id: str) -> RunRecord | None: ...
     async def append(
         self,
@@ -46,6 +56,20 @@ class RunStore(Protocol):
         after: int,
         wait_seconds: float,
     ) -> tuple[list[CanonicalEvent], bool]: ...
+    async def save_invocation(self, run_id: str, value: JsonValue) -> None: ...
+    async def recoverable(self) -> list[tuple[RunRecord, JsonValue]]: ...
+    async def clear_invocation(self, run_id: str) -> None: ...
+    async def record_evaluation(
+        self,
+        release_id: str,
+        evaluation_set_key: str,
+        evaluation_set_version: str,
+        metrics: dict[str, float],
+    ) -> str: ...
+    async def renew_lease(self, run_id: str) -> None: ...
+    async def release_lease(self, run_id: str) -> None: ...
+    async def probe(self) -> None: ...
+    async def close(self) -> None: ...
 
 
 class InMemoryRunStore:
@@ -55,8 +79,15 @@ class InMemoryRunStore:
         self._runs: dict[str, RunRecord] = {}
         self._idempotency: dict[str, str] = {}
         self._condition = asyncio.Condition()
+        self._invocations: dict[str, JsonValue] = {}
+        self._evaluations: dict[tuple[str, str, str], tuple[str, dict[str, float]]] = {}
+        self._releases: dict[str, tuple[str, JsonValue]] = {}
 
-    async def create_or_get(self, record: RunRecord) -> tuple[RunRecord, bool]:
+    async def create_or_get(
+        self,
+        record: RunRecord,
+        invocation: JsonValue = None,
+    ) -> tuple[RunRecord, bool]:
         async with self._condition:
             existing_id = self._idempotency.get(record.idempotency_key)
             if existing_id is not None:
@@ -64,8 +95,16 @@ class InMemoryRunStore:
                 if existing.fingerprint != record.fingerprint:
                     raise IdempotencyConflictError
                 return existing, False
+            release_id = release_id_of(record)
+            release = (record.operation_key, record.release)
+            existing_release = self._releases.get(release_id)
+            if existing_release is not None and existing_release != release:
+                raise IdempotencyConflictError
+            self._releases[release_id] = release
             self._runs[record.run_id] = record
             self._idempotency[record.idempotency_key] = record.run_id
+            if invocation is not None:
+                self._invocations[record.run_id] = invocation
             self._condition.notify_all()
             return record, True
 
@@ -131,14 +170,78 @@ class InMemoryRunStore:
                     return [], False
             return list(record.events[after:]), record.terminal
 
+    async def save_invocation(self, run_id: str, value: JsonValue) -> None:
+        async with self._condition:
+            if run_id not in self._runs:
+                raise KeyError(run_id)
+            self._invocations[run_id] = value
+
+    async def recoverable(self) -> list[tuple[RunRecord, JsonValue]]:
+        async with self._condition:
+            return [
+                (record, self._invocations[run_id])
+                for run_id, record in self._runs.items()
+                if not record.terminal and run_id in self._invocations
+            ]
+
+    async def clear_invocation(self, run_id: str) -> None:
+        async with self._condition:
+            self._invocations.pop(run_id, None)
+
+    async def record_evaluation(
+        self,
+        release_id: str,
+        evaluation_set_key: str,
+        evaluation_set_version: str,
+        metrics: dict[str, float],
+    ) -> str:
+        async with self._condition:
+            if release_id not in self._releases:
+                raise ReleaseNotFoundError
+            key = (release_id, evaluation_set_key, evaluation_set_version)
+            existing = self._evaluations.get(key)
+            if existing:
+                if existing[1] != metrics:
+                    raise IdempotencyConflictError
+                return existing[0]
+            evaluation_id = f"eval_{len(self._evaluations) + 1}"
+            self._evaluations[key] = (evaluation_id, metrics)
+            return evaluation_id
+
+    async def renew_lease(self, run_id: str) -> None:
+        record = self._runs.get(run_id)
+        if record is None or record.terminal:
+            raise KeyError(run_id)
+
+    async def release_lease(self, run_id: str) -> None:
+        if run_id not in self._runs:
+            raise KeyError(run_id)
+
+    async def probe(self) -> None:
+        return None
+
+    async def close(self) -> None:
+        return None
+
 
 def status_for(event_type: str, current: RunStatus) -> RunStatus:
     if event_type == "run.started":
         return "running"
     if event_type == "approval.required":
         return "awaiting-approval"
+    if event_type == "approval.resolved":
+        return "running"
     if event_type == "run.failed":
         return "failed"
     if event_type == "run.cancelled":
         return "cancelled"
     return current
+
+
+def release_id_of(record: RunRecord) -> str:
+    if not isinstance(record.release, dict):
+        raise IdempotencyConflictError
+    release_id = record.release.get("releaseId")
+    if not isinstance(release_id, str):
+        raise IdempotencyConflictError
+    return release_id

package/template/apps/agent-server/src/southwind_agent_server/store_cipher.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+from typing import cast
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from .contracts import JsonValue
+
+
+class RunStoreCipher:
+    def __init__(self, secret: str) -> None:
+        if len(secret.encode()) < 32:
+            raise ValueError("AGENT_RUN_ENCRYPTION_SECRET 至少需要 32 字节")
+        self._key = hashlib.sha256(secret.encode()).digest()
+
+    def seal(self, value: JsonValue) -> str:
+        nonce = os.urandom(12)
+        body = json.dumps(
+            value,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        ).encode()
+        encrypted = AESGCM(self._key).encrypt(nonce, body, b"agent-run-v1")
+        return base64.urlsafe_b64encode(nonce + encrypted).decode()
+
+    def open(self, value: str) -> JsonValue:
+        try:
+            raw = base64.urlsafe_b64decode(value.encode())
+            body = AESGCM(self._key).decrypt(
+                raw[:12],
+                raw[12:],
+                b"agent-run-v1",
+            )
+            return cast(JsonValue, json.loads(body))
+        except Exception as error:
+            raise ValueError("Agent Run 加密上下文无法解密") from error

package/template/apps/agent-server/src/southwind_agent_server/telemetry.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import os
+import secrets
+from contextvars import Token
+
+from opentelemetry import context, trace
+from opentelemetry.context import Context
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace import (
+    NonRecordingSpan,
+    Span,
+    SpanContext,
+    TraceFlags,
+)
+
+from .store import RunRecord
+
+_configured = False
+
+
+def configure_telemetry() -> None:
+    global _configured
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "").strip()
+    if _configured or not endpoint:
+        return
+    provider = TracerProvider(
+        resource=Resource.create({"service.name": "southwind-agent-server"})
+    )
+    provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
+    trace.set_tracer_provider(provider)
+    _configured = True
+
+
+class AgentRunTrace:
+    """OTel span carrying metadata only; prompts, inputs and outputs stay excluded."""
+
+    def __init__(self, record: RunRecord) -> None:
+        parent = parent_context(record.trace_id)
+        self._span: Span = trace.get_tracer(__name__).start_span(
+            "agent.run",
+            context=parent,
+            attributes={
+                "agent.run.id": record.run_id,
+                "agent.operation.key": record.operation_key,
+                "agent.release.id": release_id(record),
+            },
+        )
+        self._token: Token[Context] | None = None
+
+    def start(self) -> None:
+        self._token = context.attach(trace.set_span_in_context(self._span))
+
+    def finish(self, status: str) -> None:
+        self._span.set_attribute("agent.run.status", status)
+        if self._token is not None:
+            context.detach(self._token)
+        self._span.end()
+
+
+def parent_context(trace_id: str) -> context.Context | None:
+    if len(trace_id) != 32:
+        return None
+    try:
+        parsed = int(trace_id, 16)
+    except ValueError:
+        return None
+    if parsed == 0:
+        return None
+    parent = NonRecordingSpan(
+        SpanContext(
+            trace_id=parsed,
+            span_id=secrets.randbits(64) or 1,
+            is_remote=True,
+            trace_flags=TraceFlags(TraceFlags.SAMPLED),
+        )
+    )
+    return trace.set_span_in_context(parent)
+
+
+def release_id(record: RunRecord) -> str:
+    if isinstance(record.release, dict):
+        value = record.release.get("releaseId")
+        if isinstance(value, str):
+            return value
+    return "unknown"