claude-smart 0.2.42 → 0.2.44

package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/gepa_adapter.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+
+from reflexio.models.api_schema.domain import (
+    AgentPlaybook,
+    PlaybookOptimizationCandidate,
+    PlaybookOptimizationEvaluation,
+)
+from reflexio.server.services.storage.storage_base import BaseStorage
+from reflexio.server.tracing import sentry_tags
+
+from .assistant_webhook import AssistantFailedError
+from .judge import PairwiseJudge
+from .models import (
+    CandidateEvaluationOutput,
+    EvaluationTrajectory,
+    ScenarioWindow,
+)
+from .rollout import MultiTurnRollout
+
+if TYPE_CHECKING:
+    from gepa.core.adapter import EvaluationBatch, ProposalFn
+
+logger = logging.getLogger(__name__)
+
+PLAYBOOK_CONTENT_COMPONENT = "playbook_content"
+_EvaluationCacheKey = tuple[str, int | None, tuple[int, ...]]
+
+
+def _evaluation_cache_key(
+    candidate_content: str, window: ScenarioWindow
+) -> _EvaluationCacheKey:
+    return (
+        candidate_content,
+        window.user_playbook_id,
+        tuple(window.source_interaction_ids),
+    )
+
+
+class ReflexioPlaybookGEPAAdapter:
+    """GEPA adapter for paired real-assistant playbook rollouts.
+
+    GEPA proposes candidate playbook content; this adapter scores each
+    candidate by running the *same* user turns twice — once with the
+    incumbent playbook injected and once with the candidate — then asking
+    an LLM judge which conversation is better.
+
+    Per ``(candidate, window)`` evaluation it:
+
+    1. Persists (or reuses) a ``PlaybookOptimizationCandidate`` row keyed by
+       content, so duplicate proposals share an id.
+    2. Runs two ``MultiTurnRollout`` invocations against the configured
+       assistant backend.
+    3. Calls ``PairwiseJudge.judge`` to produce a verdict / score / Likert /
+       ASI rationale.
+    4. Inserts a ``PlaybookOptimizationEvaluation`` row for offline review.
+
+    Backend faults (any ``AssistantFailedError``) are caught in
+    ``_evaluate_one`` and recorded as ``verdict='aborted'`` so the GEPA
+    search loop can keep going. The optimizer reads those rows back to
+    decide whether the run as a whole should fail.
+    """
+
+    def __init__(
+        self,
+        *,
+        storage: BaseStorage,
+        job_id: int,
+        target_kind: str,
+        target_id: int,
+        incumbent: AgentPlaybook,
+        rollout: MultiTurnRollout,
+        judge: PairwiseJudge,
+        max_turns: int,
+    ) -> None:
+        self.storage = storage
+        self.job_id = job_id
+        self.target_kind = target_kind
+        self.target_id = target_id
+        self.incumbent = incumbent
+        self.rollout = rollout
+        self.judge = judge
+        self.max_turns = max_turns
+        self.propose_new_texts: ProposalFn | None = None
+        self._candidate_ids_by_content: dict[str, int] = {}
+        self._evaluation_cache: dict[
+            _EvaluationCacheKey, CandidateEvaluationOutput
+        ] = {}
+
+    def evaluate(
+        self,
+        batch: list[ScenarioWindow],
+        candidate: dict[str, str],
+        capture_traces: bool = False,
+    ) -> EvaluationBatch[EvaluationTrajectory, CandidateEvaluationOutput]:
+        from gepa.core.adapter import EvaluationBatch
+
+        candidate_content = candidate.get(PLAYBOOK_CONTENT_COMPONENT, "")
+        persisted_candidate = self._ensure_candidate(candidate_content)
+        outputs: list[CandidateEvaluationOutput] = []
+        scores: list[float] = []
+        trajectories: list[EvaluationTrajectory] = []
+
+        for window in batch:
+            cache_key = _evaluation_cache_key(candidate_content, window)
+            cached = self._evaluation_cache.get(cache_key)
+            if cached is not None:
+                output = cached
+                logger.info(
+                    "event=playbook_optimization_evaluation_cache_hit job_id=%d "
+                    "candidate_id=%d scenario_user_playbook_id=%s verdict=%s",
+                    self.job_id,
+                    persisted_candidate.candidate_id,
+                    window.user_playbook_id,
+                    output.verdict,
+                )
+            else:
+                output = self._evaluate_one(window, candidate_content)
+                self.storage.insert_playbook_optimization_evaluation(
+                    PlaybookOptimizationEvaluation(
+                        job_id=self.job_id,
+                        candidate_id=persisted_candidate.candidate_id,
+                        target_kind=self.target_kind,  # type: ignore[arg-type]
+                        target_id=self.target_id,
+                        scenario_user_playbook_id=window.user_playbook_id,
+                        source_interaction_ids=window.source_interaction_ids,
+                        score=output.score,
+                        verdict=output.verdict,
+                        likert=output.likert,
+                        rationale=output.rationale,
+                        asi_json=output.asi.model_dump_json(),
+                        incumbent_rollout_json=output.incumbent_rollout.model_dump_json(),
+                        candidate_rollout_json=output.candidate_rollout.model_dump_json(),
+                    )
+                )
+                # Persist before caching so a failed insert does not leave a
+                # phantom result behind; skip caching transient assistant-
+                # backend aborts so the next iteration retries the rollout.
+                if output.verdict != "aborted":
+                    self._evaluation_cache[cache_key] = output
+                logger.info(
+                    "event=playbook_optimization_evaluation job_id=%d candidate_id=%d "
+                    "scenario_user_playbook_id=%s verdict=%s score=%.3f likert=%d",
+                    self.job_id,
+                    persisted_candidate.candidate_id,
+                    window.user_playbook_id,
+                    output.verdict,
+                    output.score,
+                    output.likert,
+                )
+            outputs.append(output)
+            scores.append(output.score)
+            if capture_traces:
+                trajectories.append(
+                    EvaluationTrajectory(
+                        scenario=window,
+                        candidate_content=candidate_content,
+                        output=output,
+                    )
+                )
+
+        return EvaluationBatch(
+            outputs=outputs,
+            scores=scores,
+            trajectories=trajectories if capture_traces else None,
+        )
+
+    def make_reflective_dataset(
+        self,
+        candidate: dict[str, str],
+        eval_batch: EvaluationBatch[EvaluationTrajectory, CandidateEvaluationOutput],
+        components_to_update: list[str],
+    ) -> dict[str, list[dict[str, Any]]]:
+        records: list[dict[str, Any]] = [
+            {
+                "Inputs": {
+                    "source_interaction_ids": trajectory.scenario.source_interaction_ids,
+                    "current_playbook_content": candidate.get(
+                        PLAYBOOK_CONTENT_COMPONENT, ""
+                    ),
+                },
+                "Generated Outputs": {
+                    "incumbent_rollout": [
+                        message.model_dump()
+                        for message in trajectory.output.incumbent_rollout.messages
+                    ],
+                    "candidate_rollout": [
+                        message.model_dump()
+                        for message in trajectory.output.candidate_rollout.messages
+                    ],
+                },
+                "Feedback": json.dumps(
+                    {
+                        "score": trajectory.output.score,
+                        "verdict": trajectory.output.verdict,
+                        "likert": trajectory.output.likert,
+                        "rationale": trajectory.output.rationale,
+                        "asi": trajectory.output.asi.model_dump(),
+                    },
+                    ensure_ascii=False,
+                ),
+            }
+            for trajectory in eval_batch.trajectories or []
+        ]
+        return dict.fromkeys(components_to_update, records)
+
+    def _evaluate_one(
+        self, window: ScenarioWindow, candidate_content: str
+    ) -> CandidateEvaluationOutput:
+        candidate_playbook = self.incumbent.model_copy(
+            update={"content": candidate_content}
+        )
+        try:
+            incumbent_rollout = self.rollout.run(
+                window=window,
+                playbook=self.incumbent,
+                max_turns=self.max_turns,
+            )
+            candidate_rollout = self.rollout.run(
+                window=window,
+                playbook=candidate_playbook,
+                max_turns=self.max_turns,
+            )
+            judged = self.judge.judge(
+                window=window,
+                incumbent=self.incumbent,
+                candidate=candidate_playbook,
+                incumbent_rollout=incumbent_rollout,
+                candidate_rollout=candidate_rollout,
+            )
+            return CandidateEvaluationOutput(
+                score=judged.score,
+                verdict=judged.verdict,
+                likert=judged.likert,
+                rationale=judged.rationale,
+                asi=judged.asi,
+                incumbent_rollout=incumbent_rollout,
+                candidate_rollout=candidate_rollout,
+            )
+        except AssistantFailedError as exc:
+            with sentry_tags(
+                subsystem="playbook_optimizer",
+                op="assistant",
+                job_id=self.job_id,
+                target_kind=self.target_kind,
+                target_id=self.target_id,
+                error_type=type(exc).__name__,
+            ):
+                logger.exception("Playbook optimizer assistant failed")
+            return CandidateEvaluationOutput(
+                score=0.0,
+                verdict="aborted",
+                likert=0,
+                rationale=f"Assistant failed: {exc}",
+            )
+        except Exception as exc:
+            with sentry_tags(
+                subsystem="playbook_optimizer",
+                op="evaluate",
+                job_id=self.job_id,
+                target_kind=self.target_kind,
+                target_id=self.target_id,
+                error_type=type(exc).__name__,
+            ):
+                logger.exception("Playbook optimizer evaluation failed")
+            return CandidateEvaluationOutput(
+                score=0.0,
+                verdict="aborted",
+                likert=0,
+                rationale=f"Evaluation failed: {exc}",
+            )
+
+    def _ensure_candidate(self, content: str) -> PlaybookOptimizationCandidate:
+        candidate_id = self._candidate_ids_by_content.get(content)
+        if candidate_id is not None:
+            existing = self.storage.list_playbook_optimization_candidates(self.job_id)
+            for candidate in existing:
+                if candidate.candidate_id == candidate_id:
+                    return candidate
+        candidate = self.storage.insert_playbook_optimization_candidate(
+            PlaybookOptimizationCandidate(
+                job_id=self.job_id,
+                candidate_index=len(self._candidate_ids_by_content),
+                content=content,
+            )
+        )
+        self._candidate_ids_by_content[content] = candidate.candidate_id
+        return candidate

package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/judge.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import logging
+
+from reflexio.models.api_schema.domain import AgentPlaybook
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.llm.litellm_client import LiteLLMClient
+from reflexio.server.services.service_utils import log_model_response
+
+from .models import JudgeOutput, RolloutTrace, ScenarioWindow
+
+logger = logging.getLogger(__name__)
+
+PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID = "playbook_optimizer_judge"
+
+
+class PairwiseJudge:
+    """LLM-based comparator for paired playbook rollouts.
+
+    Given two rollouts that share the same user turns and differ only in
+    the playbook injected into the assistant, ``judge`` asks an LLM (using
+    the ``playbook_optimizer_judge`` prompt) to pick a winner and assign a
+    score, Likert rating, and structured rationale.
+
+    If the two playbooks have identical content, ``judge`` short-circuits
+    to a tie without spending an LLM call — the rollouts will be identical
+    by construction.
+    """
+
+    def __init__(
+        self,
+        request_context: RequestContext,
+        llm_client: LiteLLMClient,
+        model_name: str | None,
+    ) -> None:
+        self.request_context = request_context
+        self.llm_client = llm_client
+        self.model_name = model_name or llm_client.config.model
+
+    def judge(
+        self,
+        *,
+        window: ScenarioWindow,
+        incumbent: AgentPlaybook,
+        candidate: AgentPlaybook,
+        incumbent_rollout: RolloutTrace,
+        candidate_rollout: RolloutTrace,
+    ) -> JudgeOutput:
+        if incumbent.content == candidate.content:
+            return JudgeOutput(
+                verdict="tie",
+                score=0.5,
+                likert=3,
+                rationale="Candidate content is identical to incumbent content.",
+            )
+        prompt = self.request_context.prompt_manager.render_prompt(
+            PLAYBOOK_OPTIMIZER_JUDGE_PROMPT_ID,
+            {
+                "source_window_json": _json(
+                    [interaction.model_dump() for interaction in window.interactions]
+                ),
+                "incumbent_playbook_json": _json(_playbook_payload(incumbent)),
+                "candidate_playbook_json": _json(_playbook_payload(candidate)),
+                "incumbent_rollout_json": incumbent_rollout.model_dump_json(),
+                "candidate_rollout_json": candidate_rollout.model_dump_json(),
+            },
+        )
+        response = self.llm_client.generate_chat_response(
+            messages=[{"role": "user", "content": prompt}],
+            model=self.model_name,
+            response_format=JudgeOutput,
+            timeout=120,
+            max_retries=1,
+        )
+        log_model_response(logger, "Playbook optimizer judge response", response)
+        if isinstance(response, JudgeOutput):
+            return response
+        return JudgeOutput(
+            verdict="tie",
+            score=0.5,
+            likert=3,
+            rationale=f"Judge response was not parsed: {type(response).__name__}",
+        )
+
+
+def _playbook_payload(playbook: AgentPlaybook) -> dict[str, object]:
+    return {
+        "id": playbook.agent_playbook_id,
+        "content": playbook.content,
+        "trigger": playbook.trigger,
+        "rationale": playbook.rationale,
+    }
+
+
+def _json(value: object) -> str:
+    return json.dumps(value, ensure_ascii=False, indent=2, default=str)

package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/models.py

@@ -0,0 +1,96 @@
+"""In-process Pydantic models used by the playbook optimizer.
+
+These types are *not* persisted directly — they describe the inputs and
+outputs of the rollout/judge/adapter pipeline. The persisted equivalents
+live in ``reflexio.models.api_schema.domain`` (``PlaybookOptimization*``)
+and are populated from these models inside the GEPA adapter.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from reflexio.models.api_schema.domain import Interaction
+
+
+class ChatMessage(BaseModel):
+    """One conversation turn passed to / returned from the assistant backend."""
+
+    role: Literal["user", "assistant", "system"]
+    content: str
+
+
+class ScenarioWindow(BaseModel):
+    """One source-of-truth scenario the optimizer replays during rollouts.
+
+    Built by ``ScenarioResolver`` from a user playbook's
+    ``source_interaction_ids``. Only the user turns are replayed verbatim;
+    the assistant side is regenerated fresh on each rollout against the
+    playbook variant under test.
+    """
+
+    user_playbook_id: int | None = None
+    source_interaction_ids: list[int] = Field(default_factory=list)
+    interactions: list[Interaction] = Field(default_factory=list)
+
+    @property
+    def user_turns(self) -> list[ChatMessage]:
+        turns: list[ChatMessage] = []
+        for interaction in self.interactions:
+            role = _normalize_role(interaction.role)
+            if role == "user" and interaction.content:
+                turns.append(ChatMessage(role="user", content=interaction.content))
+        return turns
+
+
+class RolloutTrace(BaseModel):
+    messages: list[ChatMessage] = Field(default_factory=list)
+    playbook_content: str = ""
+
+
+class JudgeASI(BaseModel):
+    failure_modes: list[str] = Field(default_factory=list)
+    regressions: list[str] = Field(default_factory=list)
+    winning_behaviors: list[str] = Field(default_factory=list)
+    missing_instruction: str | list[str] = ""
+    recommended_mutation: str | list[str] = ""
+
+
+class JudgeOutput(BaseModel):
+    verdict: Literal["candidate", "incumbent", "tie"]
+    score: float = Field(ge=0.0, le=1.0)
+    likert: int = Field(ge=1, le=5)
+    rationale: str = ""
+    asi: JudgeASI = Field(default_factory=JudgeASI)
+
+
+class CandidateEvaluationOutput(BaseModel):
+    score: float
+    verdict: Literal["candidate", "incumbent", "tie", "aborted"]
+    likert: int
+    rationale: str
+    asi: JudgeASI = Field(default_factory=JudgeASI)
+    incumbent_rollout: RolloutTrace = Field(default_factory=RolloutTrace)
+    candidate_rollout: RolloutTrace = Field(default_factory=RolloutTrace)
+
+
+class EvaluationTrajectory(BaseModel):
+    scenario: ScenarioWindow
+    candidate_content: str
+    output: CandidateEvaluationOutput
+
+
+class GEPAEventPayload(BaseModel):
+    event: str
+    payload: dict[str, Any] = Field(default_factory=dict)
+
+
+def _normalize_role(role: str) -> Literal["user", "assistant", "system"]:
+    lowered = role.strip().lower()
+    if lowered in {"assistant", "agent"}:
+        return "assistant"
+    if lowered == "system":
+        return "system"
+    return "user"