claude-smart 0.2.41 → 0.2.43

package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_service_constants.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class PlaybookServiceConstants:
+    PLAYBOOK_EXTRACTORS_CONFIG_NAME = "user_playbook_extractor_config"
+    # ===============================
+    # prompt ids
+    # ===============================
+    PLAYBOOK_SHOULD_GENERATE_PROMPT_ID = "playbook_should_generate"
+    PLAYBOOK_EXTRACTION_CONTEXT_PROMPT_ID = "playbook_extraction_context"
+    PLAYBOOK_EXTRACTION_PROMPT_ID = "playbook_extraction_main"
+    PLAYBOOK_AGGREGATION_PROMPT_ID = "playbook_aggregation"
+
+    # ===============================
+    # expert content prompt ids
+    # ===============================
+    PLAYBOOK_SHOULD_GENERATE_EXPERT_PROMPT_ID = "playbook_should_generate_expert"
+    PLAYBOOK_EXTRACTION_CONTEXT_EXPERT_PROMPT_ID = "playbook_extraction_context_expert"
+    PLAYBOOK_EXTRACTION_EXPERT_PROMPT_ID = "playbook_extraction_main_expert"
+
+    # ===============================
+    # agent success evaluation prompt ids
+    # ===============================
+    AGENT_SUCCESS_EVALUATION_SHOULD_EVALUATE_PROMPT_ID = (
+        "agent_success_evaluation_should_evaluate"
+    )
+    AGENT_SUCCESS_EVALUATION_PROMPT_ID = "agent_success_evaluation"

package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_service_utils.py

@@ -0,0 +1,362 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from reflexio.models.api_schema.domain.entities import Interaction
+from reflexio.models.api_schema.internal_schema import RequestInteractionDataModel
+from reflexio.server.prompt.prompt_manager import PromptManager
+from reflexio.server.services.playbook.playbook_service_constants import (
+    PlaybookServiceConstants,
+)
+from reflexio.server.services.service_utils import (
+    MessageConstructionConfig,
+    PromptConfig,
+    construct_messages_from_interactions,
+    extract_interactions_from_request_interaction_data_models,
+    format_sessions_to_history_string,
+)
+
+logger = logging.getLogger(__name__)
+
+# ===============================
+# Pydantic classes for playbook_extraction_main prompt output schema
+# ===============================
+
+
+class StructuredPlaybookContent(BaseModel):
+    """
+    Structured representation of a single playbook entry from LLM output.
+
+    Field order matters for autoregressive conditioning: rationale is generated
+    first, then trigger, then content is synthesized last as a summary.
+
+    The extraction schema intentionally does not ask the model to emit a
+    polarity label. The extractor writes action rules or avoidance rules, and
+    the service derives the internal :class:`UserPlaybook` polarity from that
+    wording before storage.
+    """
+
+    rationale: str | None = Field(
+        default=None,
+        description="The reasoning behind this playbook entry — generated first for autoregressive conditioning",
+    )
+    trigger: str | None = Field(
+        default=None,
+        description="The condition or context when this rule applies",
+    )
+    content: str | None = Field(
+        default=None,
+        description="The main actionable content of the playbook entry — what to do or what to avoid",
+    )
+    source_span: str | None = Field(
+        default=None,
+        description="Verbatim excerpt from the source that most directly supports this playbook entry",
+    )
+    notes: str | None = Field(
+        default=None,
+        description="Free-form extraction notes — confidence, caveats, or alternative readings",
+    )
+    reader_angle: str | None = Field(
+        default=None,
+        description="The extraction perspective or reader role that surfaced this entry",
+    )
+    model_config = ConfigDict(
+        extra="allow",
+        json_schema_extra={"additionalProperties": False},
+    )
+
+    @property
+    def is_structured(self) -> bool:
+        """Check if this playbook entry has both a trigger and content."""
+        return bool(
+            self.trigger
+            and self.trigger.strip()
+            and self.content
+            and self.content.strip()
+        )
+
+    @property
+    def has_content(self) -> bool:
+        """Check if this output contains actual content."""
+        return bool(self.content and self.content.strip())
+
+
+class StructuredPlaybookList(BaseModel):
+    """
+    Wrapper schema for extracting zero or more playbook entries in a single LLM call.
+
+    The canonical shape is ``{"playbooks": [...]}``. An empty list means the model
+    found no valid SOPs in the window. This wrapper exists because OpenAI structured
+    output requires a JSON object at the root, so ``list[StructuredPlaybookContent]``
+    cannot be used directly as ``response_format``.
+    """
+
+    playbooks: list[StructuredPlaybookContent] = Field(
+        default_factory=list,
+        description="Extracted playbook entries — empty list when no valid SOP was found",
+    )
+
+    model_config = ConfigDict(
+        extra="forbid",
+        json_schema_extra={"additionalProperties": False},
+    )
+
+
+class PlaybookAggregationOutput(BaseModel):
+    """
+    Output schema for the playbook_aggregation prompt.
+
+    Contains the consolidated playbook entry or null if no new entry should be generated
+    (e.g., when it duplicates existing approved playbook).
+    """
+
+    playbook: StructuredPlaybookContent | None = Field(
+        default=None,
+        description="The consolidated playbook entry, or null if no new entry should be generated",
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def normalize_wrapper_key(cls, data: Any) -> Any:
+        """Accept both 'playbook' and legacy 'feedback' as the wrapper key."""
+        if isinstance(data, dict) and "feedback" in data and "playbook" not in data:
+            data["playbook"] = data.pop("feedback")
+        return data
+
+    model_config = ConfigDict(
+        extra="allow",
+        json_schema_extra={"additionalProperties": False},
+    )
+
+
+def format_structured_fields_for_display(
+    structured: StructuredPlaybookContent,
+) -> str:
+    """
+    Format structured metadata fields for display/debug purposes.
+
+    This is NOT for producing content values. Use ensure_playbook_content()
+    when you need to obtain the freeform content string.
+
+    Args:
+        structured (StructuredPlaybookContent): The structured playbook content
+
+    Returns:
+        str: Formatted structured fields string for display
+    """
+    lines = []
+
+    if structured.trigger:
+        lines.append(f'Trigger: "{structured.trigger}"')
+
+    if structured.rationale:
+        lines.append(f'Rationale: "{structured.rationale}"')
+
+    if not lines and structured.content:
+        return structured.content
+
+    return "\n".join(lines)
+
+
+def ensure_playbook_content(
+    playbook_content: str | None,
+    structured: StructuredPlaybookContent,
+) -> str:
+    """
+    Return playbook_content if present; legacy fallback from structured fields.
+
+    Args:
+        playbook_content (str | None): The freeform content from the LLM
+        structured (StructuredPlaybookContent): Structured fields for fallback
+
+    Returns:
+        str: The freeform playbook_content, or a formatted fallback from structured fields.
+    """
+    if playbook_content and playbook_content.strip():
+        return playbook_content
+    return format_structured_fields_for_display(structured)
+
+
+class PlaybookGenerationRequest(BaseModel):
+    request_id: str
+    agent_version: str
+    user_id: str | None = None  # for per-user playbook extraction
+    source: str | None = None
+    rerun_start_time: int | None = None  # Unix timestamp for rerun flows
+    rerun_end_time: int | None = None  # Unix timestamp for rerun flows
+    auto_run: bool = (
+        True  # True for regular flow (checks stride_size), False for rerun/manual
+    )
+    force_extraction: bool = False  # when True, bypass all extraction gates (stride_size, cheap pre-filter, LLM should_run)
+
+
+class PlaybookAggregatorRequest(BaseModel):
+    agent_version: str
+    rerun: bool = False
+
+
+def construct_playbook_extraction_messages_from_sessions(
+    prompt_manager: PromptManager,
+    request_interaction_data_models: list[RequestInteractionDataModel],
+    agent_context_prompt: str,
+    extraction_definition_prompt: str,
+    tool_can_use: str | None = None,
+) -> list[dict]:
+    """
+    Construct LLM messages for playbook extraction from sessions.
+
+    This function uses the shared message construction interface to build messages
+    with a system prompt and a final user prompt specific to playbook extraction.
+
+    Args:
+        prompt_manager: The prompt manager for rendering prompt templates
+        request_interaction_data_models: List of request interaction groups to extract playbook entries from
+        agent_context_prompt: Context about the agent for system message
+        extraction_definition_prompt: Definition of what the playbook should contain
+        tool_can_use: Optional formatted string of tools available to the agent
+
+    Returns:
+        list[dict]: List of messages ready for playbook extraction
+    """
+    # Configure system message (before interactions)
+    # Stable content (instructions, examples, definitions) goes in system message for token caching
+    system_config = PromptConfig(
+        prompt_id=PlaybookServiceConstants.PLAYBOOK_EXTRACTION_CONTEXT_PROMPT_ID,
+        variables={
+            "agent_context_prompt": agent_context_prompt,
+            "extraction_definition_prompt": extraction_definition_prompt,
+            "tool_can_use": tool_can_use or "",
+        },
+    )
+
+    # Configure final user message (after interactions)
+    # Only dynamic per-call data goes in user message
+    user_config = PromptConfig(
+        prompt_id=PlaybookServiceConstants.PLAYBOOK_EXTRACTION_PROMPT_ID,
+        variables={
+            "interactions": format_sessions_to_history_string(
+                request_interaction_data_models
+            ),
+        },
+    )
+
+    # Extract flat interactions for message construction
+    interactions = extract_interactions_from_request_interaction_data_models(
+        request_interaction_data_models
+    )
+
+    # Use shared message construction
+    config = MessageConstructionConfig(
+        prompt_manager=prompt_manager,
+        system_prompt_config=system_config,
+        user_prompt_config=user_config,
+    )
+
+    return construct_messages_from_interactions(interactions, config)
+
+
+# ===============================
+# Expert content utilities
+# ===============================
+
+
+def has_expert_content(interactions: list[Interaction]) -> bool:
+    """Check if any interaction has non-empty expert_content."""
+    return any(i.expert_content for i in interactions)
+
+
+def format_expert_comparison_pairs(
+    request_interaction_data_models: list[RequestInteractionDataModel],
+) -> str:
+    """
+    Format interactions with expert_content as agent-vs-expert comparison blocks.
+
+    For each agent interaction that has expert_content, includes the preceding user
+    question for context, the agent's actual response, and the expert's ideal response.
+
+    Args:
+        request_interaction_data_models: Session data models containing interactions
+
+    Returns:
+        str: Formatted comparison pairs string
+    """
+    interactions = extract_interactions_from_request_interaction_data_models(
+        request_interaction_data_models
+    )
+
+    pairs: list[str] = []
+    pair_num = 0
+    for i, interaction in enumerate(interactions):
+        if not interaction.expert_content:
+            continue
+
+        pair_num += 1
+        # Find the preceding user question for context
+        user_question = ""
+        for j in range(i - 1, -1, -1):
+            if interactions[j].role.lower() == "user":
+                user_question = interactions[j].content
+                break
+
+        parts = [f"=== Comparison {pair_num} ==="]
+        if user_question:
+            parts.append(f"User Question: ```{user_question}```")
+        parts.append(f"Agent Response: ```{interaction.content}```")
+        parts.append(f"Expert Response: ```{interaction.expert_content}```")
+        pairs.append("\n".join(parts))
+
+    return "\n\n".join(pairs)
+
+
+def construct_expert_playbook_extraction_messages(
+    prompt_manager: PromptManager,
+    request_interaction_data_models: list[RequestInteractionDataModel],
+    agent_context_prompt: str,
+    extraction_definition_prompt: str,
+) -> list[dict]:
+    """
+    Construct LLM messages for expert-content playbook extraction.
+
+    Uses expert-specific prompts that compare agent responses against expert
+    responses and extract playbook entries about alignment gaps.
+
+    Args:
+        prompt_manager: The prompt manager for rendering prompt templates
+        request_interaction_data_models: Session data with expert_content interactions
+        agent_context_prompt: Context about the agent for system message
+        extraction_definition_prompt: Definition of what the playbook should contain
+
+    Returns:
+        list[dict]: List of messages ready for expert playbook extraction
+    """
+    system_config = PromptConfig(
+        prompt_id=PlaybookServiceConstants.PLAYBOOK_EXTRACTION_CONTEXT_EXPERT_PROMPT_ID,
+        variables={
+            "agent_context_prompt": agent_context_prompt,
+            "extraction_definition_prompt": extraction_definition_prompt,
+        },
+    )
+
+    comparison_pairs = format_expert_comparison_pairs(request_interaction_data_models)
+
+    user_config = PromptConfig(
+        prompt_id=PlaybookServiceConstants.PLAYBOOK_EXTRACTION_EXPERT_PROMPT_ID,
+        variables={
+            "comparison_pairs": comparison_pairs,
+        },
+    )
+
+    interactions = extract_interactions_from_request_interaction_data_models(
+        request_interaction_data_models
+    )
+
+    config = MessageConstructionConfig(
+        prompt_manager=prompt_manager,
+        system_prompt_config=system_config,
+        user_prompt_config=user_config,
+    )
+
+    return construct_messages_from_interactions(interactions, config)

package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/__init__.py

@@ -0,0 +1,24 @@
+"""GEPA-driven playbook content optimizer.
+
+Top-level entry points exported from this package:
+
+- ``PlaybookOptimizer.optimize(target)`` — runs one paired-rollout search
+  for a single playbook and persists the result.
+- ``PlaybookOptimizationScheduler.get_instance()`` — singleton that
+  debounces enqueues from upstream services (the aggregator and the
+  user-playbook generation service) and dispatches each ``optimize`` call
+  on a worker thread.
+- ``PlaybookOptimizationTarget`` — the (kind, target_id) pair both APIs
+  consume.
+
+The full doc lives at ``docs/playbook_optimizer_assistant_backends.md``.
+"""
+
+from .optimizer import PlaybookOptimizationTarget, PlaybookOptimizer
+from .scheduler import PlaybookOptimizationScheduler
+
+__all__ = [
+    "PlaybookOptimizationScheduler",
+    "PlaybookOptimizationTarget",
+    "PlaybookOptimizer",
+]

package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/assistant_webhook.py

@@ -0,0 +1,246 @@
+"""Assistant backends for the playbook optimizer.
+
+The playbook optimizer runs paired rollouts (incumbent vs candidate playbook)
+against a "real" assistant. This module hosts the two pluggable backends that
+produce the assistant's reply for a given turn:
+
+- ``WebhookAssistant``      — POSTs JSON to a configured URL.
+- ``LocalScriptAssistant``  — spawns a subprocess and exchanges JSON via
+  stdin/stdout.
+
+Both satisfy ``AssistantCallable``, so ``MultiTurnRollout`` and the GEPA
+adapter never need to know which one is in use. Selection happens once in
+``PlaybookOptimizer._create_assistant`` based on config.
+
+Failures from either backend raise ``AssistantFailedError`` (or a subclass);
+the GEPA adapter catches that single base class and turns it into an
+``aborted`` evaluation row instead of crashing the search loop.
+
+The file is named ``assistant_webhook.py`` for historical reasons — the
+local-script backend was added later. New code should still import from this
+module.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+from typing import Any, Protocol
+
+import requests
+
+from reflexio.models.api_schema.domain import AgentPlaybook
+
+from .models import ChatMessage
+
+
+class AssistantCallable(Protocol):
+    """Shape every assistant backend must satisfy.
+
+    A backend takes the conversation so far plus the playbook(s) to inject
+    and returns the assistant's next reply as a plain string. Errors are
+    signalled by raising ``AssistantFailedError`` (or a subclass).
+    """
+
+    def __call__(
+        self, messages: list[ChatMessage], playbooks: list[AgentPlaybook]
+    ) -> str: ...
+
+
+class AssistantFailedError(Exception):
+    """Base class for any assistant-backend failure.
+
+    The GEPA adapter catches this single type to absorb backend faults
+    (network errors, subprocess crashes, malformed responses) and record
+    them as ``verdict='aborted'`` evaluations rather than aborting the
+    optimizer run.
+    """
+
+
+class WebhookFailedError(AssistantFailedError):
+    """Raised when ``WebhookAssistant`` exhausts its retries."""
+
+
+class LocalScriptFailedError(AssistantFailedError):
+    """Raised when ``LocalScriptAssistant`` exhausts its retries."""
+
+
+def _build_payload(
+    messages: list[ChatMessage], playbooks: list[AgentPlaybook]
+) -> dict[str, Any]:
+    """Build the wire payload shared by both backends.
+
+    Centralising this guarantees that the webhook body and the script's
+    stdin contain the *same* fields in the *same* shape — the only thing
+    that differs between the two transports is how the bytes are delivered.
+    """
+    return {
+        "messages": [message.model_dump() for message in messages],
+        "playbooks": [
+            {
+                "id": pb.agent_playbook_id,
+                "content": pb.content,
+                "trigger": pb.trigger,
+            }
+            for pb in playbooks
+        ],
+    }
+
+
+class WebhookAssistant:
+    """HTTP assistant backend.
+
+    POSTs the rollout payload to ``url`` and expects ``{"content": str}`` in
+    the response body. Retries any exception (network error, non-2xx,
+    malformed JSON, missing field) with exponential backoff up to
+    ``max_retries`` additional attempts before raising
+    ``WebhookFailedError``.
+
+    Args:
+        url: Destination URL. The optimizer does not validate the scheme,
+            so operators are responsible for ensuring it points at a host
+            that satisfies their data-residency requirements.
+        auth_header: Sent verbatim as the ``Authorization`` header. Never
+            logged.
+        timeout_s: Per-request timeout (passed to ``requests.post``).
+        max_retries: Number of *additional* attempts after the first try.
+            ``max_retries=0`` means a single attempt.
+        backoff_base_s: Base for the exponential delay
+            ``backoff_base_s * 2**attempt`` between retries.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        auth_header: str | None,
+        timeout_s: int,
+        max_retries: int,
+        backoff_base_s: float,
+    ) -> None:
+        self.url = url
+        self.auth_header = auth_header
+        self.timeout_s = timeout_s
+        self.max_retries = max_retries
+        self.backoff_base_s = backoff_base_s
+
+    def __call__(
+        self, messages: list[ChatMessage], playbooks: list[AgentPlaybook]
+    ) -> str:
+        headers = {"Content-Type": "application/json"}
+        if self.auth_header:
+            headers["Authorization"] = self.auth_header
+        payload = _build_payload(messages, playbooks)
+        last_error: Exception | None = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = requests.post(
+                    self.url,
+                    json=payload,
+                    headers=headers,
+                    timeout=self.timeout_s,
+                )
+                response.raise_for_status()
+                data = response.json()
+                content = data.get("content")
+                if not isinstance(content, str):
+                    raise WebhookFailedError("assistant webhook returned no content")
+                return content
+            except Exception as exc:  # noqa: PERF203
+                last_error = exc
+                if attempt >= self.max_retries:
+                    break
+                time.sleep(self.backoff_base_s * (2**attempt))
+        # Wrap-and-rethrow ensures the adapter sees a single AssistantFailedError
+        # subclass regardless of which underlying exception triggered the failure.
+        raise WebhookFailedError(str(last_error)) from last_error
+
+
+class LocalScriptAssistant:
+    """Subprocess assistant backend.
+
+    Spawns ``[script_path, *script_args]`` per turn and exchanges JSON over
+    stdin/stdout. The script must:
+
+    1. Read a single JSON object from stdin matching ``_build_payload``'s
+       shape (``{"messages": [...], "playbooks": [...]}``).
+    2. Print a single JSON object to stdout containing at least
+       ``{"content": "<assistant reply>"}``.
+    3. Exit with code 0 on success. Any non-zero exit, malformed JSON,
+       missing/non-string ``content``, or timeout is treated as a failure
+       and counted against ``max_retries``.
+
+    The command is built as a list and passed to ``subprocess.run`` *without*
+    ``shell=True``, so config values cannot inject shell metacharacters.
+    Retry/timeout/backoff semantics mirror ``WebhookAssistant``.
+    """
+
+    def __init__(
+        self,
+        script_path: str,
+        script_args: list[str],
+        timeout_s: int,
+        max_retries: int,
+        backoff_base_s: float,
+    ) -> None:
+        self.command = [script_path, *script_args]
+        self.timeout_s = timeout_s
+        self.max_retries = max_retries
+        self.backoff_base_s = backoff_base_s
+
+    def __call__(
+        self, messages: list[ChatMessage], playbooks: list[AgentPlaybook]
+    ) -> str:
+        payload = _build_payload(messages, playbooks)
+        stdin = json.dumps(payload, ensure_ascii=False)
+        last_error: Exception | None = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                # shell=False (the default) is intentional — see class docstring.
+                result = subprocess.run(  # noqa: S603
+                    self.command,
+                    input=stdin,
+                    text=True,
+                    capture_output=True,
+                    timeout=self.timeout_s,
+                    check=False,
+                )
+                if result.returncode != 0:
+                    # Truncate stderr to keep error messages bounded — long
+                    # tracebacks would otherwise blow up DB rows and logs.
+                    raise LocalScriptFailedError(
+                        "assistant script exited with code "
+                        f"{result.returncode}: {_truncate(result.stderr)}"
+                    )
+                try:
+                    data = json.loads(result.stdout)
+                except json.JSONDecodeError as exc:
+                    raise LocalScriptFailedError(
+                        f"assistant script returned invalid JSON: {_truncate(result.stdout)}"
+                    ) from exc
+                content = data.get("content") if isinstance(data, dict) else None
+                if not isinstance(content, str):
+                    raise LocalScriptFailedError("assistant script returned no content")
+                return content
+            except subprocess.TimeoutExpired:
+                # Timeouts come up via a different exception class than the
+                # validation errors above, but feed into the same retry budget.
+                last_error = LocalScriptFailedError(
+                    f"assistant script timed out after {self.timeout_s}s"
+                )
+                if attempt >= self.max_retries:
+                    break
+                time.sleep(self.backoff_base_s * (2**attempt))
+            except Exception as exc:  # noqa: PERF203
+                last_error = exc
+                if attempt >= self.max_retries:
+                    break
+                time.sleep(self.backoff_base_s * (2**attempt))
+        raise LocalScriptFailedError(str(last_error)) from last_error
+
+
+def _truncate(value: str, limit: int = 1000) -> str:
+    """Cap a string for inclusion in error messages and stored rows."""
+    if len(value) <= limit:
+        return value
+    return f"{value[:limit]}..."