claude-smart 0.2.42 → 0.2.44

package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/__init__.py

@@ -0,0 +1 @@
+"""F1 per-turn shadow comparison service."""

package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py

@@ -0,0 +1,184 @@
+"""F1 per-turn shadow comparison judge service.
+
+Stateless beyond construction. Routes every LLM call through
+``LiteLLMClient`` and every prompt through ``PromptManager`` per project
+guardrails — never imports OpenAIClient or ClaudeClient directly.
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from reflexio.models.api_schema.eval_overview_schema import (
+    ShadowComparisonOutput,
+    ShadowComparisonVerdict,
+)
+from reflexio.server.services.shadow_comparison.outcome import assign_positions
+
+if TYPE_CHECKING:
+    from reflexio.models.api_schema.domain.entities import Interaction
+    from reflexio.server.llm.litellm_client import LiteLLMClient
+    from reflexio.server.prompt.prompt_manager import PromptManager
+
+
+logger = logging.getLogger(__name__)
+
+_PROMPT_ID = "shadow_comparison"
+
+
+class ShadowComparisonJudge:
+    """Per-turn LLM-as-judge for Reflexio vs Shadow responses (F1).
+
+    The judge sees Request 1 + Request 2 (position-randomized) and decides
+    which is better, or whether they tie. The position assignment is
+    recorded on the verdict so the dashboard can derive Reflexio-relative
+    win / tie / loss via
+    :func:`reflexio.server.services.shadow_comparison.outcome.derive_reflexio_outcome`.
+    """
+
+    def __init__(
+        self,
+        llm_client: LiteLLMClient,
+        prompt_manager: PromptManager,
+        prompt_version: str,
+    ):
+        """
+        Initialize the judge.
+
+        Args:
+            llm_client (LiteLLMClient): Unified LLM client. Used to render
+                the verdict via ``generate_chat_response`` with a Pydantic
+                ``response_format``.
+            prompt_manager (PromptManager): Prompt manager for the
+                ``shadow_comparison`` prompt. Prompt-version selection is
+                governed by the manager's ``version_override`` dict;
+                ``prompt_version`` below is recorded on the verdict for
+                later filtering but does not affect rendering.
+            prompt_version (str): Semver of the active shadow_comparison
+                prompt. Stamped onto every verdict so the dashboard can
+                exclude verdicts from a stale rubric.
+        """
+        self._llm_client = llm_client
+        self._prompt_manager = prompt_manager
+        self._prompt_version = prompt_version
+
+    # ===============================
+    # public methods
+    # ===============================
+
+    def judge_turn(
+        self,
+        *,
+        interaction: Interaction,
+        session_id: str,
+        agent_version: str,
+        rng: random.Random,
+        user_message: str = "",
+    ) -> ShadowComparisonVerdict | None:
+        """
+        Grade one interaction; return ``None`` when the LLM fails or the
+        interaction has no shadow response to compare against.
+
+        Returning ``None`` (rather than raising) lets the regen worker
+        skip this turn and continue with the rest of the session — one
+        rate-limit blip should not abort an entire batch.
+
+        Args:
+            interaction (Interaction): The agent-side interaction. Its
+                ``content`` is the Reflexio response and ``shadow_content``
+                is the no-Reflexio response. When ``shadow_content`` is
+                empty the judge short-circuits with ``None``.
+            session_id (str): Denormalized onto the verdict for cheap
+                session-scoped queries.
+            agent_version (str): Pinned onto the verdict for trend slicing.
+            rng (random.Random): Position-randomization source.
+                Production callers pass a fresh ``random.Random()`` per
+                judge call; tests inject a seeded ``Random`` for
+                reproducibility.
+            user_message (str): The user turn this interaction responds
+                to. Optional — empty string is rendered into the prompt
+                if the caller does not provide a value.
+
+        Returns:
+            ShadowComparisonVerdict | None: The constructed verdict, or
+            ``None`` when shadow content is missing or the LLM call fails.
+        """
+        if not interaction.shadow_content:
+            return None
+
+        request_1, request_2, reflexio_is_request_1 = assign_positions(
+            reflexio_response=interaction.content,
+            shadow_response=interaction.shadow_content,
+            rng=rng,
+        )
+
+        prompt = self._prompt_manager.render_prompt(
+            prompt_id=_PROMPT_ID,
+            variables={
+                "user_message": user_message,
+                "request_1_response": request_1,
+                "request_2_response": request_2,
+            },
+        )
+
+        output = self._call_judge_llm(prompt, interaction.interaction_id)
+        if output is None:
+            return None
+
+        return ShadowComparisonVerdict(
+            verdict_id=0,  # storage assigns the autoincrement id
+            interaction_id=str(interaction.interaction_id),
+            session_id=session_id,
+            agent_version=agent_version,
+            reflexio_is_request_1=reflexio_is_request_1,
+            output=output,
+            judge_prompt_version=self._prompt_version,
+            created_at=datetime.now(UTC),
+        )
+
+    # ===============================
+    # private helpers
+    # ===============================
+
+    def _call_judge_llm(
+        self,
+        prompt: str,
+        interaction_id: int,
+    ) -> ShadowComparisonOutput | None:
+        """
+        Invoke the LLM with structured-output and unwrap the response.
+
+        Args:
+            prompt (str): The rendered judge prompt.
+            interaction_id (int): Interaction id for failure-logging context.
+
+        Returns:
+            ShadowComparisonOutput | None: The parsed structured output,
+            or ``None`` if the call failed or returned an unexpected type.
+        """
+        try:
+            response = self._llm_client.generate_chat_response(
+                messages=[{"role": "user", "content": prompt}],
+                response_format=ShadowComparisonOutput,
+            )
+        except Exception as exc:  # noqa: BLE001 — judge failure must not abort batch
+            logger.warning(
+                "shadow_comparison judge failed for interaction %s: %s",
+                interaction_id,
+                exc,
+            )
+            return None
+
+        if not isinstance(response, ShadowComparisonOutput):
+            logger.warning(
+                "Unexpected response type from shadow_comparison judge for "
+                "interaction %s: %s",
+                interaction_id,
+                type(response).__name__,
+            )
+            return None
+
+        return response

package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/outcome.py

@@ -0,0 +1,81 @@
+"""Pure F1 outcome helpers — position randomization + win/loss/tie derivation.
+
+These functions are deliberately DB-free and LLM-free so they can be unit
+tested in milliseconds and exercised by mutmut.
+"""
+
+from __future__ import annotations
+
+import random
+from enum import StrEnum
+
+from reflexio.models.api_schema.eval_overview_schema import ShadowComparisonVerdict
+
+
+class Outcome(StrEnum):
+    """Reflexio-relative outcome derived from a judge's position-randomized verdict.
+
+    Mirrors the GroupAssignment pattern from F2's group_aggregation module —
+    StrEnum so values serialize transparently to JSON / SQL.
+    """
+
+    WIN = "win"
+    LOSS = "loss"
+    TIE = "tie"
+
+
+def assign_positions(
+    reflexio_response: str,
+    shadow_response: str,
+    rng: random.Random,
+) -> tuple[str, str, bool]:
+    """
+    Randomize which response is shown as "Request 1" to the judge.
+
+    The judge prompt is blind to the assignment — it sees the two strings
+    labeled "REQUEST 1" and "REQUEST 2". We record the assignment so the
+    dashboard can derive the Reflexio-relative outcome via
+    `derive_reflexio_outcome`.
+
+    Args:
+        reflexio_response (str): The response generated with Reflexio rules in context.
+        shadow_response (str): The response generated without Reflexio rules.
+        rng (random.Random): Seeded random for reproducibility (tests inject a
+            seed; production uses `random.Random()` per judge call).
+
+    Returns:
+        tuple[str, str, bool]: (request_1_response, request_2_response,
+            reflexio_is_request_1). Storage records the third element on the
+            verdict row.
+    """
+    reflexio_is_request_1 = rng.random() < 0.5
+    if reflexio_is_request_1:
+        return reflexio_response, shadow_response, True
+    return shadow_response, reflexio_response, False
+
+
+def derive_reflexio_outcome(verdict: ShadowComparisonVerdict) -> Outcome:
+    """
+    Map a position-randomized verdict to a Reflexio-relative outcome.
+
+    Truth table:
+        better=='tie'                                  -> TIE
+        better=='1' AND reflexio_is_request_1==True    -> WIN
+        better=='1' AND reflexio_is_request_1==False   -> LOSS
+        better=='2' AND reflexio_is_request_1==True    -> LOSS
+        better=='2' AND reflexio_is_request_1==False   -> WIN
+
+    Args:
+        verdict (ShadowComparisonVerdict): The stored verdict, including the
+            judge's `better_request` choice and the `reflexio_is_request_1`
+            position-randomization record.
+
+    Returns:
+        Outcome: WIN / LOSS / TIE relative to the Reflexio response.
+    """
+    if verdict.output.better_request == "tie":
+        return Outcome.TIE
+    is_reflexio_better = (
+        verdict.output.better_request == "1"
+    ) == verdict.reflexio_is_request_1
+    return Outcome.WIN if is_reflexio_better else Outcome.LOSS

package/plugin/vendor/reflexio/reflexio/server/services/storage/constants.py

@@ -0,0 +1,2 @@
+class StorageConstants:
+    STORAGE_TYPE_CONFIG_NAME = "storage_type"

package/plugin/vendor/reflexio/reflexio/server/services/storage/error.py

@@ -0,0 +1,11 @@
+class StorageError(Exception):
+    """
+    Exception raised for storage errors
+    """
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"StorageError: {self.message}"

package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py

@@ -0,0 +1,154 @@
+"""Shared row-retention policy for storage backends."""
+
+from __future__ import annotations
+
+import math
+import os
+from dataclasses import dataclass
+
+DEFAULT_ROW_RETENTION_LIMIT = 250_000
+ROW_RETENTION_DELETE_FRACTION = 0.20
+
+
+@dataclass(frozen=True, slots=True)
+class RetentionTarget:
+    """A physical storage target eligible for row-count retention."""
+
+    name: str
+    table_name: str
+    order_column: str
+    id_columns: tuple[str, ...]
+
+
+RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
+    RetentionTarget("profiles", "profiles", "created_at", ("profile_id",)),
+    RetentionTarget("interactions", "interactions", "created_at", ("interaction_id",)),
+    RetentionTarget("requests", "requests", "created_at", ("request_id",)),
+    RetentionTarget(
+        "user_playbooks", "user_playbooks", "created_at", ("user_playbook_id",)
+    ),
+    RetentionTarget(
+        "agent_playbooks", "agent_playbooks", "created_at", ("agent_playbook_id",)
+    ),
+    RetentionTarget(
+        "agent_success_evaluation_result",
+        "agent_success_evaluation_result",
+        "created_at",
+        ("result_id",),
+    ),
+    RetentionTarget(
+        "profile_change_logs", "profile_change_logs", "created_at", ("id",)
+    ),
+    RetentionTarget(
+        "playbook_aggregation_change_logs",
+        "playbook_aggregation_change_logs",
+        "created_at",
+        ("id",),
+    ),
+    RetentionTarget("share_links", "share_links", "created_at", ("id",)),
+    RetentionTarget(
+        "agent_playbook_source_user_playbooks",
+        "agent_playbook_source_user_playbooks",
+        "created_at",
+        ("agent_playbook_id", "user_playbook_id"),
+    ),
+    RetentionTarget(
+        "playbook_optimization_jobs",
+        "playbook_optimization_jobs",
+        "created_at",
+        ("job_id",),
+    ),
+    RetentionTarget(
+        "playbook_optimization_candidates",
+        "playbook_optimization_candidates",
+        "created_at",
+        ("candidate_id",),
+    ),
+    RetentionTarget(
+        "playbook_optimization_evaluations",
+        "playbook_optimization_evaluations",
+        "created_at",
+        ("evaluation_id",),
+    ),
+    RetentionTarget(
+        "playbook_optimization_events",
+        "playbook_optimization_events",
+        "created_at",
+        ("event_id",),
+    ),
+    RetentionTarget("skills", "skills", "created_at", ("skill_id",)),
+)
+
+RETENTION_TARGETS_BY_NAME = {target.name: target for target in RETENTION_TARGETS}
+
+
+@dataclass(frozen=True, slots=True)
+class CascadeRef:
+    """A dependent table that must be cleaned when a retention target's rows
+    are deleted.
+
+    Attributes:
+        table_name (str): Table whose rows depend on the retention target.
+        fk_column (str): Column in ``table_name`` holding the parent target's
+            primary key. Always references the first column of the parent
+            target's ``id_columns`` (single-key parents only).
+    """
+
+    table_name: str
+    fk_column: str
+
+
+# Maps a retention target → its dependent tables that must be deleted first
+# when retention removes rows. Keep in sync with the storage backends that
+# rely on it (Postgres, Supabase, and the SQLite bespoke implementations).
+RETENTION_CASCADES: dict[str, tuple[CascadeRef, ...]] = {
+    "requests": (CascadeRef("interactions", "request_id"),),
+    "user_playbooks": (
+        CascadeRef("agent_playbook_source_user_playbooks", "user_playbook_id"),
+    ),
+    "agent_playbooks": (
+        CascadeRef("agent_playbook_source_user_playbooks", "agent_playbook_id"),
+    ),
+    "playbook_optimization_jobs": (
+        CascadeRef("playbook_optimization_evaluations", "job_id"),
+        CascadeRef("playbook_optimization_events", "job_id"),
+        CascadeRef("playbook_optimization_candidates", "job_id"),
+    ),
+    "playbook_optimization_candidates": (
+        CascadeRef("playbook_optimization_evaluations", "candidate_id"),
+    ),
+}
+
+
+def get_row_retention_limits() -> dict[str, int]:
+    """Return per-target row limits from env with code defaults.
+
+    ``REFLEXIO_ROW_LIMIT_<TARGET>`` takes precedence for every target.
+    ``INTERACTION_CLEANUP_THRESHOLD`` remains the legacy override for
+    interactions when the new variable is not present.
+    """
+    limits: dict[str, int] = {}
+    for target in RETENTION_TARGETS:
+        env_name = f"REFLEXIO_ROW_LIMIT_{target.name.upper()}"
+        default = DEFAULT_ROW_RETENTION_LIMIT
+        if target.name == "interactions":
+            default = _get_int_env("INTERACTION_CLEANUP_THRESHOLD", default)
+        limits[target.name] = _get_int_env(env_name, default)
+    return limits
+
+
+def delete_count_for_retention(current_count: int) -> int:
+    """Return how many rows to delete when a target exceeds its limit."""
+    if current_count <= 0:
+        return 0
+    return max(1, math.ceil(current_count * ROW_RETENTION_DELETE_FRACTION))
+
+
+def _get_int_env(name: str, default: int) -> int:
+    value = os.environ.get(name)
+    if value is None or not value.strip():
+        return default
+    try:
+        return int(value)
+    except ValueError:
+        return default

package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py

@@ -0,0 +1,155 @@
+"""Shared scaffolding for SQL-backed row-retention cleanup.
+
+Concrete SQL backends (SQLite, Postgres, Supabase) mix in
+``RetentionMixin`` and implement a small set of hooks. The public surface
+``count_retention_target_rows`` / ``delete_oldest_retention_target_rows``
+lives here so the dispatch — limit lookup, key selection, cascade,
+delete — cannot drift across the three backends.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator, Sequence
+from typing import Any
+
+from reflexio.server.services.storage.retention import (
+    RETENTION_TARGETS_BY_NAME,
+    RetentionTarget,
+)
+
+# Conservative chunk size for IN-list deletes. Picked to stay well under:
+#   - SQLite's SQLITE_MAX_VARIABLE_NUMBER (999 on builds before 3.32; 32766 after).
+#   - PostgREST URL length limits (gateway caps are commonly 8-16 KB).
+RETENTION_DELETE_CHUNK = 500
+
+
+def chunked(
+    values: Sequence[Any], chunk_size: int = RETENTION_DELETE_CHUNK
+) -> Iterator[list[Any]]:
+    """Yield consecutive ``chunk_size`` slices of ``values`` as lists.
+
+    Args:
+        values (Sequence[Any]): Items to split.
+        chunk_size (int): Maximum number of items per chunk.
+
+    Yields:
+        list[Any]: Successive non-empty chunks.
+    """
+    for start in range(0, len(values), chunk_size):
+        yield list(values[start : start + chunk_size])
+
+
+def get_retention_target(target_name: str) -> RetentionTarget:
+    """Resolve a retention target by name.
+
+    Args:
+        target_name (str): Registered name (e.g. ``"interactions"``).
+
+    Returns:
+        RetentionTarget: The matching target.
+
+    Raises:
+        ValueError: If ``target_name`` is not registered.
+    """
+    try:
+        return RETENTION_TARGETS_BY_NAME[target_name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown retention target: {target_name}") from exc
+
+
+class RetentionMixin(ABC):
+    """Backend-agnostic orchestration of row-retention cleanup.
+
+    SQL backends mix this in and implement the abstract hooks. The public
+    methods are intentionally defined once here so that the count/select/
+    cascade/delete ordering is identical across all three backends.
+    """
+
+    def count_retention_target_rows(self, target_name: str) -> int:
+        """Return the current row count for a retention target.
+
+        Args:
+            target_name (str): Registered retention target name.
+
+        Returns:
+            int: Row count, or 0 if the underlying table is missing.
+        """
+        target = get_retention_target(target_name)
+        if not self._retention_table_exists(target.table_name):
+            return 0
+        return self._retention_count_rows(target)
+
+    def delete_oldest_retention_target_rows(self, target_name: str, count: int) -> int:
+        """Delete up to ``count`` oldest rows for a retention target.
+
+        Calls the dependency-cascade hook before the target-row delete so
+        backends with foreign-key constraints stay consistent.
+
+        Args:
+            target_name (str): Registered retention target name.
+            count (int): Maximum number of rows to delete.
+
+        Returns:
+            int: Number of rows actually selected for deletion.
+        """
+        if count <= 0:
+            return 0
+        target = get_retention_target(target_name)
+        if not self._retention_table_exists(target.table_name):
+            return 0
+        keys = self._retention_select_oldest_keys(target, count)
+        if not keys:
+            return 0
+        self._retention_perform_delete(target, keys)
+        return len(keys)
+
+    def _retention_perform_delete(
+        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
+    ) -> None:
+        """Run dependency cleanup then target-row delete.
+
+        Default implementation runs the hooks in sequence. Backends that
+        need both steps inside one transaction (notably SQLite) should
+        override this method.
+        """
+        self._retention_delete_dependencies(target, keys)
+        self._retention_delete_target_rows(target, keys)
+
+    # -- Backend hooks --
+
+    @abstractmethod
+    def _retention_table_exists(self, table_name: str) -> bool:
+        """Return whether ``table_name`` exists in the backing store."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _retention_count_rows(self, target: RetentionTarget) -> int:
+        """Return the live row count for ``target``'s table."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _retention_select_oldest_keys(
+        self, target: RetentionTarget, count: int
+    ) -> list[tuple[Any, ...]]:
+        """Return up to ``count`` oldest key tuples for ``target``.
+
+        Each key tuple contains values aligned with ``target.id_columns``.
+        Ordering is by ``target.order_column`` ascending with the id
+        columns as a stable tiebreaker.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _retention_delete_dependencies(
+        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
+    ) -> None:
+        """Remove rows in tables that depend on ``target`` for ``keys``."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _retention_delete_target_rows(
+        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
+    ) -> None:
+        """Remove the rows for ``keys`` from ``target``'s table."""
+        raise NotImplementedError

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py

@@ -0,0 +1,59 @@
+from ._agent_run import SQLiteAgentRunMixin
+from ._base import (
+    SQLiteStorageBase,
+    _cosine_similarity,
+    _effective_search_mode,
+    _sanitize_fts_query,
+    _true_rrf_merge,
+    _vector_rank_rows,
+)
+from ._extras import ExtrasMixin
+from ._operations import OperationMixin
+from ._playbook import PlaybookMixin
+from ._profiles import ProfileMixin
+from ._requests import RequestMixin
+from ._shadow_verdicts import ShadowVerdictsMixin as SQLiteShadowVerdictsMixin
+from ._share_links import SQLiteShareLinkMixin
+from ._stall_state import (
+    SQLiteStallStateMixin,
+    StallReason,
+    StallState,
+    clear_stall_state,
+    get_stall_state,
+    init_stall_state_table,
+    mark_stall_notified,
+    upsert_stall_state,
+)
+
+
+class SQLiteStorage(
+    SQLiteAgentRunMixin,
+    ProfileMixin,
+    RequestMixin,
+    PlaybookMixin,
+    OperationMixin,
+    ExtrasMixin,
+    SQLiteShareLinkMixin,
+    SQLiteStallStateMixin,
+    SQLiteShadowVerdictsMixin,
+    SQLiteStorageBase,
+):
+    """SQLite-based storage with FTS5 and hybrid search."""
+
+    pass
+
+
+__all__ = [
+    "SQLiteStorage",
+    "_cosine_similarity",
+    "_effective_search_mode",
+    "_sanitize_fts_query",
+    "_true_rrf_merge",
+    "_vector_rank_rows",
+    "StallReason",
+    "StallState",
+    "clear_stall_state",
+    "get_stall_state",
+    "mark_stall_notified",
+    "upsert_stall_state",
+]