claude-smart 0.2.42 → 0.2.44

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/group_aggregation.py

@@ -0,0 +1,192 @@
+"""Pure aggregation helpers for F2's session-level A/B groups.
+
+These functions are deliberately DB-free and LLM-free so they can be unit
+tested in milliseconds and exercised by mutmut. The storage layer hands
+in raw tuples; we hand back SuccessRateTrendByGroup.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from collections.abc import Iterable
+from enum import StrEnum
+from typing import Any
+
+from reflexio.models.api_schema.eval_overview_schema import (
+    SuccessRateTrendByGroup,
+    TrendPoint,
+)
+
+# Z-value for a 95% Wald CI on a binomial proportion (two-sided).
+_Z_95 = 1.96
+
+# Cap on the lift CI half-width. With very small sample sizes the Wald CI
+# computes to absurd widths (e.g., ±70pp), and rendering that on a
+# dashboard implies false precision. 0.5 (= ±50pp) is the product-meaningful
+# threshold: any wider than that, the user should think "we don't know"
+# and the frontend should consider rendering a low-confidence indicator.
+_MAX_CI_HALF_WIDTH = 0.5
+
+
+class GroupAssignment(StrEnum):
+    """Group label derived from `Request.metadata.reflexio_retrieval_enabled`."""
+
+    TREATMENT = "treatment"
+    CONTROL = "control"
+    UNTAGGED = "untagged"
+
+
+def assign_group_from_metadata(metadata: dict[str, Any]) -> GroupAssignment:
+    """
+    Bucket a session by its first request's metadata.
+
+    Reads ``metadata.reflexio_retrieval_enabled`` as a true bool. Anything
+    else (missing, string, int, None) maps to UNTAGGED — we never silently
+    coerce non-bool values, so customers see how many of their sessions
+    are tagged inconsistently.
+
+    Args:
+        metadata (dict[str, Any]): The first request's metadata dict. Non-dict
+            input is tolerated and routed to UNTAGGED (defensive — the caller
+            may pass a value read from a JSON column with no type guarantee).
+
+    Returns:
+        GroupAssignment: TREATMENT, CONTROL, or UNTAGGED.
+    """
+    if not isinstance(metadata, dict):
+        return GroupAssignment.UNTAGGED
+    value = metadata.get("reflexio_retrieval_enabled")
+    if value is True:
+        return GroupAssignment.TREATMENT
+    if value is False:
+        return GroupAssignment.CONTROL
+    return GroupAssignment.UNTAGGED
+
+
+def group_session_outcomes_by_metadata(
+    outcomes: Iterable[tuple[str, int, bool, dict[str, Any]]],
+) -> dict[GroupAssignment, list[tuple[str, int, bool]]]:
+    """
+    Split (session_id, ts, is_success, metadata) tuples into three groups.
+
+    Args:
+        outcomes (Iterable[tuple[str, int, bool, dict[str, Any]]]): Tuples of
+            (session_id, ts, is_success, first_request_metadata).
+
+    Returns:
+        dict[GroupAssignment, list[tuple[str, int, bool]]]: Dict keyed by every
+        GroupAssignment member (TREATMENT, CONTROL, UNTAGGED) — always present,
+        possibly empty list — with values being (session_id, ts, is_success)
+        tuples (metadata dropped after binning).
+    """
+    out: dict[GroupAssignment, list[tuple[str, int, bool]]] = {
+        GroupAssignment.TREATMENT: [],
+        GroupAssignment.CONTROL: [],
+        GroupAssignment.UNTAGGED: [],
+    }
+    for session_id, ts, is_success, metadata in outcomes:
+        group = assign_group_from_metadata(metadata)
+        out[group].append((session_id, ts, is_success))
+    return out
+
+
+def bucket_outcomes(
+    outcomes: Iterable[tuple[str, int, bool]],
+    bucket_seconds: int,
+) -> list[dict[str, int]]:
+    """
+    Bucket outcomes by time, returning one dict per bucket.
+
+    Args:
+        outcomes (Iterable[tuple[str, int, bool]]): Tuples of (session_id, ts,
+            is_success). ``session_id`` is unused at this stage; kept for future
+            per-session breakdowns.
+        bucket_seconds (int): Bucket width in seconds (e.g., 86400 for daily,
+            604800 for weekly). Must be positive.
+
+    Returns:
+        list[dict[str, int]]: List of ``{"ts": bucket_start, "n": int,
+        "successes": int}``, sorted by ``ts`` ascending. Empty buckets are
+        omitted.
+
+    Raises:
+        ValueError: When ``bucket_seconds`` is not positive.
+    """
+    if bucket_seconds <= 0:
+        raise ValueError(f"bucket_seconds must be positive, got {bucket_seconds}")
+    counts: dict[int, dict[str, int]] = defaultdict(lambda: {"n": 0, "successes": 0})
+    for _, ts, is_success in outcomes:
+        bucket_start = (ts // bucket_seconds) * bucket_seconds
+        counts[bucket_start]["n"] += 1
+        if is_success:
+            counts[bucket_start]["successes"] += 1
+    return [
+        {"ts": bucket_start, "n": d["n"], "successes": d["successes"]}
+        for bucket_start, d in sorted(counts.items())
+    ]
+
+
+def compute_trend_by_group(
+    outcomes: Iterable[tuple[str, int, bool, dict[str, Any]]],
+    bucket_seconds: int,
+) -> SuccessRateTrendByGroup:
+    """
+    Build the dual+untagged-curve trend payload from raw session outcomes.
+
+    Args:
+        outcomes (Iterable[tuple[str, int, bool, dict[str, Any]]]): Tuples of
+            (session_id, ts, is_success, first_request_metadata).
+        bucket_seconds (int): Bucket width passed to :func:`bucket_outcomes`.
+
+    Returns:
+        SuccessRateTrendByGroup: Three curves (any of which may be empty).
+    """
+    grouped = group_session_outcomes_by_metadata(outcomes)
+    return SuccessRateTrendByGroup(
+        treatment=_to_trend_points(grouped[GroupAssignment.TREATMENT], bucket_seconds),
+        control=_to_trend_points(grouped[GroupAssignment.CONTROL], bucket_seconds),
+        untagged=_to_trend_points(grouped[GroupAssignment.UNTAGGED], bucket_seconds),
+    )
+
+
+def _to_trend_points(
+    outcomes: list[tuple[str, int, bool]],
+    bucket_seconds: int,
+) -> list[TrendPoint]:
+    """Convert raw outcomes into bucketed TrendPoints; omit empty buckets."""
+    return [
+        TrendPoint(ts=b["ts"], rate=b["successes"] / b["n"], n=b["n"])
+        for b in bucket_outcomes(outcomes, bucket_seconds)
+        if b["n"] > 0
+    ]
+
+
+def compute_lift_with_ci(
+    n_t: int, p_t: float, n_c: int, p_c: float
+) -> tuple[float | None, float | None]:
+    """
+    Compute success-rate lift (treatment − control) and a 95% CI half-width.
+
+    Uses the Wald approximation for the difference of two binomial
+    proportions. Returns ``(None, None)`` if either group has zero samples —
+    surfaces honest "no estimate" rather than fake zeros.
+
+    Args:
+        n_t (int): Treatment group size.
+        p_t (float): Treatment success rate, in ``[0.0, 1.0]``.
+        n_c (int): Control group size.
+        p_c (float): Control success rate, in ``[0.0, 1.0]``.
+
+    Returns:
+        tuple[float | None, float | None]: ``(lift, ci_half_width)`` — both in
+        proportion units (e.g., 0.14 for 14pp), or ``(None, None)`` when there
+        isn't enough data.
+    """
+    if n_t <= 0 or n_c <= 0:
+        return None, None
+    lift = p_t - p_c
+    var = (p_t * (1 - p_t)) / n_t + (p_c * (1 - p_c)) / n_c
+    ci_half = _Z_95 * math.sqrt(var)
+    ci_half = min(ci_half, _MAX_CI_HALF_WIDTH)
+    return lift, ci_half

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/hero_state.py

@@ -0,0 +1,75 @@
+"""Pure functions deriving the /evaluations hero block state and delta.
+
+These have no IO; they're invoked from EvaluationOverviewService with already-
+loaded aggregate numbers. Separated so the spec's 4-state machine can be
+unit-tested cheaply.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+_FULL_MIN_DAYS = 14
+_FULL_MIN_SHADOW_N = 500
+_SHADOW_OFF_MIN_DAYS = 7
+
+
+class HeroState(StrEnum):
+    """The four mutually exclusive hero block configurations from spec §3.2.
+
+    Members serialize to the snake_case strings the frontend switches on.
+    """
+
+    FULL = "full"
+    EARLY = "early"
+    SHADOW_OFF = "shadow_off"
+    EMPTY = "empty"
+
+
+def compute_hero_state(
+    *,
+    shadow_enabled: bool,
+    days_since_first_eval: int | None,
+    n_shadow_in_window: int,
+    total_results: int,
+) -> HeroState:
+    """Return the hero state for the current org and time window.
+
+    Args:
+        shadow_enabled (bool): Value of `Config.shadow_mode_enabled`.
+        days_since_first_eval (int | None): Wall-clock days since the first
+            ever evaluated session for this org. None when no results exist.
+        n_shadow_in_window (int): Count of evaluation results that have a
+            shadow grade inside the trend window. Caller is responsible for
+            filtering; this function only consumes the count.
+            Why graded-only: prevents the FULL gate from tripping mid-window-
+            flag-flip when sessions exist with shadow_content but no shadow
+            grade. (Note: the direct-grade `shadow_is_success` /
+            `shadow_is_escalated` columns added briefly in May 2026 were
+            retracted before any production deploy — see the deleted
+            migration pair in supabase/data/supabase/migrations/. The
+            per-turn shadow grade in F1 lives on a different surface.)
+        total_results (int): Total AgentSuccessEvaluationResult rows in the
+            trend window (used only to differentiate EMPTY from SHADOW_OFF).
+
+    Returns:
+        HeroState: The single applicable state. Empty wins over everything;
+            after that the order is shadow_off → early → full per spec.
+    """
+    if total_results == 0:
+        return HeroState.EMPTY
+    if not shadow_enabled:
+        if (
+            days_since_first_eval is not None
+            and days_since_first_eval >= _SHADOW_OFF_MIN_DAYS
+        ):
+            return HeroState.SHADOW_OFF
+        # <7 days since first eval AND shadow off → still onboarding;
+        # render as EMPTY so the frontend shows onboarding rather than a
+        # partly-formed trend.
+        return HeroState.EMPTY
+    if days_since_first_eval is None or days_since_first_eval < _FULL_MIN_DAYS:
+        return HeroState.EARLY
+    if n_shadow_in_window < _FULL_MIN_SHADOW_N:
+        return HeroState.EARLY
+    return HeroState.FULL

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/rule_attribution.py

@@ -0,0 +1,97 @@
+"""Join PlaybookApplicationStat citations with session success outcomes.
+
+Replaces the spec's proposed `rule_application` table with a join against
+the existing `Interaction.citations` data, computing "net sessions" per
+rule = successes_with_rule_fired - failures_with_rule_fired.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+CitationKey = tuple[str, str]  # (kind, real_id) — matches PlaybookApplicationStat
+
+
+@dataclass(frozen=True)
+class RuleAttribution:
+    """One row of the "rules that moved the needle" panel."""
+
+    rule_id: str
+    kind: str
+    title: str
+    successes_with: int
+    failures_with: int
+    cited_session_ids: tuple[str, ...] = ()
+    """Session IDs (within the trend window) that cited this rule. Tuple so
+    the dataclass stays hashable / frozen. The frontend uses this to filter
+    the /evaluations detail band to show only sessions where the rule
+    actually fired — answering 'which sessions did this rule help/hurt?'
+    without a second roundtrip."""
+
+    @property
+    def net_sessions(self) -> int:
+        return self.successes_with - self.failures_with
+
+
+def compute_net_sessions(
+    *,
+    citations_by_session: Mapping[str, list[CitationKey]],
+    is_success_by_session: Mapping[str, bool],
+    rule_titles: Mapping[CitationKey, str],
+    top_n: int,
+) -> list[RuleAttribution]:
+    """Aggregate net sessions per rule and return the top N by net_sessions.
+
+    Args:
+        citations_by_session (Mapping[str, list[CitationKey]]): For each
+            session in the window, the list of `(kind, real_id)` citations
+            harvested from its interactions.
+        is_success_by_session (Mapping[str, bool]): The session's evaluated
+            outcome. Sessions not in this map are skipped on both sides.
+        rule_titles (Mapping[CitationKey, str]): Pre-loaded display titles
+            for each rule (empty string when the underlying row is gone).
+        top_n (int): How many rows to return at most. Ordering: net_sessions
+            desc, then total fires desc.
+
+    Returns:
+        list[RuleAttribution]: At most `top_n` rows in the order described.
+        Each row carries the session ids that cited the rule so the
+        frontend can drill from a rule into the sessions it fired in.
+    """
+    successes: dict[CitationKey, int] = {}
+    failures: dict[CitationKey, int] = {}
+    # Map (kind, real_id) -> ordered list of session_ids that cited this rule.
+    # Ordering preserves caller-provided iteration order; the frontend sorts
+    # the detail band by created_at when displaying.
+    sessions_by_rule: dict[CitationKey, list[str]] = {}
+    for session_id, citations in citations_by_session.items():
+        outcome = is_success_by_session.get(session_id)
+        if outcome is None:
+            continue
+        # A rule cited multiple times in the same session counts once.
+        unique_cites = set(citations)
+        for key in unique_cites:
+            if outcome:
+                successes[key] = successes.get(key, 0) + 1
+            else:
+                failures[key] = failures.get(key, 0) + 1
+            sessions_by_rule.setdefault(key, []).append(session_id)
+
+    all_keys = set(successes) | set(failures)
+    rows = [
+        RuleAttribution(
+            rule_id=real_id,
+            kind=kind,
+            title=rule_titles.get((kind, real_id), ""),
+            successes_with=successes.get((kind, real_id), 0),
+            failures_with=failures.get((kind, real_id), 0),
+            cited_session_ids=tuple(sessions_by_rule.get((kind, real_id), [])),
+        )
+        for (kind, real_id) in all_keys
+    ]
+
+    rows.sort(
+        key=lambda r: (-r.net_sessions, -(r.successes_with + r.failures_with)),
+    )
+    return rows[:top_n]