claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py

@@ -1,27 +1,31 @@
 """Aggregator that composes hero, tiles, rule attribution, and distribution.
 
-The service does three reads (results, citations, playbook stats)
-and returns a GetEvaluationOverviewResponse. It's invoked from the FastAPI
-route handler; the storage is the same BaseStorage the rest of the server
-uses, so the same instance is reused via request_context.
+The service bulk-loads the requested evaluation window plus related source,
+citation, Braintrust, and optional shadow verdict rows, then returns a
+GetEvaluationOverviewResponse. It's invoked from the FastAPI route handler; the
+storage is the same BaseStorage the rest of the server uses, so the same
+instance is reused via request_context.
 """
 
 from __future__ import annotations
 
+import logging
+import time
 from collections import defaultdict
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import UTC, datetime
+from typing import Any
 
 from reflexio.models.api_schema.braintrust_schema import ImportedScore
 from reflexio.models.api_schema.domain.entities import (
     AgentSuccessEvaluationResult,
-    Request,
 )
 from reflexio.models.api_schema.eval_overview_schema import (
     BraintrustTileRow,
     BucketLiteral,
     ContextTile,
+    EvaluationSourceSetRequest,
     GetEvaluationOverviewRequest,
     GetEvaluationOverviewResponse,
     HeroBlock,
@@ -31,29 +35,29 @@ from reflexio.models.api_schema.eval_overview_schema import (
     RuleAttributionRow,
     ScoreDistribution,
     ShadowWinRateTrend,
-    SuccessRateTrendByGroup,
+    SourceSetComparison,
+    SourceSetEvaluationMetrics,
 )
 from reflexio.models.config_schema import Config
-from reflexio.server.services.evaluation_overview.distribution import (
+from reflexio.server.services.evaluation_overview.components.distribution import (
     BUCKET_LABELS,
     bucket_corrections,
 )
-from reflexio.server.services.evaluation_overview.group_aggregation import (
-    compute_trend_by_group,
-)
-from reflexio.server.services.evaluation_overview.hero_state import (
+from reflexio.server.services.evaluation_overview.components.hero_state import (
     compute_hero_state,
 )
-from reflexio.server.services.evaluation_overview.rule_attribution import (
+from reflexio.server.services.evaluation_overview.components.rule_attribution import (
     compute_net_sessions,
 )
-from reflexio.server.services.evaluation_overview.shadow_aggregation import (
+from reflexio.server.services.evaluation_overview.components.shadow_aggregation import (
     compute_shadow_win_rate_trend,
 )
 
 _DAY_SECONDS = 24 * 60 * 60
 _WEEK_SECONDS = 7 * 24 * 60 * 60
 _TOP_N_RULES = 5
+_LOGGER = logging.getLogger(__name__)
+ResultKey = tuple[str, str]
 
 
 @dataclass
@@ -65,7 +69,7 @@ class EvaluationOverviewService:
     Stateless across calls — safe to reuse the instance across requests.
     """
 
-    storage: object
+    storage: Any
     config: Config
 
     def run(
@@ -85,18 +89,25 @@ class EvaluationOverviewService:
             so every tile would display "no baseline" regardless of how much
             data the org has.
         """
-        all_results = self.storage.get_agent_success_evaluation_results(  # type: ignore[attr-defined]
-            agent_version=None, limit=10_000
+        # Tile + distribution baselines are always last-7d-vs-prior-7d,
+        # anchored to ``request.to_ts`` (which is "now" from the frontend's
+        # perspective).
+        cur_7d_from = max(request.from_ts, request.to_ts - _WEEK_SECONDS)
+        prev_to = cur_7d_from
+        prev_from = max(0, prev_to - _WEEK_SECONDS)
+        load_from = min(request.from_ts, prev_from)
+
+        start = time.perf_counter()
+        all_results = self.storage.get_agent_success_evaluation_results_in_window(  # type: ignore[attr-defined]
+            from_ts=load_from,
+            to_ts=request.to_ts,
+            agent_version=None,
         )
+        self._log_phase("eval_results", start, rows=len(all_results))
+
         results = [
             r for r in all_results if request.from_ts <= r.created_at <= request.to_ts
         ]
-
-        # Tile + distribution baselines are always last-7d-vs-prior-7d, anchored
-        # to ``request.to_ts`` (which is "now" from the frontend's perspective).
-        cur_7d_from = max(request.from_ts, request.to_ts - _WEEK_SECONDS)
-        prev_to = cur_7d_from
-        prev_from = max(0, prev_to - _WEEK_SECONDS)
         results_current_7d = [
             r for r in all_results if cur_7d_from <= r.created_at <= request.to_ts
         ]
@@ -105,23 +116,76 @@ class EvaluationOverviewService:
         ]
 
         earliest_eval_ts = min((r.created_at for r in all_results), default=None)
+        result_keys = list(
+            dict.fromkeys((r.user_id, r.session_id) for r in results if r.session_id)
+        )
+        if request.source_sets:
+            source_keys = list(
+                dict.fromkeys(
+                    (r.user_id, r.session_id)
+                    for r in (*results, *results_current_7d, *results_prev_7d)
+                    if r.session_id
+                )
+            )
+        else:
+            source_keys = result_keys
+        start = time.perf_counter()
+        session_sources = self._build_first_request_sources(source_keys)
+        self._log_phase(
+            "session_sources",
+            start,
+            sessions=len(set(source_keys)),
+            rows=len(session_sources),
+        )
+
+        start = time.perf_counter()
+        citations_by_session, rule_titles = self._load_citations(result_keys)
+        self._log_phase(
+            "citations",
+            start,
+            sessions=len(set(result_keys)),
+            rows=sum(len(v) for v in citations_by_session.values()),
+        )
+
         hero = self._build_hero(request, results, earliest_eval_ts)
         tiles = self._build_tiles(results_current_7d, results_prev_7d)
-        attribution = self._build_attribution(results)
+        attribution = self._build_attribution(
+            results, citations_by_session, rule_titles
+        )
         distribution = self._build_distribution(results_current_7d, results_prev_7d)
-        braintrust_tiles = self._build_braintrust_tiles(
+        start = time.perf_counter()
+        current_scores, prior_scores = self._load_braintrust_scores(
             cur_7d_from, request.to_ts, prev_from, prev_to
         )
-        # F2: group-aware success-rate trend curves. Joins each eval result
-        # with its session's first request metadata to bucket sessions into
-        # treatment / control / untagged, then time-buckets each curve at
-        # the same granularity as the hero chart.
-        success_rate_trend_by_group = self._build_group_trend(results, request.bucket)
-        # F1: per-turn shadow win-rate trend. Filters verdicts to the org's
-        # pinned judge prompt version so a future rubric bump doesn't
-        # silently mix epochs into the headline.
-        shadow_win_rate_trend = self._build_shadow_win_rate_trend(
-            request.from_ts, request.to_ts
+        self._log_phase(
+            "braintrust",
+            start,
+            current_rows=len(current_scores),
+            prior_rows=len(prior_scores),
+        )
+        braintrust_tiles = self._build_braintrust_tiles_from_scores(
+            current_scores, prior_scores
+        )
+        start = time.perf_counter()
+        shadow_win_rate_trend = (
+            self._build_shadow_win_rate_trend(request.from_ts, request.to_ts)
+            if request.include_shadow
+            else ShadowWinRateTrend(
+                judge_prompt_version=self.config.shadow_comparison_judge_prompt_version
+            )
+        )
+        self._log_phase("shadow", start, enabled=request.include_shadow)
+        source_set_comparison = self._build_source_set_comparison(
+            source_sets=request.source_sets,
+            results=results,
+            current=results_current_7d,
+            previous=results_prev_7d,
+            session_sources=session_sources,
+            bucket=request.bucket,
+            citations_by_session=citations_by_session,
+            rule_titles=rule_titles,
+            current_scores=current_scores,
+            prior_scores=prior_scores,
         )
 
         return GetEvaluationOverviewResponse(
@@ -130,8 +194,8 @@ class EvaluationOverviewService:
             rule_attribution=attribution,
             score_distribution=distribution,
             braintrust_tiles=braintrust_tiles,
-            success_rate_trend_by_group=success_rate_trend_by_group,
             shadow_win_rate_trend=shadow_win_rate_trend,
+            source_set_comparison=source_set_comparison,
         )
 
     # --- private helpers ---
@@ -198,12 +262,14 @@ class EvaluationOverviewService:
         )
 
     def _build_attribution(
-        self, results: list[AgentSuccessEvaluationResult]
+        self,
+        results: list[AgentSuccessEvaluationResult],
+        citations_by_session: dict[ResultKey, list[tuple[str, str]]],
+        rule_titles: dict[tuple[str, str], str],
     ) -> list[RuleAttributionRow]:
-        is_success_by_session = {r.session_id: r.is_success for r in results}
-        citations_by_session, rule_titles = self._load_citations(
-            list(is_success_by_session.keys())
-        )
+        is_success_by_session = {
+            (r.user_id, r.session_id): r.is_success for r in results
+        }
         rows = compute_net_sessions(
             citations_by_session=citations_by_session,
             is_success_by_session=is_success_by_session,
@@ -224,51 +290,57 @@ class EvaluationOverviewService:
         ]
 
     def _load_citations(
-        self, session_ids: list[str]
-    ) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str], str]]:
-        """Pull `Interaction.citations` keyed by session, with title lookup.
+        self, result_keys: list[ResultKey]
+    ) -> tuple[dict[ResultKey, list[tuple[str, str]]], dict[tuple[str, str], str]]:
+        """Pull `Interaction.citations` keyed by user/session, with title lookup.
 
         Falls back to empty data when the underlying storage method returns
         no interactions (default behavior on backends that haven't yet
         implemented `get_interactions_by_session`).
         """
-        citations_by_session: dict[str, list[tuple[str, str]]] = defaultdict(list)
-        for sid in session_ids:
-            interactions = self.storage.get_interactions_by_session(sid)  # type: ignore[attr-defined]
-            for interaction in interactions:
-                for cite in getattr(interaction, "citations", []) or []:
-                    # Citations may arrive as Pydantic Citation objects (from
-                    # the normal storage path) or as plain dicts (e.g. when
-                    # tests stub the storage). Handle both shapes.
-                    if isinstance(cite, dict):
-                        kind = cite.get("kind")
-                        rid = cite.get("real_id")
-                    else:
-                        kind = getattr(cite, "kind", None)
-                        rid = getattr(cite, "real_id", None)
-                    if kind and rid:
-                        citations_by_session[sid].append((kind, str(rid)))
-        # Titles via existing playbook_application_stats lookup
-        stats = self.storage.get_playbook_application_stats(days_back=30)  # type: ignore[attr-defined]
-        rule_titles = {(s.kind, s.real_id): s.title for s in stats}
+        wanted = set(result_keys)
+        session_ids = sorted({session_id for _, session_id in result_keys})
+        citations_by_session: dict[ResultKey, list[tuple[str, str]]] = defaultdict(list)
+        rule_titles: dict[tuple[str, str], str] = {}
+        for citation in self.storage.get_citations_by_session_ids(session_ids):  # type: ignore[attr-defined]
+            result_key = (citation.user_id, citation.session_id)
+            if result_key not in wanted:
+                continue
+            key = (citation.kind, citation.real_id)
+            citations_by_session[result_key].append(key)
+            if citation.title and key not in rule_titles:
+                rule_titles[key] = citation.title
         return citations_by_session, rule_titles
 
-    def _build_braintrust_tiles(
-        self, from_ts: int, to_ts: int, prev_from: int, prev_to: int
-    ) -> list[BraintrustTileRow]:
-        """Aggregate imported_score rows per scorer_name for current + prior windows.
-
-        Returns [] when the org has no Braintrust connection (default no-op
-        storage returns []). The frontend treats an empty list as "not
-        connected" and hides the Braintrust strip entirely.
-        """
+    def _load_braintrust_scores(
+        self,
+        from_ts: int,
+        to_ts: int,
+        prev_from: int,
+        prev_to: int,
+    ) -> tuple[list[ImportedScore], list[ImportedScore]]:
         org_id = self._org_id()
         if not org_id:
-            return []
+            return [], []
         current = self.storage.get_imported_scores(org_id, from_ts, to_ts)  # type: ignore[attr-defined]
         if not current:
-            return []
+            return [], []
         prior = self.storage.get_imported_scores(org_id, prev_from, prev_to)  # type: ignore[attr-defined]
+        return current, prior
+
+    def _build_braintrust_tiles_from_scores(
+        self,
+        current: list[ImportedScore],
+        prior: list[ImportedScore],
+        session_ids: set[str] | None = None,
+    ) -> list[BraintrustTileRow]:
+        """Aggregate imported_score rows per scorer_name for current + prior windows."""
+        if session_ids is not None:
+            current = [s for s in current if s.session_id in session_ids]
+        if not current:
+            return []
+        if session_ids is not None:
+            prior = [s for s in prior if s.session_id in session_ids]
         cur_agg = _aggregate_imported_scores(current)
         prior_agg = _aggregate_imported_scores(prior)
         rows: list[BraintrustTileRow] = []
@@ -288,6 +360,81 @@ class EvaluationOverviewService:
             )
         return rows
 
+    def _build_source_set_comparison(
+        self,
+        *,
+        source_sets: list[EvaluationSourceSetRequest],
+        results: list[AgentSuccessEvaluationResult],
+        current: list[AgentSuccessEvaluationResult],
+        previous: list[AgentSuccessEvaluationResult],
+        session_sources: dict[ResultKey, str],
+        bucket: BucketLiteral,
+        citations_by_session: dict[ResultKey, list[tuple[str, str]]],
+        rule_titles: dict[tuple[str, str], str],
+        current_scores: list[ImportedScore],
+        prior_scores: list[ImportedScore],
+    ) -> SourceSetComparison:
+        """Build request-source cohort metrics for the evaluation page."""
+        available_sources = sorted(
+            {session_sources.get((r.user_id, r.session_id), "") for r in results}
+        )
+        if not source_sets:
+            return SourceSetComparison(available_sources=available_sources)
+
+        requested_sources = {source for s in source_sets for source in s.sources}
+        unmatched = sum(
+            1
+            for r in results
+            if session_sources.get((r.user_id, r.session_id), "")
+            not in requested_sources
+        )
+        rows: list[SourceSetEvaluationMetrics] = []
+        for source_set in source_sets:
+            source_values = set(source_set.sources)
+            set_results = [
+                r
+                for r in results
+                if session_sources.get((r.user_id, r.session_id), "") in source_values
+            ]
+            set_current = [
+                r
+                for r in current
+                if session_sources.get((r.user_id, r.session_id), "") in source_values
+            ]
+            set_previous = [
+                r
+                for r in previous
+                if session_sources.get((r.user_id, r.session_id), "") in source_values
+            ]
+            session_ids = {r.session_id for r in set_results if r.session_id}
+            rows.append(
+                SourceSetEvaluationMetrics(
+                    label=source_set.label,
+                    sources=list(source_set.sources),
+                    session_count=len(set_results),
+                    session_ids=sorted(session_ids),
+                    success_rate_pp=_success_rate(set_results) * 100,
+                    buckets=_buckets(set_results, bucket),
+                    context_tiles=self._build_tiles(set_current, set_previous),
+                    score_distribution=self._build_distribution(
+                        set_current, set_previous
+                    ),
+                    rule_attribution=self._build_attribution(
+                        set_results, citations_by_session, rule_titles
+                    ),
+                    braintrust_tiles=self._build_braintrust_tiles_from_scores(
+                        current_scores,
+                        prior_scores,
+                        session_ids=session_ids,
+                    ),
+                )
+            )
+        return SourceSetComparison(
+            available_sources=available_sources,
+            sets=rows,
+            unmatched_session_count=unmatched,
+        )
+
     def _org_id(self) -> str:
         """Resolve org_id from request_context when available; else empty string."""
         # The service is constructed with `storage` + `config`; org_id isn't a
@@ -331,97 +478,14 @@ class EvaluationOverviewService:
             verdicts, judge_prompt_version=pinned_version
         )
 
-    def _build_group_trend(
-        self,
-        results: list[AgentSuccessEvaluationResult],
-        bucket: str,
-    ) -> SuccessRateTrendByGroup:
-        """Build the dual+untagged-curve trend payload for the window.
-
-        Joins each eval result with the first request of its session to read
-        ``metadata.reflexio_retrieval_enabled``, then delegates to the pure
-        ``compute_trend_by_group`` aggregator. Returns the default empty
-        ``SuccessRateTrendByGroup`` when ``results`` is empty.
-
-        Args:
-            results (list[AgentSuccessEvaluationResult]): Eval results in the
-                trend window.
-            bucket (str): Bucket granularity literal (``"week"`` or ``"day"``).
-
-        Returns:
-            SuccessRateTrendByGroup: Three curves (any of which may be empty).
-        """
-        if not results:
-            return SuccessRateTrendByGroup()
-        bucket_seconds = _WEEK_SECONDS if bucket == "week" else _DAY_SECONDS
-        outcomes = self._build_group_outcomes(results)
-        return compute_trend_by_group(outcomes, bucket_seconds)
-
-    def _build_group_outcomes(
-        self,
-        results: list[AgentSuccessEvaluationResult],
-    ) -> list[tuple[str, int, bool, dict]]:
-        """Pair each eval result with its session's first-request metadata.
-
-        Caches ``session_id → first_request_metadata`` so storage is hit
-        once per distinct session, not once per result. Sessions without a
-        matching request fall through with an empty-dict metadata, which the
-        downstream aggregator routes to the UNTAGGED bucket.
-
-        Args:
-            results (list[AgentSuccessEvaluationResult]): Eval results to
-                annotate with their session's first-request metadata.
-
-        Returns:
-            list[tuple[str, int, bool, dict]]: Tuples of
-            ``(session_id, created_at, is_success, first_request_metadata)``
-            suitable for :func:`compute_trend_by_group`.
-        """
-        session_ids = {r.session_id for r in results if r.session_id}
-        first_request_metadata: dict[str, dict] = {}
-        for sid in session_ids:
-            reqs = self._get_session_requests(sid)
-            if reqs:
-                first = min(reqs, key=lambda r: r.created_at)
-                first_request_metadata[sid] = first.metadata or {}
-            else:
-                first_request_metadata[sid] = {}
-
-        return [
-            (
-                r.session_id or "",
-                r.created_at,
-                r.is_success,
-                first_request_metadata.get(r.session_id or "", {}),
-            )
-            for r in results
-        ]
-
-    def _get_session_requests(self, session_id: str) -> list[Request]:
-        """Return every request in ``session_id`` regardless of user.
-
-        Uses ``BaseStorage.get_sessions(session_id=...)`` because the
-        per-session, user-id-required ``get_requests_by_session`` doesn't
-        fit our caller (eval results don't carry ``user_id``). All locally
-        testable backends ignore the ``user_id`` filter when it's ``None``,
-        and ``top_k`` is raised well above realistic per-session counts so
-        we don't truncate.
-
-        Args:
-            session_id (str): The session whose requests to fetch.
-
-        Returns:
-            list[Request]: All requests in the session, in storage order.
-        """
-        grouped = self.storage.get_sessions(  # type: ignore[attr-defined]
-            session_id=session_id, top_k=1000
-        )
-        return [
-            entry.request
-            for entries in grouped.values()
-            for entry in entries
-            if entry.request is not None
-        ]
+    def _build_first_request_sources(
+        self, result_keys: list[ResultKey]
+    ) -> dict[ResultKey, str]:
+        """Map each user/session slice to its earliest request's source."""
+        first_requests = self.storage.get_first_requests_by_user_session_pairs(
+            result_keys
+        )  # type: ignore[attr-defined]
+        return {key: row.source or "" for key, row in first_requests.items()}
 
     def _build_distribution(
         self,
@@ -440,6 +504,16 @@ class EvaluationOverviewService:
             labels=list(BUCKET_LABELS),
         )
 
+    def _log_phase(self, phase: str, started_at: float, **metadata: object) -> None:
+        duration_ms = int((time.perf_counter() - started_at) * 1000)
+        fields = " ".join(f"{key}={value}" for key, value in metadata.items())
+        _LOGGER.info(
+            "evaluation_overview phase=%s duration_ms=%d %s",
+            phase,
+            duration_ms,
+            fields,
+        )
+
 
 # --- module-level helpers (pure) ---
 

package/plugin/vendor/reflexio/reflexio/server/services/extraction/README.md

@@ -0,0 +1,31 @@
+# services/extraction
+
+Shared async extraction runtime for profile and playbook pipelines.
+
+This package is intentionally domain-neutral. Profile and playbook modules own
+their prompts, schemas, extractor semantics, and storage decisions; this package
+owns the reusable runtime needed when extraction pauses, waits for more
+information, and resumes outside the request path.
+
+## Files
+
+| File | Responsibility |
+|------|----------------|
+| `resumable_agent.py` | Runs extraction agents, records agent usage, injects resolved pending-tool context, and exposes pending tool-call helpers. |
+| `pending_tool_call_dispatch.py` | Implements the `ask_human` and pending-info tool dispatch flow. |
+| `prior_answer_search.py` | Finds and formats previous human answers for async extraction context. |
+| `agent_run_records.py` | Builds durable extraction-agent run records and source interaction identity. |
+| `resume_scheduler.py` | Schedules due paused extraction runs in a background singleton. |
+| `resume_worker.py` | Resumes paused runs, rebuilds request context, and records retry state. |
+| `outcome.py` | Provides the generic extraction outcome wrapper used by callers. |
+
+## Boundary Rules
+
+- Keep profile-specific and playbook-specific extraction behavior in their own
+  modules; call this package only for shared async runtime concerns.
+- Add a new file here when the behavior is shared by more than one extraction
+  caller or is part of the resumable runtime itself.
+- Split into subpackages only when a responsibility grows large enough that a
+  flat package stops being easier to scan.
+- Do not recreate removed legacy modules such as `tools.py`, `plan.py`, or
+  `invariants.py`; use the current focused files above.

package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py

@@ -3,7 +3,10 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Literal
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    from reflexio.server.llm.token_accounting import RunTokenTotals
 
 
 @dataclass(frozen=True)
@@ -13,12 +16,19 @@ class ExtractionOutcome[T]:
     status: Literal["completed", "empty"]
     items: list[T] = field(default_factory=list)
     run_id: str | None = None
+    token_totals: RunTokenTotals | None = None
 
     @classmethod
     def completed(
-        cls, items: list[T], *, run_id: str | None = None
+        cls,
+        items: list[T],
+        *,
+        run_id: str | None = None,
+        token_totals: RunTokenTotals | None = None,
     ) -> ExtractionOutcome[T]:
-        return cls(status="completed", items=items, run_id=run_id)
+        return cls(
+            status="completed", items=items, run_id=run_id, token_totals=token_totals
+        )
 
     @classmethod
     def empty(cls, *, run_id: str | None = None) -> ExtractionOutcome[T]:

package/plugin/vendor/reflexio/reflexio/server/services/extraction/pending_tool_call_dispatch.py

@@ -11,6 +11,7 @@ from typing import Annotated, Any
 from pydantic import BaseModel, Field, field_validator
 
 from reflexio.models.config_schema import PendingToolCallConfig
+from reflexio.models.structured_output import StrictStructuredOutput
 from reflexio.server.llm.tools import (
     AsyncAccepted,
     AsyncInfoTool,
@@ -31,7 +32,7 @@ from reflexio.server.usage_metrics import record_usage_event
 logger = logging.getLogger(__name__)
 
 
-class AskHumanArgs(BaseModel):
+class AskHumanArgs(StrictStructuredOutput):
     """Ask Agent Builder for missing extraction information and continue now."""
 
     question: Annotated[str, Field(min_length=1)]
@@ -46,7 +47,7 @@ class AskHumanArgs(BaseModel):
         return value
 
 
-class AttachPendingInfoRequestArgs(BaseModel):
+class AttachPendingInfoRequestArgs(StrictStructuredOutput):
     """Attach this run to a relevant pending Prior Knowledge request."""
 
     pending_tool_call_id: Annotated[str, Field(min_length=1)]