claude-smart 0.2.45 → 0.2.47

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{agent_success_evaluation_service.py → service.py}

@@ -10,10 +10,14 @@ from reflexio.server.services.agent_success_evaluation import _eval_health
 from reflexio.server.services.agent_success_evaluation.agent_success_evaluation_utils import (
     AgentSuccessEvaluationRequest,
 )
-from reflexio.server.services.agent_success_evaluation.agent_success_evaluator import (
+from reflexio.server.services.agent_success_evaluation.components.evaluator import (
     AgentSuccessEvaluator,
 )
 from reflexio.server.services.base_generation_service import BaseGenerationService
+from reflexio.server.services.extractor_interaction_utils import (
+    filter_interactions_by_source,
+    get_effective_source_filter,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -23,12 +27,14 @@ class AgentSuccessGenerationServiceConfig:
     """Runtime configuration for agent success evaluation service shared across all extractors.
 
     Attributes:
+        user_id: The user whose session slice is being evaluated
         session_id: The session being evaluated
         agent_version: The agent version
         request_interaction_data_models: The interactions to evaluate
         source: Source of the interactions
     """
 
+    user_id: str
     session_id: str
     agent_version: str
     request_interaction_data_models: list[RequestInteractionDataModel]
@@ -76,6 +82,7 @@ class AgentSuccessEvaluationService(
             AgentSuccessGenerationServiceConfig object
         """
         return AgentSuccessGenerationServiceConfig(
+            user_id=request.user_id,
             session_id=request.session_id,
             agent_version=request.agent_version,
             request_interaction_data_models=request.request_interaction_data_models,
@@ -115,6 +122,34 @@ class AgentSuccessEvaluationService(
             agent_context=self.configurator.get_agent_context(),
         )
 
+    def _collect_scoped_interactions_for_precheck(
+        self, extractor_config: AgentSuccessConfig
+    ) -> tuple[list[RequestInteractionDataModel], AgentSuccessConfig]:
+        """Use the session slice already loaded by the group runner.
+
+        The generic pre-check queries storage for learnable interaction windows,
+        which intentionally excludes evaluation-only requests. Agent-success
+        evaluation is allowed to run on evaluation-only sessions, so its
+        pre-check must inspect the request models passed to this service.
+        """
+        service_config = self.service_config
+        if service_config is None:
+            raise RuntimeError("Agent success pre-check requires service_config")
+        session_data_models = service_config.request_interaction_data_models
+        should_skip, source_filter = get_effective_source_filter(
+            extractor_config, service_config.source
+        )
+        if should_skip:
+            self._last_precheck_sessions = []
+            return [], extractor_config
+
+        scoped_models = filter_interactions_by_source(
+            session_data_models,
+            source_filter,
+        )
+        self._last_precheck_sessions = scoped_models
+        return scoped_models, extractor_config
+
     def _process_results(self, results: list) -> None:
         """
         Process and save agent success evaluation results.

package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py

@@ -20,6 +20,7 @@ from reflexio.models.api_schema.internal_schema import RequestInteractionDataMod
 from reflexio.models.api_schema.service_schemas import Status
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
+from reflexio.server.llm.token_accounting import RunTokenTotals
 from reflexio.server.services.extraction.outcome import ExtractionOutcome
 from reflexio.server.services.extractor_config_utils import (
     filter_extractor_configs,
@@ -165,6 +166,10 @@ class BaseGenerationService(
     ABC,
     Generic[TExtractorConfig, TExtractor, TGenerationServiceConfig, TRequest],  # noqa: UP046
 ):
+    # Only profile/playbook GENERATION services bill the ② Learning line;
+    # reflection/consolidation/aggregation/evaluation are bundled, not metered.
+    # Default is False so any future subclass is safe by default (opt-IN).
+    EMITS_LEARNING_BILLING: bool = False
     """
     Base class for generation services that run one configured extractor.
 
@@ -209,6 +214,11 @@ class BaseGenerationService(
             "timed_out": 0,
         }
         self._last_extraction_run_ids: list[str] = []
+        self._last_token_totals: RunTokenTotals | None = None
+        # Window fetched by the should-run gate (_collect_scoped_interactions_for_precheck),
+        # stashed so the billing path (_extraction_input_text) can reuse it instead of
+        # re-querying storage. None when the gate did not run (bypass paths).
+        self._last_precheck_sessions: list[Any] | None = None
 
     def _usage_pipeline(self) -> str | None:
         service_name = self._get_service_name()
@@ -252,6 +262,139 @@ class BaseGenerationService(
             metadata=metadata,
         )
 
+    def _extraction_input_text(self, prepared: "PreparedGenerationRun[Any]") -> str:
+        """Return the interaction content fed to the extractor as a single string.
+
+        Replays the same storage query the extractor uses (same user_id, source,
+        window params) so ``count_input_tokens`` sees the same text the LLM saw.
+        Returns an empty string when storage is unavailable or no interactions exist.
+
+        Args:
+            prepared: The prepared generation run, used to access extractor_config
+                for window/source parameters.
+
+        Returns:
+            str: Formatted interaction history string, or "" if nothing is found.
+        """
+        from reflexio.server.services.service_utils import (
+            format_sessions_to_history_string,
+        )
+
+        if self.storage is None or self.service_config is None:
+            return ""
+        # Reuse the window the should-run gate already fetched
+        # (_collect_scoped_interactions_for_precheck stashed it). Same query params,
+        # so the token count is byte-identical — and we avoid a second storage read.
+        if self._last_precheck_sessions is not None:
+            return format_sessions_to_history_string(self._last_precheck_sessions)
+        # Fallback storage read: used ONLY when the gate did not pre-fetch
+        # (bypass paths: auto_run=False, force_extraction, skip_should_run_check,
+        # mock mode). Re-fetches the extractor's input window so billing token
+        # counting sees exactly the text the LLM saw.
+        try:
+            root_config = self.request_context.configurator.get_config()
+            global_window_size = (
+                getattr(root_config, "window_size", None) if root_config else None
+            )
+            global_stride_size = (
+                getattr(root_config, "stride_size", None) if root_config else None
+            )
+            window_size, _ = get_extractor_window_params(
+                prepared.extractor_config, global_window_size, global_stride_size
+            )
+            should_skip, effective_source = get_effective_source_filter(
+                prepared.extractor_config, getattr(self.service_config, "source", None)
+            )
+            if should_skip:
+                return ""
+            session_data_models, _ = self.storage.get_last_k_interactions_grouped(
+                user_id=getattr(self.service_config, "user_id", None),
+                k=window_size,
+                sources=effective_source,
+                start_time=getattr(self.service_config, "rerun_start_time", None),
+                end_time=getattr(self.service_config, "rerun_end_time", None),
+                **self._get_precheck_interaction_query_kwargs(),
+            )
+            return format_sessions_to_history_string(session_data_models)
+        except Exception:
+            logger.warning(
+                "_extraction_input_text failed; billing_input_tokens will be 0",
+                exc_info=True,
+            )
+            return ""
+
+    def _record_billing_learning_events(
+        self, *, prepared: "PreparedGenerationRun[Any]", generated_count: int
+    ) -> None:
+        """Emit the ② Learning billing events. Called ONLY when extraction fired.
+
+        Emits ``learnings_generated`` (value facet) and ``extraction_tokens``
+        (cost facet) via the OSS emission helpers. ``platform_storage`` is left
+        ``None`` here and resolved enterprise-side at rollup (Phase 1).
+
+        Gated by ``EMITS_LEARNING_BILLING`` — only profile/playbook generation
+        services opt in. Evaluation, reflection, consolidation and aggregation
+        are bundled and must NOT separately meter the ② Learning line.
+
+        Args:
+            prepared: The prepared generation run (used for input-text computation).
+            generated_count: Number of learnings produced by this extraction run.
+        """
+        if not self.EMITS_LEARNING_BILLING:
+            return
+
+        try:
+            from reflexio.server.billing_meter import (
+                record_extraction_tokens,
+                record_learnings_generated,
+            )
+            from reflexio.server.billing_signals import (
+                count_input_tokens,
+                platform_llm_from_config,
+            )
+
+            config = self.request_context.configurator.get_config()
+            platform_llm = platform_llm_from_config(config)
+            ctx = self._usage_context()
+
+            # session_id is intentionally not passed: the generation path has no
+            # session_id source. _usage_context() never includes it, and neither
+            # the Profile/Playbook service configs nor their requests carry a
+            # session_id (unlike the Application-line path in server/api.py, which
+            # reads request_id/session_id off the search request payload). Learning
+            # events therefore meter without session attribution by design.
+
+            # ② Learning — value: learnings generated (helper no-ops on count <= 0).
+            record_learnings_generated(
+                org_id=ctx["org_id"],
+                count=generated_count,
+                platform_llm=platform_llm,
+                platform_storage=None,
+                pipeline=ctx.get("pipeline"),
+                request_id=ctx.get("request_id"),
+            )
+
+            # ② Learning — cost: input-anchored extraction tokens + real provider tokens.
+            totals = self._last_token_totals or RunTokenTotals()
+            billing_input_tokens = count_input_tokens(
+                self._extraction_input_text(prepared)
+            )
+            record_extraction_tokens(
+                org_id=ctx["org_id"],
+                billing_input_tokens=billing_input_tokens,
+                prompt_tokens=totals.prompt_tokens,
+                completion_tokens=totals.completion_tokens,
+                platform_llm=platform_llm,
+                platform_storage=None,
+                pipeline=ctx.get("pipeline"),
+                request_id=ctx.get("request_id"),
+            )
+        except Exception:
+            logger.warning(
+                "_record_billing_learning_events failed; billing learning events not emitted",
+                exc_info=True,
+            )
+
     @staticmethod
     def _count_generated_results(result: Any) -> int:
         if isinstance(result, list):
@@ -675,6 +818,7 @@ class BaseGenerationService(
                 },
             )
             self._last_extraction_run_ids = []
+            self._last_token_totals = None
             result = self._execute_extractor(
                 prepared.extractor_config, prepared.identifier
             )
@@ -704,6 +848,9 @@ class BaseGenerationService(
                     ),
                 },
             )
+            self._record_billing_learning_events(
+                prepared=prepared, generated_count=generated_count
+            )
 
         except Exception as e:
             self._record_generation_event(
@@ -737,6 +884,13 @@ class BaseGenerationService(
         Returns:
             PreparedGenerationRun when generation should proceed, otherwise None.
         """
+        # Reset BEFORE the should-run gate runs. The gate
+        # (_collect_scoped_interactions_for_precheck) stashes its fetched window
+        # here for the billing path to reuse. On bypass paths (auto_run=False,
+        # force_extraction, skip_should_run_check, mock mode) the gate never runs,
+        # so this stays None and billing falls back to its own fetch.
+        self._last_precheck_sessions = None
+
         self.service_config = self._load_generation_service_config(request)
 
         extractor_config = self._load_extractor_config()
@@ -834,6 +988,7 @@ class BaseGenerationService(
             if isinstance(result, ExtractionOutcome):
                 if result.run_id:
                     self._last_extraction_run_ids.append(result.run_id)
+                self._last_token_totals = result.token_totals
                 if result.status == "completed" and result.items:
                     return result.items
                 logger.info(
@@ -1128,6 +1283,8 @@ class BaseGenerationService(
             extractor_config, getattr(self.service_config, "source", None)
         )
         if should_skip:
+            # Stash for the billing path to reuse (same window the gate saw).
+            self._last_precheck_sessions = []
             return [], extractor_config
 
         window_size, _ = get_extractor_window_params(
@@ -1142,6 +1299,9 @@ class BaseGenerationService(
             **extra_kwargs,
         )
 
+        # Stash for the billing path (_extraction_input_text) to reuse instead of
+        # re-querying storage for billing_input_tokens.
+        self._last_precheck_sessions = session_data_models
         return session_data_models, extractor_config
 
     def _get_precheck_interaction_query_kwargs(self) -> dict:

package/plugin/vendor/reflexio/reflexio/server/services/configurator/base_configurator.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Callable
+from pathlib import Path
 from typing import Any, ClassVar
 
 from pydantic import BaseModel
@@ -81,6 +82,10 @@ class BaseConfigurator(ABC):
         """Normalize raw API config payloads before Pydantic validation."""
         return config
 
+    def get_prompt_bank_paths(self) -> list[Path]:
+        """Return additional prompt banks this configurator contributes."""
+        return []
+
     def get_agent_context(self) -> str:
         context = self.get_config().agent_context_prompt
         if not context:

package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py

@@ -2,7 +2,7 @@
 Shared utilities for deduplication services.
 
 This module contains base classes and utility functions used by both
-ProfileDeduplicator and PlaybookConsolidator.
+ProfileConsolidator and PlaybookConsolidator.
 """
 
 import logging

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/README.md

@@ -0,0 +1,9 @@
+# evaluation_overview
+
+Read-side aggregation module for `POST /api/get_evaluation_overview`.
+
+- `service.py` is the request-path entry point. It loads evaluation, citation, Braintrust, and optional shadow verdict data, then composes `GetEvaluationOverviewResponse`.
+- `components/` contains pure read-side aggregation helpers used by the service and focused tests.
+- `eval_sampler.py` stays at the package root because regenerate jobs also use it to sample evaluation sessions.
+
+This module mutates no core state. Keep response-shape changes in API schema tests and service integration tests.

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/{rule_attribution.py → components/rule_attribution.py}

@@ -11,6 +11,7 @@ from collections.abc import Mapping
 from dataclasses import dataclass
 
 CitationKey = tuple[str, str]  # (kind, real_id) — matches PlaybookApplicationStat
+SessionIdentity = tuple[str, str]  # (user_id, session_id)
 
 
 @dataclass(frozen=True)
@@ -36,19 +37,19 @@ class RuleAttribution:
 
 def compute_net_sessions(
     *,
-    citations_by_session: Mapping[str, list[CitationKey]],
-    is_success_by_session: Mapping[str, bool],
+    citations_by_session: Mapping[SessionIdentity, list[CitationKey]],
+    is_success_by_session: Mapping[SessionIdentity, bool],
     rule_titles: Mapping[CitationKey, str],
     top_n: int,
 ) -> list[RuleAttribution]:
     """Aggregate net sessions per rule and return the top N by net_sessions.
 
     Args:
-        citations_by_session (Mapping[str, list[CitationKey]]): For each
-            session in the window, the list of `(kind, real_id)` citations
+        citations_by_session (Mapping[SessionIdentity, list[CitationKey]]): For
+            each user/session slice in the window, the list of `(kind, real_id)` citations
             harvested from its interactions.
-        is_success_by_session (Mapping[str, bool]): The session's evaluated
-            outcome. Sessions not in this map are skipped on both sides.
+        is_success_by_session (Mapping[SessionIdentity, bool]): The evaluated
+            outcome. User/session slices not in this map are skipped on both sides.
         rule_titles (Mapping[CitationKey, str]): Pre-loaded display titles
             for each rule (empty string when the underlying row is gone).
         top_n (int): How many rows to return at most. Ordering: net_sessions
@@ -65,8 +66,8 @@ def compute_net_sessions(
     # Ordering preserves caller-provided iteration order; the frontend sorts
     # the detail band by created_at when displaying.
     sessions_by_rule: dict[CitationKey, list[str]] = {}
-    for session_id, citations in citations_by_session.items():
-        outcome = is_success_by_session.get(session_id)
+    for session_key, citations in citations_by_session.items():
+        outcome = is_success_by_session.get(session_key)
         if outcome is None:
             continue
         # A rule cited multiple times in the same session counts once.
@@ -76,7 +77,7 @@ def compute_net_sessions(
                 successes[key] = successes.get(key, 0) + 1
             else:
                 failures[key] = failures.get(key, 0) + 1
-            sessions_by_rule.setdefault(key, []).append(session_id)
+            sessions_by_rule.setdefault(key, []).append(session_key[1])
 
     all_keys = set(successes) | set(failures)
     rows = [

package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/eval_sampler.py

@@ -3,10 +3,9 @@
 DB-free and LLM-free. Operates on SampleCandidate tuples emitted by the
 regen worker's candidate-discovery step; returns the sampled subset.
 
-Stratification key: (day_bucket, F2 group). Day buckets are aligned to
-86400-second boundaries. F2 group is derived via the existing
-`assign_group_from_metadata` so the per-day cap is honored separately
-for treatment / control / untagged.
+Stratification key: (day_bucket, first_request_source). Day buckets are
+aligned to 86400-second boundaries. The source value is sticky per session
+and comes from the session's earliest request.
 """
 
 from __future__ import annotations
@@ -15,12 +14,6 @@ import random
 from collections import defaultdict
 from collections.abc import Iterable
 from dataclasses import dataclass
-from typing import Any
-
-from reflexio.server.services.evaluation_overview.group_aggregation import (
-    GroupAssignment,
-    assign_group_from_metadata,
-)
 
 # Day bucket width in seconds. Strata are aligned to (ts // _DAY) * _DAY.
 _DAY = 86_400
@@ -31,8 +24,7 @@ class SampleCandidate:
     """One candidate for the regen pipeline's sampling step.
 
     Carries everything the regen worker needs to dispatch a per-session
-    evaluation, plus the first-request metadata used to assign the F2
-    group for stratification.
+    evaluation, plus the first-request source used for stratification.
 
     Attributes:
         session_id: The session to evaluate.
@@ -41,8 +33,8 @@ class SampleCandidate:
         source: Optional source tag (passed through to run_group_evaluation).
         created_at: Unix epoch seconds of the session's most relevant
             timestamp; used for day-bucket alignment.
-        first_request_metadata: The metadata dict on the session's first
-            request, read once during candidate discovery.
+        first_request_source: Source on the session's first request, read once
+            during candidate discovery. Empty string is a real stratum.
     """
 
     session_id: str
@@ -50,27 +42,26 @@ class SampleCandidate:
     agent_version: str
     source: str | None
     created_at: int
-    first_request_metadata: dict[str, Any]
+    first_request_source: str
 
 
-def stratify_by_day_and_group(
+def stratify_by_day_and_source(
     candidates: Iterable[SampleCandidate],
-) -> dict[tuple[int, GroupAssignment], list[SampleCandidate]]:
-    """Bucket candidates by (day_bucket, F2 group).
+) -> dict[tuple[int, str], list[SampleCandidate]]:
+    """Bucket candidates by (day_bucket, first_request_source).
 
     Args:
         candidates: Candidate tuples emitted by the regen candidate-discovery
             step.
 
     Returns:
-        Dict from (day_bucket_start, group) to the candidate list for that
+        Dict from (day_bucket_start, source) to the candidate list for that
         stratum. Empty strata are absent; the dict has only populated keys.
     """
-    strata: dict[tuple[int, GroupAssignment], list[SampleCandidate]] = defaultdict(list)
+    strata: dict[tuple[int, str], list[SampleCandidate]] = defaultdict(list)
     for c in candidates:
         day_bucket = (c.created_at // _DAY) * _DAY
-        group = assign_group_from_metadata(c.first_request_metadata)
-        strata[(day_bucket, group)].append(c)
+        strata[(day_bucket, c.first_request_source)].append(c)
     return dict(strata)
 
 
@@ -105,7 +96,7 @@ def sample_candidates(
     n_per_stratum: int,
     rng: random.Random,
 ) -> list[SampleCandidate]:
-    """Stratify by (day x group), then sample per stratum, then flatten.
+    """Stratify by (day x source), then sample per stratum, then flatten.
 
     Predictable cost across runs: at most ``n_per_stratum * num_strata``
     items returned, regardless of traffic volume.
@@ -119,7 +110,7 @@ def sample_candidates(
     Returns:
         Flattened sampled list.
     """
-    strata = stratify_by_day_and_group(candidates)
+    strata = stratify_by_day_and_source(candidates)
     sampled: list[SampleCandidate] = []
     for stratum_candidates in strata.values():
         sampled.extend(sample_per_stratum(stratum_candidates, n_per_stratum, rng))