claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/server/api.py

@@ -2,11 +2,13 @@ import asyncio
 import inspect
 import logging
 import os
+import time
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from datetime import UTC, datetime
 from typing import Any
 
+from anyio.to_thread import current_default_thread_limiter
 from fastapi import (
     APIRouter,
     BackgroundTasks,
@@ -52,6 +54,7 @@ from reflexio.models.api_schema.retriever_schema import (
     GetEvaluationResultsViewResponse,
     GetInteractionsRequest,
     GetInteractionsViewResponse,
+    GetLearningProvenanceRequest,
     GetPlaybookApplicationStatsRequest,
     GetPlaybookApplicationStatsResponse,
     GetProfileStatisticsResponse,
@@ -61,6 +64,7 @@ from reflexio.models.api_schema.retriever_schema import (
     GetUserPlaybooksRequest,
     GetUserPlaybooksViewResponse,
     GetUserProfilesRequest,
+    LearningProvenanceViewResponse,
     ProfileChangeLogViewResponse,
     RequestDataView,
     RerankUserProfilesRequest,
@@ -74,6 +78,7 @@ from reflexio.models.api_schema.retriever_schema import (
     SearchUserProfileRequest,
     SessionView,
     SetConfigResponse,
+    SourceUserPlaybookProvenanceView,
     StorageStatsRequest,
     StorageStatsResponse,
     UnifiedSearchRequest,
@@ -153,10 +158,16 @@ from reflexio.models.api_schema.ui.converters import (
     to_user_playbook_view,
 )
 from reflexio.models.config_schema import (
+    DEFAULT_WINDOW_SIZE,
     SINGLETON_AGENT_SUCCESS_EVALUATION_NAME,
     Config,
 )
-from reflexio.server._auth import DEFAULT_ORG_ID, default_get_org_id
+from reflexio.server._auth import (
+    DEFAULT_ORG_ID,
+    default_billing_gate,
+    default_get_caller_type,
+    default_get_org_id,
+)
 from reflexio.server.api_endpoints import (
     account_api,
     health_api,
@@ -172,22 +183,34 @@ from reflexio.server.correlation import correlation_id_var, generate_correlation
 from reflexio.server.operation_limiter import (
     OperationName,
     limiter_http_exception,
+    log_publish_hardware_capacity,
     run_with_operation_limit,
 )
-from reflexio.server.services.agent_success_evaluation.group_evaluation_runner import (
-    run_group_evaluation,
-)
 from reflexio.server.services.agent_success_evaluation.regen_jobs import (
     REGEN_JOBS,
     run_regen,
 )
+from reflexio.server.services.agent_success_evaluation.runner import (
+    run_group_evaluation,
+)
+from reflexio.server.services.extractor_interaction_utils import (
+    get_effective_source_filter,
+    get_extractor_window_params,
+)
+from reflexio.server.tracing import profile_step
 
 logger = logging.getLogger(__name__)
 
 # Re-exported for backwards compatibility — callers that did
 # ``from reflexio.server.api import default_get_org_id`` or ``DEFAULT_ORG_ID``
 # continue to work.
-__all__ = ["DEFAULT_ORG_ID", "create_app", "default_get_org_id"]
+__all__ = [
+    "DEFAULT_ORG_ID",
+    "create_app",
+    "default_billing_gate",
+    "default_get_caller_type",
+    "default_get_org_id",
+]
 
 # Bot protection configuration
 REQUEST_TIMEOUT_SECONDS = 60
@@ -255,8 +278,41 @@ def get_rate_limit_key(request: Request) -> str:
     return get_remote_address(request)
 
 
+def _storage_backend_name(limiter_obj: Limiter) -> str:
+    storage = getattr(limiter_obj, "_storage", None)
+    if storage is None:
+        return "unknown"
+    return storage.__class__.__name__
+
+
+def _trace_external_rate_limit_backend(limiter_obj: Limiter) -> None:
+    """Trace rate-limit storage hits when the backend is an external service."""
+    backend = _storage_backend_name(limiter_obj)
+    if backend == "MemoryStorage":
+        return
+
+    strategy = getattr(limiter_obj, "limiter", None)
+    if strategy is None or getattr(strategy, "_reflexio_traced", False):
+        return
+
+    original_hit = strategy.hit
+
+    def traced_hit(item: Any, *identifiers: str, cost: int = 1) -> bool:
+        with profile_step(
+            "rate_limit.backend_hit",
+            storage_backend=backend,
+            strategy=strategy.__class__.__name__,
+            cost=cost,
+        ):
+            return original_hit(item, *identifiers, cost=cost)
+
+    strategy.hit = traced_hit
+    strategy._reflexio_traced = True
+
+
 # Initialize rate limiter
 limiter = Limiter(key_func=get_rate_limit_key)
+_trace_external_rate_limit_backend(limiter)
 
 
 def _run_limited_api[T](
@@ -448,6 +504,16 @@ class CorrelationIdMiddleware(BaseHTTPMiddleware):
     ) -> Response:
         cid = generate_correlation_id()
         correlation_id_var.set(cid)
+        try:
+            stats = current_default_thread_limiter().statistics()
+            request.state.tp_borrowed = stats.borrowed_tokens
+            request.state.tp_total = stats.total_tokens
+            request.state.tp_waiting = stats.tasks_waiting
+        except Exception as exc:  # noqa: BLE001
+            logger.debug("Failed to snapshot threadpool limiter stats: %s", exc)
+            request.state.tp_borrowed = None
+            request.state.tp_total = None
+            request.state.tp_waiting = None
         response = await call_next(request)
         response.headers["X-Correlation-ID"] = cid
         return response
@@ -456,6 +522,86 @@ class CorrelationIdMiddleware(BaseHTTPMiddleware):
 core_router = APIRouter()
 
 
+def _stamp_search_dependencies_done(request: Request) -> None:
+    """Stamp when dependency resolution reaches its final search dependency."""
+    request.state.search_deps_done_monotonic = time.monotonic()
+
+
+def _meter_applied_learnings(
+    *,
+    org_id: str,
+    caller_type: str,
+    surfaced_count: int,
+    request_id: str | None = None,
+    session_id: str | None = None,
+) -> None:
+    """Emit the ③ Application event via the OSS emission helper.
+
+    No-op unless a production-agent caller surfaced >= 1 result.  The cheap
+    caller-type and count guard runs first so the get_reflexio / config lookup
+    is skipped on the free paths (dashboard / empty result).
+
+    Args:
+        org_id: Organization ID for the requesting caller.
+        caller_type: Resolved caller classification (e.g. ``"production_agent"``).
+        surfaced_count: Total number of learnings returned to the caller.
+        request_id: Optional request correlation ID from the payload.
+        session_id: Optional session ID from the payload.
+    """
+    if caller_type != "production_agent" or surfaced_count <= 0:
+        return
+    try:
+        from reflexio.server.billing_meter import record_applied_learnings
+        from reflexio.server.billing_signals import platform_llm_from_config
+
+        config = get_reflexio(org_id=org_id).request_context.configurator.get_config()
+        record_applied_learnings(
+            org_id=org_id,
+            surfaced_count=surfaced_count,
+            caller_type=caller_type,
+            platform_llm=platform_llm_from_config(config),
+            platform_storage=None,  # resolved enterprise-side at rollup (Phase 1)
+            request_id=request_id,
+            session_id=session_id,
+        )
+    except Exception:
+        logger.warning(
+            "applied-learnings metering failed for org %s", org_id, exc_info=True
+        )
+
+
+def _meter_search_request(
+    *,
+    org_id: str,
+    caller_type: str,
+    request_id: str | None = None,
+    session_id: str | None = None,
+) -> None:
+    """Emit one production-agent search request metric.
+
+    Args:
+        org_id: Organization ID for the requesting caller.
+        caller_type: Resolved caller classification.
+        request_id: Optional request correlation ID from the payload.
+        session_id: Optional session ID from the payload.
+    """
+    if caller_type != "production_agent":
+        return
+    try:
+        from reflexio.server.billing_meter import record_search_request
+
+        record_search_request(
+            org_id=org_id,
+            caller_type=caller_type,
+            request_id=request_id,
+            session_id=session_id,
+        )
+    except Exception:
+        logger.warning(
+            "search-request metering failed for org %s", org_id, exc_info=True
+        )
+
+
 @core_router.get("/")
 def root() -> dict[str, str]:
     return {
@@ -466,7 +612,7 @@ def root() -> dict[str, str]:
 
 
 @core_router.get("/health")
-def health_check() -> dict[str, str]:
+async def health_check() -> dict[str, str]:
     """Health check endpoint for ECS/container orchestration."""
     return {"status": "healthy"}
 
@@ -534,6 +680,7 @@ def publish_user_interaction(
     background_tasks: BackgroundTasks,
     org_id: str = Depends(default_get_org_id),
     wait_for_response: bool = False,
+    _gate: None = Depends(default_billing_gate("learnings_generated")),  # noqa: B008
 ) -> PublishUserInteractionResponse:
     if wait_for_response:
         # Process synchronously so the caller gets the real result
@@ -551,6 +698,7 @@ def publish_user_interaction(
                 fn=lambda: publisher_api.add_user_interaction(
                     org_id=org_id, request=payload
                 ),
+                wait_forever=False,
             )
         except TimeoutError:
             logger.warning(
@@ -647,17 +795,33 @@ def search_user_profiles(
     request: Request,
     payload: SearchUserProfileRequest,
     org_id: str = Depends(default_get_org_id),
+    caller_type: str = Depends(default_get_caller_type),
+    _gate: None = Depends(default_billing_gate("application")),  # noqa: B008
 ) -> SearchProfilesViewResponse:
     response = _run_limited_api(
         org_id,
         "search",
         lambda: get_reflexio(org_id=org_id).search_user_profiles(payload),
     )
-    return SearchProfilesViewResponse(
+    resp = SearchProfilesViewResponse(
         success=response.success,
         user_profiles=[to_profile_view(p) for p in response.user_profiles],
         msg=response.msg,
     )
+    _meter_search_request(
+        org_id=org_id,
+        caller_type=caller_type,
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    _meter_applied_learnings(
+        org_id=org_id,
+        caller_type=caller_type,
+        surfaced_count=len(resp.user_profiles),
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    return resp
 
 
 @core_router.post(
@@ -757,6 +921,8 @@ def search_user_playbooks_endpoint(
     request: Request,
     payload: SearchUserPlaybookRequest,
     org_id: str = Depends(default_get_org_id),
+    caller_type: str = Depends(default_get_caller_type),
+    _gate: None = Depends(default_billing_gate("application")),  # noqa: B008
 ) -> SearchUserPlaybooksViewResponse:
     """Search user playbooks with semantic search and advanced filtering.
 
@@ -767,6 +933,7 @@ def search_user_playbooks_endpoint(
         request (Request): The HTTP request object (for rate limiting)
         payload (SearchUserPlaybookRequest): The search request
         org_id (str): Organization ID
+        caller_type (str): Billing caller classification (injected via dependency).
 
     Returns:
         SearchUserPlaybooksViewResponse: Response containing matching user playbooks
@@ -776,11 +943,25 @@ def search_user_playbooks_endpoint(
         "search",
         lambda: get_reflexio(org_id=org_id).search_user_playbooks(payload),
     )
-    return SearchUserPlaybooksViewResponse(
+    resp = SearchUserPlaybooksViewResponse(
         success=response.success,
         user_playbooks=[to_user_playbook_view(rf) for rf in response.user_playbooks],
         msg=response.msg,
     )
+    _meter_search_request(
+        org_id=org_id,
+        caller_type=caller_type,
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    _meter_applied_learnings(
+        org_id=org_id,
+        caller_type=caller_type,
+        surfaced_count=len(resp.user_playbooks),
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    return resp
 
 
 @core_router.post(
@@ -793,6 +974,8 @@ def search_agent_playbooks_endpoint(
     request: Request,
     payload: SearchAgentPlaybookRequest,
     org_id: str = Depends(default_get_org_id),
+    caller_type: str = Depends(default_get_caller_type),
+    _gate: None = Depends(default_billing_gate("application")),  # noqa: B008
 ) -> SearchAgentPlaybooksViewResponse:
     """Search agent playbooks with semantic search and advanced filtering.
 
@@ -803,6 +986,7 @@ def search_agent_playbooks_endpoint(
         request (Request): The HTTP request object (for rate limiting)
         payload (SearchAgentPlaybookRequest): The search request
         org_id (str): Organization ID
+        caller_type (str): Billing caller classification (injected via dependency).
 
     Returns:
         SearchAgentPlaybooksViewResponse: Response containing matching agent playbooks
@@ -812,11 +996,25 @@ def search_agent_playbooks_endpoint(
         "search",
         lambda: get_reflexio(org_id=org_id).search_agent_playbooks(payload),
     )
-    return SearchAgentPlaybooksViewResponse(
+    resp = SearchAgentPlaybooksViewResponse(
         success=response.success,
         agent_playbooks=[to_agent_playbook_view(fb) for fb in response.agent_playbooks],
         msg=response.msg,
     )
+    _meter_search_request(
+        org_id=org_id,
+        caller_type=caller_type,
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    _meter_applied_learnings(
+        org_id=org_id,
+        caller_type=caller_type,
+        surfaced_count=len(resp.agent_playbooks),
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    return resp
 
 
 @core_router.post(
@@ -828,7 +1026,11 @@ def search_agent_playbooks_endpoint(
 def unified_search_endpoint(
     request: Request,
     payload: UnifiedSearchRequest,
+    background_tasks: BackgroundTasks,
     org_id: str = Depends(default_get_org_id),
+    caller_type: str = Depends(default_get_caller_type),
+    _gate: None = Depends(default_billing_gate("application")),  # noqa: B008
+    _deps_done: None = Depends(_stamp_search_dependencies_done),
 ) -> UnifiedSearchViewResponse:
     """Search across all entity types (profiles, agent playbooks, user playbooks).
 
@@ -840,31 +1042,76 @@ def unified_search_endpoint(
         request (Request): The HTTP request object (for rate limiting)
         payload (UnifiedSearchRequest): The unified search request
         org_id (str): Organization ID
+        caller_type (str): Billing caller classification (injected via dependency).
 
     Returns:
         UnifiedSearchViewResponse: Combined search results
     """
-    response = _run_limited_api(
-        org_id,
-        "search",
-        lambda: get_reflexio(org_id=org_id).unified_search(payload, org_id=org_id),
-    )
-    return UnifiedSearchViewResponse(
-        success=response.success,
-        profiles=[to_profile_view(p) for p in response.profiles],
-        agent_playbooks=[to_agent_playbook_view(fb) for fb in response.agent_playbooks],
-        user_playbooks=[to_user_playbook_view(rf) for rf in response.user_playbooks],
-        reformulated_query=response.reformulated_query,
-        msg=response.msg,
-        agent_trace=response.agent_trace,
-        rehydrated_text=response.rehydrated_text,
+    deps_done = getattr(request.state, "search_deps_done_monotonic", None)
+    deps_to_body_ms = (
+        int((time.monotonic() - deps_done) * 1000) if deps_done is not None else None
     )
+    with profile_step(
+        "search.endpoint",
+        enabled=bool(payload.enable_reformulation),
+        has_conversation_history=bool(payload.conversation_history),
+        search_mode=payload.search_mode,
+    ) as endpoint_span:
+        endpoint_span.set_data("deps_to_body_ms", deps_to_body_ms)
+        endpoint_span.set_data(
+            "tp_borrowed", getattr(request.state, "tp_borrowed", None)
+        )
+        endpoint_span.set_data("tp_total", getattr(request.state, "tp_total", None))
+        endpoint_span.set_data("tp_waiting", getattr(request.state, "tp_waiting", None))
+
+        def run_search() -> Any:
+            with profile_step("search.reflexio_cache"):
+                reflexio = get_reflexio(org_id=org_id)
+            return reflexio.unified_search(payload, org_id=org_id)
+
+        response = _run_limited_api(org_id, "search", run_search)
+        with profile_step("search.response_view"):
+            resp = UnifiedSearchViewResponse(
+                success=response.success,
+                profiles=[to_profile_view(p) for p in response.profiles],
+                agent_playbooks=[
+                    to_agent_playbook_view(fb) for fb in response.agent_playbooks
+                ],
+                user_playbooks=[
+                    to_user_playbook_view(rf) for rf in response.user_playbooks
+                ],
+                reformulated_query=response.reformulated_query,
+                msg=response.msg,
+                agent_trace=response.agent_trace,
+                rehydrated_text=response.rehydrated_text,
+            )
+        background_tasks.add_task(
+            _meter_search_request,
+            org_id=org_id,
+            caller_type=caller_type,
+            request_id=getattr(payload, "request_id", None),
+            session_id=getattr(payload, "session_id", None),
+        )
+        background_tasks.add_task(
+            _meter_applied_learnings,
+            org_id=org_id,
+            caller_type=caller_type,
+            surfaced_count=len(resp.profiles)
+            + len(resp.agent_playbooks)
+            + len(resp.user_playbooks),
+            request_id=getattr(payload, "request_id", None),
+            session_id=getattr(payload, "session_id", None),
+        )
+    return resp
 
 
 @core_router.get("/api/profile_change_log", response_model=ProfileChangeLogViewResponse)
 def get_profile_change_log(
     org_id: str = Depends(default_get_org_id),
 ) -> ProfileChangeLogViewResponse:
+    # Serves the reconstructed profile change log (rebuilt from lineage events). The
+    # legacy `profile_change_logs` table is no longer written; see
+    # reconstruct_profile_change_log in lib/_profiles.py.
     response = get_reflexio(org_id=org_id).get_profile_change_logs()
     return ProfileChangeLogViewResponse(
         success=response.success,
@@ -1269,6 +1516,307 @@ def get_requests_endpoint(
     )
 
 
+def _learning_provenance_error(
+    payload: GetLearningProvenanceRequest,
+    msg: str,
+) -> LearningProvenanceViewResponse:
+    return LearningProvenanceViewResponse(
+        success=False,
+        target_kind=payload.kind,
+        target_id=payload.id,
+        provenance_status="unavailable",
+        msg=msg,
+    )
+
+
+def _parse_learning_target_int(
+    payload: GetLearningProvenanceRequest,
+) -> int | LearningProvenanceViewResponse:
+    try:
+        return int(payload.id)
+    except ValueError:
+        return _learning_provenance_error(
+            payload,
+            f"{payload.kind} id must be an integer",
+        )
+
+
+def _sort_interactions_by_time(interactions: list[Any]) -> list[Any]:
+    return sorted(
+        interactions,
+        key=lambda i: (
+            getattr(i, "created_at", 0) or 0,
+            getattr(i, "interaction_id", 0),
+        ),
+    )
+
+
+def _interaction_views(interactions: list[Any]) -> list[Any]:
+    return [to_interaction_view(i) for i in _sort_interactions_by_time(interactions)]
+
+
+def _effective_profile_provenance_window(
+    reflexio: Any, trigger_source: str | None
+) -> tuple[int, list[str] | None]:
+    config = reflexio.request_context.configurator.get_config()
+    profile_config = getattr(config, "profile_extractor_config", None)
+    if profile_config is None:
+        return getattr(config, "window_size", DEFAULT_WINDOW_SIZE), None
+
+    window_size, _ = get_extractor_window_params(
+        profile_config,
+        getattr(config, "window_size", None),
+        getattr(config, "stride_size", None),
+    )
+    should_skip, source_filter = get_effective_source_filter(
+        profile_config,
+        trigger_source,
+    )
+    if should_skip:
+        return window_size, None
+    return window_size, source_filter
+
+
+def _get_profile_learning_provenance(
+    payload: GetLearningProvenanceRequest,
+    reflexio: Any,
+) -> LearningProvenanceViewResponse:
+    storage = reflexio.request_context.storage
+    profile = storage.get_profile_by_id(payload.id)
+    if profile is None:
+        return _learning_provenance_error(payload, "Profile not found")
+
+    if profile.source_interaction_ids:
+        interactions = storage.get_interactions_by_ids(profile.source_interaction_ids)
+        return LearningProvenanceViewResponse(
+            success=True,
+            target_kind=payload.kind,
+            target_id=payload.id,
+            provenance_status="exact",
+            trigger_request_id=profile.generated_from_request_id,
+            interactions=_interaction_views(interactions),
+        )
+
+    if not profile.generated_from_request_id:
+        return _learning_provenance_error(
+            payload,
+            "Profile has no generation request for provenance reconstruction",
+        )
+
+    trigger_request = storage.get_request(profile.generated_from_request_id)
+    if trigger_request is None:
+        return _learning_provenance_error(
+            payload,
+            "Profile generation request was not found",
+        )
+
+    trigger_interactions = storage.get_interactions_by_request_ids(
+        [profile.generated_from_request_id]
+    )
+    anchor_time = (
+        max(i.created_at for i in trigger_interactions)
+        if trigger_interactions
+        else trigger_request.created_at
+    )
+    window_size, source_filter = _effective_profile_provenance_window(
+        reflexio,
+        trigger_request.source,
+    )
+    _, interactions = storage.get_last_k_interactions_grouped(
+        user_id=profile.user_id,
+        k=window_size,
+        sources=source_filter,
+        end_time=anchor_time,
+    )
+    return LearningProvenanceViewResponse(
+        success=True,
+        target_kind=payload.kind,
+        target_id=payload.id,
+        provenance_status="best_effort" if interactions else "unavailable",
+        trigger_request_id=profile.generated_from_request_id,
+        interactions=_interaction_views(interactions),
+        msg=None if interactions else "No interactions found for reconstructed window",
+    )
+
+
+def _get_user_playbook_learning_provenance(
+    payload: GetLearningProvenanceRequest,
+    reflexio: Any,
+) -> LearningProvenanceViewResponse:
+    parsed_id = _parse_learning_target_int(payload)
+    if isinstance(parsed_id, LearningProvenanceViewResponse):
+        return parsed_id
+
+    storage = reflexio.request_context.storage
+    playbook = storage.get_user_playbook_by_id(parsed_id)
+    if playbook is None:
+        return _learning_provenance_error(payload, "User playbook not found")
+
+    status = "exact"
+    if playbook.source_interaction_ids:
+        interactions = storage.get_interactions_by_ids(playbook.source_interaction_ids)
+    elif playbook.request_id:
+        status = "best_effort"
+        interactions = storage.get_interactions_by_request_ids([playbook.request_id])
+    else:
+        status = "unavailable"
+        interactions = []
+
+    return LearningProvenanceViewResponse(
+        success=True,
+        target_kind=payload.kind,
+        target_id=payload.id,
+        provenance_status=status if interactions else "unavailable",
+        trigger_request_id=playbook.request_id,
+        interactions=_interaction_views(interactions),
+        msg=None if interactions else "No interactions found for this user playbook",
+    )
+
+
+def _get_agent_playbook_learning_provenance(
+    payload: GetLearningProvenanceRequest,
+    reflexio: Any,
+) -> LearningProvenanceViewResponse:
+    parsed_id = _parse_learning_target_int(payload)
+    if isinstance(parsed_id, LearningProvenanceViewResponse):
+        return parsed_id
+
+    storage = reflexio.request_context.storage
+    agent_playbook = storage.get_agent_playbook_by_id(parsed_id)
+    if agent_playbook is None:
+        return _learning_provenance_error(payload, "Agent playbook not found")
+
+    windows = storage.get_source_windows_for_agent_playbook(parsed_id)
+    if not windows:
+        return LearningProvenanceViewResponse(
+            success=True,
+            target_kind=payload.kind,
+            target_id=payload.id,
+            provenance_status="unavailable",
+            msg="No source user playbooks are recorded for this agent playbook",
+        )
+
+    user_playbook_ids = [window.user_playbook_id for window in windows]
+    user_playbooks = storage.get_user_playbooks_by_ids_any_user(
+        user_playbook_ids,
+        status_filter=None,
+    )
+    user_playbooks_by_id = {
+        playbook.user_playbook_id: playbook for playbook in user_playbooks
+    }
+
+    interaction_ids_to_fetch: set[int] = set()
+    request_ids_to_fetch: set[str] = set()
+    group_sources: dict[int, tuple[list[int], str | None, bool]] = {}
+    for window in windows:
+        source_user_playbook = user_playbooks_by_id.get(window.user_playbook_id)
+        if source_user_playbook is None:
+            continue
+
+        source_ids = list(window.source_interaction_ids)
+        request_id: str | None = None
+        uses_fallback = False
+        if not source_ids and source_user_playbook.source_interaction_ids:
+            source_ids = list(source_user_playbook.source_interaction_ids)
+        elif not source_ids and source_user_playbook.request_id:
+            request_id = source_user_playbook.request_id
+            uses_fallback = True
+        elif not source_ids:
+            uses_fallback = True
+
+        interaction_ids_to_fetch.update(source_ids)
+        if request_id:
+            request_ids_to_fetch.add(request_id)
+        group_sources[window.user_playbook_id] = (source_ids, request_id, uses_fallback)
+
+    interactions_by_id = (
+        {
+            interaction.interaction_id: interaction
+            for interaction in storage.get_interactions_by_ids(
+                sorted(interaction_ids_to_fetch)
+            )
+        }
+        if interaction_ids_to_fetch
+        else {}
+    )
+    request_interactions: dict[str, list[Any]] = {}
+    if request_ids_to_fetch:
+        for interaction in storage.get_interactions_by_request_ids(
+            sorted(request_ids_to_fetch)
+        ):
+            request_interactions.setdefault(interaction.request_id, []).append(
+                interaction
+            )
+
+    groups: list[SourceUserPlaybookProvenanceView] = []
+    used_fallback = False
+    found_interactions = False
+    for window in windows:
+        source_user_playbook = user_playbooks_by_id.get(window.user_playbook_id)
+        if source_user_playbook is None:
+            used_fallback = True
+            continue
+
+        source_ids, request_id, uses_fallback = group_sources.get(
+            window.user_playbook_id,
+            ([], None, True),
+        )
+        used_fallback = used_fallback or uses_fallback
+        interactions = (
+            [interactions_by_id[i] for i in source_ids if i in interactions_by_id]
+            if source_ids
+            else request_interactions.get(request_id or "", [])
+        )
+
+        found_interactions = found_interactions or bool(interactions)
+        groups.append(
+            SourceUserPlaybookProvenanceView(
+                user_playbook=to_user_playbook_view(source_user_playbook),
+                interactions=_interaction_views(interactions),
+                source_interaction_ids=source_ids,
+            )
+        )
+
+    if not groups:
+        return _learning_provenance_error(
+            payload,
+            "Recorded source user playbooks were not found",
+        )
+
+    return LearningProvenanceViewResponse(
+        success=True,
+        target_kind=payload.kind,
+        target_id=payload.id,
+        provenance_status=(
+            "exact"
+            if found_interactions and not used_fallback
+            else "best_effort"
+            if found_interactions
+            else "unavailable"
+        ),
+        source_user_playbooks=groups,
+        msg=None if found_interactions else "No source interactions were found",
+    )
+
+
+@core_router.post(
+    "/api/get_learning_provenance",
+    response_model=LearningProvenanceViewResponse,
+    response_model_exclude_none=True,
+)
+def get_learning_provenance(
+    payload: GetLearningProvenanceRequest,
+    org_id: str = Depends(default_get_org_id),
+) -> LearningProvenanceViewResponse:
+    """Return read-only learning provenance for a generated learning row."""
+    reflexio = get_reflexio(org_id=org_id)
+    if payload.kind == "profile":
+        return _get_profile_learning_provenance(payload, reflexio)
+    if payload.kind == "user_playbook":
+        return _get_user_playbook_learning_provenance(payload, reflexio)
+    return _get_agent_playbook_learning_provenance(payload, reflexio)
+
+
 @core_router.post(
     "/api/get_profiles",
     response_model=GetProfilesViewResponse,
@@ -1296,6 +1844,8 @@ def get_all_profiles(
     request: Request,
     limit: int = 100,
     status_filter: str | None = None,
+    profile_id: str | None = None,
+    include_tombstones: bool = False,
     org_id: str = Depends(default_get_org_id),
 ) -> GetProfilesViewResponse:
     """Get all user profiles across all users.
@@ -1303,6 +1853,9 @@ def get_all_profiles(
     Args:
         limit (int, optional): Maximum number of profiles to return. Defaults to 100.
         status_filter (str, optional): Filter by profile status. Can be "current", "pending", or "archived".
+        profile_id (str, optional): Exact profile ID to retrieve.
+        include_tombstones (bool, optional): Include merged/superseded rows when
+            looking up a specific profile_id.
         org_id (str): Organization ID
 
     Returns:
@@ -1310,6 +1863,24 @@ def get_all_profiles(
     """
     reflexio = get_reflexio(org_id=org_id)
 
+    if profile_id:
+        storage = reflexio.request_context.storage
+        if storage is None:
+            return GetProfilesViewResponse(
+                success=True,
+                user_profiles=[],
+                msg="Storage is not configured",
+            )
+        profile = storage.get_profile_by_id(
+            profile_id, include_tombstones=include_tombstones
+        )
+        profiles = [profile] if profile else []
+        return GetProfilesViewResponse(
+            success=True,
+            user_profiles=[to_profile_view(p) for p in profiles],
+            msg=f"Found {len(profiles)} profile(s)",
+        )
+
     # Map status_filter string to Status list
     status_filter_list = None
     if status_filter == "current":
@@ -1378,17 +1949,30 @@ def set_config(
     """Set configuration for the organization.
 
     Args:
-        config (Config): The configuration to set
+        config (dict[str, Any]): The configuration payload to set
         org_id (str): Organization ID
 
     Returns:
         dict: Response containing success status and message
     """
+    from pydantic import ValidationError
+
     # Create Reflexio instance to access the configurator through request_context
     reflexio = get_reflexio(org_id=org_id)
+    configurator = reflexio.request_context.configurator
+    normalized_config = configurator.normalize_config_payload(config)
+    if not isinstance(normalized_config, dict):
+        normalized_config = config
+    try:
+        Config.model_validate(normalized_config)
+    except ValidationError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            detail=exc.errors(),
+        ) from exc
 
     # Set the config using Reflexio's set_config method
-    response = reflexio.set_config(config)
+    response = reflexio.set_config(normalized_config)
 
     # Invalidate cache on successful config change to ensure fresh instance next request
     if response.success:
@@ -1568,26 +2152,40 @@ def get_user_playbooks(
     response_model=GetAgentPlaybooksViewResponse,
     response_model_exclude_none=True,
 )
+@limiter.limit("120/minute")  # Rate limit for read operations
 def get_agent_playbooks(
-    request: GetAgentPlaybooksRequest,
+    request: Request,
+    payload: GetAgentPlaybooksRequest,
     org_id: str = Depends(default_get_org_id),
+    caller_type: str = Depends(default_get_caller_type),
+    _gate: None = Depends(default_billing_gate("application")),  # noqa: B008
 ) -> GetAgentPlaybooksViewResponse:
     """Get agent playbooks with internal fields filtered out.
 
     Args:
-        request (GetAgentPlaybooksRequest): The get request
+        request (Request): The HTTP request object (for rate limiting)
+        payload (GetAgentPlaybooksRequest): The get request
         org_id (str): Organization ID
+        caller_type (str): Billing caller classification (injected via dependency).
 
     Returns:
         GetAgentPlaybooksViewResponse: Response containing agent playbooks without internal fields
     """
     reflexio = get_reflexio(org_id=org_id)
-    response = reflexio.get_agent_playbooks(request)
-    return GetAgentPlaybooksViewResponse(
+    response = reflexio.get_agent_playbooks(payload)
+    resp = GetAgentPlaybooksViewResponse(
         success=response.success,
         agent_playbooks=[to_agent_playbook_view(fb) for fb in response.agent_playbooks],
         msg=response.msg,
     )
+    _meter_applied_learnings(
+        org_id=org_id,
+        caller_type=caller_type,
+        surfaced_count=len(resp.agent_playbooks),
+        request_id=getattr(payload, "request_id", None),
+        session_id=getattr(payload, "session_id", None),
+    )
+    return resp
 
 
 @core_router.post(
@@ -2094,10 +2692,8 @@ def _read_grade_on_demand_cache(
 def _resolve_session_user_id(storage: Any, session_id: str) -> str | None:
     """Look up the user_id that owns a session_id without requiring it as input.
 
-    Uses ``get_sessions(session_id=...)`` because it's the only storage
-    method on ``BaseStorage`` that accepts session_id alone — every other
-    request-fetcher requires a user_id paired with it. Returns None when
-    the session has no requests, signalling NO_REQUESTS to the caller.
+    Uses the first-request bulk helper even for this single-session path so the
+    lookup can use the same indexed query shape as evaluation overview.
 
     Args:
         storage: The request's storage backend.
@@ -2107,17 +2703,17 @@ def _resolve_session_user_id(storage: Any, session_id: str) -> str | None:
         str | None: The user_id of the earliest request in the session,
         or None when no requests exist.
     """
-    sessions = storage.get_sessions(session_id=session_id, top_k=100)
-    rows = sessions.get(session_id) or []
-    if not rows:
+    first_requests = storage.get_first_requests_by_session_ids([session_id])
+    first = first_requests.get(session_id)
+    if first is None:
         return None
-    earliest = min(rows, key=lambda r: r.request.created_at)
-    return earliest.request.user_id
+    return first.user_id
 
 
 def _find_fresh_result_id(
     storage: Any,
     *,
+    user_id: str,
     session_id: str,
     agent_version: str,
     evaluation_name: str,
@@ -2125,12 +2721,12 @@ def _find_fresh_result_id(
 ) -> int | None:
     """Locate the result_id written by the most-recent grade for this session.
 
-    The runner writes rows but doesn't return the id. Reading back through
-    ``get_agent_success_evaluation_results`` matches the pattern used by
-    the regen worker's prior-row capture (group_evaluation_runner step 5).
+    The runner writes rows but doesn't return the id. Use the targeted result-id
+    lookup so this path does not scan broad evaluation windows.
 
     Args:
         storage: The request's storage backend.
+        user_id (str): The user whose session slice was graded.
         session_id (str): The graded session.
         agent_version (str): The version dimension.
         evaluation_name (str): Evaluator/result namespace to isolate readback.
@@ -2140,20 +2736,16 @@ def _find_fresh_result_id(
         int | None: result_id of the latest matching row, or None if the
         runner wrote nothing.
     """
-    rows = storage.get_agent_success_evaluation_results(
-        limit=1000, agent_version=agent_version
-    )
-    matched = [
-        r
-        for r in rows
-        if r.session_id == session_id
-        and r.evaluation_name == evaluation_name
-        and r.result_id not in previous_result_ids
-    ]
-    if not matched:
+    result_ids = storage.get_agent_success_evaluation_result_ids(
+        user_id=user_id,
+        session_id=session_id,
+        evaluation_name=evaluation_name,
+        agent_version=agent_version,
+    )
+    fresh_result_ids = [rid for rid in result_ids if rid not in previous_result_ids]
+    if not fresh_result_ids:
         return None
-    latest = max(matched, key=lambda r: r.created_at)
-    return latest.result_id
+    return max(fresh_result_ids)
 
 
 @core_router.post(
@@ -2224,13 +2816,14 @@ def grade_on_demand(
             skipped_reason="NO_REQUESTS",
         )
 
-    previous_result_ids = {
-        r.result_id
-        for r in storage.get_agent_success_evaluation_results(
-            limit=1000, agent_version=payload.agent_version
+    previous_result_ids = set(
+        storage.get_agent_success_evaluation_result_ids(
+            user_id=user_id,
+            session_id=payload.session_id,
+            evaluation_name=evaluation_name,
+            agent_version=payload.agent_version,
         )
-        if r.session_id == payload.session_id and r.evaluation_name == evaluation_name
-    }
+    )
 
     # Two operation_state rows are intentionally written for this session:
     #   1) `grade_on_demand::{org_id}::{session_id}::{agent_version}::{evaluation_name}`
@@ -2255,6 +2848,7 @@ def grade_on_demand(
 
     result_id = _find_fresh_result_id(
         storage,
+        user_id=user_id,
         session_id=payload.session_id,
         agent_version=payload.agent_version,
         evaluation_name=evaluation_name,
@@ -2307,8 +2901,8 @@ def get_recent_shadow_comparisons(
     Filters to the org's currently pinned
     ``Config.shadow_comparison_judge_prompt_version`` so verdicts produced
     under an older rubric do not mix into the drawer or the Top 10
-    disagreements widget. Storage returns verdicts in ascending ``created_at``
-    order; we reverse to "newest first" and cap at ``limit``.
+    disagreements widget. Storage returns verdicts newest-first and caps the
+    read at ``limit``.
 
     Args:
         limit (int): Max verdicts to return. Clamped to ``[1, 100]``.
@@ -2340,10 +2934,11 @@ def get_recent_shadow_comparisons(
 
     now = int(datetime.now(UTC).timestamp())
     try:
-        verdicts = storage.get_shadow_comparison_verdicts(
+        verdicts = storage.get_recent_shadow_comparison_verdicts(
             from_ts=now - _RECENT_SHADOW_COMPARISONS_LOOKBACK_SECONDS,
             to_ts=now,
             judge_prompt_version=pinned_version,
+            limit=clamped_limit,
         )
     except NotImplementedError:
         # Backends that don't support shadow verdicts (e.g., Disk) should
@@ -2351,9 +2946,7 @@ def get_recent_shadow_comparisons(
         # "no data yet" in the UI.
         return GetRecentShadowComparisonsResponse(verdicts=[])
 
-    # Storage contract returns ascending — flip to "newest first" and cap.
-    newest_first = list(reversed(verdicts))[:clamped_limit]
-    return GetRecentShadowComparisonsResponse(verdicts=newest_first)
+    return GetRecentShadowComparisonsResponse(verdicts=verdicts)
 
 
 @core_router.post(
@@ -2714,6 +3307,9 @@ def create_app(
     additional_routers: list[APIRouter] | None = None,
     middleware_config: dict | None = None,
     require_auth: bool = False,
+    get_caller_type: Callable[..., str] | None = None,
+    get_billing_gate: Callable[[str], Callable[..., None]] | None = None,
+    mount_data_plane: bool = True,
 ) -> FastAPI:
     """Factory to create a FastAPI app.
 
@@ -2724,6 +3320,23 @@ def create_app(
         middleware_config: Optional middleware overrides (not used yet, reserved for future).
         require_auth: When True, declares a Bearer security scheme in the OpenAPI spec
             so Swagger UI shows lock icons and the Authorize button works.
+        get_caller_type: Custom dependency for classifying the caller (e.g., production
+            agent vs dashboard).  When provided, overrides the default_get_caller_type
+            dependency globally, exactly mirroring the get_org_id override.
+        get_billing_gate: Optional factory ``(line: str) -> FastAPI dependency`` that
+            replaces the default no-op gate for each billable billing line.  When
+            provided, for every line used in the app (``"application"`` and
+            ``"learnings_generated"``) the returned dependency overrides the
+            ``default_billing_gate(line)`` sentinel in ``dependency_overrides``,
+            exactly mirroring the ``get_caller_type`` override pattern.
+        mount_data_plane: When True (default), include the data-plane routers
+            (core, stall-state, pending-tool-call) and run the data-plane
+            lifespan work (LLM availability check, cross-encoder prewarm,
+            resume scheduler). When False, skip both so a control-plane host
+            can build an app without requiring LLM/storage or starting the
+            scheduler, while keeping all other scaffolding (middleware, CORS,
+            auth overrides, OpenAPI security, health, ``/meta/version``,
+            ``additional_routers``).
 
     Returns:
         Configured FastAPI application.
@@ -2731,12 +3344,19 @@ def create_app(
     from collections.abc import AsyncIterator
     from contextlib import asynccontextmanager
 
-    from reflexio.server._auth import default_get_org_id
+    from reflexio.server._auth import (
+        default_billing_gate,
+        default_get_caller_type,
+        default_get_org_id,
+    )
     from reflexio.server.api_endpoints.request_context import RequestContext
     from reflexio.server.llm.model_defaults import validate_llm_availability
     from reflexio.server.services.extraction.resume_scheduler import (
         maybe_start_resume_scheduler,
     )
+    from reflexio.server.services.lineage.gc_scheduler import (
+        maybe_start_lineage_gc,
+    )
 
     def _lifespan_org_id() -> str:
         if get_org_id is None:
@@ -2755,23 +3375,33 @@ def create_app(
 
     @asynccontextmanager
     async def lifespan(app: FastAPI) -> AsyncIterator[None]:  # noqa: ARG001
-        validate_llm_availability()
-        from reflexio.server.llm.rerank import prewarm as _prewarm_cross_encoder
-
-        _prewarm_cross_encoder()
-        # The scheduler discovers every org with resumable work each tick and
-        # drives a per-org worker with org-scoped claims, so it is not limited
-        # to the bootstrap org. The bootstrap org is only used to read config
-        # and to seed cross-org discovery.
-        scheduler = maybe_start_resume_scheduler(
-            lambda org_id: RequestContext(org_id=org_id),
-            bootstrap_org_id=_lifespan_org_id(),
-        )
+        scheduler = None
+        gc_scheduler = None
+        if mount_data_plane:
+            log_publish_hardware_capacity()
+            validate_llm_availability()
+            from reflexio.server.llm.rerank import prewarm as _prewarm_cross_encoder
+
+            _prewarm_cross_encoder()
+            # The scheduler discovers every org with resumable work each tick and
+            # drives a per-org worker with org-scoped claims, so it is not limited
+            # to the bootstrap org. The bootstrap org is only used to read config
+            # and to seed cross-org discovery.
+            scheduler = maybe_start_resume_scheduler(
+                lambda org_id: RequestContext(org_id=org_id),
+                bootstrap_org_id=_lifespan_org_id(),
+            )
+            gc_scheduler = maybe_start_lineage_gc(
+                lambda org_id: RequestContext(org_id=org_id),
+                bootstrap_org_id=_lifespan_org_id(),
+            )
         try:
             yield
         finally:
             if scheduler is not None:
                 scheduler.stop()
+            if gc_scheduler is not None:
+                gc_scheduler.stop()
 
     app = FastAPI(docs_url="/docs", lifespan=lifespan)
 
@@ -2837,6 +3467,21 @@ def create_app(
     if get_org_id is not None:
         app.dependency_overrides[default_get_org_id] = get_org_id
 
+    # Override get_caller_type dependency if custom one provided
+    if get_caller_type is not None:
+        app.dependency_overrides[default_get_caller_type] = get_caller_type
+
+    # Override billing gate dependencies if a custom gate factory is provided.
+    # Each billing line needs its own override because dependency_overrides is
+    # keyed by callable identity.  ``default_billing_gate`` uses lru_cache so
+    # the same sentinel object is returned for the same line — which is why
+    # the overrides reliably fire at request time.
+    if get_billing_gate is not None:
+        for line in ("application", "learnings_generated"):
+            app.dependency_overrides[default_billing_gate(line)] = get_billing_gate(
+                line
+            )
+
     # When a custom get_org_id is provided together with require_auth,
     # auth is enforced on every route — mark this app instance so the
     # token-gated my_config endpoint becomes reachable. Using
@@ -2845,14 +3490,18 @@ def create_app(
     # multi-tenant embeddings) can coexist without leaking state.
     app.state.my_config_enabled = bool(get_org_id is not None and require_auth)
 
-    # Include core routes
-    app.include_router(core_router)
+    # Include data-plane routes (core, stall-state, pending-tool-call). A
+    # control-plane host sets mount_data_plane=False to skip these while
+    # keeping every other piece of scaffolding below.
+    if mount_data_plane:
+        # Include core routes
+        app.include_router(core_router)
 
-    # Include stall_state routes
-    app.include_router(stall_state_api.router)
+        # Include stall_state routes
+        app.include_router(stall_state_api.router)
 
-    # Include pending tool call routes
-    app.include_router(pending_tool_call_api.router)
+        # Include pending tool call routes
+        app.include_router(pending_tool_call_api.router)
 
     # Include additional routers
     for router in additional_routers or []: