claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_manager.py

@@ -4,6 +4,7 @@ Prompt management using file system prompt bank with markdown frontmatter files.
 
 import logging
 import re
+from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
@@ -75,28 +76,37 @@ class PromptManager:
 
     def __init__(
         self,
-        prompt_bank_path: str | None = None,
+        prompt_bank_path: str | Path | None = None,
         version_override: dict[str, str] | None = None,
+        extra_prompt_bank_paths: Sequence[str | Path] | None = None,
     ):
         """
         Initialize the PromptManager.
 
         Args:
-            prompt_bank_path (str, optional): Path to the prompt bank directory.
+            prompt_bank_path (str | Path, optional): Path to the primary prompt bank directory.
             version_override (dict[str, str], optional): Map of prompt_id → version string to override the active version.
+            extra_prompt_bank_paths (Sequence[str | Path], optional): Additional prompt bank directories.
         """
         if prompt_bank_path is None:
             current_dir = Path(__file__).parent
             self.prompt_bank_path = current_dir / "prompt_bank"
         else:
             self.prompt_bank_path = Path(prompt_bank_path)
+        self.prompt_bank_paths = [self.prompt_bank_path]
+        if extra_prompt_bank_paths:
+            self.prompt_bank_paths.extend(
+                Path(path) for path in extra_prompt_bank_paths
+            )
 
         self.version_override = version_override
 
-        if not self.prompt_bank_path.exists():
-            logger.warning("Prompt bank path does not exist: %s", self.prompt_bank_path)
+        for path in self.prompt_bank_paths:
+            if not path.exists():
+                logger.warning("Prompt bank path does not exist: %s", path)
 
         self._cache: dict[str, Prompt] = {}
+        self._validate_unique_prompt_ids()
 
     # ==============================
     # Public methods
@@ -148,13 +158,13 @@ class PromptManager:
         Returns:
             list[str]: List of version strings.
         """
-        prompt_dir = self.prompt_bank_path / prompt_id
-        if not prompt_dir.is_dir():
-            return []
-        return [
-            p.name.removeprefix("v").removesuffix(".prompt.md")
-            for p in sorted(prompt_dir.glob("v*.prompt.md"))
-        ]
+        versions: list[str] = []
+        for prompt_dir in self._prompt_dirs(prompt_id):
+            versions.extend(
+                p.name.removeprefix("v").removesuffix(".prompt.md")
+                for p in sorted(prompt_dir.glob("v*.prompt.md"))
+            )
+        return versions
 
     def get_active_version(self, prompt_id: str) -> str | None:
         """
@@ -177,21 +187,65 @@ class PromptManager:
         Returns:
             list[str]: List of prompt IDs.
         """
-        if not self.prompt_bank_path.exists():
-            return []
-        return [
-            item.name
-            for item in self.prompt_bank_path.iterdir()
-            if item.is_dir() and any(item.glob("v*.prompt.md"))
-        ]
+        prompt_ids: list[str] = []
+        seen: set[str] = set()
+        for prompt_bank_path in self.prompt_bank_paths:
+            if not prompt_bank_path.exists():
+                continue
+            for item in prompt_bank_path.iterdir():
+                if (
+                    item.is_dir()
+                    and item.name not in seen
+                    and any(item.glob("v*.prompt.md"))
+                ):
+                    prompt_ids.append(item.name)
+                    seen.add(item.name)
+        return prompt_ids
 
     # ==============================
     # Private methods
     # ==============================
 
+    def _prompt_dirs(self, prompt_id: str) -> list[Path]:
+        """Return configured prompt directories for a prompt ID."""
+        return [
+            prompt_bank_path / prompt_id
+            for prompt_bank_path in self.prompt_bank_paths
+            if (prompt_bank_path / prompt_id).is_dir()
+        ]
+
+    def _validate_unique_prompt_ids(self) -> None:
+        """Reject prompt IDs that appear in more than one configured prompt bank."""
+        locations: dict[str, list[Path]] = {}
+        for prompt_bank_path in self.prompt_bank_paths:
+            if not prompt_bank_path.exists():
+                continue
+            for item in prompt_bank_path.iterdir():
+                if item.is_dir() and any(item.glob("v*.prompt.md")):
+                    locations.setdefault(item.name, []).append(item)
+
+        duplicates = {
+            prompt_id: paths for prompt_id, paths in locations.items() if len(paths) > 1
+        }
+        if duplicates:
+            details = "; ".join(
+                f"{prompt_id}: {', '.join(str(path) for path in paths)}"
+                for prompt_id, paths in sorted(duplicates.items())
+            )
+            raise ValueError(f"Duplicate prompt_id found in prompt banks: {details}")
+
     def _load_prompt(self, prompt_id: str, version: str) -> Prompt | None:
         """Load a single prompt file by prompt_id and version string."""
-        path = self.prompt_bank_path / prompt_id / f"v{version}.prompt.md"
+        path: Path | None = None
+        for prompt_dir in self._prompt_dirs(prompt_id):
+            candidate = prompt_dir / f"v{version}.prompt.md"
+            if candidate.is_file():
+                path = candidate
+                break
+        if path is None:
+            logger.warning("Prompt file not found: %s v%s", prompt_id, version)
+            return None
+
         try:
             raw = path.read_text(encoding="utf-8")
         except FileNotFoundError:
@@ -217,8 +271,8 @@ class PromptManager:
 
     def _find_active_version(self, prompt_id: str) -> str | None:
         """Scan .prompt.md files to find the latest one with active: true."""
-        prompt_dir = self.prompt_bank_path / prompt_id
-        if not prompt_dir.is_dir():
+        prompt_dirs = self._prompt_dirs(prompt_id)
+        if not prompt_dirs:
             return None
 
         def _semver_key(p: Path) -> tuple[int, ...]:
@@ -239,17 +293,20 @@ class PromptManager:
                 return (0,)
 
         latest: str | None = None
-        for path in sorted(
-            prompt_dir.glob("v*.prompt.md"), key=_semver_key, reverse=True
-        ):
-            try:
-                raw = path.read_text(encoding="utf-8")
-                meta, _ = _parse_frontmatter(raw)
-                if meta.get("active"):
-                    latest = path.name.removeprefix("v").removesuffix(".prompt.md")
-                    break
-            except (ValueError, OSError):
-                continue
+        for prompt_dir in prompt_dirs:
+            for path in sorted(
+                prompt_dir.glob("v*.prompt.md"), key=_semver_key, reverse=True
+            ):
+                try:
+                    raw = path.read_text(encoding="utf-8")
+                    meta, _ = _parse_frontmatter(raw)
+                    if meta.get("active"):
+                        latest = path.name.removeprefix("v").removesuffix(".prompt.md")
+                        break
+                except (ValueError, OSError):
+                    continue
+            if latest:
+                break
         return latest
 
     def _get_prompt(self, prompt_id: str, version: str | None = None) -> Prompt | None:

package/plugin/vendor/reflexio/reflexio/server/services/README.md

@@ -5,6 +5,19 @@ Description: Core business-logic layer — LLM orchestration, extraction, evalua
 
 **Service Boundary**: services own the LLM/extraction/evaluation/storage logic; API endpoints only authenticate, build `RequestContext`, and delegate into `Reflexio` or a focused service helper.
 
+## LLM Pipeline Module Contract
+
+LLM/pipeline modules use a shared vocabulary across OSS and enterprise:
+`service.py` for request-path entry points, `runner.py` for manual/background
+workflow entry points, `scheduler.py` for periodic/deferred execution,
+`config.py` for module-owned config, `models.py` for module-local data shapes,
+and `components/` for internal extractors, judges, proposers, aggregators,
+evaluators, gates, and resolvers.
+
+Files are optional. Do not create empty files only to satisfy the vocabulary.
+Complete cutover migrations update consumers, tests, docs, and monkeypatch
+strings before deleting old import paths in the same PR.
+
 ## Orchestration & Base Infrastructure
 
 | File | Purpose |
@@ -13,32 +26,34 @@ Description: Core business-logic layer — LLM orchestration, extraction, evalua
 | `base_generation_service.py` | `BaseGenerationService` — abstract base; the **Service Pattern** (load configs → create actors → run in parallel → save results). Per-extractor timeout `EXTRACTOR_TIMEOUT_SECONDS = 300`. |
 | `operation_state_utils.py` | `OperationStateManager` — all `_operation_state` access (progress, concurrency locks, extractor/aggregator bookmarks, cluster fingerprints, cancellation). |
 | `extractor_config_utils.py`, `extractor_interaction_utils.py` | Filter extractors by source / `allow_manual_trigger` / names; per-extractor stride + window + bookmark handling. |
-| `deduplication_utils.py`, `service_utils.py`, `embedding_text.py` | LLM dedup helpers (used by `ProfileDeduplicator` + `PlaybookConsolidator`), message construction / JSON extraction / response logging, embedding text builders. |
+| `deduplication_utils.py`, `service_utils.py`, `embedding_text.py` | LLM dedup helpers (used by `ProfileConsolidator` + `PlaybookConsolidator`), message construction / JSON extraction / response logging, embedding text builders. |
 
 ## Generation Services
 
 | Directory | Entry class | Key files |
 |-----------|-------------|-----------|
-| `profile/` | `ProfileGenerationService` | `profile_extractor.py`, `profile_deduplicator.py`, `profile_updater.py` |
-| `playbook/` | `PlaybookGenerationService` | `playbook_extractor.py`, `playbook_consolidator.py`, `playbook_aggregator.py` (cluster-fingerprint change detection) — has its own [README](playbook/README.md) |
-| `agent_success_evaluation/` | `AgentSuccessEvaluationService` | `agent_success_evaluator.py` (session-level), `delayed_group_evaluator.py` (`GroupEvaluationScheduler`, 10-min defer), `group_evaluation_runner.py`, `regen_jobs.py` |
-| `reflection/` | `ReflectionService` | `reflection_extractor.py` — post-horizon reflection; runs **before** extraction so extractors read post-reflection state |
+| `profile/` | `ProfileGenerationService` | `service.py`, `components/extractor.py`, `components/consolidator.py` |
+| `playbook/` | `PlaybookGenerationService` | `components/extractor.py`, `components/consolidator.py`, `components/aggregator.py` (cluster-fingerprint change detection) — has its own [README](playbook/README.md) |
+| `agent_success_evaluation/` | `AgentSuccessEvaluationService` | `service.py` (session-level service), `runner.py` (`run_group_evaluation`), `scheduler.py` (`GroupEvaluationScheduler`, 10-min defer), `regen_jobs.py`, `components/evaluator.py` |
+| `reflection/` | `ReflectionService` | `service.py`, `components/extractor.py` — post-horizon reflection; runs **before** extraction so extractors read post-reflection state |
 
 ## Async Extraction
 
 | Directory | Purpose |
 |-----------|---------|
-| `extraction/` | Resumable extraction agent: `resumable_agent.py`, `resume_scheduler.py`, `resume_worker.py`, `pending_tool_call_dispatch.py` (`ask_human`), `tools.py`, `plan.py`, `agent_run_records.py`, `invariants.py`. Long-horizon / tool-mediated extraction continues outside the request path. |
+| `extraction/` | Shared async extraction runtime: `resumable_agent.py`, `resume_scheduler.py`, `resume_worker.py`, `pending_tool_call_dispatch.py` (`ask_human`), `prior_answer_search.py`, `agent_run_records.py`, and `outcome.py`. Long-horizon / tool-mediated extraction continues outside the request path. See [README](extraction/README.md). |
 
 ## Evaluation, Search & Integrations
 
 | Path | Purpose |
 |------|---------|
-| `shadow_comparison/` | `ShadowComparisonJudge` — per-turn regular-vs-shadow verdicts written to a separate table (session-level shadow was retracted due to trajectory contamination). |
-| `evaluation_overview/` | Dashboard rollups: `service.py`, `hero_state.py`, `distribution.py`, `group_aggregation.py`, `rule_attribution.py`, `shadow_aggregation.py`, `eval_sampler.py`. |
-| `playbook_optimizer/` | Scenario-based playbook optimization: `optimizer.py`, `scheduler.py`, `rollout.py`, `judge.py`, `scenario_resolver.py`, `gepa_adapter.py`, `assistant_webhook.py`. |
+| `shadow_comparison/` | `ShadowComparisonJudge` (`judge.py`) plus pure outcome helpers (`outcome.py`) - per-turn regular-vs-shadow verdicts written to a separate table. Compact by design; see [README](shadow_comparison/README.md). |
+| `evaluation_overview/` | Dashboard/read-side rollups: `service.py` entry point, `components/` aggregation helpers, and root `eval_sampler.py` shared with regenerate jobs. See [README](evaluation_overview/README.md). |
+| `playbook_optimizer/` | Scenario-based playbook optimization: mature flat package with `optimizer.py`, `scheduler.py`, `rollout.py`, `judge.py`, `models.py`, `scenario_resolver.py`, `gepa_adapter.py`, and `assistant_webhook.py`. See [README](playbook_optimizer/README.md). |
 | `braintrust/` | Braintrust export/sync: `service.py`, `client.py`, `_cron.py`, `_encryption.py`. |
-| `pre_retrieval/` | `QueryReformulator` (`_query_reformulator.py`) + `_document_expander.py` — query rewrite & doc expansion for recall. |
+| `lineage/` | Current-record resolution and tombstone GC: `resolve.py`, `gc_scheduler.py`. |
+| `pre_retrieval/` | `QueryReformulator` (`_query_reformulator.py`) + `DocumentExpander` (`_document_expander.py`) - query rewrite and doc expansion for recall. Compact by design; see [README](pre_retrieval/README.md). |
+| `tagging/` | `TaggingService` (`service.py`) + deferred `tagging_scheduler.py` - post-generation profile/playbook tagging. Compact by design; see [README](tagging/README.md). |
 | `unified_search_service.py` | `run_unified_search()` — two-phase parallel search across profiles / agent playbooks / user playbooks. |
 | `retrieval/` | `relevance_floor.py` — result relevance thresholding. |
 
@@ -46,7 +61,7 @@ Description: Core business-logic layer — LLM orchestration, extraction, evalua
 
 | Path | Purpose |
 |------|---------|
-| `storage/` | `storage_base/` (`BaseStorage` split by domain) + `sqlite_storage/` + `retention*.py`. Access via `request_context.storage` only. |
+| `storage/` | `storage_base/` (`BaseStorage` split by domain, including `_lineage.py`) + `sqlite_storage/` (including lineage/tombstones) + `retention*.py`. Access via `request_context.storage` only. |
 | `configurator/` | `DefaultConfigurator` — loads YAML config and creates the storage backend. |
 
 ## Key Rules

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md

@@ -0,0 +1,22 @@
+# agent_success_evaluation
+
+Session-level agent success evaluation module.
+
+## Module Shape
+
+- `service.py`: `AgentSuccessEvaluationService`, the request-path service that runs configured evaluators and saves result rows.
+- `runner.py`: `run_group_evaluation(...)`, the background/manual workflow entry point that loads a session, runs the service, dispatches shadow comparison, and marks operation state.
+- `scheduler.py`: `GroupEvaluationScheduler`, the deferred inactivity scheduler.
+- `components/evaluator.py`: `AgentSuccessEvaluator`, the LLM evaluator component.
+- `regen_jobs.py`: regeneration job planning and execution; remains root-level because API/admin regenerate flows import it directly.
+- `_eval_health.py`: producer/scheduler health counters.
+- `agent_success_evaluation_constants.py`: prompt/model output constants.
+- `agent_success_evaluation_utils.py`: request DTO and prompt-message construction helpers.
+
+## Prompt IDs
+
+- Owns `agent_success_evaluation`.
+- Keeps historical/configured prompt ID `agent_success_evaluation_with_comparison` stable where prompt mapping tests require it.
+
+Do not reintroduce the deleted service/evaluator/runner/scheduler legacy
+module files.

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/__init__.py

@@ -0,0 +1 @@
+"""Agent success evaluation service package."""

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_constants.py

@@ -3,7 +3,9 @@
 from dataclasses import dataclass
 from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
+
+from reflexio.models.structured_output import StrictStructuredOutput
 
 
 @dataclass(frozen=True)
@@ -14,7 +16,7 @@ class AgentSuccessEvaluationConstants:
     AGENT_SUCCESS_EVALUATION_PROMPT_ID = "agent_success_evaluation"
 
 
-class AgentSuccessEvaluationOutput(BaseModel):
+class AgentSuccessEvaluationOutput(StrictStructuredOutput):
     """
     Unified output schema for agent success evaluation.
 

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_utils.py

@@ -19,6 +19,7 @@ from reflexio.server.services.service_utils import (
 class AgentSuccessEvaluationRequest(BaseModel):
     """Request schema for agent success evaluation"""
 
+    user_id: str = ""
     session_id: str
     agent_version: str
     request_interaction_data_models: list[RequestInteractionDataModel]

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/__init__.py

@@ -0,0 +1,7 @@
+"""Internal components for agent success evaluation."""
+
+from reflexio.server.services.agent_success_evaluation.components.evaluator import (
+    AgentSuccessEvaluator,
+)
+
+__all__ = ["AgentSuccessEvaluator"]

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{agent_success_evaluator.py → components/evaluator.py}

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-import random
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
@@ -34,7 +33,7 @@ from reflexio.server.services.service_utils import (
 from reflexio.server.site_var.site_var_manager import SiteVarManager
 
 if TYPE_CHECKING:
-    from reflexio.server.services.agent_success_evaluation.agent_success_evaluation_service import (
+    from reflexio.server.services.agent_success_evaluation.service import (
         AgentSuccessGenerationServiceConfig,
     )
 
@@ -104,7 +103,7 @@ class AgentSuccessEvaluator:
 
         Treats all request_interaction_data_models as a single conversation.
         Applies source filtering based on extractor config.
-        Applies sampling rate once per group.
+        Sampling is handled by callers before invoking the evaluator.
 
         Returns:
             List of AgentSuccessEvaluationResult objects (single result for the group)
@@ -130,19 +129,6 @@ class AgentSuccessEvaluator:
             # No matching interactions after source filter
             return []
 
-        # Check sampling rate once per group
-        if self.config.sampling_rate < 1.0:
-            random_value = random.random()  # noqa: S311
-            if random_value >= self.config.sampling_rate:
-                logger.info(
-                    "Skipping evaluation for session %s due to sampling rate. "
-                    "sampling_rate=%s, random_value=%.3f",
-                    self.service_config.session_id,
-                    self.config.sampling_rate,
-                    random_value,
-                )
-                return []
-
         result = self._evaluate_group(request_interaction_data_models)
         return [result] if result else []
 
@@ -288,6 +274,7 @@ class AgentSuccessEvaluator:
             request_interaction_data_models
         )
         return AgentSuccessEvaluationResult(
+            user_id=self.service_config.user_id,
             session_id=self.service_config.session_id,
             agent_version=self.service_config.agent_version,
             evaluation_name=get_extractor_name(self.config),

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py

@@ -13,12 +13,12 @@ import time
 import uuid
 from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
-from typing import Any, Literal
+from typing import Literal
 
 from reflexio.models.api_schema.internal_schema import SessionDescriptor
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
-from reflexio.server.services.agent_success_evaluation.group_evaluation_runner import (
+from reflexio.server.services.agent_success_evaluation.runner import (
     run_group_evaluation,
 )
 from reflexio.server.services.evaluation_overview.eval_sampler import (
@@ -170,74 +170,16 @@ class RegenJobRegistry:
 REGEN_JOBS = RegenJobRegistry()
 
 
-def _load_first_request(
-    storage: BaseStorage,
-    user_id: str,
-    session_id: str,
-    cache: dict[str, tuple[int, dict[str, Any]]],
-) -> tuple[int, dict[str, Any]]:
-    """Return the (created_at, metadata) of a session's earliest request, memoized.
-
-    The regen worker can see multiple SessionDescriptors for the same
-    session_id (one per distinct agent_version/source tuple). This
-    helper guarantees each session_id triggers at most one storage call,
-    mirroring the amortized pattern F2's evaluation overview service
-    uses.
-
-    Args:
-        storage (BaseStorage): Storage backend bound to the request_context.
-        user_id (str): Owner of the session's requests.
-        session_id (str): Session whose first-request data to return.
-        cache (dict[str, tuple[int, dict[str, Any]]]): Memoization dict
-            keyed by session_id; updated in place.
-
-    Returns:
-        tuple[int, dict[str, Any]]: ``(first_created_at, first_metadata)``.
-        Falls back to ``(0, {})`` when the session has no requests (the
-        descriptor is still kept so the regen worker can attempt
-        evaluation; the sampler simply treats it as the epoch-zero day
-        bucket and the untagged group).
-    """
-    cached = cache.get(session_id)
-    if cached is not None:
-        return cached
-    try:
-        requests = storage.get_requests_by_session(user_id, session_id)
-    except Exception as e:  # noqa: BLE001 — per-session resilience boundary
-        logger.warning(
-            "Failed to fetch requests for session %s during F3 sampler candidate "
-            "discovery (%s); falling back to (created_at=0, metadata={}). The "
-            "session will be retried in the per-session loop below.",
-            session_id,
-            e,
-        )
-        cache[session_id] = (0, {})
-        return cache[session_id]
-    if not requests:
-        logger.warning(
-            "Session %s has no requests in storage despite being returned by "
-            "get_session_ids_in_window (possible race or contract violation); "
-            "falling back to (created_at=0, metadata={}).",
-            session_id,
-        )
-        cache[session_id] = (0, {})
-        return cache[session_id]
-    first = min(requests, key=lambda r: r.created_at)
-    cache[session_id] = (first.created_at, first.metadata or {})
-    return cache[session_id]
-
-
 def _build_sample_candidates(
     storage: BaseStorage,
     descriptors: list[SessionDescriptor],
 ) -> list[SampleCandidate]:
-    """Convert raw SessionDescriptors into SampleCandidates with metadata.
+    """Convert raw SessionDescriptors into SampleCandidates.
 
-    Reads the first-request data for each distinct session_id at most
-    once via ``_load_first_request``'s cache, then assembles one
-    ``SampleCandidate`` per descriptor. ``created_at`` is sourced from
-    the first request's timestamp so the day-bucket stratum aligns with
-    the session's wall clock (not the regen window's edges).
+    Reads first-request data for all distinct session_ids in one storage call,
+    then assembles one ``SampleCandidate`` per descriptor. ``created_at`` is
+    sourced from the first request's timestamp so the day-bucket stratum aligns
+    with the session's wall clock (not the regen window's edges).
 
     Args:
         storage (BaseStorage): Storage backend used to load per-session data.
@@ -248,20 +190,34 @@ def _build_sample_candidates(
         list[SampleCandidate]: One candidate per descriptor, ready for the
         pure ``sample_candidates`` function.
     """
-    cache: dict[str, tuple[int, dict[str, Any]]] = {}
+    pairs = sorted({(sd.user_id, sd.session_id) for sd in descriptors})
+    try:
+        first_requests = storage.get_first_requests_by_user_session_pairs(pairs)
+    except Exception as e:  # noqa: BLE001 — discovery should not abort the whole job
+        logger.warning(
+            "Failed to bulk fetch first requests during F3 sampler candidate "
+            "discovery (%s); falling back to epoch-zero source buckets.",
+            e,
+        )
+        first_requests = {}
+
     candidates: list[SampleCandidate] = []
     for sd in descriptors:
-        created_at, metadata = _load_first_request(
-            storage, sd.user_id, sd.session_id, cache
-        )
+        first = first_requests.get((sd.user_id, sd.session_id))
+        if first is None:
+            logger.warning(
+                "Session %s has no first request in storage despite being returned by "
+                "get_session_ids_in_window; falling back to (created_at=0, source='').",
+                sd.session_id,
+            )
         candidates.append(
             SampleCandidate(
                 session_id=sd.session_id,
                 user_id=sd.user_id,
                 agent_version=sd.agent_version,
                 source=sd.source,
-                created_at=created_at,
-                first_request_metadata=metadata,
+                created_at=first.created_at if first else 0,
+                first_request_source=(first.source or "") if first else "",
             )
         )
     return candidates
@@ -328,7 +284,7 @@ def run_regen(
 
     Step 1 enumerates all candidate sessions in ``[from_ts, to_ts]`` via
     ``storage.get_session_ids_in_window``. Step 2 stratifies them by
-    (day-bucket x F2 group) and samples up to
+    (day-bucket x first request source) and samples up to
     ``config.eval_sample_n_per_stratum`` per stratum — giving the regen
     pipeline predictable cost regardless of traffic volume. Step 3
     dispatches the sampled subset through a ``ThreadPoolExecutor`` whose

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{group_evaluation_runner.py → runner.py}

@@ -15,15 +15,16 @@ from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
 from reflexio.server.services.agent_success_evaluation import _eval_health
 from reflexio.server.services.agent_success_evaluation._eval_health import SkipReason
-from reflexio.server.services.agent_success_evaluation.agent_success_evaluation_service import (
-    AgentSuccessEvaluationService,
-)
 from reflexio.server.services.agent_success_evaluation.agent_success_evaluation_utils import (
     AgentSuccessEvaluationRequest,
 )
-from reflexio.server.services.agent_success_evaluation.delayed_group_evaluator import (
+from reflexio.server.services.agent_success_evaluation.scheduler import (
     _EFFECTIVE_DELAY_SECONDS,
 )
+from reflexio.server.services.agent_success_evaluation.service import (
+    AgentSuccessEvaluationService,
+)
+from reflexio.server.services.extractor_config_utils import get_extractor_name
 from reflexio.server.services.shadow_comparison.judge import ShadowComparisonJudge
 
 logger = logging.getLogger(__name__)
@@ -166,13 +167,13 @@ def run_group_evaluation(
     # afterwards cannot remove the new rows.
     old_result_ids: list[int] = []
     if force_regenerate:
-        # Eval rows per session are tiny (usually 1), so pulling a
-        # generous limit and filtering in-process is cheap and avoids adding
-        # a third query shape to the storage contract.
-        prior_rows = storage.get_agent_success_evaluation_results(  # type: ignore[reportOptionalMemberAccess]
-            limit=10000, agent_version=agent_version
+        config = request_context.configurator.get_config()
+        old_result_ids = storage.get_agent_success_evaluation_result_ids(  # type: ignore[reportOptionalMemberAccess]
+            user_id=user_id,
+            session_id=session_id,
+            evaluation_name=get_extractor_name(config),
+            agent_version=agent_version,
         )
-        old_result_ids = [r.result_id for r in prior_rows if r.session_id == session_id]
 
     logger.info(
         "Running group evaluation for session=%s with %d requests and %d interactions"
@@ -185,6 +186,7 @@ def run_group_evaluation(
     )
 
     evaluation_request = AgentSuccessEvaluationRequest(
+        user_id=user_id,
         session_id=session_id,
         agent_version=agent_version,
         source=source,

package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{delayed_group_evaluator.py → scheduler.py}

@@ -18,9 +18,17 @@ from reflexio.server.services.agent_success_evaluation import _eval_health
 
 logger = logging.getLogger(__name__)
 
-# Delay in seconds before evaluating a group after the last request
-GROUP_EVALUATION_DELAY_SECONDS = 600  # 10 minutes
+# Default delay in seconds before evaluating a group after the last request.
+_DEFAULT_DELAY_SECONDS = 600  # 10 minutes
 
+# Inactivity delay before a session is evaluated. Override the default via the
+# GROUP_EVALUATION_DELAY_SECONDS environment variable (in seconds).
+GROUP_EVALUATION_DELAY_SECONDS = int(
+    os.environ.get("GROUP_EVALUATION_DELAY_SECONDS", str(_DEFAULT_DELAY_SECONDS))
+)
+
+# Kept as a patch point for tests. Production should use the full inactivity
+# delay unless a test deliberately lowers this constant.
 IS_TEST_ENV = os.environ.get("IS_TEST_ENV", "false").strip() == "true"
 _EFFECTIVE_DELAY_SECONDS = 30 if IS_TEST_ENV else GROUP_EVALUATION_DELAY_SECONDS