claude-smart 0.2.48 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py

@@ -7,12 +7,12 @@ import os
 import threading
 import time
 import uuid
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import TimeoutError as FuturesTimeoutError
 from contextlib import nullcontext
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from reflexio.defaults import resolve_agent_version
 from reflexio.models.api_schema.service_schemas import (
@@ -22,6 +22,7 @@ from reflexio.models.api_schema.service_schemas import (
 )
 from reflexio.models.config_schema import Config
 from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.env_utils import env_str, env_truthy
 from reflexio.server.llm.litellm_client import LiteLLMClient
 from reflexio.server.operation_limiter import operation_limit
 from reflexio.server.services.agent_success_evaluation.runner import (
@@ -30,6 +31,7 @@ from reflexio.server.services.agent_success_evaluation.runner import (
 from reflexio.server.services.agent_success_evaluation.scheduler import (
     GroupEvaluationScheduler,
 )
+from reflexio.server.services.deferred_learning_plan import DeferredLearningPlan
 from reflexio.server.services.operation_state_utils import OperationStateManager
 from reflexio.server.services.playbook.playbook_service_utils import (
     PlaybookGenerationRequest,
@@ -47,6 +49,10 @@ from reflexio.server.services.reflection.reflection_service_utils import (
     ReflectionServiceRequest,
 )
 from reflexio.server.services.reflection.service import ReflectionService
+from reflexio.server.services.shadow_comparison.worker import (
+    ShadowComparisonJob,
+    enqueue_shadow_comparison,
+)
 from reflexio.server.services.storage.retention import (
     delete_count_for_retention,
     get_row_retention_limits,
@@ -56,6 +62,7 @@ from reflexio.server.tracing import sentry_tags
 from reflexio.server.usage_metrics import record_usage_event
 
 if TYPE_CHECKING:
+    from reflexio.server.services.deferred_learning_plan import ReflectionWritePlan
     from reflexio.server.services.unified_search_service import UnifiedSearchService
 
 logger = logging.getLogger(__name__)
@@ -65,6 +72,24 @@ CLEANUP_STALE_LOCK_SECONDS = 600
 GENERATION_SERVICE_TIMEOUT_SECONDS = 600
 _STALL_WARNING_PREFIX = "Reflexio learning is paused"
 
+# ── Durable-learning same-user guard (F4) ──
+# Service prefix for the per-user in-progress lock that serializes concurrent
+# durable-learning jobs for one user (compute → persist across the fence).
+_DURABLE_LEARNING_LOCK_SERVICE = "durable_learning"
+# Stale timeout for the per-user durable lock. Deliberately set ABOVE the 300s
+# durable-job claim lease (V4): a job's claim can be re-leased after 300s, so the
+# lock must outlive one claim window and NOT expire in lockstep with the lease
+# under compute-time throttling — 900s = 3x the lease (>= the required 2x).
+_DURABLE_LOCK_STALE_SECONDS = 900
+# Operation-state payload written when releasing the per-user lock (mirrors the
+# clean-slate state OperationStateManager.clear_lock_if_owner writes).
+_DURABLE_LOCK_CLEARED_STATE = {
+    "in_progress": False,
+    "current_request_id": None,
+    "pending_request_id": None,
+    "pending_request_queue": [],
+}
+
 
 def _retention_cleanup_interval_seconds() -> float:
     raw = os.getenv("REFLEXIO_RETENTION_CLEANUP_INTERVAL_SECONDS", "300") or "300"
@@ -92,6 +117,22 @@ def _stable_group_sampling_fraction(
     return int.from_bytes(digest[:8], "big") / 2**64
 
 
+def _org_in_durable_allowlist(org_id: str | None) -> bool:
+    """Whether ``org_id`` may use the durable learning queue.
+
+    Reads ``REFLEXIO_DURABLE_LEARNING_QUEUE_ORG_ALLOWLIST`` (comma-separated org
+    IDs). An empty/whitespace-only value means the allowlist is unset — the
+    default global behavior, so every org is eligible. The allowlist can only
+    *narrow* the durable path; it never enables it on its own (the
+    ``REFLEXIO_DURABLE_LEARNING_QUEUE`` flag still gates activation).
+    """
+    raw = env_str("REFLEXIO_DURABLE_LEARNING_QUEUE_ORG_ALLOWLIST", "")
+    allowed = {s.strip() for s in raw.split(",") if s.strip()}
+    if not allowed:
+        return True
+    return str(org_id) in allowed
+
+
 @dataclass
 class GenerationServiceResult:
     """Result of a GenerationService.run call.
@@ -249,35 +290,51 @@ class GenerationService:
                 session_id=publish_user_interaction_request.session_id,
                 evaluation_only=publish_user_interaction_request.evaluation_only,
             )
-            self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
 
-            # Add interactions to storage (bulk insert with batched embedding generation)
-            self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
-                user_id=user_id, interactions=new_interactions
+            # When the durable queue is enabled and this is a deferred publish, the
+            # request + interactions + job row are written together inside a single
+            # commit_scope (below).  Every other path persists here unconditionally.
+            use_durable_queue = env_truthy(
+                env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")
+            )
+            _durable_defer = (
+                defer_learning
+                and use_durable_queue
+                and _org_in_durable_allowlist(self.org_id)
             )
 
+            if not _durable_defer:
+                self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                # Add interactions to storage (bulk insert with batched embedding generation)
+                self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                    user_id=user_id, interactions=new_interactions
+                )
+
             # Extract source (empty string treated as None)
             source = publish_user_interaction_request.source or None
 
             if publish_user_interaction_request.evaluation_only:
-                self._schedule_group_evaluation_if_needed(
+                if _durable_defer:
+                    # evaluation_only returns early; ensure data lands even on durable path.
+                    self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                    self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                        user_id=user_id, interactions=new_interactions
+                    )
+                self._schedule_post_publish_evaluations(
                     new_request=new_request,
+                    interactions=new_interactions,
                     user_id=user_id,
                     agent_version=agent_version,
                     source=source,
                 )
-                record_usage_event(
-                    org_id=self.org_id,
+                self._emit_publish_success_events(
+                    interactions=new_interactions,
                     user_id=user_id,
                     request_id=request_id,
                     session_id=new_request.session_id,
                     source=source,
                     agent_version=agent_version,
                     backend="evaluation_only",
-                    event_name="publish_request_succeeded",
-                    event_category="publish",
-                    outcome="success",
-                    count_value=len(new_interactions),
                     duration_ms=int((time.perf_counter() - publish_start) * 1000),
                     metadata={
                         "evaluation_only": True,
@@ -290,32 +347,87 @@ class GenerationService:
                 not publish_user_interaction_request.override_learning_stall
                 and (stall_warning := self._active_learning_stall_warning()) is not None
             ):
+                if _durable_defer:
+                    # Stall skips learning but data must still land.
+                    self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                    self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                        user_id=user_id, interactions=new_interactions
+                    )
                 result.warnings.append(stall_warning)
                 logger.warning("%s; skipping automatic extraction", stall_warning)
-                record_usage_event(
-                    org_id=self.org_id,
+                self._schedule_post_publish_evaluations(
+                    new_request=new_request,
+                    interactions=new_interactions,
+                    user_id=user_id,
+                    agent_version=agent_version,
+                    source=source,
+                )
+                self._emit_publish_success_events(
+                    interactions=new_interactions,
                     user_id=user_id,
                     request_id=request_id,
                     session_id=new_request.session_id,
                     source=source,
                     agent_version=agent_version,
-                    event_name="publish_request_succeeded",
-                    event_category="publish",
-                    outcome="success",
-                    count_value=len(new_interactions),
                     duration_ms=int((time.perf_counter() - publish_start) * 1000),
                     metadata={"warning_count": len(result.warnings)},
                 )
                 return result
 
             if defer_learning:
+                if _durable_defer:
+                    # Durable atomic path: pre-generate embeddings OUTSIDE the
+                    # transaction (network call), then persist request + interactions
+                    # + job in one commit_scope.
+                    self.storage.prepare_interaction_embeddings(new_interactions)  # type: ignore[reportOptionalMemberAccess]
+                    covers_through = max(i.created_at for i in new_interactions)
+                    with self.storage.commit_scope():  # type: ignore[reportOptionalMemberAccess]
+                        self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                        self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                            user_id=user_id,
+                            interactions=new_interactions,
+                            embeddings_prepared=True,
+                        )
+                        self.storage.enqueue_learning_job(  # type: ignore[reportOptionalMemberAccess]
+                            org_id=self.org_id,
+                            user_id=user_id,
+                            request_id=request_id,
+                            covers_through=covers_through,
+                            force_extraction=publish_user_interaction_request.force_extraction,
+                            skip_aggregation=publish_user_interaction_request.skip_aggregation,
+                        )
+                    self._schedule_post_publish_evaluations(
+                        new_request=new_request,
+                        interactions=new_interactions,
+                        user_id=user_id,
+                        agent_version=agent_version,
+                        source=source,
+                    )
+                    self._emit_publish_success_events(
+                        interactions=new_interactions,
+                        user_id=user_id,
+                        request_id=request_id,
+                        session_id=new_request.session_id,
+                        source=source,
+                        agent_version=agent_version,
+                        backend="durable",
+                        duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                        metadata={
+                            "defer_learning": True,
+                            "durable_queue": True,
+                            "warning_count": len(result.warnings),
+                        },
+                    )
+                    return result
+
                 from reflexio.server.services.publish_learning_worker import (
                     PublishLearningJob,
                     enqueue_publish_learning,
                 )
 
-                self._schedule_group_evaluation_if_needed(
+                self._schedule_post_publish_evaluations(
                     new_request=new_request,
+                    interactions=new_interactions,
                     user_id=user_id,
                     agent_version=agent_version,
                     source=source,
@@ -332,18 +444,14 @@ class GenerationService:
                         skip_aggregation=publish_user_interaction_request.skip_aggregation,
                     )
                 )
-                record_usage_event(
-                    org_id=self.org_id,
+                self._emit_publish_success_events(
+                    interactions=new_interactions,
                     user_id=user_id,
                     request_id=request_id,
                     session_id=new_request.session_id,
                     source=source,
                     agent_version=agent_version,
                     backend="classic",
-                    event_name="publish_request_succeeded",
-                    event_category="publish",
-                    outcome="success",
-                    count_value=len(new_interactions),
                     duration_ms=int((time.perf_counter() - publish_start) * 1000),
                     metadata={
                         "defer_learning": True,
@@ -377,30 +485,26 @@ class GenerationService:
                     result=result,
                 )
 
-                # Schedule delayed group evaluation for the required session.
-                self._schedule_group_evaluation_if_needed(
-                    new_request=new_request,
-                    user_id=user_id,
-                    agent_version=agent_version,
-                    source=source,
-                )
+            self._schedule_post_publish_evaluations(
+                new_request=new_request,
+                interactions=new_interactions,
+                user_id=user_id,
+                agent_version=agent_version,
+                source=source,
+            )
 
-                record_usage_event(
-                    org_id=self.org_id,
-                    user_id=user_id,
-                    request_id=request_id,
-                    session_id=new_request.session_id,
-                    source=source,
-                    agent_version=agent_version,
-                    backend="classic",
-                    event_name="publish_request_succeeded",
-                    event_category="publish",
-                    outcome="success",
-                    count_value=len(new_interactions),
-                    duration_ms=int((time.perf_counter() - publish_start) * 1000),
-                    metadata={"warning_count": len(result.warnings)},
-                )
-                return result
+            self._emit_publish_success_events(
+                interactions=new_interactions,
+                user_id=user_id,
+                request_id=request_id,
+                session_id=new_request.session_id,
+                source=source,
+                agent_version=agent_version,
+                backend="classic",
+                duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                metadata={"warning_count": len(result.warnings)},
+            )
+            return result
 
         except Exception as e:
             record_usage_event(
@@ -430,6 +534,47 @@ class GenerationService:
                 )
             raise
 
+    def _emit_publish_success_events(
+        self,
+        *,
+        interactions: list[Interaction],
+        user_id: str,
+        request_id: str,
+        session_id: str,
+        source: str | None,
+        agent_version: str,
+        duration_ms: int,
+        metadata: Mapping[str, Any],
+        backend: str | None = None,
+    ) -> None:
+        """Emit one ``publish_request_succeeded`` event per interaction.
+
+        Replaces the old single aggregate event (``count_value=len(interactions)``)
+        with one entity-backed event per interaction (``count_value=1``,
+        ``event_key=f"pub:{interaction_id}"``, ``entity_id=interaction_id``) so
+        downstream dedup can key on the interaction id. The summed
+        ``count_value`` across the emitted events equals the old aggregate
+        count, so totals are unchanged.
+        """
+        for interaction in interactions:
+            record_usage_event(
+                org_id=self.org_id,
+                user_id=user_id,
+                request_id=request_id,
+                session_id=session_id,
+                source=source,
+                agent_version=agent_version,
+                backend=backend,
+                event_name="publish_request_succeeded",
+                event_category="publish",
+                outcome="success",
+                count_value=1,
+                event_key=f"pub:{interaction.interaction_id}",
+                entity_id=str(interaction.interaction_id),
+                duration_ms=duration_ms,
+                metadata=metadata,
+            )
+
     # ===============================
     # deferred learning
     # ===============================
@@ -445,6 +590,13 @@ class GenerationService:
         force_extraction: bool,
         skip_aggregation: bool,
     ) -> GenerationServiceResult:
+        """Run the learning steps for a deferred job (synchronous / non-durable
+        callers: ``PublishLearningWorker`` and manual reruns).
+
+        Runs compute + persist + side-effects together, with profile and playbook
+        generation in parallel — no external ``commit_scope`` is held on this
+        path (the durable worker uses the compute/persist/emit split instead).
+        """
         result = GenerationServiceResult(request_id=request_id)
         self._run_learning_steps(
             user_id=user_id,
@@ -454,49 +606,97 @@ class GenerationService:
             skip_aggregation=skip_aggregation,
             source=source,
             result=result,
+            parallel=True,
         )
         return result
 
     # ===============================
-    # private methods
+    # deferred learning — compute / persist / emit split (gate b)
     # ===============================
 
-    def _run_learning_steps(
+    def compute_deferred_learning(
         self,
         *,
         user_id: str,
         request_id: str,
+        session_id: str | None,  # noqa: ARG002 - kept for worker call symmetry
+        source: str | None,
         agent_version: str,
         force_extraction: bool,
         skip_aggregation: bool,
-        source: str | None,
-        result: GenerationServiceResult,
-    ) -> None:
-        # Reflection runs as its own sliding-window step BEFORE the extractor
-        # pool spins up, so replacements are visible to extractors.
-        self._maybe_run_reflection(
+    ) -> DeferredLearningPlan:
+        """Compute half of one durable-learning job — NO ``commit_scope`` held.
+
+        Runs the same-user guard (F4) first, then reflection + profile + playbook
+        compute (LLM extraction, dedup, embeddings) entirely OUTSIDE any writer
+        transaction, and assembles a :class:`DeferredLearningPlan` of the held
+        ``(service, plan)`` pairs. Issues NO learning DB write (F3): the only
+        writes are the extractor's own ``agent_run`` rows + the per-user
+        coordination lock (``try_acquire_in_progress_lock``, not a learning
+        terminal).
+
+        F4 GUARD FIRST: acquires a DB-backed per-user in-progress lock BEFORE any
+        LLM work. On contention returns ``DeferredLearningPlan(lock_acquired=
+        False, reflection/profile/playbook=None)`` immediately (no compute) — the
+        worker must then leave the job reclaimable (Task 9), NOT complete it.
+
+        Reflection runs first on the calling thread (it reads the pre-persist
+        snapshot); profile + playbook ``compute_generation`` then run in parallel
+        (no ``commit_scope`` is held, so the old ``sequential=True`` RLock
+        avoidance is unnecessary). Per-half failures are best-effort — captured
+        into ``warnings`` and the half dropped — mirroring
+        ``_run_learning_steps``.
+        """
+        warnings: list[str] = []
+
+        # F4 GUARD — before any LLM. On contention, no compute runs.
+        if not self._acquire_durable_learning_lock(
+            user_id=user_id, request_id=request_id
+        ):
+            logger.info(
+                "durable-learning same-user lock held; leaving job reclaimable "
+                "(user_id=%s request_id=%s)",
+                user_id,
+                request_id,
+            )
+            return DeferredLearningPlan(
+                request_id=request_id,
+                user_id=user_id,
+                agent_version=agent_version,
+                lock_acquired=False,
+                reflection=None,
+                profile=None,
+                playbook=None,
+                warnings=warnings,
+            )
+
+        # Reflection first (calling thread) — one pre-persist read snapshot.
+        reflection_pair = self._compute_reflection(
             user_id=user_id,
             request_id=request_id,
             agent_version=agent_version,
             source=source,
+            warnings=warnings,
         )
 
-        profile_generation_service = ProfileGenerationService(
+        # Profile + playbook compute in parallel — no scope held, so no RLock
+        # contention (the pre-split sequential mode only existed to avoid the
+        # SQLite commit_scope RLock, which compute never takes).
+        profile_service = ProfileGenerationService(
             llm_client=self.client, request_context=self.request_context
         )
-        profile_generation_request = ProfileGenerationRequest(
+        profile_request = ProfileGenerationRequest(
             user_id=user_id,
             request_id=request_id,
             source=source,
             force_extraction=force_extraction,
         )
-
-        playbook_generation_service = PlaybookGenerationService(
+        playbook_service = PlaybookGenerationService(
             llm_client=self.client,
             request_context=self.request_context,
             skip_aggregation=skip_aggregation,
         )
-        playbook_generation_request = PlaybookGenerationRequest(
+        playbook_request = PlaybookGenerationRequest(
             request_id=request_id,
             agent_version=agent_version,
             user_id=user_id,
@@ -504,25 +704,26 @@ class GenerationService:
             force_extraction=force_extraction,
         )
 
+        profile_pair: tuple = None  # type: ignore[assignment]
+        playbook_pair: tuple = None  # type: ignore[assignment]
         executor = ThreadPoolExecutor(max_workers=2)
         try:
-            futures = [
-                executor.submit(
-                    contextvars.copy_context().run,
-                    profile_generation_service.run,
-                    profile_generation_request,
-                ),
-                executor.submit(
-                    contextvars.copy_context().run,
-                    playbook_generation_service.run,
-                    playbook_generation_request,
-                ),
-            ]
-
-            service_names = ["profile_generation", "playbook_generation"]
-            for future, service_name in zip(futures, service_names, strict=True):
+            profile_future = executor.submit(
+                contextvars.copy_context().run,
+                profile_service.compute_generation,
+                profile_request,
+            )
+            playbook_future = executor.submit(
+                contextvars.copy_context().run,
+                playbook_service.compute_generation,
+                playbook_request,
+            )
+            for future, service_name, service in (
+                (profile_future, "profile_generation", profile_service),
+                (playbook_future, "playbook_generation", playbook_service),
+            ):
                 try:
-                    future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
+                    plan = future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
                 except FuturesTimeoutError:  # noqa: PERF203
                     msg = (
                         f"{service_name} timed out after "
@@ -535,7 +736,8 @@ class GenerationService:
                         error_type="timeout",
                     ):
                         logger.error("%s for request %s", msg, request_id)
-                    result.warnings.append(msg)
+                    warnings.append(msg)
+                    continue
                 except Exception as e:
                     msg = f"{service_name} failed: {e}"
                     with sentry_tags(
@@ -548,10 +750,334 @@ class GenerationService:
                             "Generation service failed for request %s",
                             request_id,
                         )
-                    result.warnings.append(msg)
+                    warnings.append(msg)
+                    continue
+                if plan is None:
+                    continue
+                if service_name == "profile_generation":
+                    profile_pair = (service, plan)
+                else:
+                    playbook_pair = (service, plan)
         finally:
             executor.shutdown(wait=False, cancel_futures=True)
 
+        return DeferredLearningPlan(
+            request_id=request_id,
+            user_id=user_id,
+            agent_version=agent_version,
+            lock_acquired=True,
+            reflection=reflection_pair,
+            profile=profile_pair,
+            playbook=playbook_pair,
+            warnings=warnings,
+        )
+
+    def persist_deferred_learning(self, plan: DeferredLearningPlan) -> None:
+        """Persist half — apply the fence-critical writes for a computed job.
+
+        This is the ONLY part the durable worker runs inside its fenced
+        ``commit_scope``. For each present half it calls the held instance's
+        persist (reflection apply loop + bookmark; profile/playbook row writes +
+        extractor bookmark advance). Issues NO side-effects (telemetry, billing,
+        tagging, off-thread schedulers, lock release) — those are post-commit
+        (``emit_deferred_learning_side_effects``) so a fence-lost job never fires
+        them.
+        """
+        if plan.reflection is not None:
+            reflection_service, reflection_plan = plan.reflection
+            reflection_service.persist(reflection_plan)
+        if plan.profile is not None:
+            profile_service, profile_plan = plan.profile
+            profile_service.persist_generation(profile_plan)
+        if plan.playbook is not None:
+            playbook_service, playbook_plan = plan.playbook
+            playbook_service.persist_generation(playbook_plan)
+
+    def emit_deferred_learning_side_effects(self, plan: DeferredLearningPlan) -> None:
+        """Post-commit side-effects — telemetry, billing, tagging, lock release.
+
+        Runs only for a fence-winning job (the durable worker calls it after the
+        scope commits). Fires each held half's post-commit emit + schedules the
+        deferred tagging pass, then ALWAYS releases the per-user F4 lock (even if
+        an emit raised) so a completed job never strands the lock.
+
+        A ``lock_acquired=False`` plan never reaches here (the worker skips emit
+        on contention), so releasing here is safe — this instance owns the lock.
+
+        Each emit half is isolated in its own try/except (mirroring how
+        ``schedule_tagging`` already self-guards): the halves are independent
+        post-commit side effects (billing / telemetry / off-thread schedulers),
+        so a failure in an early half must NOT starve the later halves. The
+        per-user lock release always runs in ``finally`` so a completed job never
+        strands the lock.
+        """
+        try:
+            if plan.reflection is not None:
+                reflection_service, reflection_plan = plan.reflection
+                try:
+                    reflection_service.emit_side_effects(reflection_plan)
+                except Exception:
+                    logger.exception(
+                        "Failed to emit reflection side effects for deferred "
+                        "learning request %s",
+                        plan.request_id,
+                    )
+            if plan.profile is not None:
+                profile_service, profile_plan = plan.profile
+                try:
+                    profile_service.emit_generation_side_effects(profile_plan)
+                except Exception:
+                    logger.exception(
+                        "Failed to emit profile side effects for deferred "
+                        "learning request %s",
+                        plan.request_id,
+                    )
+            if plan.playbook is not None:
+                playbook_service, playbook_plan = plan.playbook
+                try:
+                    playbook_service.emit_generation_side_effects(playbook_plan)
+                except Exception:
+                    logger.exception(
+                        "Failed to emit playbook side effects for deferred "
+                        "learning request %s",
+                        plan.request_id,
+                    )
+            try:
+                schedule_tagging(
+                    org_id=self.org_id,
+                    user_id=plan.user_id,
+                    agent_version=plan.agent_version,
+                    request_context=self.request_context,
+                    llm_client=self.client,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to schedule tagging for deferred learning request %s",
+                    plan.request_id,
+                )
+        finally:
+            self._release_durable_learning_lock(
+                user_id=plan.user_id, request_id=plan.request_id
+            )
+
+    # ===============================
+    # private methods
+    # ===============================
+
+    def _durable_learning_lock_key(self, user_id: str) -> str:
+        """Per-user durable-learning in-progress lock key.
+
+        Mirrors ``OperationStateManager._lock_key`` shape
+        (``{service}::{org}::{scope}::lock``) but on a DEDICATED
+        ``durable_learning`` service prefix, distinct from the
+        profile/playbook generation locks and from any extractor bookmark row.
+        """
+        return f"{_DURABLE_LEARNING_LOCK_SERVICE}::{self.org_id}::{user_id}::lock"
+
+    def _acquire_durable_learning_lock(self, *, user_id: str, request_id: str) -> bool:
+        """Try to acquire the per-user durable-learning lock (F4).
+
+        Uses the atomic DB-backed ``try_acquire_in_progress_lock`` (a distinct
+        storage method, NOT one of the learning-write terminals the compute
+        purity contract forbids), so the guard does not trip the compute
+        write-tripwire. Returns ``True`` when acquired (proceed with compute),
+        ``False`` on contention (another same-user durable job holds it).
+        """
+        if self.storage is None:
+            return True
+        result = self.storage.try_acquire_in_progress_lock(
+            self._durable_learning_lock_key(user_id),
+            request_id,
+            stale_lock_seconds=_DURABLE_LOCK_STALE_SECONDS,
+        )
+        return bool(result.get("acquired", False))
+
+    def _release_durable_learning_lock(self, *, user_id: str, request_id: str) -> None:
+        """Release the per-user durable-learning lock iff we still own it.
+
+        Uses ``clear_in_progress_lock_if_owner`` (CAS on the holder) so a job
+        whose lock was already stolen after a stale-timeout does not clobber the
+        new holder's lock.
+        """
+        if self.storage is None:
+            return
+        self.storage.clear_in_progress_lock_if_owner(
+            self._durable_learning_lock_key(user_id),
+            request_id,
+            dict(_DURABLE_LOCK_CLEARED_STATE),
+        )
+
+    def _compute_reflection(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        agent_version: str,
+        source: str | None,
+        warnings: list[str],
+    ) -> tuple[ReflectionService, ReflectionWritePlan] | None:
+        """Run reflection ``compute`` best-effort; return its held pair or None.
+
+        Mirrors ``_maybe_run_reflection`` (a reflection failure must never break
+        the publish) but returns the ``(service, plan)`` pair so the caller can
+        defer ``persist`` / ``emit_side_effects`` across the worker fence.
+        Returns ``None`` when reflection has nothing to persist (gate closed /
+        disabled / storage None) or a compute error (logged + appended to
+        ``warnings``).
+        """
+        try:
+            service = ReflectionService(
+                request_context=self.request_context,
+                llm_client=self.client,
+            )
+            reflection_plan = service.compute(
+                ReflectionServiceRequest(
+                    user_id=user_id,
+                    request_id=request_id,
+                    agent_version=agent_version,
+                    source=source,
+                )
+            )
+            if reflection_plan is None:
+                return None
+            return (service, reflection_plan)
+        except Exception as exc:  # noqa: BLE001 — reflection must not break publish
+            with sentry_tags(
+                subsystem="generation",
+                op="reflection",
+                org_id=self.org_id,
+                user_id=user_id,
+                error_type=type(exc).__name__,
+            ):
+                logger.exception("reflection compute failed for user %s", user_id)
+            warnings.append(f"reflection failed: {exc}")
+            return None
+
+    def _run_learning_steps(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        agent_version: str,
+        force_extraction: bool,
+        skip_aggregation: bool,
+        source: str | None,
+        result: GenerationServiceResult,
+        parallel: bool = True,
+    ) -> None:
+        # Reflection runs as its own sliding-window step BEFORE the extractor
+        # pool spins up, so replacements are visible to extractors.
+        self._maybe_run_reflection(
+            user_id=user_id,
+            request_id=request_id,
+            agent_version=agent_version,
+            source=source,
+        )
+
+        profile_generation_service = ProfileGenerationService(
+            llm_client=self.client, request_context=self.request_context
+        )
+        source_request_id = request_id
+        profile_generation_request = ProfileGenerationRequest(
+            user_id=user_id,
+            request_id=source_request_id,
+            source=source,
+            force_extraction=force_extraction,
+        )
+
+        playbook_generation_service = PlaybookGenerationService(
+            llm_client=self.client,
+            request_context=self.request_context,
+            skip_aggregation=skip_aggregation,
+        )
+        playbook_generation_request = PlaybookGenerationRequest(
+            request_id=source_request_id,
+            agent_version=agent_version,
+            user_id=user_id,
+            source=source,
+            force_extraction=force_extraction,
+        )
+
+        if not parallel:
+            # Sequential mode: run both services on the calling thread.
+            # Required when the caller holds a commit_scope lock — spawning
+            # child threads inside that scope would deadlock the SQLite RLock.
+            for svc_name, _run in (
+                (
+                    "profile_generation",
+                    lambda: profile_generation_service.run(profile_generation_request),
+                ),
+                (
+                    "playbook_generation",
+                    lambda: playbook_generation_service.run(
+                        playbook_generation_request
+                    ),
+                ),
+            ):
+                try:
+                    _run()
+                except Exception as e:
+                    msg = f"{svc_name} failed: {e}"
+                    with sentry_tags(
+                        subsystem="generation",
+                        service=svc_name,
+                        request_id=request_id,
+                        error_type=type(e).__name__,
+                    ):
+                        logger.exception(
+                            "Generation service failed for request %s",
+                            request_id,
+                        )
+                    result.warnings.append(msg)
+        else:
+            executor = ThreadPoolExecutor(max_workers=2)
+            try:
+                futures = [
+                    executor.submit(
+                        contextvars.copy_context().run,
+                        profile_generation_service.run,
+                        profile_generation_request,
+                    ),
+                    executor.submit(
+                        contextvars.copy_context().run,
+                        playbook_generation_service.run,
+                        playbook_generation_request,
+                    ),
+                ]
+
+                service_names = ["profile_generation", "playbook_generation"]
+                for future, service_name in zip(futures, service_names, strict=True):
+                    try:
+                        future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
+                    except FuturesTimeoutError:  # noqa: PERF203
+                        msg = (
+                            f"{service_name} timed out after "
+                            f"{GENERATION_SERVICE_TIMEOUT_SECONDS}s"
+                        )
+                        with sentry_tags(
+                            subsystem="generation",
+                            service=service_name,
+                            request_id=request_id,
+                            error_type="timeout",
+                        ):
+                            logger.error("%s for request %s", msg, request_id)
+                        result.warnings.append(msg)
+                    except Exception as e:
+                        msg = f"{service_name} failed: {e}"
+                        with sentry_tags(
+                            subsystem="generation",
+                            service=service_name,
+                            request_id=request_id,
+                            error_type=type(e).__name__,
+                        ):
+                            logger.exception(
+                                "Generation service failed for request %s",
+                                request_id,
+                            )
+                        result.warnings.append(msg)
+            finally:
+                executor.shutdown(wait=False, cancel_futures=True)
+
         try:
             schedule_tagging(
                 org_id=self.org_id,
@@ -566,6 +1092,39 @@ class GenerationService:
                 request_id,
             )
 
+    def _schedule_post_publish_evaluations(
+        self,
+        *,
+        new_request: Request,
+        interactions: list[Interaction],
+        user_id: str,
+        agent_version: str,
+        source: str | None,
+    ) -> None:
+        """Schedule all best-effort evaluation work after publish persistence."""
+        if any(interaction.shadow_content for interaction in interactions):
+            try:
+                enqueue_shadow_comparison(
+                    ShadowComparisonJob(
+                        org_id=self.org_id,
+                        interactions=interactions,
+                        session_id=new_request.session_id,
+                        agent_version=agent_version,
+                    )
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to enqueue shadow comparison for session %s",
+                    new_request.session_id,
+                )
+
+        self._schedule_group_evaluation_if_needed(
+            new_request=new_request,
+            user_id=user_id,
+            agent_version=agent_version,
+            source=source,
+        )
+
     def _schedule_group_evaluation_if_needed(
         self,
         *,
@@ -845,6 +1404,7 @@ class GenerationService:
                 expert_content=interaction_data.expert_content,
                 tools_used=interaction_data.tools_used,
                 citations=interaction_data.citations,
+                retrieved_learnings=interaction_data.retrieved_learnings,
             )
             for interaction_data in interaction_data_list
         ]