claude-smart 0.2.48 → 0.2.49

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py

@@ -0,0 +1,193 @@
+"""DurableLearningWorker: claim jobs from the durable queue, process in a fenced
+commit_scope so exactly one of N racing workers commits its outputs.
+
+Design constraints (v1 — do not remove):
+- LLM calls are inside the commit_scope (compute/persist separation is deferred).
+- No heartbeat thread (deferred; would deadlock a SQLite scope anyway).
+- Exactly-once is guaranteed by the fenced complete_learning_job:
+    rowcount == 0  -> lease was stolen -> _SupersededError -> rollback.
+    rowcount == 1  -> we own the lease -> commit succeeds.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from collections.abc import Callable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.cache.reflexio_cache import get_reflexio
+from reflexio.server.services.generation_service import GenerationService
+from reflexio.server.services.storage.storage_base._learning_jobs import LearningJob
+
+logger = logging.getLogger(__name__)
+
+
+class _SupersededError(Exception):
+    """Raised inside commit_scope when complete_learning_job returns 0.
+
+    Propagating this exception causes commit_scope to roll back all writes
+    (profiles, playbooks) made by run_deferred_learning for the superseded
+    worker — guaranteeing exactly-once side effects.
+    """
+
+    def __init__(self, job_id: str) -> None:
+        super().__init__(f"learning job {job_id} superseded (lease stolen)")
+
+
+class DurableLearningWorker:
+    """Claim durable learning jobs and process each in a fenced commit_scope.
+
+    Two instances racing the same job produce profile/playbook side effects
+    identical to a single run: the loser's complete_learning_job returns 0,
+    which raises _SupersededError inside the scope, rolling back its writes.
+
+    Args:
+        request_context_factory: Callable that returns a RequestContext for a
+            given org_id.  Called once per drain_org invocation.
+        instance_id: Stable label written to claimed_by on each claim.
+            Defaults to a random short UUID suffix.
+    """
+
+    def __init__(
+        self,
+        request_context_factory: Callable[[str], RequestContext],
+        *,
+        instance_id: str | None = None,
+    ) -> None:
+        self._factory = request_context_factory
+        self._instance_id = instance_id or str(uuid.uuid4())[:8]
+
+    def drain_org(self, org_id: str, batch_size: int, lease_seconds: int) -> int:
+        """Claim up to batch_size of this org's jobs and process each.
+
+        Args:
+            org_id: Organisation to drain.
+            batch_size: Maximum number of jobs to claim in one call.
+            lease_seconds: Lease duration passed to claim_learning_jobs.
+
+        Returns:
+            Number of jobs successfully completed (not counting superseded or failed).
+        """
+        ctx = self._factory(org_id)
+        storage = ctx.storage
+        if storage is None:
+            logger.error("event=learning_worker_no_storage org_id=%s", org_id)
+            return 0
+        jobs = storage.claim_learning_jobs(
+            claimed_by=self._instance_id,
+            limit=batch_size,
+            lease_seconds=lease_seconds,
+        )
+        processed = 0
+        for job in jobs:
+            if self._process_job(ctx, job):
+                processed += 1
+        return processed
+
+    def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool:
+        """Process a single claimed job.
+
+        Args:
+            ctx: RequestContext for the job's org.
+            job: The claimed LearningJob (claim_token set).
+
+        Returns:
+            True if the job was successfully completed; False if superseded or failed.
+        """
+        storage = ctx.storage
+        if storage is None:
+            logger.error(
+                "event=learning_job_no_storage job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        # claim_learning_jobs always sets claim_token on a returned job.
+        claim_token = job.claim_token
+        if claim_token is None:
+            logger.error(
+                "event=learning_job_no_claim_token job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        dead = job.attempts >= job.max_attempts
+
+        # The job does not carry agent_version / session_id / source.
+        # These must be read from the durably-persisted Request.
+        if job.latest_request_id is None:
+            logger.error(
+                "event=learning_job_no_request_id job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        request = storage.get_request(job.latest_request_id)
+        if request is None:
+            logger.error(
+                "event=learning_job_missing_request job_id=%s request_id=%s",
+                job.job_id,
+                job.latest_request_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        try:
+            reflexio = get_reflexio(
+                org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir
+            )
+            gen = GenerationService(llm_client=reflexio.llm_client, request_context=ctx)
+
+            with storage.commit_scope():
+                gen.run_deferred_learning(
+                    user_id=job.user_id,
+                    request_id=job.latest_request_id,
+                    session_id=request.session_id,
+                    source=request.source,
+                    agent_version=request.agent_version,
+                    force_extraction=job.force_extraction,
+                    skip_aggregation=job.skip_aggregation,
+                    sequential=True,  # prevent ThreadPoolExecutor deadlock on commit_scope RLock
+                )
+                rows = storage.complete_learning_job(
+                    job_id=job.job_id, claim_token=claim_token
+                )
+                if rows == 0:
+                    raise _SupersededError(job.job_id)
+            logger.info(
+                "event=learning_job_done job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            return True
+        except _SupersededError:
+            logger.info(
+                "event=learning_job_superseded job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+        except Exception:
+            logger.exception(
+                "event=learning_job_failed job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            # attempts was incremented by claim_learning_jobs; go dead when we've
+            # exhausted max_attempts total claim attempts.
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py

@@ -107,9 +107,7 @@ class ExtractionResumeScheduler(ThreadedScheduler):
         try:
             bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
             config = bootstrap_ctx.configurator.get_config()
-            poll_interval = (
-                config.pending_tool_call_config.resume_poll_interval_seconds
-            )
+            poll_interval = config.pending_tool_call_config.resume_poll_interval_seconds
             self._expire_pending_tool_calls(bootstrap_ctx)
             for org_id in self._discover_org_ids(bootstrap_ctx):
                 if self._stop_event.is_set():

package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py

@@ -22,6 +22,7 @@ from reflexio.models.api_schema.service_schemas import (
 )
 from reflexio.models.config_schema import Config
 from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.env_utils import env_str, env_truthy
 from reflexio.server.llm.litellm_client import LiteLLMClient
 from reflexio.server.operation_limiter import operation_limit
 from reflexio.server.services.agent_success_evaluation.runner import (
@@ -249,17 +250,32 @@ class GenerationService:
                 session_id=publish_user_interaction_request.session_id,
                 evaluation_only=publish_user_interaction_request.evaluation_only,
             )
-            self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
 
-            # Add interactions to storage (bulk insert with batched embedding generation)
-            self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
-                user_id=user_id, interactions=new_interactions
+            # When the durable queue is enabled and this is a deferred publish, the
+            # request + interactions + job row are written together inside a single
+            # commit_scope (below).  Every other path persists here unconditionally.
+            use_durable_queue = env_truthy(
+                env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")
             )
+            _durable_defer = defer_learning and use_durable_queue
+
+            if not _durable_defer:
+                self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                # Add interactions to storage (bulk insert with batched embedding generation)
+                self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                    user_id=user_id, interactions=new_interactions
+                )
 
             # Extract source (empty string treated as None)
             source = publish_user_interaction_request.source or None
 
             if publish_user_interaction_request.evaluation_only:
+                if _durable_defer:
+                    # evaluation_only returns early; ensure data lands even on durable path.
+                    self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                    self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                        user_id=user_id, interactions=new_interactions
+                    )
                 self._schedule_group_evaluation_if_needed(
                     new_request=new_request,
                     user_id=user_id,
@@ -290,6 +306,12 @@ class GenerationService:
                 not publish_user_interaction_request.override_learning_stall
                 and (stall_warning := self._active_learning_stall_warning()) is not None
             ):
+                if _durable_defer:
+                    # Stall skips learning but data must still land.
+                    self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                    self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                        user_id=user_id, interactions=new_interactions
+                    )
                 result.warnings.append(stall_warning)
                 logger.warning("%s; skipping automatic extraction", stall_warning)
                 record_usage_event(
@@ -309,6 +331,54 @@ class GenerationService:
                 return result
 
             if defer_learning:
+                if _durable_defer:
+                    # Durable atomic path: pre-generate embeddings OUTSIDE the
+                    # transaction (network call), then persist request + interactions
+                    # + job in one commit_scope.
+                    self.storage.prepare_interaction_embeddings(new_interactions)  # type: ignore[reportOptionalMemberAccess]
+                    covers_through = max(i.created_at for i in new_interactions)
+                    with self.storage.commit_scope():  # type: ignore[reportOptionalMemberAccess]
+                        self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
+                        self.storage.add_user_interactions_bulk(  # type: ignore[reportOptionalMemberAccess]
+                            user_id=user_id,
+                            interactions=new_interactions,
+                            embeddings_prepared=True,
+                        )
+                        self.storage.enqueue_learning_job(  # type: ignore[reportOptionalMemberAccess]
+                            org_id=self.org_id,
+                            user_id=user_id,
+                            request_id=request_id,
+                            covers_through=covers_through,
+                            force_extraction=publish_user_interaction_request.force_extraction,
+                            skip_aggregation=publish_user_interaction_request.skip_aggregation,
+                        )
+                    self._schedule_group_evaluation_if_needed(
+                        new_request=new_request,
+                        user_id=user_id,
+                        agent_version=agent_version,
+                        source=source,
+                    )
+                    record_usage_event(
+                        org_id=self.org_id,
+                        user_id=user_id,
+                        request_id=request_id,
+                        session_id=new_request.session_id,
+                        source=source,
+                        agent_version=agent_version,
+                        backend="durable",
+                        event_name="publish_request_succeeded",
+                        event_category="publish",
+                        outcome="success",
+                        count_value=len(new_interactions),
+                        duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                        metadata={
+                            "defer_learning": True,
+                            "durable_queue": True,
+                            "warning_count": len(result.warnings),
+                        },
+                    )
+                    return result
+
                 from reflexio.server.services.publish_learning_worker import (
                     PublishLearningJob,
                     enqueue_publish_learning,
@@ -444,7 +514,16 @@ class GenerationService:
         agent_version: str,
         force_extraction: bool,
         skip_aggregation: bool,
+        sequential: bool = False,
     ) -> GenerationServiceResult:
+        """Run the learning steps for a deferred job.
+
+        Args:
+            sequential: When True, run profile and playbook generation on the
+                calling thread (no ThreadPoolExecutor).  Required when the
+                caller already holds the storage commit_scope lock — spawning
+                worker threads inside that scope would deadlock on the RLock.
+        """
         result = GenerationServiceResult(request_id=request_id)
         self._run_learning_steps(
             user_id=user_id,
@@ -454,6 +533,7 @@ class GenerationService:
             skip_aggregation=skip_aggregation,
             source=source,
             result=result,
+            parallel=not sequential,
         )
         return result
 
@@ -471,6 +551,7 @@ class GenerationService:
         skip_aggregation: bool,
         source: str | None,
         result: GenerationServiceResult,
+        parallel: bool = True,
     ) -> None:
         # Reflection runs as its own sliding-window step BEFORE the extractor
         # pool spins up, so replacements are visible to extractors.
@@ -504,43 +585,29 @@ class GenerationService:
             force_extraction=force_extraction,
         )
 
-        executor = ThreadPoolExecutor(max_workers=2)
-        try:
-            futures = [
-                executor.submit(
-                    contextvars.copy_context().run,
-                    profile_generation_service.run,
-                    profile_generation_request,
+        if not parallel:
+            # Sequential mode: run both services on the calling thread.
+            # Required when the caller holds a commit_scope lock — spawning
+            # child threads inside that scope would deadlock the SQLite RLock.
+            for svc_name, _run in (
+                (
+                    "profile_generation",
+                    lambda: profile_generation_service.run(profile_generation_request),
                 ),
-                executor.submit(
-                    contextvars.copy_context().run,
-                    playbook_generation_service.run,
-                    playbook_generation_request,
+                (
+                    "playbook_generation",
+                    lambda: playbook_generation_service.run(
+                        playbook_generation_request
+                    ),
                 ),
-            ]
-
-            service_names = ["profile_generation", "playbook_generation"]
-            for future, service_name in zip(futures, service_names, strict=True):
+            ):
                 try:
-                    future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
-                except FuturesTimeoutError:  # noqa: PERF203
-                    msg = (
-                        f"{service_name} timed out after "
-                        f"{GENERATION_SERVICE_TIMEOUT_SECONDS}s"
-                    )
-                    with sentry_tags(
-                        subsystem="generation",
-                        service=service_name,
-                        request_id=request_id,
-                        error_type="timeout",
-                    ):
-                        logger.error("%s for request %s", msg, request_id)
-                    result.warnings.append(msg)
+                    _run()
                 except Exception as e:
-                    msg = f"{service_name} failed: {e}"
+                    msg = f"{svc_name} failed: {e}"
                     with sentry_tags(
                         subsystem="generation",
-                        service=service_name,
+                        service=svc_name,
                         request_id=request_id,
                         error_type=type(e).__name__,
                     ):
@@ -549,8 +616,54 @@ class GenerationService:
                             request_id,
                         )
                     result.warnings.append(msg)
-        finally:
-            executor.shutdown(wait=False, cancel_futures=True)
+        else:
+            executor = ThreadPoolExecutor(max_workers=2)
+            try:
+                futures = [
+                    executor.submit(
+                        contextvars.copy_context().run,
+                        profile_generation_service.run,
+                        profile_generation_request,
+                    ),
+                    executor.submit(
+                        contextvars.copy_context().run,
+                        playbook_generation_service.run,
+                        playbook_generation_request,
+                    ),
+                ]
+
+                service_names = ["profile_generation", "playbook_generation"]
+                for future, service_name in zip(futures, service_names, strict=True):
+                    try:
+                        future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
+                    except FuturesTimeoutError:  # noqa: PERF203
+                        msg = (
+                            f"{service_name} timed out after "
+                            f"{GENERATION_SERVICE_TIMEOUT_SECONDS}s"
+                        )
+                        with sentry_tags(
+                            subsystem="generation",
+                            service=service_name,
+                            request_id=request_id,
+                            error_type="timeout",
+                        ):
+                            logger.error("%s for request %s", msg, request_id)
+                        result.warnings.append(msg)
+                    except Exception as e:
+                        msg = f"{service_name} failed: {e}"
+                        with sentry_tags(
+                            subsystem="generation",
+                            service=service_name,
+                            request_id=request_id,
+                            error_type=type(e).__name__,
+                        ):
+                            logger.exception(
+                                "Generation service failed for request %s",
+                                request_id,
+                            )
+                        result.warnings.append(msg)
+            finally:
+                executor.shutdown(wait=False, cancel_futures=True)
 
         try:
             schedule_tagging(

package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py

@@ -477,29 +477,43 @@ class ProfileGenerationService(
 
     def _get_generated_count(
         self,
-        request: RerunProfileGenerationRequest,
+        request: RerunProfileGenerationRequest | ManualProfileGenerationRequest,
         processed_user_ids: list[str] | None = None,
     ) -> int:
-        """Get the count of profiles generated during rerun.
+        """Get the count of profiles generated during batch generation.
 
-        Counts profiles with PENDING status for each processed user.
+        Counts PENDING profiles for rerun requests and CURRENT profiles for manual
+        regular requests, scoped to users the batch runner actually processed.
 
         Args:
-            request: The rerun request object
+            request: The rerun or manual generation request object
             processed_user_ids: List of user IDs processed in the batch
 
         Returns:
             Number of profiles generated
         """
-        user_ids = processed_user_ids or ([request.user_id] if request.user_id else [])
-        total = 0
-        for user_id in user_ids:
-            profiles = self.storage.get_user_profile(  # type: ignore[reportOptionalMemberAccess]
-                user_id=user_id,
-                status_filter=[Status.PENDING],
+        if isinstance(request, ManualProfileGenerationRequest):
+            return self._count_manual_generated(
+                request, processed_user_ids=processed_user_ids
             )
-            total += len(profiles)
-        return total
+
+        user_ids = self._count_user_ids(request.user_id, processed_user_ids)
+        if not user_ids:
+            return 0
+        return self.storage.count_user_profiles_by_status(  # type: ignore[reportOptionalMemberAccess]
+            user_ids=user_ids,
+            status=Status.PENDING,
+        )
+
+    @staticmethod
+    def _count_user_ids(
+        request_user_id: str | None, processed_user_ids: list[str] | None
+    ) -> list[str]:
+        """Resolve users to count without treating an empty processed batch as missing."""
+
+        if processed_user_ids is not None:
+            return processed_user_ids
+        return [request_user_id] if request_user_id else []
 
     # ===============================
     # Upgrade/Downgrade hook implementations (override base class methods)
@@ -676,16 +690,15 @@ class ProfileGenerationService(
                 "source": request.source,
                 "mode": "manual_regular",
             }
-            users_processed, _ = self._run_batch_with_progress(
+            # total_profiles is computed inside the batch runner via
+            # _get_generated_count(processed_user_ids=...).
+            users_processed, total_profiles = self._run_batch_with_progress(
                 user_ids=user_ids,
                 request=request,  # type: ignore[reportArgumentType]
                 request_params=request_params,
                 state_manager=state_manager,
             )
 
-            # 3. Count generated profiles (CURRENT status = None)
-            total_profiles = self._count_manual_generated(request)
-
             return ManualProfileGenerationResponse(
                 success=True,
                 msg=f"Generated {total_profiles} profiles for {users_processed} user(s)",
@@ -700,20 +713,29 @@ class ProfileGenerationService(
                 profiles_generated=0,
             )
 
-    def _count_manual_generated(self, request: ManualProfileGenerationRequest) -> int:
+    def _count_manual_generated(
+        self,
+        request: ManualProfileGenerationRequest,
+        processed_user_ids: list[str] | None = None,
+    ) -> int:
         """
         Count profiles generated during manual regular generation.
 
-        Counts profiles with CURRENT status (None), optionally filtered by user_id.
+        Counts profiles with CURRENT status (None) for each processed user.
 
         Args:
             request: The manual generation request object
+            processed_user_ids: User IDs processed by the manual batch. When
+                the request omitted user_id, this prevents passing None through
+                to storage methods that require a concrete user.
 
         Returns:
             Number of profiles with CURRENT status
         """
-        profiles = self.storage.get_user_profile(  # type: ignore[reportOptionalMemberAccess]
-            user_id=request.user_id,  # type: ignore[reportArgumentType]
-            status_filter=[None],  # CURRENT profiles
+        user_ids = self._count_user_ids(request.user_id, processed_user_ids)
+        if not user_ids:
+            return 0
+        return self.storage.count_user_profiles_by_status(  # type: ignore[reportOptionalMemberAccess]
+            user_ids=user_ids,
+            status=None,
         )
-        return len(profiles)

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py

@@ -1,3 +1,4 @@
+from ._learning_jobs import SQLiteLearningJobStoreMixin
 from ._base import (
     SQLiteStorageBase,
     _cosine_similarity,
@@ -48,6 +49,7 @@ from .profiles import InteractionStoreMixin, ProfileSearchMixin, ProfileStoreMix
 
 
 class SQLiteStorage(
+    SQLiteLearningJobStoreMixin,
     SQLiteAgentRunStoreMixin,
     SQLitePendingToolCallStoreMixin,
     SQLiteRunToolDependencyStoreMixin,