claude-smart 0.2.46 → 0.2.48

package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py

@@ -48,6 +48,10 @@ def parse_item_id(item_id: str) -> tuple[str, int] | None:
     """
     Parse a prompt-format item ID like 'NEW-0' or 'EXISTING-1' into its prefix and index.
 
+    Weak models sometimes echo the rendered display label with its brackets
+    (``[NEW-0]``); strip a single surrounding pair so those outputs parse
+    instead of being silently dropped.
+
     Args:
         item_id (str): Item ID string in the format 'PREFIX-N' (e.g., 'NEW-0', 'EXISTING-1')
 
@@ -55,7 +59,10 @@ def parse_item_id(item_id: str) -> tuple[str, int] | None:
         Optional[tuple[str, int]]: A tuple of (prefix, index) where prefix is 'NEW' or 'EXISTING',
             or None if the item ID is invalid
     """
-    parts = item_id.rsplit("-", 1)
+    stripped = item_id.strip()
+    if stripped.startswith("[") and stripped.endswith("]"):
+        stripped = stripped[1:-1].strip()
+    parts = stripped.rsplit("-", 1)
     if len(parts) != 2:
         logger.warning("Invalid item ID format: %s", item_id)
         return None

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py

@@ -11,11 +11,11 @@ with — never another tenant's runs.
 from __future__ import annotations
 
 import logging
-import threading
 from collections.abc import Callable
 from datetime import UTC, datetime
 
 from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.scheduling import ThreadedScheduler
 from reflexio.server.services.extraction.resumable_agent import (
     pending_tool_calls_enabled,
 )
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 _DEFAULT_POLL_INTERVAL_SECONDS = 5.0
 
 
-class ExtractionResumeScheduler:
+class ExtractionResumeScheduler(ThreadedScheduler):
     """Small polling wrapper that drives :class:`ExtractionResumeWorker` per org."""
 
     def __init__(
@@ -37,29 +37,15 @@ class ExtractionResumeScheduler:
         bootstrap_org_id: str,
         max_runs_per_tick: int = 10,
     ) -> None:
+        super().__init__(thread_name="reflexio-extraction-resume-scheduler")
         self.request_context_factory = request_context_factory
         self.bootstrap_org_id = bootstrap_org_id
         self.max_runs_per_tick = max_runs_per_tick
-        self._stop_event = threading.Event()
-        self._thread: threading.Thread | None = None
 
-    def start(self) -> None:
-        if self._thread is not None and self._thread.is_alive():
-            return
-        self._stop_event.clear()
-        self._thread = threading.Thread(
-            target=self._run_loop,
-            name="reflexio-extraction-resume-scheduler",
-            daemon=True,
-        )
-        self._thread.start()
+    def _on_started(self) -> None:
         logger.info("event=extraction_resume_scheduler_started")
 
-    def stop(self, *, timeout_seconds: float = 5.0) -> None:
-        self._stop_event.set()
-        if self._thread is not None:
-            self._thread.join(timeout=timeout_seconds)
-        self._thread = None
+    def _on_stopped(self) -> None:
         logger.info("event=extraction_resume_scheduler_stopped")
 
     def _discover_org_ids(self, bootstrap_ctx: RequestContext) -> list[str]:
@@ -116,28 +102,27 @@ class ExtractionResumeScheduler:
                     org_id,
                 )
 
-    def _run_loop(self) -> None:
-        while not self._stop_event.is_set():
-            poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
-            try:
-                bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
-                config = bootstrap_ctx.configurator.get_config()
-                poll_interval = (
-                    config.pending_tool_call_config.resume_poll_interval_seconds
-                )
-                self._expire_pending_tool_calls(bootstrap_ctx)
-                for org_id in self._discover_org_ids(bootstrap_ctx):
-                    if self._stop_event.is_set():
-                        break
-                    self._drain_org(org_id)
-            except Exception as exc:
-                with sentry_tags(
-                    subsystem="extraction",
-                    op="scheduler_tick",
-                    error_type=type(exc).__name__,
-                ):
-                    logger.exception("event=extraction_resume_scheduler_tick_failed")
-            self._stop_event.wait(poll_interval)
+    def _run_once(self) -> float:
+        poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
+        try:
+            bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
+            config = bootstrap_ctx.configurator.get_config()
+            poll_interval = (
+                config.pending_tool_call_config.resume_poll_interval_seconds
+            )
+            self._expire_pending_tool_calls(bootstrap_ctx)
+            for org_id in self._discover_org_ids(bootstrap_ctx):
+                if self._stop_event.is_set():
+                    break
+                self._drain_org(org_id)
+        except Exception as exc:
+            with sentry_tags(
+                subsystem="extraction",
+                op="scheduler_tick",
+                error_type=type(exc).__name__,
+            ):
+                logger.exception("event=extraction_resume_scheduler_tick_failed")
+        return poll_interval
 
 
 def maybe_start_resume_scheduler(

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py

@@ -484,11 +484,18 @@ class ExtractionResumeWorker:
         raw_profiles = [
             profile.model_dump() for profile in result.output.profiles or []
         ]
+        source_interaction_ids = [
+            interaction.interaction_id
+            for request_model in request_interaction_data_models
+            for interaction in request_model.interactions
+            if interaction.interaction_id
+        ]
         return (
             extractor._convert_raw_to_user_profiles(
                 raw_profiles=raw_profiles,
                 user_id=run.binding.user_id,
                 request_id=run.binding.request_id,
+                source_interaction_ids=source_interaction_ids,
             ),
             result.pending_tool_call_ids,
         )
@@ -698,6 +705,7 @@ class ExtractionResumeWorker:
                 raw_profiles=raw_profiles,
                 user_id=run.binding.user_id,
                 request_id=run.binding.request_id,
+                source_interaction_ids=list(run.binding.source_interaction_ids),
             ),
             run.pending_tool_call_ids,
         )

package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py

@@ -10,6 +10,7 @@ import uuid
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import TimeoutError as FuturesTimeoutError
+from contextlib import nullcontext
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -22,6 +23,7 @@ from reflexio.models.api_schema.service_schemas import (
 from reflexio.models.config_schema import Config
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
+from reflexio.server.operation_limiter import operation_limit
 from reflexio.server.services.agent_success_evaluation.runner import (
     run_group_evaluation,
 )
@@ -140,7 +142,12 @@ class GenerationService:
     # ===============================
 
     def run(
-        self, publish_user_interaction_request: PublishUserInteractionRequest
+        self,
+        publish_user_interaction_request: PublishUserInteractionRequest,
+        *,
+        use_publish_limiter: bool = True,
+        publish_limiter_wait_forever: bool = True,
+        defer_learning: bool = False,
     ) -> GenerationServiceResult:
         """
         Process a user interaction request by storing interactions and triggering generation services.
@@ -156,6 +163,14 @@ class GenerationService:
 
         Args:
             publish_user_interaction_request: The incoming user interaction request
+            use_publish_limiter: Whether this service should throttle the post-write
+                learning pipeline. Server sync calls acquire the bounded API
+                limiter before invoking this service, so they disable the inner
+                limiter to avoid waiting after storage side effects.
+            publish_limiter_wait_forever: Whether to queue indefinitely for the
+                post-write learning limiter when ``use_publish_limiter`` is true.
+            defer_learning: Whether to persist the publish and enqueue learning
+                for a process-local worker instead of running it inline.
 
         Returns:
             GenerationServiceResult: The request_id assigned to this publish call
@@ -293,137 +308,99 @@ class GenerationService:
                 )
                 return result
 
-            # Reflection runs as its own sliding-window step BEFORE the
-            # extractor pool spins up, so any replacements it makes are
-            # visible to the extractors when they retrieve existing
-            # profile/playbook context. Wrapped in a broad except so a
-            # reflection bug never breaks the publish.
-            self._maybe_run_reflection(
-                user_id=user_id,
-                request_id=request_id,
-                agent_version=agent_version,
-                source=source,
-            )
+            if defer_learning:
+                from reflexio.server.services.publish_learning_worker import (
+                    PublishLearningJob,
+                    enqueue_publish_learning,
+                )
 
-            # Create generation services and requests
-            # Each service writes to separate storage tables and has no dependencies on others
-            profile_generation_service = ProfileGenerationService(
-                llm_client=self.client, request_context=self.request_context
-            )
-            profile_generation_request = ProfileGenerationRequest(
-                user_id=user_id,
-                request_id=request_id,
-                source=source,
-                force_extraction=publish_user_interaction_request.force_extraction,
-            )
+                self._schedule_group_evaluation_if_needed(
+                    new_request=new_request,
+                    user_id=user_id,
+                    agent_version=agent_version,
+                    source=source,
+                )
+                learning_queued = enqueue_publish_learning(
+                    PublishLearningJob(
+                        org_id=self.org_id,
+                        user_id=user_id,
+                        request_id=request_id,
+                        session_id=new_request.session_id,
+                        source=source,
+                        agent_version=agent_version,
+                        force_extraction=publish_user_interaction_request.force_extraction,
+                        skip_aggregation=publish_user_interaction_request.skip_aggregation,
+                    )
+                )
+                record_usage_event(
+                    org_id=self.org_id,
+                    user_id=user_id,
+                    request_id=request_id,
+                    session_id=new_request.session_id,
+                    source=source,
+                    agent_version=agent_version,
+                    backend="classic",
+                    event_name="publish_request_succeeded",
+                    event_category="publish",
+                    outcome="success",
+                    count_value=len(new_interactions),
+                    duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                    metadata={
+                        "defer_learning": True,
+                        "learning_queued": learning_queued,
+                        "warning_count": len(result.warnings),
+                    },
+                )
+                return result
 
-            playbook_generation_service = PlaybookGenerationService(
-                llm_client=self.client,
-                request_context=self.request_context,
-                skip_aggregation=publish_user_interaction_request.skip_aggregation,
-            )
-            playbook_generation_request = PlaybookGenerationRequest(
-                request_id=request_id,
-                agent_version=agent_version,
-                user_id=user_id,
-                source=source,
-                force_extraction=publish_user_interaction_request.force_extraction,
+            # The durable write above must not sit behind the publish limiter:
+            # async API callers have already received success=True. Throttle only
+            # the post-write learning pipeline so a hung LLM cannot silently drop
+            # later acknowledged publishes before they reach storage.
+            learning_limit = (
+                operation_limit(
+                    self.org_id,
+                    "publish",
+                    wait_forever=publish_limiter_wait_forever,
+                )
+                if use_publish_limiter
+                else nullcontext()
             )
+            with learning_limit:
+                self._run_learning_steps(
+                    user_id=user_id,
+                    request_id=request_id,
+                    agent_version=agent_version,
+                    force_extraction=publish_user_interaction_request.force_extraction,
+                    skip_aggregation=publish_user_interaction_request.skip_aggregation,
+                    source=source,
+                    result=result,
+                )
 
-            # Run profile and playbook generation services in parallel
-            # Each service creates its own internal ThreadPoolExecutor for extractors
-            # This is safe because we create separate, independent pool instances
-            # Uses manual executor management to avoid blocking on shutdown(wait=True)
-            # when threads are hung on LLM calls
-            executor = ThreadPoolExecutor(max_workers=2)
-            try:
-                # Each thread needs its own context copy — Context.run() is non-reentrant
-                futures = [
-                    executor.submit(
-                        contextvars.copy_context().run,
-                        profile_generation_service.run,
-                        profile_generation_request,
-                    ),
-                    executor.submit(
-                        contextvars.copy_context().run,
-                        playbook_generation_service.run,
-                        playbook_generation_request,
-                    ),
-                ]
-
-                # Collect results and handle any exceptions
-                # Each service failure is logged but doesn't block others
-                service_names = ["profile_generation", "playbook_generation"]
-                for future, service_name in zip(futures, service_names, strict=True):
-                    try:
-                        future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
-                    except FuturesTimeoutError:  # noqa: PERF203
-                        msg = f"{service_name} timed out after {GENERATION_SERVICE_TIMEOUT_SECONDS}s"
-                        with sentry_tags(
-                            subsystem="generation",
-                            service=service_name,
-                            request_id=request_id,
-                            error_type="timeout",
-                        ):
-                            logger.error("%s for request %s", msg, request_id)
-                        result.warnings.append(msg)
-                    except Exception as e:
-                        msg = f"{service_name} failed: {e}"
-                        with sentry_tags(
-                            subsystem="generation",
-                            service=service_name,
-                            request_id=request_id,
-                            error_type=type(e).__name__,
-                        ):
-                            logger.exception(
-                                "Generation service failed for request %s",
-                                request_id,
-                            )
-                        result.warnings.append(msg)
-            finally:
-                executor.shutdown(wait=False, cancel_futures=True)
+                # Schedule delayed group evaluation for the required session.
+                self._schedule_group_evaluation_if_needed(
+                    new_request=new_request,
+                    user_id=user_id,
+                    agent_version=agent_version,
+                    source=source,
+                )
 
-            # Tagging runs an LLM call per newly generated entity, so defer it off
-            # the publish path. The scheduler debounces bursts and is idempotent
-            # (already-tagged entities are skipped).
-            try:
-                schedule_tagging(
+                record_usage_event(
                     org_id=self.org_id,
                     user_id=user_id,
+                    request_id=request_id,
+                    session_id=new_request.session_id,
+                    source=source,
                     agent_version=agent_version,
-                    request_context=self.request_context,
-                    llm_client=self.client,
-                )
-            except Exception:
-                logger.exception(
-                    "Failed to schedule tagging for publish request %s",
-                    request_id,
+                    backend="classic",
+                    event_name="publish_request_succeeded",
+                    event_category="publish",
+                    outcome="success",
+                    count_value=len(new_interactions),
+                    duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                    metadata={"warning_count": len(result.warnings)},
                 )
-
-            # Schedule delayed group evaluation for the required session.
-            self._schedule_group_evaluation_if_needed(
-                new_request=new_request,
-                user_id=user_id,
-                agent_version=agent_version,
-                source=source,
-            )
-
-            record_usage_event(
-                org_id=self.org_id,
-                user_id=user_id,
-                request_id=request_id,
-                session_id=new_request.session_id,
-                source=source,
-                agent_version=agent_version,
-                backend="classic",
-                event_name="publish_request_succeeded",
-                event_category="publish",
-                outcome="success",
-                count_value=len(new_interactions),
-                duration_ms=int((time.perf_counter() - publish_start) * 1000),
-                metadata={"warning_count": len(result.warnings)},
-            )
-            return result
+                return result
 
         except Exception as e:
             record_usage_event(
@@ -453,10 +430,142 @@ class GenerationService:
                 )
             raise
 
+    # ===============================
+    # deferred learning
+    # ===============================
+
+    def run_deferred_learning(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        session_id: str | None,  # noqa: ARG002 - kept for worker telemetry symmetry
+        source: str | None,
+        agent_version: str,
+        force_extraction: bool,
+        skip_aggregation: bool,
+    ) -> GenerationServiceResult:
+        result = GenerationServiceResult(request_id=request_id)
+        self._run_learning_steps(
+            user_id=user_id,
+            request_id=request_id,
+            agent_version=agent_version,
+            force_extraction=force_extraction,
+            skip_aggregation=skip_aggregation,
+            source=source,
+            result=result,
+        )
+        return result
+
     # ===============================
     # private methods
     # ===============================
 
+    def _run_learning_steps(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        agent_version: str,
+        force_extraction: bool,
+        skip_aggregation: bool,
+        source: str | None,
+        result: GenerationServiceResult,
+    ) -> None:
+        # Reflection runs as its own sliding-window step BEFORE the extractor
+        # pool spins up, so replacements are visible to extractors.
+        self._maybe_run_reflection(
+            user_id=user_id,
+            request_id=request_id,
+            agent_version=agent_version,
+            source=source,
+        )
+
+        profile_generation_service = ProfileGenerationService(
+            llm_client=self.client, request_context=self.request_context
+        )
+        profile_generation_request = ProfileGenerationRequest(
+            user_id=user_id,
+            request_id=request_id,
+            source=source,
+            force_extraction=force_extraction,
+        )
+
+        playbook_generation_service = PlaybookGenerationService(
+            llm_client=self.client,
+            request_context=self.request_context,
+            skip_aggregation=skip_aggregation,
+        )
+        playbook_generation_request = PlaybookGenerationRequest(
+            request_id=request_id,
+            agent_version=agent_version,
+            user_id=user_id,
+            source=source,
+            force_extraction=force_extraction,
+        )
+
+        executor = ThreadPoolExecutor(max_workers=2)
+        try:
+            futures = [
+                executor.submit(
+                    contextvars.copy_context().run,
+                    profile_generation_service.run,
+                    profile_generation_request,
+                ),
+                executor.submit(
+                    contextvars.copy_context().run,
+                    playbook_generation_service.run,
+                    playbook_generation_request,
+                ),
+            ]
+
+            service_names = ["profile_generation", "playbook_generation"]
+            for future, service_name in zip(futures, service_names, strict=True):
+                try:
+                    future.result(timeout=GENERATION_SERVICE_TIMEOUT_SECONDS)
+                except FuturesTimeoutError:  # noqa: PERF203
+                    msg = (
+                        f"{service_name} timed out after "
+                        f"{GENERATION_SERVICE_TIMEOUT_SECONDS}s"
+                    )
+                    with sentry_tags(
+                        subsystem="generation",
+                        service=service_name,
+                        request_id=request_id,
+                        error_type="timeout",
+                    ):
+                        logger.error("%s for request %s", msg, request_id)
+                    result.warnings.append(msg)
+                except Exception as e:
+                    msg = f"{service_name} failed: {e}"
+                    with sentry_tags(
+                        subsystem="generation",
+                        service=service_name,
+                        request_id=request_id,
+                        error_type=type(e).__name__,
+                    ):
+                        logger.exception(
+                            "Generation service failed for request %s",
+                            request_id,
+                        )
+                    result.warnings.append(msg)
+        finally:
+            executor.shutdown(wait=False, cancel_futures=True)
+
+        try:
+            schedule_tagging(
+                org_id=self.org_id,
+                user_id=user_id,
+                agent_version=agent_version,
+                request_context=self.request_context,
+                llm_client=self.client,
+            )
+        except Exception:
+            logger.exception(
+                "Failed to schedule tagging for publish request %s",
+                request_id,
+            )
+
     def _schedule_group_evaluation_if_needed(
         self,
         *,

package/plugin/vendor/reflexio/reflexio/server/services/governance/config.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from reflexio.server.env_utils import env_str, env_truthy
+from reflexio.server.services.governance.subject_refs import (
+    actor_ref,
+    request_ref,
+    subject_ref,
+)
+
+LOCAL_GOVERNANCE_REF_SECRET = "reflexio-local-governance-ref-secret"  # noqa: S105
+_PRODUCTION_DEPLOYMENT_MODES = {"platform", "self_host"}
+_PRODUCTION_ENVS = {"production", "staging"}
+
+
+def _is_local_dev_or_test() -> bool:
+    deployment_mode = env_str("DEPLOYMENT_MODE").lower()
+    reflexio_env = env_str("REFLEXIO_ENV", "development").lower()
+    if env_truthy(env_str("REFLEXIO_TEST_MODE")):
+        return True
+    return (
+        deployment_mode not in _PRODUCTION_DEPLOYMENT_MODES
+        and reflexio_env not in _PRODUCTION_ENVS
+    )
+
+
+def get_governance_ref_secret() -> str:
+    secret = env_str("REFLEXIO_GOVERNANCE_REF_SECRET")
+    if secret:
+        return secret
+    if _is_local_dev_or_test():
+        return LOCAL_GOVERNANCE_REF_SECRET
+    raise RuntimeError("REFLEXIO_GOVERNANCE_REF_SECRET is required")
+
+
+def governance_subject_ref(org_id: str, user_id: str, secret: str) -> str:
+    return subject_ref(f"{org_id}:subject:{user_id}", secret)
+
+
+def governance_request_ref(org_id: str, request_id: str, secret: str) -> str:
+    return request_ref(f"{org_id}:request:{request_id}", secret)
+
+
+def governance_actor_ref(
+    org_id: str,
+    credential_kind: str,
+    principal_or_fingerprint: str,
+    secret: str,
+) -> str:
+    return actor_ref(
+        f"{org_id}:actor:{credential_kind}:{principal_or_fingerprint}",
+        secret,
+    )