claude-smart 0.2.49 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py

@@ -7,7 +7,6 @@ import logging
 import os
 import uuid
 from datetime import UTC, datetime
-from typing import Any, cast
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -20,8 +19,8 @@ from reflexio.server.services.deduplication_utils import (
     BaseDeduplicator,
     format_dedup_timestamp,
     parse_item_id,
+    resolve_dedup_query_embeddings,
 )
-from reflexio.server.services.embedding_text import embedding_input
 from reflexio.server.services.profile.profile_generation_service_utils import (
     ProfileTimeToLive,
     calculate_expiration_timestamp,
@@ -306,42 +305,9 @@ class ProfileConsolidator(BaseDeduplicator):
 
         # Generate embeddings with the request storage backend so dedup search
         # uses the same model/prefix/routing as normal profile search.
-        embeddings: list[list[float] | None]
-        try:
-            get_storage_embedding = getattr(storage, "_get_embedding", None)
-            if callable(get_storage_embedding):
-                logger.info(
-                    "Profile dedup query embeddings: source=storage model=%s",
-                    getattr(storage, "embedding_model_name", "unknown"),
-                )
-                embeddings = [
-                    cast(
-                        list[float],
-                        get_storage_embedding(query_text, purpose="query"),
-                    )
-                    for query_text in query_texts
-                ]
-            else:
-                storage_with_embeddings = cast(Any, storage)
-                embedding_model_name = storage_with_embeddings.embedding_model_name
-                embedding_dimensions = storage_with_embeddings.embedding_dimensions
-                logger.info(
-                    "Profile dedup query embeddings: source=llm_client model=%s",
-                    embedding_model_name,
-                )
-                embeddings = list(
-                    self.client.get_embeddings(
-                        [
-                            embedding_input(query_text, purpose="query")
-                            for query_text in query_texts
-                        ],
-                        model=embedding_model_name,
-                        dimensions=embedding_dimensions,
-                    )
-                )
-        except Exception as e:
-            logger.warning("Failed to generate embeddings for dedup search: %s", e)
-            embeddings = [None] * len(query_texts)
+        embeddings = resolve_dedup_query_embeddings(
+            storage, self.client, query_texts, entity_label="Profile"
+        )
 
         # Search for each new profile.
         #
@@ -465,7 +431,14 @@ class ProfileConsolidator(BaseDeduplicator):
 
             dedup_output = response
         except Exception as e:
-            logger.error("Failed to identify duplicates: %s", str(e))
+            # Graceful degradation: dedup is best-effort — on any failure
+            # (commonly a transient minimax timeout/overload) keep the profiles
+            # un-deduped rather than failing the caller. WARNING, not ERROR, so a
+            # flaky provider doesn't flood error alerts for a handled fallback.
+            logger.warning(
+                "Failed to identify duplicates (%s); keeping profiles un-deduped",
+                str(e),
+            )
             return _strip_deletion_markers(new_profiles), [], []
 
         if not dedup_output.duplicate_groups and not dedup_output.deletions:

package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py

@@ -14,6 +14,7 @@ from reflexio.models.config_schema import ProfileExtractorConfig
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
 from reflexio.server.llm.token_accounting import RunTokenTotals, sum_trace_tokens
+from reflexio.server.services.deferred_learning_plan import ExtractorBookmarkAdvance
 from reflexio.server.services.extraction.outcome import ExtractionOutcome
 from reflexio.server.services.extraction.resumable_agent import (
     run_resumable_extraction_agent,
@@ -181,25 +182,6 @@ class ProfileExtractor:
         )
         return session_data_models
 
-    def _update_operation_state(
-        self, request_interaction_data_models: list[RequestInteractionDataModel]
-    ) -> None:
-        """
-        Update operation state after processing interactions.
-
-        Args:
-            request_interaction_data_models: The interactions that were processed
-        """
-        all_interactions = extract_interactions_from_request_interaction_data_models(
-            request_interaction_data_models
-        )
-        mgr = self._create_state_manager()
-        mgr.update_extractor_bookmark(
-            extractor_name=get_extractor_name(self.config),
-            processed_interactions=all_interactions,
-            user_id=self.service_config.user_id,
-        )
-
     def run(self) -> list[UserProfile] | ExtractionOutcome[UserProfile] | None:
         """
         Extract profiles from request interaction groups.
@@ -209,10 +191,12 @@ class ProfileExtractor:
         2. Applies time range filter for rerun flows
         3. Calls LLM to extract profiles
         4. Converts raw extraction to UserProfile objects
-        5. Updates operation state after processing
+        5. Defers the stride-bookmark advance onto the outcome (applied in persist)
 
         Returns:
-            Optional[list[UserProfile]]: List of extracted profiles, or None if no profiles found
+            None when there are no interactions to process; otherwise an
+            ExtractionOutcome carrying the extracted profiles, the resumable
+            run_id (when set), and the deferred bookmark advance.
         """
         # Collect interactions using extractor's own window_size/stride_size settings
         request_interaction_data_models = self._get_interactions()
@@ -257,22 +241,33 @@ class ProfileExtractor:
             request_id=self.service_config.request_id,
             source_interaction_ids=source_interaction_ids,
         )
+        # Defer the stride-bookmark advance onto the outcome instead of
+        # self-advancing here (F1): the advance is applied downstream — inside
+        # the persist fence on the durable path, or in ``.run()``'s persist
+        # half — so it stays atomic with the profile row writes it corresponds
+        # to. Only produced when output was generated (bookmark-iff-rows).
+        bookmark_advance: ExtractorBookmarkAdvance | None = None
         if raw_profiles:
-            # Update operation state (bookmark) only when output was produced.
-            self._update_operation_state(request_interaction_data_models)
-
-        # A resumable run must always surface its run_id so the generation
-        # service can finalize the _agent_runs row (FINALIZED_PENDING_TOOL when
-        # the agent created a follow-up ask and finished with empty output).
-        # Dropping the run_id here would orphan the run in AGENT_COMPLETED and
-        # sever the resolve -> resume chain. Mirrors PlaybookExtractor.run().
-        if self._last_resumable_run_id:
-            return ExtractionOutcome.completed(
-                user_profiles,
-                run_id=self._last_resumable_run_id,
-                token_totals=self._last_resumable_token_totals,
+            bookmark_advance = ExtractorBookmarkAdvance(
+                extractor_name=get_extractor_name(self.config),
+                processed_interactions=extract_interactions_from_request_interaction_data_models(
+                    request_interaction_data_models
+                ),
+                user_id=self.service_config.user_id,
             )
-        return user_profiles or None
+
+        # Always return an ExtractionOutcome so the bookmark advance rides along
+        # even in the non-resumable case. A resumable run must also surface its
+        # run_id so the generation service can finalize the _agent_runs row
+        # (FINALIZED_PENDING_TOOL when the agent created a follow-up ask and
+        # finished with empty output); dropping it would orphan the run in
+        # AGENT_COMPLETED and sever the resolve -> resume chain.
+        return ExtractionOutcome.completed(
+            user_profiles,
+            run_id=self._last_resumable_run_id,
+            token_totals=self._last_resumable_token_totals,
+            bookmark_advance=bookmark_advance,
+        )
 
     def _convert_raw_to_user_profiles(
         self,

package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py

@@ -27,6 +27,7 @@ from reflexio.server.services.base_generation_service import (
     BaseGenerationService,
     StatusChangeOperation,
 )
+from reflexio.server.services.deferred_learning_plan import ProfileWritePlan
 from reflexio.server.services.profile.components.extractor import ProfileExtractor
 from reflexio.server.services.profile.profile_generation_service_utils import (
     ProfileGenerationRequest,
@@ -123,9 +124,10 @@ class ProfileGenerationService(
         else:
             existing_profiles = self.storage.get_user_profile(request.user_id)  # type: ignore[reportOptionalMemberAccess]
 
+        generation_request_id = request.request_id
         return ProfileGenerationServiceConfig(
             user_id=request.user_id,
-            request_id=request.request_id,
+            request_id=generation_request_id,
             source=request.source,
             existing_data=existing_profiles,
             allow_manual_trigger=self.allow_manual_trigger,
@@ -147,12 +149,24 @@ class ProfileGenerationService(
             [p for result in results if result for p in result]
         )
 
-    def _finalize_extracted_items(self, all_new_profiles: list[UserProfile]) -> None:
-        """Deduplicate, persist, and changelog extracted profile items."""
+    def _resolve_write_plan(
+        self, results: list[list[UserProfile]]
+    ) -> ProfileWritePlan | None:
+        """Compute-half of profile finalization — NO learning DB write.
+
+        Flattens the extractor results, runs the deduplicator (the 2nd LLM call
+        + reads of existing rows), assigns ``source``/``status``, resolves the
+        missing-``request_id`` guard (dropping unreconstructable supersede ids),
+        and **precomputes embeddings** on the new rows. Returns a
+        :class:`ProfileWritePlan` for the persist half, or ``None`` when there is
+        nothing to write. Issues no ``add_user_profile``/``supersede_*`` — the
+        write is the persist half's job (compute is write-free).
+        """
         user_id = self.service_config.user_id  # type: ignore[reportOptionalMemberAccess]
         source = self.service_config.source  # type: ignore[reportOptionalMemberAccess]
-        request_id = self.service_config.request_id  # type: ignore[reportOptionalMemberAccess]
+        generation_request_id = self.service_config.request_id  # type: ignore[reportOptionalMemberAccess]
 
+        all_new_profiles = [p for result in results if result for p in result]
         existing_ids_to_delete: list[str] = []
 
         # Always run deduplicator when enabled and there are new profiles
@@ -170,7 +184,9 @@ class ProfileGenerationService(
                     output_pending_status=self.output_pending_status,
                 )
                 all_new_profiles, existing_ids_to_delete, _superseded_profiles = (
-                    consolidator.deduplicate(all_new_profiles, user_id, request_id)
+                    consolidator.deduplicate(
+                        all_new_profiles, user_id, generation_request_id
+                    )
                 )
                 logger.info(
                     "Profile updates after deduplication: %d profiles, %d existing to delete",
@@ -183,60 +199,120 @@ class ProfileGenerationService(
             profile.source = source
             profile.status = Status.PENDING if self.output_pending_status else None
 
-        # Save new profiles
+        # Missing-request_id guard (moved here, in compute). An empty request_id
+        # makes the supersede unreconstructable (the lineage events are keyed on
+        # it). Fail loud and drop those ids entirely — never silently
+        # hard-delete. Persist then only supersedes with a non-empty request_id.
+        if existing_ids_to_delete and not generation_request_id:
+            capture_anomaly(
+                "lineage.dedup.missing_request_id",
+                level="error",
+                org_id=self.org_id,
+                user_id=user_id,
+            )
+            existing_ids_to_delete = []
+
+        if not all_new_profiles and not existing_ids_to_delete:
+            return None
+
+        # Precompute embeddings on the new rows (compute-side, NO DB write). The
+        # persist half passes skip_embedding=True so no embedding runs in the fence.
         if all_new_profiles:
+            self.storage.precompute_profile_embeddings(all_new_profiles)  # type: ignore[reportOptionalMemberAccess]
+
+        return ProfileWritePlan(
+            user_id=user_id,
+            request_id=generation_request_id,
+            new_profiles=all_new_profiles,
+            superseded_ids=existing_ids_to_delete,
+        )
+
+    def _persist_write_plan(self, plan: ProfileWritePlan) -> None:
+        """Persist-half of profile finalization — apply the resolved write-plan.
+
+        Issues only the fence-critical row writes: inserts the new profiles
+        (``skip_embedding=True`` — embeddings were precomputed in compute) then
+        soft-supersedes the dedup'd existing ids. NO LLM / embedding / dedup.
+        The soft-supersede emits the lineage events the profile change log is
+        reconstructed from (the legacy ``profile_change_logs`` table is no longer
+        written — see reconstruct_profile_change_log).
+
+        On a write failure this **re-raises** (symmetric with playbook
+        ``_persist_write_plan``): on the durable path the raise rolls back the
+        fenced ``commit_scope`` so the rows AND the extractor bookmark advance
+        (applied by ``persist_generation`` only if persist returns) are discarded
+        together — never a "write failed but bookmark advanced" window. On the
+        synchronous ``.run()`` path ``_run_generation`` catches it, records
+        ``generation_failed``, and leaves the bookmark un-advanced so the next
+        publish retries the window.
+        """
+        user_id = plan.user_id
+        generation_request_id = plan.request_id
+
+        # Save new profiles (embeddings already set → skip re-embedding).
+        if plan.new_profiles:
             try:
-                self.storage.add_user_profile(user_id, all_new_profiles)  # type: ignore[reportOptionalMemberAccess]
+                self.storage.add_user_profile(  # type: ignore[reportOptionalMemberAccess]
+                    user_id, plan.new_profiles, skip_embedding=True
+                )
             except Exception as e:
                 with sentry_tags(
                     subsystem="profile_generation",
                     op="save_profiles",
                     org_id=self.org_id,
                     user_id=user_id,
-                    request_id=request_id,
+                    request_id=generation_request_id,
                     error_type=type(e).__name__,
                 ):
                     logger.exception(
                         "Failed to save profiles for user id: %s",
                         user_id,
                     )
-                return
-
-        # Always soft-supersede superseded existing profiles (never hard-delete on
-        # the dedup path). This emits the lineage events that the profile change log
-        # is reconstructed from (the legacy `profile_change_logs` table is no longer
-        # written — see reconstruct_profile_change_log).
-        if existing_ids_to_delete:
-            if not request_id:
-                # An empty request_id makes the removal unreconstructable (the lineage
-                # events are keyed on it). Fail loud and skip removal entirely — never
-                # silently hard-delete.
-                capture_anomaly(
-                    "lineage.dedup.missing_request_id",
-                    level="error",
-                    org_id=self.org_id,
+                # Re-raise so the bookmark advance is skipped / the fence rolls
+                # back (F1 symmetry with playbook persist) — never advance the
+                # extractor bookmark over a window whose rows failed to write.
+                raise
+
+        # Always soft-supersede superseded existing profiles (never hard-delete
+        # on the dedup path). Compute already dropped these when request_id was
+        # empty, so any ids here carry a valid lineage key.
+        if plan.superseded_ids:
+            try:
+                self.storage.supersede_profiles_by_ids(  # type: ignore[reportOptionalMemberAccess]
                     user_id=user_id,
+                    profile_ids=plan.superseded_ids,
+                    request_id=generation_request_id,
                 )
-            else:
-                try:
-                    self.storage.supersede_profiles_by_ids(  # type: ignore[reportOptionalMemberAccess]
-                        user_id=user_id,
-                        profile_ids=existing_ids_to_delete,
-                        request_id=request_id,
+            except Exception as e:
+                with sentry_tags(
+                    subsystem="profile_generation",
+                    op="supersede_profiles",
+                    org_id=self.org_id,
+                    user_id=user_id,
+                    request_id=generation_request_id,
+                    error_type=type(e).__name__,
+                ):
+                    logger.exception(
+                        "Failed to soft-delete superseded profiles for user %s",
+                        user_id,
                     )
-                except Exception as e:
-                    with sentry_tags(
-                        subsystem="profile_generation",
-                        op="supersede_profiles",
-                        org_id=self.org_id,
-                        user_id=user_id,
-                        request_id=request_id,
-                        error_type=type(e).__name__,
-                    ):
-                        logger.exception(
-                            "Failed to soft-delete superseded profiles for user %s",
-                            user_id,
-                        )
+                # Re-raise for the same reason: a half-applied persist (new rows
+                # in, supersede failed) must not advance the bookmark. Playbook's
+                # _apply_consolidation_lineage raises here too.
+                raise
+
+    def _finalize_extracted_items(self, all_new_profiles: list[UserProfile]) -> None:
+        """Permanent V3 wrapper: compute-then-persist together (no external fence).
+
+        Kept for the synchronous resume/manual callers
+        (``ExtractionResumeWorker`` calls this directly). Routes them through the
+        same ``_resolve_write_plan`` (compute) + ``_persist_write_plan``
+        (persist) split the durable worker uses — with no external
+        ``commit_scope`` — so the result is identical to the pre-split monolith.
+        """
+        plan = self._resolve_write_plan([all_new_profiles])
+        if plan is not None:
+            self._persist_write_plan(plan)
 
     def check_and_update_profiles(self, profiles: list[UserProfile]) -> None:
         """check if the profiles are expired and update them if they are"""
@@ -436,9 +512,10 @@ class ProfileGenerationService(
         """
         # Handle rerun requests (have start_time/end_time datetime objects)
         if isinstance(request, RerunProfileGenerationRequest):
+            operation_request_id = f"rerun_{uuid.uuid4().hex[:8]}"
             return ProfileGenerationRequest(
                 user_id=user_id,
-                request_id=f"rerun_{uuid.uuid4().hex[:8]}",
+                request_id=operation_request_id,
                 source=request.source,
                 rerun_start_time=(
                     int(request.start_time.timestamp()) if request.start_time else None
@@ -449,9 +526,10 @@ class ProfileGenerationService(
                 auto_run=False,
             )
         # Handle manual requests (ManualProfileGenerationRequest)
+        operation_request_id = f"manual_{uuid.uuid4().hex[:8]}"
         return ProfileGenerationRequest(
             user_id=user_id,
-            request_id=f"manual_{uuid.uuid4().hex[:8]}",
+            request_id=operation_request_id,
             source=request.source,
             auto_run=False,
         )