claude-smart 0.2.48 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py

@@ -27,6 +27,16 @@ from .._base import (
 logger = logging.getLogger(__name__)
 
 
+def _embed_text_for_interaction(content: str | None, action_desc: str | None) -> str:
+    """Derive the text embedded for an interaction.
+
+    Kept byte-for-byte identical to the ingest path
+    (``add_user_interactions_bulk`` / ``prepare_interaction_embeddings``) so a
+    backfilled vector matches a freshly-ingested one.
+    """
+    return "\n".join([content or "", action_desc or ""])
+
+
 class InteractionStoreMixin:
     """Mixin providing interaction-store CRUD for SQLite storage."""
 
@@ -45,6 +55,7 @@ class InteractionStoreMixin:
     embedding_dimensions: int
     _subject_ref_for_user_id: Any
     _assert_subject_writable_locked: Any
+    _own_transaction: Any
 
     # ------------------------------------------------------------------
     # CRUD — Interactions
@@ -71,6 +82,12 @@ class InteractionStoreMixin:
 
     @SQLiteStorageBase.handle_exceptions
     def add_user_interaction(self, user_id: str, interaction: Interaction) -> None:  # noqa: ARG002
+        # NOTE: distinct from the bulk/backfill derivation
+        # (_embed_text_for_interaction). This legacy single-insert path embeds via
+        # the purpose-prefixed _get_embedding ("search_document: ...") and its own
+        # f-string, so it is intentionally NOT routed through the shared helper —
+        # aligning it would silently change stored embedding values (e.g. null ->
+        # literal "None"). The real bulk ingest + backfill share the helper.
         embedding = self._get_embedding(
             f"{interaction.content}\n{interaction.user_action_description}"
         )
@@ -81,8 +98,10 @@ class InteractionStoreMixin:
         created_at_iso = _epoch_to_iso(interaction.created_at)
         subject_ref = self._subject_ref_for_user_id(interaction.user_id)
         with self._lock:
+            own_txn = self._own_transaction()
             try:
-                self.conn.execute("BEGIN IMMEDIATE")
+                if own_txn:
+                    self.conn.execute("BEGIN IMMEDIATE")
                 self._assert_subject_writable_locked(subject_ref)
                 if interaction.interaction_id:
                     self.conn.execute(
@@ -90,8 +109,9 @@ class InteractionStoreMixin:
                            (interaction_id, user_id, content, request_id, created_at,
                             role, user_action, user_action_description,
                             interacted_image_url, image_encoding, shadow_content,
-                            expert_content, tools_used, citations, embedding, governance_subject_ref)
-                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                            expert_content, tools_used, citations, retrieved_learnings,
+                            embedding, governance_subject_ref)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                         (
                             interaction.interaction_id,
                             interaction.user_id,
@@ -111,6 +131,12 @@ class InteractionStoreMixin:
                             _json_dumps(
                                 [c.model_dump() for c in interaction.citations]
                             ),
+                            _json_dumps(
+                                [
+                                    c.model_dump()
+                                    for c in interaction.retrieved_learnings
+                                ]
+                            ),
                             _json_dumps(interaction.embedding),
                             subject_ref,
                         ),
@@ -122,8 +148,9 @@ class InteractionStoreMixin:
                            (user_id, content, request_id, created_at,
                             role, user_action, user_action_description,
                             interacted_image_url, image_encoding, shadow_content,
-                            expert_content, tools_used, citations, embedding, governance_subject_ref)
-                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                            expert_content, tools_used, citations, retrieved_learnings,
+                            embedding, governance_subject_ref)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                         (
                             interaction.user_id,
                             interaction.content,
@@ -142,15 +169,23 @@ class InteractionStoreMixin:
                             _json_dumps(
                                 [c.model_dump() for c in interaction.citations]
                             ),
+                            _json_dumps(
+                                [
+                                    c.model_dump()
+                                    for c in interaction.retrieved_learnings
+                                ]
+                            ),
                             _json_dumps(interaction.embedding),
                             subject_ref,
                         ),
                     )
                     iid = cur.lastrowid or 0
                     interaction.interaction_id = iid
-                self.conn.commit()
+                if own_txn:
+                    self.conn.commit()
             except Exception:
-                self.conn.rollback()
+                if own_txn:
+                    self.conn.rollback()
                 raise
         # Update FTS and vec
         self._fts_upsert(
@@ -168,12 +203,53 @@ class InteractionStoreMixin:
         self,
         user_id: str,  # noqa: ARG002
         interactions: list[Interaction],
+        *,
+        embeddings_prepared: bool = False,
     ) -> None:
         if not interactions:
             return
+        if not embeddings_prepared:
+            # Only generate embeddings for interactions that do not already have them.
+            # This allows callers to pre-populate embeddings (e.g. via
+            # prepare_interaction_embeddings) before opening a commit_scope so that no
+            # network I/O occurs inside the transaction.
+            to_embed = [i for i in interactions if not i.embedding]
+            if to_embed:
+                texts = [
+                    _embed_text_for_interaction(i.content, i.user_action_description)
+                    for i in to_embed
+                ]
+                try:
+                    embeddings = self.llm_client.get_embeddings(
+                        texts, self.embedding_model_name, self.embedding_dimensions
+                    )
+                except EmbeddingUnavailableError as exc:
+                    logger.warning(
+                        "Embedding unavailable for interaction bulk insert; "
+                        "continuing without vectors: %s",
+                        exc,
+                    )
+                    embeddings = [[] for _ in texts]
+                for interaction, embedding in zip(to_embed, embeddings, strict=False):
+                    interaction.embedding = embedding
+        for interaction in interactions:
+            self._insert_interaction(interaction)
+
+    @SQLiteStorageBase.handle_exceptions
+    def prepare_interaction_embeddings(self, interactions: list[Interaction]) -> None:
+        """Pre-populate interaction.embedding for each interaction (no DB write).
+
+        Generates embeddings in one batch call so the write path inside a
+        commit_scope can skip the network round-trip.
+        """
+        if not interactions:
+            return
+        to_embed = [i for i in interactions if not i.embedding]
+        if not to_embed:
+            return
         texts = [
-            "\n".join([i.content or "", i.user_action_description or ""])
-            for i in interactions
+            _embed_text_for_interaction(i.content, i.user_action_description)
+            for i in to_embed
         ]
         try:
             embeddings = self.llm_client.get_embeddings(
@@ -181,14 +257,123 @@ class InteractionStoreMixin:
             )
         except EmbeddingUnavailableError as exc:
             logger.warning(
-                "Embedding unavailable for interaction bulk insert; "
+                "Embedding unavailable during prepare_interaction_embeddings; "
                 "continuing without vectors: %s",
                 exc,
             )
             embeddings = [[] for _ in texts]
-        for interaction, embedding in zip(interactions, embeddings, strict=False):
+        for interaction, embedding in zip(to_embed, embeddings, strict=False):
             interaction.embedding = embedding
-            self._insert_interaction(interaction)
+
+    # ------------------------------------------------------------------
+    # Missing-vector backfill (durability sweep)
+    # ------------------------------------------------------------------
+
+    @SQLiteStorageBase.handle_exceptions
+    def iter_interactions_missing_vectors(self, limit: int) -> list[tuple[int, str]]:
+        """Enumerate interactions whose embedding was never persisted.
+
+        A degraded/failed embedding leaves the ``embedding`` column empty
+        (``'[]'`` or NULL) and writes no ``interactions_vec`` row — equivalent
+        conditions on the write path. Detecting on the ``embedding`` column is
+        both the root-cause signal and portable across backends (the column
+        exists everywhere; ``interactions_vec`` is a SQLite-vec detail).
+
+        Args:
+            limit: Maximum number of interactions to return.
+
+        Returns:
+            list[tuple[int, str]]: ``(interaction_id, embed_text)`` pairs, at
+            most ``limit`` long, ordered by ``interaction_id`` for stable paging.
+        """
+        if limit <= 0:
+            return []
+        rows = self._fetchall(
+            """SELECT interaction_id, content, user_action_description
+               FROM interactions
+               WHERE embedding IS NULL OR embedding = '[]'
+               ORDER BY interaction_id
+               LIMIT ?""",
+            (limit,),
+        )
+        return [
+            (
+                r["interaction_id"],
+                _embed_text_for_interaction(r["content"], r["user_action_description"]),
+            )
+            for r in rows
+        ]
+
+    @SQLiteStorageBase.handle_exceptions
+    def backfill_missing_interaction_vectors(self, limit: int) -> int:
+        """Re-embed and persist vectors for interactions missing them.
+
+        Bounded by ``limit`` and idempotent: once the ``embedding`` column and
+        ``interactions_vec`` row are written, the row no longer matches
+        detection and is skipped next time. Embeds via the same batch call and
+        text derivation the ingest path uses so backfilled vectors match
+        freshly-ingested ones.
+
+        Fail-safe: if the embedder is unavailable the batch call raises
+        ``EmbeddingUnavailableError``; we log a single bounded WARN and return 0,
+        leaving the rows for the next tick rather than crashing the caller or
+        hot-looping a down embedder.
+
+        Args:
+            limit: Maximum number of interactions to re-embed this call.
+
+        Returns:
+            int: Number of interactions whose vector was backfilled.
+        """
+        if limit <= 0:
+            return 0
+        pairs = self.iter_interactions_missing_vectors(limit)
+        if not pairs:
+            return 0
+        texts = [text for _, text in pairs]
+        try:
+            embeddings = self.llm_client.get_embeddings(
+                texts, self.embedding_model_name, self.embedding_dimensions
+            )
+        except EmbeddingUnavailableError as exc:
+            logger.warning(
+                "Embedding unavailable during missing-vector backfill; leaving "
+                "%d interaction(s) for the next tick: %s",
+                len(pairs),
+                exc,
+            )
+            return 0
+        backfilled = 0
+        for (iid, _text), embedding in zip(pairs, embeddings, strict=False):
+            if not embedding:
+                # Embedder returned empty for this row — leave it for next tick.
+                continue
+            self._persist_backfilled_vector(iid, embedding)
+            backfilled += 1
+        return backfilled
+
+    def _persist_backfilled_vector(
+        self, interaction_id: int, embedding: list[float]
+    ) -> None:
+        """Write a re-embedded vector for one interaction (column + vec row).
+
+        Updates the ``embedding`` TEXT column (read by the Python vector-rank
+        search path) and upserts the ``interactions_vec`` row via the same
+        ``_vec_upsert`` helper the insert path uses.
+
+        Only commits when it owns the transaction (mirroring ``_insert_interaction``)
+        so it can never flush a partial outer transaction if ever called inside an
+        open ``commit_scope``; ``_vec_upsert`` likewise defers under an open scope.
+        """
+        with self._lock:
+            own_txn = self._own_transaction()
+            self.conn.execute(
+                "UPDATE interactions SET embedding = ? WHERE interaction_id = ?",
+                (_json_dumps(embedding), interaction_id),
+            )
+            if own_txn:
+                self.conn.commit()
+        self._vec_upsert("interactions_vec", interaction_id, embedding)
 
     @SQLiteStorageBase.handle_exceptions
     def delete_user_interaction(self, request: DeleteUserInteractionRequest) -> None:

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py

@@ -16,6 +16,9 @@ from reflexio.models.api_schema.service_schemas import (
     Status,
     UserProfile,
 )
+from reflexio.server.services.storage.lifecycle_filters import (
+    validate_include_inactive,
+)
 
 from .._base import (
     _PROFILE_TOMBSTONE_STATUS_VALUES,
@@ -77,6 +80,7 @@ class ProfileStoreMixin:
     _has_sqlite_vec: bool
     _subject_ref_for_user_id: Any
     _assert_subject_writable_locked: Any
+    _own_transaction: Any
 
     def _subject_ref_from_profile_row(self, row: sqlite3.Row) -> str:
         subject_ref = row["governance_subject_ref"]
@@ -207,12 +211,14 @@ class ProfileStoreMixin:
         sql = f"SELECT * FROM profiles WHERE {' AND '.join(conditions)}"
         return [_row_to_profile(r) for r in self._fetchall(sql, all_params)]
 
-    @SQLiteStorageBase.handle_exceptions
-    def add_user_profile(self, user_id: str, user_profiles: list[UserProfile]) -> None:  # noqa: ARG002
-        for profile in user_profiles:
-            subject_ref = self._subject_ref_for_user_id(profile.user_id)
-            with self._lock:
-                self._assert_subject_writable_locked(subject_ref)
+    def precompute_profile_embeddings(self, profiles: list[UserProfile]) -> None:
+        """Populate ``.embedding`` / ``.expanded_terms`` in place; no DB write.
+
+        Extracted verbatim from the former ``add_user_profile`` prelude so the
+        durable compute/persist split can embed outside the writer transaction
+        and then persist with ``skip_embedding=True``.
+        """
+        for profile in profiles:
             embedding_text = "\n".join([profile.content, str(profile.custom_features)])
             if self._should_expand_documents():
                 with ThreadPoolExecutor(max_workers=2) as executor:
@@ -222,10 +228,31 @@ class ProfileStoreMixin:
                     profile.expanded_terms = exp_future.result(timeout=15)
             else:
                 profile.embedding = self._get_embedding(embedding_text)
+
+    @SQLiteStorageBase.handle_exceptions
+    def add_user_profile(
+        self,
+        user_id: str,  # noqa: ARG002
+        user_profiles: list[UserProfile],
+        *,
+        skip_embedding: bool = False,
+    ) -> None:
+        for profile in user_profiles:
+            subject_ref = self._subject_ref_for_user_id(profile.user_id)
+            with self._lock:
+                self._assert_subject_writable_locked(subject_ref)
+            # Default (skip_embedding=False) recomputes unconditionally, exactly
+            # as before — model_copy callers that change content while keeping
+            # the old embedding depend on this. The durable persist path opts
+            # out (embedding already set by precompute_profile_embeddings).
+            if not skip_embedding:
+                self.precompute_profile_embeddings([profile])
             embedding = profile.embedding
             with self._lock:
+                own_txn = self._own_transaction()
                 try:
-                    self.conn.execute("BEGIN IMMEDIATE")
+                    if own_txn:
+                        self.conn.execute("BEGIN IMMEDIATE")
                     self._assert_subject_writable_locked(subject_ref)
                     self.conn.execute(
                         """INSERT OR REPLACE INTO profiles
@@ -261,9 +288,11 @@ class ProfileStoreMixin:
                             subject_ref,
                         ),
                     )
-                    self.conn.commit()
+                    if own_txn:
+                        self.conn.commit()
                 except Exception:
-                    self.conn.rollback()
+                    if own_txn:
+                        self.conn.rollback()
                     raise
             fts_parts = [profile.content or ""]
             if profile.custom_features:
@@ -473,6 +502,24 @@ class ProfileStoreMixin:
         row = self._fetchone("SELECT COUNT(*) as cnt FROM profiles")
         return row["cnt"] if row else 0
 
+    @SQLiteStorageBase.handle_exceptions
+    def count_user_profiles_by_status(
+        self, user_ids: list[str], status: Status | None
+    ) -> int:
+        if not user_ids:
+            return 0
+
+        current_ts = _epoch_now()
+        status_frag, status_params = _build_status_sql([status])
+        ph = ",".join("?" for _ in user_ids)
+        sql = (
+            f"SELECT COUNT(*) as cnt FROM profiles "
+            f"WHERE user_id IN ({ph}) "
+            f"AND expiration_timestamp >= ? AND {status_frag}"
+        )
+        row = self._fetchone(sql, [*user_ids, current_ts, *status_params])
+        return row["cnt"] if row else 0
+
     @SQLiteStorageBase.handle_exceptions
     def update_all_profiles_status(
         self,
@@ -616,14 +663,25 @@ class ProfileStoreMixin:
         user_id: str,
         profile_ids: list[str],
         status_filter: list[Status | None] | None = None,
+        *,
+        include_inactive: bool = False,
     ) -> list[UserProfile]:
+        validate_include_inactive(
+            include_inactive=include_inactive, status_filter=status_filter
+        )
         if not profile_ids:
             return []
+        ph = ",".join("?" for _ in profile_ids)
+        if include_inactive:
+            rows = self._fetchall(
+                f"SELECT * FROM profiles WHERE user_id = ? AND profile_id IN ({ph})",
+                (user_id, *profile_ids),
+            )
+            return [_row_to_profile(r) for r in rows]
         if status_filter is None:
             status_filter = [None]
         current_ts = _epoch_now()
         frag, sparams = _build_status_sql(status_filter)
-        ph = ",".join("?" for _ in profile_ids)
         sql = (
             f"SELECT * FROM profiles "
             f"WHERE user_id = ? AND profile_id IN ({ph}) "
@@ -809,7 +867,8 @@ class ProfileStoreMixin:
                         status_namespace="lifecycle_status",
                     )
                     committed_ids.append(pid)
-            self.conn.commit()
+            if self._own_transaction():
+                self.conn.commit()
         return committed_ids
 
     @SQLiteStorageBase.handle_exceptions

package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py

@@ -22,7 +22,9 @@ from ._agent_run import (
     not_applicable_tool_result,
 )
 from ._base import BaseStorageCore, matches_status_filter
+from ._commit_scope import CommitScopeMixin
 from ._extras import ExtrasMixin
+from ._learning_jobs import LearningJob, LearningJobStatus, LearningJobStoreABC
 from ._lineage import EntityType, LineageEventMixin
 from ._operations import OperationMixin
 from ._requests import RequestMixin
@@ -48,6 +50,8 @@ from .profiles import InteractionStoreMixin, ProfileSearchMixin, ProfileStoreMix
 
 
 class BaseStorage(
+    LearningJobStoreABC,
+    CommitScopeMixin,
     AgentRunMixin,
     ProfileStoreMixin,
     InteractionStoreMixin,
@@ -260,8 +264,21 @@ class BaseStorage(
             "purged_user_playbooks": len(purge_upb_ids),
         }
 
+    def learning_jobs_columns(self) -> list[str]:
+        """Return the column names of the learning_jobs table.
+
+        Each backend overrides this to query its own schema introspection
+        mechanism (SQLite: PRAGMA table_info; Postgres/Supabase:
+        information_schema.columns).
+        """
+        raise NotImplementedError
+
 
 __all__ = [
+    "LearningJob",
+    "LearningJobStatus",
+    "LearningJobStoreABC",
+    "CommitScopeMixin",
     "AgentBinding",
     "AgentRunMixin",
     "EntityType",

package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+from contextlib import AbstractContextManager
+
+
+class CommitScopeMixin(ABC):
+    @abstractmethod
+    def commit_scope(self) -> AbstractContextManager[None]:
+        """Atomic multi-write transaction; see Workstream A plan Task 1."""
+        raise NotImplementedError

package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py

@@ -1,13 +1,20 @@
 from abc import abstractmethod
+from typing import cast
 
 from reflexio.models.api_schema.braintrust_schema import (
     BraintrustConnection,
     ImportedScore,
 )
 from reflexio.models.api_schema.domain import (
+    CitationKind,
     Interaction,
 )
 from reflexio.models.api_schema.internal_schema import SessionCitation
+
+# Stored citation dicts are untyped JSON; only these kinds are valid.
+CITATION_KINDS: frozenset[str] = frozenset(
+    {"playbook", "profile", "user_playbook", "agent_playbook"}
+)
 from reflexio.models.api_schema.retriever_schema import PlaybookApplicationStat
 
 
@@ -142,12 +149,12 @@ class ExtrasMixin:
                         kind = getattr(cite, "kind", None)
                         real_id = getattr(cite, "real_id", None)
                         title = getattr(cite, "title", "") or ""
-                    if kind and real_id:
+                    if str(kind) in CITATION_KINDS and real_id:
                         out.append(
                             SessionCitation(
                                 user_id="",
                                 session_id=session_id,
-                                kind=str(kind),
+                                kind=cast("CitationKind", str(kind)),
                                 real_id=str(real_id),
                                 title=str(title),
                             )