claude-smart 0.2.48 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass
 
 DEFAULT_ROW_RETENTION_LIMIT = 250_000
 ROW_RETENTION_DELETE_FRACTION = 0.20
+TOMBSTONE_STATUSES = ("archived", "merged", "superseded", "expired")
 
 
 @dataclass(frozen=True, slots=True)
@@ -18,17 +19,32 @@ class RetentionTarget:
     table_name: str
     order_column: str
     id_columns: tuple[str, ...]
+    priority_statuses: tuple[str, ...] = ()
 
 
 RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
-    RetentionTarget("profiles", "profiles", "created_at", ("profile_id",)),
+    RetentionTarget(
+        "profiles",
+        "profiles",
+        "created_at",
+        ("profile_id",),
+        priority_statuses=TOMBSTONE_STATUSES,
+    ),
     RetentionTarget("interactions", "interactions", "created_at", ("interaction_id",)),
     RetentionTarget("requests", "requests", "created_at", ("request_id",)),
     RetentionTarget(
-        "user_playbooks", "user_playbooks", "created_at", ("user_playbook_id",)
+        "user_playbooks",
+        "user_playbooks",
+        "created_at",
+        ("user_playbook_id",),
+        priority_statuses=TOMBSTONE_STATUSES,
     ),
     RetentionTarget(
-        "agent_playbooks", "agent_playbooks", "created_at", ("agent_playbook_id",)
+        "agent_playbooks",
+        "agent_playbooks",
+        "created_at",
+        ("agent_playbook_id",),
+        priority_statuses=TOMBSTONE_STATUSES,
     ),
     RetentionTarget(
         "agent_success_evaluation_result",
@@ -36,6 +52,25 @@ RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
         "created_at",
         ("result_id",),
     ),
+    # Grouped session target: keyed on (user_id, session_id) — not result_id —
+    # so retention always removes whole session snapshots, never a partial
+    # per-learning subset. Rows within a session share created_at (earliest
+    # request timestamp), so ordering keeps groups adjacent. The session's
+    # retrieved-eval _operation_state row is intentionally left in place: it
+    # is content-free (digest + counters) and self-heals on the next
+    # publish/forced evaluation.
+    RetentionTarget(
+        "retrieved_learning_evaluation",
+        "retrieved_learning_evaluation",
+        "created_at",
+        ("user_id", "session_id"),
+    ),
+    RetentionTarget(
+        "offline_tuner_reward_label",
+        "offline_tuner_reward_label",
+        "label_created_at",
+        ("reward_label_id",),
+    ),
     RetentionTarget("share_links", "share_links", "created_at", ("id",)),
     RetentionTarget(
         "agent_playbook_source_user_playbooks",
@@ -117,6 +152,9 @@ RETENTION_CASCADES: dict[str, tuple[CascadeRef, ...]] = {
     "playbook_retrieval_logs": (
         CascadeRef("playbook_retrieval_log_items", "retrieval_log_id"),
     ),
+    "offline_tuner_reward_label": (
+        CascadeRef("offline_tuner_reward_label_target", "reward_label_id"),
+    ),
 }
 
 

package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py

@@ -98,12 +98,40 @@ class RetentionMixin(ABC):
         target = get_retention_target(target_name)
         if not self._retention_table_exists(target.table_name):
             return 0
-        keys = self._retention_select_oldest_keys(target, count)
+        keys = self._retention_select_keys(target, count)
         if not keys:
             return 0
         self._retention_perform_delete(target, keys)
         return len(keys)
 
+    def _retention_select_keys(
+        self, target: RetentionTarget, count: int
+    ) -> list[tuple[Any, ...]]:
+        """Select tombstones first, then oldest rows when a target opts in.
+
+        The tombstone pass can never under-delete: the fallback select returns
+        ``min(count, table_rows)`` keys and ``seen`` only ever holds tombstones
+        already counted, so the union still reaches ``count`` whenever the
+        table holds that many rows.
+        """
+        if not target.priority_statuses:
+            return self._retention_select_oldest_keys(target, count)
+
+        keys = self._retention_select_oldest_keys(
+            target, count, statuses=target.priority_statuses
+        )
+        if len(keys) >= count:
+            return keys
+
+        seen = set(keys)
+        for key in self._retention_select_oldest_keys(target, count):
+            if key not in seen:
+                keys.append(key)
+                seen.add(key)
+            if len(keys) == count:
+                break
+        return keys
+
     def _retention_perform_delete(
         self, target: RetentionTarget, keys: list[tuple[Any, ...]]
     ) -> None:
@@ -130,13 +158,23 @@ class RetentionMixin(ABC):
 
     @abstractmethod
     def _retention_select_oldest_keys(
-        self, target: RetentionTarget, count: int
+        self,
+        target: RetentionTarget,
+        count: int,
+        statuses: tuple[str, ...] | None = None,
     ) -> list[tuple[Any, ...]]:
         """Return up to ``count`` oldest key tuples for ``target``.
 
         Each key tuple contains values aligned with ``target.id_columns``.
         Ordering is by ``target.order_column`` ascending with the id
         columns as a stable tiebreaker.
+
+        Args:
+            target (RetentionTarget): Target whose keys are being selected.
+            count (int): Maximum number of key tuples to return.
+            statuses (tuple[str, ...] | None): When not None, restrict the
+                select to rows whose ``status`` is one of these values. An
+                empty tuple matches nothing (never every row).
         """
         raise NotImplementedError
 

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py

@@ -9,6 +9,7 @@ from ._base import (
 )
 from ._extras import ExtrasMixin
 from ._governance import SQLiteGovernanceMixin
+from ._learning_jobs import SQLiteLearningJobStoreMixin
 from ._lineage import SQLiteLineageMixin
 from ._operations import OperationMixin
 from ._requests import RequestMixin
@@ -48,6 +49,7 @@ from .profiles import InteractionStoreMixin, ProfileSearchMixin, ProfileStoreMix
 
 
 class SQLiteStorage(
+    SQLiteLearningJobStoreMixin,
     SQLiteAgentRunStoreMixin,
     SQLitePendingToolCallStoreMixin,
     SQLiteRunToolDependencyStoreMixin,

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py

@@ -7,6 +7,7 @@ are available.
 
 """
 
+import contextlib
 import functools
 import json
 import logging
@@ -14,7 +15,7 @@ import math
 import re
 import sqlite3
 import threading
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Generator, Sequence
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, ClassVar, Literal
@@ -29,6 +30,7 @@ from reflexio.models.api_schema.service_schemas import (
     ProfileTimeToLive,
     RegularVsShadow,
     Request,
+    RetrievedLearning,
     Status,
     ToolUsed,
     UserActionType,
@@ -455,6 +457,12 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
         if citations_raw and isinstance(citations_raw, list)
         else []
     )
+    retrieved_learnings_raw = _json_loads(d.get("retrieved_learnings"))
+    retrieved_learnings = (
+        [RetrievedLearning(**c) for c in retrieved_learnings_raw if isinstance(c, dict)]
+        if retrieved_learnings_raw and isinstance(retrieved_learnings_raw, list)
+        else []
+    )
     return Interaction(
         interaction_id=d["interaction_id"],
         user_id=d["user_id"],
@@ -470,6 +478,7 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
         expert_content=d.get("expert_content") or "",
         tools_used=tools_used,
         citations=citations,
+        retrieved_learnings=retrieved_learnings,
     )
 
 
@@ -590,6 +599,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
     # FTS/vec index helpers provided by SQLiteFtsVecMixin via the composed
     # SQLiteStorage MRO; declared here for _migrate_vec_tables's benefit.
     _vec_upsert: Any
+    _flush_index_op: Any
 
     @staticmethod
     def handle_exceptions(func: Callable[..., Any]) -> Callable[..., Any]:
@@ -633,6 +643,10 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
 
         self.db_path = db_path
         self._lock = threading.RLock()
+        self._scope_depth = 0
+        self._deferred_index_ops: list[
+            tuple[str, Any]
+        ] = []  # (kind, args) flushed post-commit
 
         logger.info("SQLite Storage for org %s using db_path: %s", org_id, db_path)
 
@@ -656,6 +670,16 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
             api_key_config=self.api_key_config,
         )
         self.embedding_dimensions = EMBEDDING_DIMENSIONS
+        # Text-generation model for storage-time document expansion. The
+        # shared self.llm_client is pinned to the EMBEDDING model, which is
+        # not a chat model — expansion calls must override the model or they
+        # fail (e.g. local/minilm-l6-v2 cannot serve completions).
+        self._expansion_model_name = resolve_model_name(
+            ModelRole.GENERATION,
+            site_var_value=site_var.get("default_generation_model_name"),
+            config_override=llm_config.generation_model_name if llm_config else None,
+            api_key_config=self.api_key_config,
+        )
 
         litellm_config = LiteLLMConfig(
             model=self.embedding_model_name,
@@ -670,6 +694,45 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         # Create tables
         self.migrate()
 
+    # ------------------------------------------------------------------
+    # Transaction scope
+    # ------------------------------------------------------------------
+
+    def _own_transaction(self) -> bool:
+        """True when the caller is NOT inside a commit_scope (owns its BEGIN/commit)."""
+        return self._scope_depth == 0
+
+    @contextlib.contextmanager
+    def commit_scope(self) -> Generator[None, None, None]:
+        """Group writes into one atomic commit; defer FTS/vec index ops.
+
+        Nested scopes join the outermost: only the outer scope commits. On
+        exception the whole transaction rolls back and deferred index ops
+        are discarded.
+        """
+        with self._lock:
+            if self._scope_depth > 0:
+                self._scope_depth += 1
+                try:
+                    yield
+                finally:
+                    self._scope_depth -= 1
+                return
+            self.conn.execute("BEGIN IMMEDIATE")
+            self._scope_depth = 1
+            try:
+                yield
+                self.conn.commit()
+                ops, self._deferred_index_ops = self._deferred_index_ops, []
+                for kind, args in ops:
+                    self._flush_index_op(kind, args)  # self-commits — fine post-commit
+            except Exception:
+                self.conn.rollback()
+                self._deferred_index_ops = []
+                raise
+            finally:
+                self._scope_depth = 0
+
     # ------------------------------------------------------------------
     # DDL / migration
     # ------------------------------------------------------------------
@@ -714,6 +777,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         self._migrate_retire_profile_change_logs()
         self._migrate_retire_playbook_aggregation_change_logs()
         init_stall_state_table(self.conn)
+        self._migrate_learning_jobs()
         return True
 
     def _try_load_sqlite_vec(self) -> bool:
@@ -809,6 +873,14 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
                 )
                 self.conn.commit()
 
+        if "retrieved_learnings" not in columns:
+            logger.info("Adding retrieved_learnings column to interactions table.")
+            with self._lock:
+                self.conn.execute(
+                    "ALTER TABLE interactions ADD COLUMN retrieved_learnings TEXT"
+                )
+                self.conn.commit()
+
     def _migrate_feedback_schema(self) -> None:
         """Drop old-schema feedback/playbook tables so _DDL can recreate them.
 
@@ -1261,6 +1333,70 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
             )
             self.conn.commit()
 
+    def _migrate_learning_jobs(self) -> None:
+        """Create the learning_jobs table + partial indexes if missing (idempotent).
+
+        Runs ``CREATE TABLE IF NOT EXISTS`` and ``CREATE INDEX IF NOT EXISTS`` only —
+        both are no-ops when the table / indexes already exist.  New columns added in
+        subsequent tasks are backfilled via ``PRAGMA table_info`` + ``ALTER TABLE … ADD
+        COLUMN`` (mirroring ``_migrate_lineage_event_table``), because
+        ``CREATE TABLE IF NOT EXISTS`` silently skips the DDL on existing databases.
+
+        Called at the end of migrate() so the table is always present on startup.
+        """
+        with self._lock:
+            self.conn.executescript("""
+                CREATE TABLE IF NOT EXISTS learning_jobs (
+                    job_id TEXT PRIMARY KEY,
+                    org_id TEXT NOT NULL,
+                    user_id TEXT NOT NULL,
+                    job_type TEXT NOT NULL DEFAULT 'learning',
+                    latest_request_id TEXT,
+                    status TEXT NOT NULL DEFAULT 'pending',
+                    attempts INTEGER NOT NULL DEFAULT 0,
+                    max_attempts INTEGER NOT NULL DEFAULT 3,
+                    claimed_by TEXT,
+                    claim_token TEXT,
+                    claim_expires_at TEXT,
+                    covers_through TEXT,
+                    force_extraction INTEGER NOT NULL DEFAULT 0,
+                    skip_aggregation INTEGER NOT NULL DEFAULT 0,
+                    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+                    updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
+                    ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
+                -- Recreate the poll index with 'failed' in the predicate. CREATE
+                -- INDEX IF NOT EXISTS is a no-op on an existing index with the old
+                -- ('pending','claimed') predicate, so DROP first to migrate it.
+                DROP INDEX IF EXISTS learning_jobs_poll;
+                CREATE INDEX IF NOT EXISTS learning_jobs_poll
+                    ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
+            """)
+            # Backfill columns added after the initial release (existing DBs skip
+            # CREATE TABLE IF NOT EXISTS so these must be applied separately).
+            existing_cols = {
+                row["name"]
+                for row in self.conn.execute(
+                    "PRAGMA table_info(learning_jobs)"
+                ).fetchall()
+            }
+            if "force_extraction" not in existing_cols:
+                self.conn.execute(
+                    "ALTER TABLE learning_jobs ADD COLUMN force_extraction INTEGER NOT NULL DEFAULT 0"
+                )
+            if "skip_aggregation" not in existing_cols:
+                self.conn.execute(
+                    "ALTER TABLE learning_jobs ADD COLUMN skip_aggregation INTEGER NOT NULL DEFAULT 0"
+                )
+            self.conn.commit()
+
+    def learning_jobs_columns(self) -> list[str]:
+        """Return the column names of the learning_jobs table."""
+        with self._lock:
+            rows = self.conn.execute("PRAGMA table_info(learning_jobs)").fetchall()
+        return [row["name"] for row in rows]
+
     def _migrate_agent_playbook_source_windows(self) -> None:
         """Add source window snapshots to existing agent source mappings."""
         cols = {
@@ -1451,7 +1587,8 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
     ) -> sqlite3.Cursor:
         with self._lock:
             cur = self.conn.execute(sql, params)
-            self.conn.commit()
+            if self._own_transaction():
+                self.conn.commit()
             return cur
 
     def _fetchone(
@@ -1519,6 +1656,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
             expander = DocumentExpander(
                 llm_client=self.llm_client,
                 prompt_manager=PromptManager(),
+                model_name=self._expansion_model_name,
             )
             result = expander.expand(content)
             return result.expanded_text or None
@@ -1737,6 +1875,7 @@ CREATE TABLE IF NOT EXISTS interactions (
     expert_content TEXT NOT NULL DEFAULT '',
     tools_used TEXT,
     citations TEXT,
+    retrieved_learnings TEXT,
     embedding TEXT
 );
 CREATE INDEX IF NOT EXISTS idx_interactions_user_id ON interactions(user_id);
@@ -1838,6 +1977,28 @@ CREATE INDEX IF NOT EXISTS idx_eval_agent_version_created_at_desc
 CREATE INDEX IF NOT EXISTS idx_eval_identity_created_at_desc
     ON agent_success_evaluation_result(user_id, session_id, evaluation_name, agent_version, created_at DESC);
 
+CREATE TABLE IF NOT EXISTS retrieved_learning_evaluation (
+    result_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id TEXT NOT NULL,
+    session_id TEXT NOT NULL,
+    agent_version TEXT NOT NULL DEFAULT '',
+    kind TEXT NOT NULL,
+    learning_id TEXT NOT NULL,
+    is_relevant INTEGER,
+    relevance_reason TEXT NOT NULL DEFAULT '',
+    impact TEXT,
+    impact_reason TEXT NOT NULL DEFAULT '',
+    created_at INTEGER NOT NULL,
+    governance_subject_ref TEXT,
+    UNIQUE (user_id, session_id, kind, learning_id)
+);
+CREATE INDEX IF NOT EXISTS idx_rle_created_at_result_id
+    ON retrieved_learning_evaluation(created_at DESC, result_id DESC);
+CREATE INDEX IF NOT EXISTS idx_rle_session_id
+    ON retrieved_learning_evaluation(session_id);
+CREATE INDEX IF NOT EXISTS idx_rle_subject_ref
+    ON retrieved_learning_evaluation(governance_subject_ref);
+
 CREATE TABLE IF NOT EXISTS agent_playbook_source_user_playbooks (
     agent_playbook_id INTEGER NOT NULL,
     user_playbook_id INTEGER NOT NULL,
@@ -2094,4 +2255,31 @@ CREATE TABLE IF NOT EXISTS lineage_event (
 );
 CREATE INDEX IF NOT EXISTS idx_lineage_entity ON lineage_event (entity_type, entity_id);
 
+-- ============================================================================
+-- Durable learning pipeline — cross-org job queue
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS learning_jobs (
+    job_id TEXT PRIMARY KEY,
+    org_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    job_type TEXT NOT NULL DEFAULT 'learning',
+    latest_request_id TEXT,
+    status TEXT NOT NULL DEFAULT 'pending',
+    attempts INTEGER NOT NULL DEFAULT 0,
+    max_attempts INTEGER NOT NULL DEFAULT 3,
+    claimed_by TEXT,
+    claim_token TEXT,
+    claim_expires_at TEXT,
+    covers_through TEXT,
+    force_extraction INTEGER NOT NULL DEFAULT 0,
+    skip_aggregation INTEGER NOT NULL DEFAULT 0,
+    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+    updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+);
+CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
+    ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
+CREATE INDEX IF NOT EXISTS learning_jobs_poll
+    ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
+
 """

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py

@@ -25,7 +25,14 @@ from ._base import (
     _row_to_interaction,
 )
 
-type _CitationKind = Literal["playbook", "profile"]
+type _CitationKind = Literal["playbook", "profile", "user_playbook", "agent_playbook"]
+
+_SUPPORTED_CITATION_KINDS: tuple[_CitationKind, ...] = (
+    "playbook",
+    "profile",
+    "user_playbook",
+    "agent_playbook",
+)
 
 
 class ExtrasMixin:
@@ -291,7 +298,7 @@ class ExtrasMixin:
                     continue
                 kind = c.get("kind")
                 real_id = c.get("real_id")
-                if kind not in ("playbook", "profile") or not real_id:
+                if kind not in _SUPPORTED_CITATION_KINDS or not real_id:
                     continue
                 key: tuple[_CitationKind, str] = (
                     cast(_CitationKind, kind),
@@ -453,12 +460,12 @@ class ExtrasMixin:
                         continue
                     kind = citation.get("kind")
                     real_id = citation.get("real_id")
-                    if kind and real_id:
+                    if kind in _SUPPORTED_CITATION_KINDS and real_id:
                         out.append(
                             SessionCitation(
                                 user_id=row["user_id"],
                                 session_id=row["session_id"],
-                                kind=str(kind),
+                                kind=cast(_CitationKind, kind),
                                 real_id=str(real_id),
                                 title=str(citation.get("title") or ""),
                             )

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py

@@ -392,11 +392,27 @@ class SQLiteGovernanceMixin:
                WHERE user_id = ?""",
             (user_id,),
         ).fetchone()
+        rle_result_row = self.conn.execute(
+            """SELECT COUNT(*) AS cnt
+               FROM retrieved_learning_evaluation
+               WHERE user_id = ?""",
+            (user_id,),
+        ).fetchone()
+        session_count_row = self.conn.execute(
+            "SELECT COUNT(DISTINCT session_id) AS cnt FROM requests WHERE user_id = ?",
+            (user_id,),
+        ).fetchone()
         profile_rows = self.conn.execute(
             "SELECT profile_id FROM profiles WHERE user_id = ?",
             (user_id,),
         ).fetchall()
-        if request_row is None or interaction_row is None or eval_result_row is None:
+        if (
+            request_row is None
+            or interaction_row is None
+            or eval_result_row is None
+            or rle_result_row is None
+            or session_count_row is None
+        ):
             raise ValueError("Missing governance count rows")
         profile_ids = [str(row["profile_id"]) for row in profile_rows]
         purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete(
@@ -420,6 +436,17 @@ class SQLiteGovernanceMixin:
             "profile_purge": len(purge_profile_ids),
             "user_playbook": len(delete_playbook_ids),
             "agent_success_evaluation_result": int(eval_result_row["cnt"]),
+            # Offline tuner tables are enterprise-only (tenant stream); the
+            # OSS SQLite backend has no such tables, so the planned and
+            # deleted counts are structurally zero. The delete-target matrix
+            # validator still requires the target rows to exist.
+            "offline_tuner_reward_label": 0,
+            "offline_tuner_reward_label_target_by_target_owner": 0,
+            "retrieved_learning_evaluation_result": int(rle_result_row["cnt"]),
+            # Planned as an upper bound: up to 3 evaluation state namespaces
+            # per session (retrieved-eval state, agent-success marker,
+            # grade-cache rows). The delete phase reports the exact count.
+            "evaluation_operation_state": 3 * int(session_count_row["cnt"]),
             "user_playbook_purge": len(purge_playbook_ids),
         }