claude-smart 0.2.46 → 0.2.48

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py

@@ -53,15 +53,11 @@ from reflexio.server.services.storage.error import (
     StorageError,
     require_non_empty_session_id,
 )
-from reflexio.server.services.storage.retention import RetentionTarget
-from reflexio.server.services.storage.retention_mixin import (
-    RETENTION_DELETE_CHUNK,
-    RetentionMixin,
-    chunked,
-)
+from reflexio.server.services.storage.retention_mixin import RetentionMixin
 from reflexio.server.services.storage.storage_base import BaseStorage
 from reflexio.server.site_var.site_var_manager import SiteVarManager
 
+from ._governance import init_governance_tables
 from ._stall_state import init_stall_state_table
 
 logger = logging.getLogger(__name__)
@@ -287,6 +283,42 @@ def _true_rrf_merge(
 # explicit status_filter on list/count methods.
 _TOMBSTONE_STATUS_VALUES = (Status.MERGED.value, Status.SUPERSEDED.value)
 
+# Profile reads also treat EXPIRED (TTL-expired tombstone) as non-current. Kept
+# separate from _TOMBSTONE_STATUS_VALUES because that tuple is shared with playbook
+# queries (which never carry EXPIRED) via hardcoded placeholders.
+_PROFILE_TOMBSTONE_STATUS_VALUES = (
+    Status.MERGED.value,
+    Status.SUPERSEDED.value,
+    Status.EXPIRED.value,
+)
+
+
+def parse_status(value: str | None) -> Status | None:
+    """Parse a stored status string into a Status, tolerating unknown values.
+
+    Unknown non-null values (e.g. a status written by a newer build after a
+    rollback) map to a non-current tombstone sentinel and log an anomaly, rather
+    than raising ValueError. Never maps to None (which models CURRENT/live).
+
+    Args:
+        value: Raw status string from the database, or None/empty for CURRENT.
+
+    Returns:
+        The matching Status enum member, Status.SUPERSEDED for unknown values,
+        or None for falsy input (representing the CURRENT/live state).
+    """
+    if not value:
+        return None
+    try:
+        return Status(value)
+    except ValueError:
+        from reflexio.server.tracing import capture_anomaly
+
+        capture_anomaly(
+            "storage.status.unknown_value", level="warning", status_value=value
+        )
+        return Status.SUPERSEDED
+
 
 def _status_value(status: Status | None) -> str | None:
     """Convert a Status enum (or None) to its DB string value."""
@@ -362,14 +394,19 @@ def _iso_to_epoch(iso_str: str | None) -> int:
 # stored row) with a valid value.
 _MAX_SAFE_EPOCH_TS = 253_402_300_799  # 9999-12-31T23:59:59Z
 _MIN_SAFE_EPOCH_TS = 0  # 1970-01-01T00:00:00Z
+_MIN_CONTEMPORARY_MILLISECOND_EPOCH_TS = 1_500_000_000_000
 
 
 def _epoch_to_iso(ts: int) -> str:
     """Convert a Unix timestamp (seconds) to an ISO 8601 string.
 
-    Out-of-range sentinel bounds are clamped to the representable range so that
-    callers passing "open" window bounds never trigger a ``ValueError``.
+    Plausible contemporary millisecond timestamps are normalized to seconds.
+    Lower values stay in seconds-space so documented sentinel bounds such as
+    ``to_ts=10**12`` still clamp to the representable range instead of being
+    treated as a 2001 millisecond timestamp.
     """
+    if _MIN_CONTEMPORARY_MILLISECOND_EPOCH_TS <= ts <= _MAX_SAFE_EPOCH_TS * 1000:
+        ts = ts // 1000
     clamped = max(_MIN_SAFE_EPOCH_TS, min(ts, _MAX_SAFE_EPOCH_TS))
     return datetime.fromtimestamp(clamped, tz=UTC).isoformat()
 
@@ -391,7 +428,7 @@ def _row_to_profile(row: sqlite3.Row) -> UserProfile:
         expiration_timestamp=d["expiration_timestamp"],
         custom_features=_json_loads(d.get("custom_features")),
         source=d.get("source") or "",
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         extractor_names=_json_loads(d.get("extractor_names")),
         expanded_terms=d.get("expanded_terms"),
         source_span=d.get("source_span"),
@@ -471,7 +508,7 @@ def _row_to_user_playbook(
         blocking_issue=BlockingIssue(**json.loads(d["blocking_issue"]))
         if d.get("blocking_issue")
         else None,
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         source=d.get("source"),
         source_interaction_ids=_json_loads(d.get("source_interaction_ids")) or [],
         tags=_json_loads(d.get("tags")),
@@ -504,7 +541,7 @@ def _row_to_agent_playbook(row: sqlite3.Row) -> AgentPlaybook:
         playbook_metadata=d.get("playbook_metadata") or "",
         tags=_json_loads(d.get("tags")),
         embedding=[],
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         expanded_terms=d.get("expanded_terms"),
         merged_into=d.get("merged_into"),
         superseded_by=d.get("superseded_by"),
@@ -545,6 +582,15 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
 
     supports_embedding: ClassVar[bool] = True
 
+    # Chunked bulk-delete helpers provided by SQLiteDeletionMixin via the
+    # composed SQLiteStorage MRO; declared here for clear_user_data's benefit.
+    _delete_in_chunks: Any
+    _delete_source_windows_for_user_playbook_ids: Any
+
+    # FTS/vec index helpers provided by SQLiteFtsVecMixin via the composed
+    # SQLiteStorage MRO; declared here for _migrate_vec_tables's benefit.
+    _vec_upsert: Any
+
     @staticmethod
     def handle_exceptions(func: Callable[..., Any]) -> Callable[..., Any]:
         @functools.wraps(func)
@@ -642,6 +688,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         with self._lock:
             cur = self.conn.cursor()
             cur.executescript(_DDL)
+            init_governance_tables(self.conn)
             self.conn.commit()
         if self._has_sqlite_vec:
             self._create_vec_tables()
@@ -669,239 +716,6 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         init_stall_state_table(self.conn)
         return True
 
-    # -- Retention hooks (see RetentionMixin) --
-
-    @handle_exceptions
-    def _retention_table_exists(self, table_name: str) -> bool:
-        row = self._fetchone(
-            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
-            (table_name,),
-        )
-        return row is not None
-
-    @handle_exceptions
-    def _retention_count_rows(self, target: RetentionTarget) -> int:
-        row = self._fetchone(f"SELECT COUNT(*) as cnt FROM {target.table_name}")  # noqa: S608
-        return int(row["cnt"]) if row else 0
-
-    @handle_exceptions
-    def _retention_select_oldest_keys(
-        self, target: RetentionTarget, count: int
-    ) -> list[tuple[Any, ...]]:
-        id_sql = ", ".join(target.id_columns)
-        tiebreak_sql = id_sql
-        rows = self._fetchall(
-            f"SELECT {id_sql} FROM {target.table_name} "  # noqa: S608
-            f"ORDER BY {target.order_column} ASC, {tiebreak_sql} ASC LIMIT ?",
-            (count,),
-        )
-        return [tuple(row[col] for col in target.id_columns) for row in rows]
-
-    @handle_exceptions
-    def _retention_perform_delete(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        # Wrap dependency + target deletes in a single critical section so
-        # concurrent writers see either both or neither.
-        with self._lock:
-            try:
-                self._retention_delete_dependencies(target, keys)
-                self._retention_delete_target_rows(target, keys)
-                self.conn.commit()
-            except Exception:
-                self.conn.rollback()
-                raise
-
-    def _retention_delete_dependencies(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        ids = [key[0] for key in keys]
-        target_name = target.name
-        if target_name == "requests":
-            self._delete_interactions_for_request_ids([str(v) for v in ids])
-        elif target_name == "interactions":
-            self._delete_interaction_search_rows([int(v) for v in ids])
-        elif target_name == "profiles":
-            self._delete_profile_search_rows([str(v) for v in ids])
-        elif target_name == "user_playbooks":
-            self._delete_source_windows_for_user_playbook_ids([int(v) for v in ids])
-            self._delete_playbook_search_rows(
-                "user", [int(v) for v in ids], commit=False
-            )
-        elif target_name == "agent_playbooks":
-            self._delete_source_windows_for_agent_playbook_ids([int(v) for v in ids])
-            self._delete_playbook_search_rows(
-                "agent", [int(v) for v in ids], commit=False
-            )
-        elif target_name == "playbook_optimization_jobs":
-            self._delete_optimizer_rows_for_job_ids([int(v) for v in ids])
-        elif target_name == "playbook_optimization_candidates":
-            self._delete_optimizer_evaluations_for_candidate_ids([int(v) for v in ids])
-
-    def _retention_delete_target_rows(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        if len(target.id_columns) == 1:
-            self._delete_in_chunks(
-                target.table_name,
-                target.id_columns[0],
-                [key[0] for key in keys],
-            )
-            return
-        # Composite-key delete: chunk by row to bound parameter count.
-        params_per_key = len(target.id_columns)
-        rows_per_chunk = max(1, RETENTION_DELETE_CHUNK // params_per_key)
-        for chunk in chunked(keys, rows_per_chunk):
-            where = " OR ".join(
-                "("
-                + " AND ".join(f"{column} = ?" for column in target.id_columns)
-                + ")"
-                for _ in chunk
-            )
-            params = [value for key in chunk for value in key]
-            self.conn.execute(
-                f"DELETE FROM {target.table_name} WHERE {where}",  # noqa: S608
-                params,
-            )
-
-    # -- Chunked-delete primitives shared by the cascade helpers --
-
-    def _delete_in_chunks(
-        self, table_name: str, column_name: str, values: list[Any]
-    ) -> None:
-        """Chunked ``DELETE FROM table WHERE col IN (...)``.
-
-        Chunking keeps parameter count under ``SQLITE_MAX_VARIABLE_NUMBER``
-        on older sqlite builds (default 999) and avoids degenerate plans
-        on very large IN lists.
-        """
-        if not values:
-            return
-        for chunk in chunked(values):
-            placeholders = ",".join("?" for _ in chunk)
-            self.conn.execute(
-                f"DELETE FROM {table_name} WHERE {column_name} IN ({placeholders})",  # noqa: S608
-                chunk,
-            )
-
-    def _select_in_chunks(self, sql_template: str, values: list[Any]) -> list[Any]:
-        """Run ``sql_template`` (containing ``{placeholders}``) over chunks of
-        ``values`` and aggregate the result rows."""
-        results: list[Any] = []
-        for chunk in chunked(values):
-            placeholders = ",".join("?" for _ in chunk)
-            stmt = sql_template.format(placeholders=placeholders)
-            results.extend(self.conn.execute(stmt, chunk).fetchall())
-        return results
-
-    def _delete_interactions_for_request_ids(self, request_ids: list[str]) -> None:
-        if not request_ids:
-            return
-        rows = self._select_in_chunks(
-            "SELECT interaction_id FROM interactions WHERE request_id IN ({placeholders})",
-            request_ids,
-        )
-        self._delete_interaction_search_rows(
-            [int(row["interaction_id"]) for row in rows]
-        )
-        self._delete_in_chunks("interactions", "request_id", request_ids)
-
-    def _delete_interaction_search_rows(self, interaction_ids: list[int]) -> None:
-        """Remove fts + vec index rows for the given interaction IDs.
-
-        Non-committing: participates in the caller's transaction.  Only called
-        from inside the retention atomic block (_retention_perform_delete).
-        """
-        if not interaction_ids:
-            return
-        self._delete_in_chunks("interactions_fts", "rowid", interaction_ids)
-        if self._has_sqlite_vec:
-            self._delete_in_chunks("interactions_vec", "rowid", interaction_ids)
-
-    def _delete_profile_search_rows(self, profile_ids: list[str]) -> None:
-        """Remove fts + vec index rows for the given profile IDs.
-
-        Non-committing: participates in the caller's transaction.  Only called
-        from inside the retention atomic block (_retention_perform_delete).
-        profiles_fts is keyed by profile_id (TEXT); profiles_vec by rowid (INT).
-        """
-        if not profile_ids:
-            return
-        self._delete_in_chunks("profiles_fts", "profile_id", profile_ids)
-        if self._has_sqlite_vec:
-            rows = self._select_in_chunks(
-                "SELECT rowid FROM profiles WHERE profile_id IN ({placeholders})",
-                profile_ids,
-            )
-            rowids = [row["rowid"] for row in rows]
-            if rowids:
-                self._delete_in_chunks("profiles_vec", "rowid", rowids)
-
-    def _delete_playbook_search_rows(
-        self, kind: str, ids: list[int], *, commit: bool = True
-    ) -> None:
-        """Remove fts + vec index rows for the given playbook IDs.
-
-        Args:
-            kind: ``"user"`` or ``"agent"``.
-            ids: Playbook row IDs to remove from the search indexes.
-            commit: When ``True`` (default) commits after the deletes so the
-                after-commit callers in ``_playbook.py`` get a clean, durable
-                cleanup.  Pass ``commit=False`` from inside the retention atomic
-                block so the deletes participate in the single block-level commit
-                (``_retention_perform_delete``).
-
-        Note: callers may already hold ``self._lock`` when calling this (the
-        ``commit=False`` retention/atomic-delete call sites do). The internal
-        ``with self._lock:`` re-acquire is safe ONLY because ``self._lock`` is a
-        reentrant ``threading.RLock``; a non-reentrant lock would deadlock here.
-        """
-        if not ids:
-            return
-        with self._lock:
-            self._delete_in_chunks(f"{kind}_playbooks_fts", "rowid", ids)
-            if self._has_sqlite_vec:
-                self._delete_in_chunks(f"{kind}_playbooks_vec", "rowid", ids)
-            if commit:
-                self.conn.commit()
-
-    def _delete_source_windows_for_agent_playbook_ids(
-        self, agent_playbook_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "agent_playbook_source_user_playbooks",
-            "agent_playbook_id",
-            agent_playbook_ids,
-        )
-
-    def _delete_source_windows_for_user_playbook_ids(
-        self, user_playbook_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "agent_playbook_source_user_playbooks",
-            "user_playbook_id",
-            user_playbook_ids,
-        )
-
-    def _delete_optimizer_rows_for_job_ids(self, job_ids: list[int]) -> None:
-        if not job_ids:
-            return
-        for table in (
-            "playbook_optimization_evaluations",
-            "playbook_optimization_events",
-            "playbook_optimization_candidates",
-        ):
-            self._delete_in_chunks(table, "job_id", job_ids)
-
-    def _delete_optimizer_evaluations_for_candidate_ids(
-        self, candidate_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "playbook_optimization_evaluations",
-            "candidate_id",
-            candidate_ids,
-        )
-
     def _try_load_sqlite_vec(self) -> bool:
         """Attempt to load the sqlite-vec extension for native KNN search.
 
@@ -1715,113 +1529,6 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
     def _current_timestamp(self) -> str:
         return datetime.now(UTC).isoformat()
 
-    # FTS helpers
-    def _fts_upsert(self, table: str, rowid: int, **text_fields: str | None) -> None:
-        """Insert or update an FTS row.  Deletes old entry first to avoid duplicates."""
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            cols = list(text_fields.keys())
-            vals = [text_fields[c] or "" for c in cols]
-            placeholders = ",".join("?" for _ in cols)
-            col_str = ",".join(cols)
-            self.conn.execute(
-                f"INSERT INTO {table}(rowid, {col_str}) VALUES (?, {placeholders})",
-                [rowid, *vals],
-            )
-            self.conn.commit()
-
-    def _fts_delete(self, table: str, rowid: int) -> None:
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.commit()
-
-    def _fts_upsert_profile(self, profile_id: str, content: str) -> None:
-        """FTS for profiles uses profile_id TEXT as key column."""
-        with self._lock:
-            self.conn.execute(
-                "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
-            )
-            self.conn.execute(
-                "INSERT INTO profiles_fts(profile_id, content) VALUES (?, ?)",
-                (profile_id, content),
-            )
-            self.conn.commit()
-
-    def _fts_delete_profile(self, profile_id: str) -> None:
-        with self._lock:
-            self.conn.execute(
-                "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
-            )
-            self.conn.commit()
-
-    # Vec helpers (sqlite-vec)
-    def _vec_upsert(self, table: str, rowid: int, embedding: list[float]) -> None:
-        """Insert or update a vec table row. No-op when sqlite-vec is unavailable."""
-        if not self._has_sqlite_vec:
-            return
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.execute(
-                f"INSERT INTO {table}(rowid, embedding) VALUES (?, ?)",
-                (rowid, json.dumps(embedding)),
-            )
-            self.conn.commit()
-
-    def _vec_delete(self, table: str, rowid: int) -> None:
-        """Delete a vec table row. No-op when sqlite-vec is unavailable."""
-        if not self._has_sqlite_vec:
-            return
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.commit()
-
-    def _vec_knn_search(
-        self,
-        vec_table: str,
-        main_table: str,
-        query_embedding: list[float],
-        match_count: int,
-        conditions: list[str] | None = None,
-        params: list[Any] | None = None,
-    ) -> list[sqlite3.Row]:
-        """Run a native KNN search via sqlite-vec and join back to the main table.
-
-        Over-fetches from the KNN index (5x ``match_count``) so that post-filter
-        WHERE conditions (org, user, status, etc.) don't silently reduce the
-        result set below the requested count.
-
-        Args:
-            vec_table: Name of the vec0 virtual table.
-            main_table: Name of the main data table.
-            query_embedding: Query embedding vector.
-            match_count: Number of results to return.
-            conditions: Optional WHERE conditions for the main table.
-            params: Parameters for the conditions.
-
-        Returns:
-            Up to ``match_count`` rows from the main table, ordered by vector
-            distance (ascending).
-        """
-        knn_overfetch = match_count * 5
-        where_clause = " AND ".join(conditions) if conditions else "1=1"
-        sql = f"""SELECT m.* FROM {main_table} m
-                  JOIN (
-                      SELECT rowid, distance FROM {vec_table}
-                      WHERE embedding MATCH ?
-                      ORDER BY distance
-                      LIMIT ?
-                  ) v ON m.rowid = v.rowid
-                  WHERE {where_clause}
-                  ORDER BY v.distance
-                  LIMIT ?"""
-        all_params = [
-            json.dumps(query_embedding),
-            knn_overfetch,
-            *(params or []),
-            match_count,
-        ]
-        return self._fetchall(sql, all_params)
-
     # ------------------------------------------------------------------
     # Per-user data clear
     # ------------------------------------------------------------------

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py

@@ -2,6 +2,7 @@
 
 import sqlite3
 from collections import defaultdict
+from datetime import UTC, datetime
 from typing import Any, Literal, cast
 
 from reflexio.models.api_schema.braintrust_schema import (
@@ -179,31 +180,60 @@ class ExtrasMixin:
             (current_start_iso, current_time_iso),
         )
 
+        def day_bucket(timestamp: int) -> int:
+            return int(
+                datetime.fromtimestamp(timestamp, tz=UTC)
+                .replace(hour=0, minute=0, second=0, microsecond=0)
+                .timestamp()
+            )
+
+        def count_series(timestamps: list[int]) -> list[dict[str, int]]:
+            buckets: dict[int, int] = defaultdict(int)
+            for timestamp in timestamps:
+                buckets[day_bucket(timestamp)] += 1
+            return [
+                {"timestamp": timestamp, "value": value}
+                for timestamp, value in sorted(buckets.items())
+            ]
+
+        def rate_series(rows: list[sqlite3.Row]) -> list[dict[str, int | float]]:
+            buckets: dict[int, dict[str, int]] = defaultdict(
+                lambda: {"success": 0, "count": 0}
+            )
+            for row in rows:
+                bucket = day_bucket(_iso_to_epoch(row["created_at"]))
+                buckets[bucket]["count"] += 1
+                if row["is_success"]:
+                    buckets[bucket]["success"] += 1
+            return [
+                {
+                    "timestamp": timestamp,
+                    "value": (
+                        100.0 * aggregate["success"] / aggregate["count"]
+                        if aggregate["count"]
+                        else 0.0
+                    ),
+                    "count": aggregate["count"],
+                }
+                for timestamp, aggregate in sorted(buckets.items())
+            ]
+
         return {
             "current_period": current_stats,
             "previous_period": previous_stats,
-            "interactions_time_series": [
-                {"timestamp": _iso_to_epoch(r["created_at"]), "value": 1}
-                for r in interactions_ts
-            ],
-            "profiles_time_series": [
-                {"timestamp": r["last_modified_timestamp"], "value": 1}
-                for r in profiles_ts
-            ],
-            "playbooks_time_series": sorted(
+            "interactions_time_series": count_series(
+                [_iso_to_epoch(r["created_at"]) for r in interactions_ts]
+            ),
+            "profiles_time_series": count_series(
+                [r["last_modified_timestamp"] for r in profiles_ts]
+            ),
+            "playbooks_time_series": count_series(
                 [
-                    {"timestamp": _iso_to_epoch(r["created_at"]), "value": 1}
+                    _iso_to_epoch(r["created_at"])
                     for r in (*user_playbooks_ts, *agent_playbooks_ts)
-                ],
-                key=lambda x: x["timestamp"],
+                ]
             ),
-            "evaluations_time_series": [
-                {
-                    "timestamp": _iso_to_epoch(r["created_at"]),
-                    "value": 100 if r["is_success"] else 0,
-                }
-                for r in evals_ts
-            ],
+            "evaluations_time_series": rate_series(evals_ts),
         }
 
     @SQLiteStorageBase.handle_exceptions