claude-smart 0.2.45 → 0.2.47

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_profiles.py

@@ -2,6 +2,7 @@
 
 import logging
 import sqlite3
+import uuid
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any
 
@@ -24,6 +25,7 @@ from reflexio.server.llm.providers.embedding_service_provider import (
 )
 
 from ._base import (
+    _TOMBSTONE_STATUS_VALUES,
     SQLiteStorageBase,
     _build_status_sql,
     _effective_search_mode,
@@ -37,6 +39,44 @@ from ._base import (
     _true_rrf_merge,
     _vector_rank_rows,
 )
+from ._lineage import _GC_ELIGIBLE_STATUSES, _append_event_stmt
+
+
+def _emit_hard_delete_profile(
+    conn: sqlite3.Connection,
+    *,
+    org_id: str,
+    entity_id: str,
+    request_id: str,
+    actor: str = "api",
+) -> None:
+    """Emit a single hard_delete lineage event for a profile entity."""
+    _append_event_stmt(
+        conn,
+        org_id=org_id,
+        entity_type="profile",
+        entity_id=entity_id,
+        op="hard_delete",
+        prov="wasInvalidatedBy",
+        source_ids=[],
+        actor=actor,
+        request_id=request_id,
+        reason="erasure",
+    )
+
+
+def _build_tags_sql(alias: str, tags: list[str] | None) -> tuple[str, list[Any]]:
+    if not tags:
+        return "", []
+    placeholders = ",".join("?" for _ in tags)
+    return (
+        f"EXISTS (SELECT 1 FROM json_each({alias}.tags) WHERE value IN ({placeholders}))",
+        list(tags),
+    )
+
+
+def _escape_like_pattern(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 
 
 class ProfileMixin:
@@ -45,6 +85,7 @@ class ProfileMixin:
     # Type hints for instance attributes/methods provided by SQLiteStorageBase via MRO
     _lock: Any
     conn: sqlite3.Connection
+    org_id: str
     _execute: Any
     _fetchone: Any
     _fetchall: Any
@@ -57,25 +98,93 @@ class ProfileMixin:
     _fts_delete_profile: Any
     _vec_upsert: Any
     _vec_delete: Any
+    _delete_profile_search_rows: Any
+    _delete_in_chunks: Any
     _has_sqlite_vec: bool
     llm_client: Any
     embedding_model_name: str
     embedding_dimensions: int
+    _subject_ref_for_user_id: Any
+    _assert_subject_writable_locked: Any
 
     # ------------------------------------------------------------------
     # CRUD — Profiles
     # ------------------------------------------------------------------
 
+    def _subject_ref_from_profile_row(self, row: sqlite3.Row) -> str:
+        subject_ref = row["governance_subject_ref"]
+        return (
+            str(subject_ref)
+            if subject_ref
+            else self._subject_ref_for_user_id(row["user_id"])
+        )
+
+    def _assert_profile_writable_locked(
+        self,
+        profile_id: str,
+        *,
+        user_id: str | None = None,
+    ) -> sqlite3.Row | None:
+        if user_id is None:
+            row = self.conn.execute(
+                "SELECT user_id, governance_subject_ref FROM profiles WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+        else:
+            row = self.conn.execute(
+                "SELECT user_id, governance_subject_ref FROM profiles WHERE profile_id = ? AND user_id = ?",
+                (profile_id, user_id),
+            ).fetchone()
+        if row is None:
+            return None
+        self._assert_subject_writable_locked(self._subject_ref_from_profile_row(row))
+        return row
+
     @SQLiteStorageBase.handle_exceptions
     def get_all_profiles(
         self,
         limit: int = 100,
         status_filter: list[Status | None] | None = None,
+        user_id: str | None = None,
+        profile_id: str | None = None,
+        query: str | None = None,
+        source: str | None = None,
+        profile_time_to_live: str | None = None,
+        start_time: int | None = None,
+        end_time: int | None = None,
     ) -> list[UserProfile]:
         if status_filter is None:
             status_filter = [None]
         frag, params = _build_status_sql(status_filter)
-        sql = f"SELECT * FROM profiles WHERE {frag} ORDER BY last_modified_timestamp DESC LIMIT ?"
+        conditions = [frag]
+        if user_id:
+            conditions.append("user_id = ?")
+            params.append(user_id)
+        if profile_id:
+            conditions.append("LOWER(profile_id) = LOWER(?)")
+            params.append(profile_id)
+        if query:
+            like = f"%{_escape_like_pattern(query.lower())}%"
+            conditions.append(
+                "(LOWER(content) LIKE ? ESCAPE '\\' OR LOWER(profile_id) LIKE ? ESCAPE '\\' OR LOWER(user_id) LIKE ? ESCAPE '\\')"
+            )
+            params.extend([like, like, like])
+        if source is not None:
+            conditions.append("source = ?")
+            params.append(source)
+        if profile_time_to_live:
+            conditions.append("profile_time_to_live = ?")
+            params.append(profile_time_to_live)
+        if start_time is not None:
+            conditions.append("last_modified_timestamp >= ?")
+            params.append(start_time)
+        if end_time is not None:
+            conditions.append("last_modified_timestamp <= ?")
+            params.append(end_time)
+        sql = (
+            f"SELECT * FROM profiles WHERE {' AND '.join(conditions)} "
+            "ORDER BY last_modified_timestamp DESC LIMIT ?"
+        )
         params.append(limit)
         return [_row_to_profile(r) for r in self._fetchall(sql, params)]
 
@@ -84,18 +193,54 @@ class ProfileMixin:
         self,
         user_id: str,
         status_filter: list[Status | None] | None = None,
+        tags: list[str] | None = None,
+        profile_id: str | None = None,
+        query: str | None = None,
+        source: str | None = None,
+        profile_time_to_live: str | None = None,
+        start_time: int | None = None,
+        end_time: int | None = None,
     ) -> list[UserProfile]:
         if status_filter is None:
             status_filter = [None]
         current_ts = _epoch_now()
         frag, params = _build_status_sql(status_filter)
-        sql = f"SELECT * FROM profiles WHERE user_id = ? AND expiration_timestamp >= ? AND {frag}"
+        conditions = ["user_id = ?", "expiration_timestamp >= ?", frag]
         all_params: list[Any] = [user_id, current_ts, *params]
+        if profile_id:
+            conditions.append("LOWER(profile_id) = LOWER(?)")
+            all_params.append(profile_id)
+        if query:
+            like = f"%{_escape_like_pattern(query.lower())}%"
+            conditions.append(
+                "(LOWER(content) LIKE ? ESCAPE '\\' OR LOWER(profile_id) LIKE ? ESCAPE '\\' OR LOWER(user_id) LIKE ? ESCAPE '\\')"
+            )
+            all_params.extend([like, like, like])
+        if source is not None:
+            conditions.append("source = ?")
+            all_params.append(source)
+        if profile_time_to_live:
+            conditions.append("profile_time_to_live = ?")
+            all_params.append(profile_time_to_live)
+        if start_time is not None:
+            conditions.append("last_modified_timestamp >= ?")
+            all_params.append(start_time)
+        if end_time is not None:
+            conditions.append("last_modified_timestamp <= ?")
+            all_params.append(end_time)
+        tag_frag, tag_params = _build_tags_sql("profiles", tags)
+        if tag_frag:
+            conditions.append(tag_frag)
+            all_params.extend(tag_params)
+        sql = f"SELECT * FROM profiles WHERE {' AND '.join(conditions)}"
         return [_row_to_profile(r) for r in self._fetchall(sql, all_params)]
 
     @SQLiteStorageBase.handle_exceptions
     def add_user_profile(self, user_id: str, user_profiles: list[UserProfile]) -> None:  # noqa: ARG002
         for profile in user_profiles:
+            subject_ref = self._subject_ref_for_user_id(profile.user_id)
+            with self._lock:
+                self._assert_subject_writable_locked(subject_ref)
             embedding_text = "\n".join([profile.content, str(profile.custom_features)])
             if self._should_expand_documents():
                 with ThreadPoolExecutor(max_workers=2) as executor:
@@ -106,34 +251,48 @@ class ProfileMixin:
             else:
                 profile.embedding = self._get_embedding(embedding_text)
             embedding = profile.embedding
-            self._execute(
-                """INSERT OR REPLACE INTO profiles
-                   (profile_id, user_id, content, last_modified_timestamp,
-                    generated_from_request_id, profile_time_to_live,
-                    expiration_timestamp, custom_features, embedding, source,
-                    status, extractor_names, expanded_terms,
-                    source_span, notes, reader_angle, created_at)
-                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
-                (
-                    profile.profile_id,
-                    profile.user_id,
-                    profile.content,
-                    profile.last_modified_timestamp,
-                    profile.generated_from_request_id,
-                    profile.profile_time_to_live.value,
-                    profile.expiration_timestamp,
-                    _json_dumps(profile.custom_features),
-                    _json_dumps(profile.embedding),
-                    profile.source,
-                    profile.status.value if profile.status else None,
-                    _json_dumps(profile.extractor_names),
-                    profile.expanded_terms,
-                    profile.source_span,
-                    profile.notes,
-                    profile.reader_angle,
-                    _iso_now(),
-                ),
-            )
+            with self._lock:
+                try:
+                    self.conn.execute("BEGIN IMMEDIATE")
+                    self._assert_subject_writable_locked(subject_ref)
+                    self.conn.execute(
+                        """INSERT OR REPLACE INTO profiles
+                           (profile_id, user_id, content, last_modified_timestamp,
+                            generated_from_request_id, profile_time_to_live,
+                            expiration_timestamp, custom_features, embedding, source,
+                            status, extractor_names, expanded_terms,
+                            source_span, notes, reader_angle, tags, source_interaction_ids, created_at,
+                            merged_into, superseded_by, governance_subject_ref)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                        (
+                            profile.profile_id,
+                            profile.user_id,
+                            profile.content,
+                            profile.last_modified_timestamp,
+                            profile.generated_from_request_id,
+                            profile.profile_time_to_live.value,
+                            profile.expiration_timestamp,
+                            _json_dumps(profile.custom_features),
+                            _json_dumps(profile.embedding),
+                            profile.source,
+                            profile.status.value if profile.status else None,
+                            _json_dumps(profile.extractor_names),
+                            profile.expanded_terms,
+                            profile.source_span,
+                            profile.notes,
+                            profile.reader_angle,
+                            _json_dumps(profile.tags),
+                            _json_dumps(profile.source_interaction_ids),
+                            _iso_now(),
+                            profile.merged_into,
+                            profile.superseded_by,
+                            subject_ref,
+                        ),
+                    )
+                    self.conn.commit()
+                except Exception:
+                    self.conn.rollback()
+                    raise
             fts_parts = [profile.content or ""]
             if profile.custom_features:
                 fts_parts.extend(str(v) for v in profile.custom_features.values() if v)
@@ -152,88 +311,188 @@ class ProfileMixin:
     def update_user_profile_by_id(
         self, user_id: str, profile_id: str, new_profile: UserProfile
     ) -> None:
-        current_ts = _epoch_now()
-        row = self._fetchone(
-            "SELECT profile_id FROM profiles WHERE user_id = ? AND profile_id = ? AND expiration_timestamp >= ?",
-            (user_id, profile_id, current_ts),
-        )
-        if not row:
-            import logging
+        """Replace a profile's content in-place and emit a revise lineage event.
 
-            logger = logging.getLogger(__name__)
-            logger.warning("User profile not found for user id: %s", user_id)
-            return
+        Each call generates a fresh request_id so every edit is a distinct audit
+        event (not collapsed by the idempotency key).  The UPDATE, lineage event,
+        FTS sync, and vec sync are all executed inside a single lock acquisition;
+        self._lock is an RLock so the inner _fts_upsert_profile/_vec_upsert calls
+        that re-acquire it are safe.
+        """
+        current_ts = _epoch_now()
+        with self._lock:
+            row = self.conn.execute(
+                "SELECT user_id, governance_subject_ref FROM profiles WHERE user_id = ? AND profile_id = ? AND expiration_timestamp >= ?",
+                (user_id, profile_id, current_ts),
+            ).fetchone()
+            if not row:
+                logger.warning("User profile not found for user id: %s", user_id)
+                return
+            self._assert_subject_writable_locked(
+                self._subject_ref_from_profile_row(row)
+            )
         embedding = self._get_embedding(
             "\n".join([new_profile.content, str(new_profile.custom_features)])
         )
         new_profile.embedding = embedding
-        self._execute(
-            """UPDATE profiles SET content=?, last_modified_timestamp=?,
-               generated_from_request_id=?, profile_time_to_live=?,
-               expiration_timestamp=?, custom_features=?, embedding=?,
-               source=?, status=?, extractor_names=?, expanded_terms=?,
-               source_span=?, notes=?, reader_angle=?
-               WHERE profile_id=?""",
-            (
-                new_profile.content,
-                new_profile.last_modified_timestamp,
-                new_profile.generated_from_request_id,
-                new_profile.profile_time_to_live.value,
-                new_profile.expiration_timestamp,
-                _json_dumps(new_profile.custom_features),
-                _json_dumps(new_profile.embedding),
-                new_profile.source,
-                new_profile.status.value if new_profile.status else None,
-                _json_dumps(new_profile.extractor_names),
-                new_profile.expanded_terms,
-                new_profile.source_span,
-                new_profile.notes,
-                new_profile.reader_angle,
-                profile_id,
-            ),
-        )
-        fts_parts = [new_profile.content or ""]
-        if new_profile.custom_features:
-            fts_parts.extend(str(v) for v in new_profile.custom_features.values() if v)
-        if new_profile.expanded_terms:
-            fts_parts.append(new_profile.expanded_terms)
-        self._fts_upsert_profile(profile_id, " ".join(fts_parts))
-        rowid_row = self._fetchone(
-            "SELECT rowid FROM profiles WHERE profile_id = ?", (profile_id,)
-        )
-        if rowid_row and embedding:
-            self._vec_upsert("profiles_vec", rowid_row["rowid"], embedding)
+        with self._lock:
+            if (
+                self._assert_profile_writable_locked(profile_id, user_id=user_id)
+                is None
+            ):
+                logger.warning("User profile not found for user id: %s", user_id)
+                return
+            cur = self.conn.execute(
+                """UPDATE profiles SET content=?, last_modified_timestamp=?,
+                   generated_from_request_id=?, profile_time_to_live=?,
+                   expiration_timestamp=?, custom_features=?, embedding=?,
+                   source=?, status=?, extractor_names=?, expanded_terms=?,
+                   source_span=?, notes=?, reader_angle=?, tags=?, source_interaction_ids=?
+                   WHERE profile_id=?""",
+                (
+                    new_profile.content,
+                    new_profile.last_modified_timestamp,
+                    new_profile.generated_from_request_id,
+                    new_profile.profile_time_to_live.value,
+                    new_profile.expiration_timestamp,
+                    _json_dumps(new_profile.custom_features),
+                    _json_dumps(new_profile.embedding),
+                    new_profile.source,
+                    new_profile.status.value if new_profile.status else None,
+                    _json_dumps(new_profile.extractor_names),
+                    new_profile.expanded_terms,
+                    new_profile.source_span,
+                    new_profile.notes,
+                    new_profile.reader_angle,
+                    _json_dumps(new_profile.tags),
+                    _json_dumps(new_profile.source_interaction_ids),
+                    profile_id,
+                ),
+            )
+            if cur.rowcount > 0:
+                _append_event_stmt(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_type="profile",
+                    entity_id=str(profile_id),
+                    op="revise",
+                    prov="wasRevisionOf",
+                    source_ids=[],
+                    actor="api",
+                    request_id=uuid.uuid4().hex,
+                    reason="in-place update",
+                )
+            self.conn.commit()
+            fts_parts = [new_profile.content or ""]
+            if new_profile.custom_features:
+                fts_parts.extend(
+                    str(v) for v in new_profile.custom_features.values() if v
+                )
+            if new_profile.expanded_terms:
+                fts_parts.append(new_profile.expanded_terms)
+            self._fts_upsert_profile(profile_id, " ".join(fts_parts))
+            rowid_row = self._fetchone(
+                "SELECT rowid FROM profiles WHERE profile_id = ?", (profile_id,)
+            )
+            if rowid_row and embedding:
+                self._vec_upsert("profiles_vec", rowid_row["rowid"], embedding)
+
+    @SQLiteStorageBase.handle_exceptions
+    def update_user_profile_tags(
+        self, user_id: str, profile_id: str, tags: list[str]
+    ) -> None:
+        with self._lock:
+            if (
+                self._assert_profile_writable_locked(profile_id, user_id=user_id)
+                is None
+            ):
+                return
+            self.conn.execute(
+                "UPDATE profiles SET tags=? WHERE user_id=? AND profile_id=?",
+                (_json_dumps(tags), user_id, profile_id),
+            )
+            self.conn.commit()
 
     @SQLiteStorageBase.handle_exceptions
     def delete_user_profile(self, request: DeleteUserProfileRequest) -> None:
-        rowid_row = self._fetchone(
-            "SELECT rowid FROM profiles WHERE profile_id = ?",
-            (request.profile_id,),
-        )
-        self._fts_delete_profile(request.profile_id)
-        if rowid_row:
-            self._vec_delete("profiles_vec", rowid_row["rowid"])
-        self._execute(
-            "DELETE FROM profiles WHERE user_id = ? AND profile_id = ?",
-            (request.user_id, request.profile_id),
-        )
+        # Atomic: fts + vec + row + lineage in ONE lock/commit to prevent rowid reuse
+        # race. profiles uses implicit (reusable) rowid keyed by TEXT PK — a cleanup
+        # running after commit could race with a concurrent INSERT reusing the freed
+        # rowid and delete the NEW profile's vec row. (#196)
+        with self._lock:
+            rowid_row = self.conn.execute(
+                "SELECT rowid FROM profiles WHERE user_id = ? AND profile_id = ?",
+                (request.user_id, request.profile_id),
+            ).fetchone()
+            if rowid_row is None:
+                return
+            self.conn.execute(
+                "DELETE FROM profiles_fts WHERE profile_id = ?",
+                (request.profile_id,),
+            )
+            if self._has_sqlite_vec and rowid_row:
+                self.conn.execute(
+                    "DELETE FROM profiles_vec WHERE rowid = ?",
+                    (rowid_row["rowid"],),
+                )
+            cur = self.conn.execute(
+                "DELETE FROM profiles WHERE user_id = ? AND profile_id = ?",
+                (request.user_id, request.profile_id),
+            )
+            if cur.rowcount > 0:
+                _emit_hard_delete_profile(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_id=str(request.profile_id),
+                    request_id=uuid.uuid4().hex,
+                )
+            self.conn.commit()
 
     @SQLiteStorageBase.handle_exceptions
     def delete_all_profiles_for_user(self, user_id: str) -> None:
-        pids = [
-            r["profile_id"]
-            for r in self._fetchall(
-                "SELECT profile_id FROM profiles WHERE user_id = ?", (user_id,)
-            )
-        ]
-        for pid in pids:
-            self._fts_delete_profile(pid)
-        self._execute("DELETE FROM profiles WHERE user_id = ?", (user_id,))
+        # Atomic: fts + vec + row + lineage in ONE lock/commit — rowid reuse race
+        # prevention (see delete_user_profile comment, #196).
+        batch_request_id = uuid.uuid4().hex
+        with self._lock:
+            rows = self.conn.execute(
+                "SELECT rowid, profile_id FROM profiles WHERE user_id = ?", (user_id,)
+            ).fetchall()
+            if not rows:
+                return
+            pids = [r["profile_id"] for r in rows]
+            rowids = [r["rowid"] for r in rows]
+            self._delete_in_chunks("profiles_fts", "profile_id", pids)
+            if self._has_sqlite_vec and rowids:
+                self._delete_in_chunks("profiles_vec", "rowid", rowids)
+            self.conn.execute("DELETE FROM profiles WHERE user_id = ?", (user_id,))
+            for pid in pids:
+                _emit_hard_delete_profile(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_id=str(pid),
+                    request_id=batch_request_id,
+                )
+            self.conn.commit()
 
     @SQLiteStorageBase.handle_exceptions
     def delete_all_profiles(self) -> None:
+        # Also wipe profiles_vec (full-wipe variant of the rowid-race fix, #196).
+        batch_request_id = uuid.uuid4().hex
         with self._lock:
+            pids = [
+                r["profile_id"]
+                for r in self.conn.execute("SELECT profile_id FROM profiles").fetchall()
+            ]
+            for pid in pids:
+                _emit_hard_delete_profile(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_id=str(pid),
+                    request_id=batch_request_id,
+                )
             self.conn.execute("DELETE FROM profiles_fts")
+            if self._has_sqlite_vec:
+                self.conn.execute("DELETE FROM profiles_vec")
             self.conn.execute("DELETE FROM profiles")
             self.conn.commit()
 
@@ -251,25 +510,64 @@ class ProfileMixin:
     ) -> int:
         new_val = new_status.value if new_status else None
         now_ts = _epoch_now()
-        params: list[Any] = [new_val, now_ts]
+        old_val_str = old_status.value if old_status else "None"
+        new_val_str = new_status.value if new_status else "None"
+        reason = f"{old_val_str}->{new_val_str}"
 
         if old_status is None or (
             hasattr(old_status, "value") and old_status.value is None
         ):
             where = "status IS NULL"
+            select_params: list[Any] = []
         else:
             where = "status = ?"
-            params.append(old_status.value)
+            select_params = [old_status.value]
 
+        extra_params: list[Any] = []
         if user_ids is not None:
             placeholders = ",".join("?" for _ in user_ids)
             where += f" AND user_id IN ({placeholders})"
-            params.extend(user_ids)
+            extra_params.extend(user_ids)
 
-        cur = self._execute(
-            f"UPDATE profiles SET status = ?, last_modified_timestamp = ? WHERE {where}",
-            params,
-        )
+        # Set retired_at = now when transitioning to a GC-eligible status; clear to NULL otherwise.
+        retired_at_val = now_ts if new_val in _GC_ELIGIBLE_STATUSES else None
+
+        batch_request_id = uuid.uuid4().hex
+        with self._lock:
+            affected = list(
+                self.conn.execute(
+                    f"SELECT profile_id, user_id, governance_subject_ref FROM profiles WHERE {where}",
+                    select_params + extra_params,
+                ).fetchall()
+            )
+            for row in affected:
+                self._assert_subject_writable_locked(
+                    self._subject_ref_from_profile_row(row)
+                )
+            cur = self.conn.execute(
+                f"UPDATE profiles SET status = ?, last_modified_timestamp = ?, retired_at = ? WHERE {where}",
+                [new_val, now_ts, retired_at_val] + select_params + extra_params,
+            )
+            from_val = old_status.value if old_status else None
+            to_val = new_status.value if new_status else None
+            for row in affected:
+                pid = row["profile_id"]
+                _append_event_stmt(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_type="profile",
+                    entity_id=str(pid),
+                    op="status_change",
+                    prov="wasInvalidatedBy",
+                    source_ids=[],
+                    actor="api",
+                    request_id=batch_request_id,
+                    reason=reason,
+                    from_status=from_val,
+                    to_status=to_val,
+                    status_namespace="lifecycle_status",
+                )
+            self.conn.commit()
         return cur.rowcount
 
     @SQLiteStorageBase.handle_exceptions
@@ -295,26 +593,214 @@ class ProfileMixin:
         return [_row_to_profile(r) for r in self._fetchall(sql, params)]
 
     @SQLiteStorageBase.handle_exceptions
-    def archive_profile_by_id(self, user_id: str, profile_id: str) -> bool:
-        cur = self._execute(
-            "UPDATE profiles SET status = ?, last_modified_timestamp = ? "
-            "WHERE profile_id = ? AND user_id = ? AND status IS NULL",
-            (Status.ARCHIVED.value, _epoch_now(), profile_id, user_id),
+    def get_profile_by_id(
+        self, profile_id: str, *, include_tombstones: bool = False
+    ) -> UserProfile | None:
+        """Fetch a single profile by primary key.
+
+        Args:
+            profile_id: The profile's primary key.
+            include_tombstones: When False (default), MERGED/SUPERSEDED profiles
+                return None. Set to True for lineage resolution (resolve_current).
+
+        Returns:
+            The UserProfile if found and not filtered, otherwise None.
+        """
+        sql = "SELECT * FROM profiles WHERE profile_id = ?"
+        if not include_tombstones:
+            sql += " AND (status IS NULL OR status NOT IN (?, ?))"
+            row = self._fetchone(sql, (profile_id, *_TOMBSTONE_STATUS_VALUES))
+        else:
+            row = self._fetchone(sql, (profile_id,))
+        return _row_to_profile(row) if row else None
+
+    @SQLiteStorageBase.handle_exceptions
+    def get_distinct_generated_from_request_ids(self) -> list[str]:
+        """Return DISTINCT non-empty generated_from_request_id values, including tombstones.
+
+        Returns:
+            list[str]: Distinct non-empty ``generated_from_request_id`` values.
+        """
+        rows = self._fetchall(
+            "SELECT DISTINCT generated_from_request_id FROM profiles"
+            " WHERE generated_from_request_id IS NOT NULL"
+            " AND generated_from_request_id != ''",
+            (),
         )
+        return [row[0] for row in rows]
+
+    @SQLiteStorageBase.handle_exceptions
+    def get_profiles_by_generated_from_request_id(
+        self,
+        request_id: str,
+    ) -> list[UserProfile]:
+        """Return all profiles for a generated_from_request_id, including tombstones.
+
+        Args:
+            request_id (str): The generated_from_request_id to filter on.
+
+        Returns:
+            list[UserProfile]: All matching profiles (any status).
+        """
+        rows = self._fetchall(
+            "SELECT * FROM profiles WHERE generated_from_request_id = ?",
+            (request_id,),
+        )
+        return [_row_to_profile(r) for r in rows]
+
+    def get_all_generated_profiles(self) -> list[UserProfile]:
+        """All profiles (any status) with a non-empty generated_from_request_id."""
+        rows = self._fetchall(
+            "SELECT * FROM profiles "
+            "WHERE generated_from_request_id IS NOT NULL "
+            "AND generated_from_request_id <> ''",
+        )
+        return [_row_to_profile(r) for r in rows]
+
+    @SQLiteStorageBase.handle_exceptions
+    def archive_profile_by_id(self, user_id: str, profile_id: str) -> bool:
+        with self._lock:
+            if (
+                self._assert_profile_writable_locked(profile_id, user_id=user_id)
+                is None
+            ):
+                return False
+            now_ts = _epoch_now()
+            cur = self.conn.execute(
+                "UPDATE profiles SET status = ?, last_modified_timestamp = ?, retired_at = ? "
+                "WHERE profile_id = ? AND user_id = ? AND status IS NULL",
+                (Status.ARCHIVED.value, now_ts, now_ts, profile_id, user_id),
+            )
+            if cur.rowcount > 0:
+                _append_event_stmt(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_type="profile",
+                    entity_id=str(profile_id),
+                    op="status_change",
+                    prov="wasInvalidatedBy",
+                    source_ids=[],
+                    actor="api",
+                    request_id=uuid.uuid4().hex,
+                    reason="None->archived",
+                    from_status=None,
+                    to_status="archived",
+                    status_namespace="lifecycle_status",
+                )
+            self.conn.commit()
         return cur.rowcount > 0
 
+    @SQLiteStorageBase.handle_exceptions
+    def supersede_profiles_by_ids(
+        self,
+        user_id: str,
+        profile_ids: list[str],
+        request_id: str,
+    ) -> list[str]:
+        """Soft-delete profiles by setting status to SUPERSEDED, emitting set-based lineage.
+
+        For each matching id (user_id scoped, currently CURRENT), updates status to
+        SUPERSEDED and emits one ``status_change`` event under the shared ``request_id``.
+        Atomic: one ``conn.commit()`` at the end, guarded on rowcount per id.
+        FTS/vec rows are NOT removed — reads exclude tombstones by status filter.
+
+        Args:
+            user_id (str): Owning user id.
+            profile_ids (list[str]): Profile ids to supersede.
+            request_id (str): Shared request id for all emitted lineage events.
+
+        Returns:
+            list[str]: The profile ids actually superseded by this call, in input order
+                (already-superseded or absent ids are omitted).
+        """
+        if not profile_ids:
+            return []
+        if not request_id:
+            raise ValueError("request_id must be non-empty for supersede")
+        now_ts = _epoch_now()
+        # Eligibility: CURRENT (NULL) or PENDING — the two live statuses dedup can target.
+        eligible = (None, Status.PENDING.value)
+        committed_ids: list[str] = []
+        with self._lock:
+            for pid in profile_ids:
+                # Read current status for from_status derivation (user_id scoped)
+                row = self.conn.execute(
+                    "SELECT status, user_id, governance_subject_ref FROM profiles WHERE profile_id = ? AND user_id = ?",
+                    (pid, user_id),
+                ).fetchone()
+                if row is None:
+                    continue
+                self._assert_subject_writable_locked(
+                    self._subject_ref_from_profile_row(row)
+                )
+                old_status_val = (
+                    row[0] if isinstance(row, (tuple, list)) else row["status"]
+                )
+                if old_status_val not in eligible:
+                    continue
+                cur = self.conn.execute(
+                    "UPDATE profiles SET status = ?, last_modified_timestamp = ?, retired_at = ? "
+                    "WHERE profile_id = ? AND user_id = ? "
+                    "AND (status IS NULL OR status = ?)",
+                    (
+                        Status.SUPERSEDED.value,
+                        now_ts,
+                        now_ts,
+                        pid,
+                        user_id,
+                        Status.PENDING.value,
+                    ),
+                )
+                if cur.rowcount > 0:
+                    _append_event_stmt(
+                        self.conn,
+                        org_id=self.org_id,
+                        entity_type="profile",
+                        entity_id=str(pid),
+                        op="status_change",
+                        prov="wasInvalidatedBy",
+                        source_ids=[],
+                        actor="dedup",
+                        request_id=request_id,
+                        reason=f"{old_status_val}->superseded",
+                        from_status=old_status_val,
+                        to_status=Status.SUPERSEDED.value,
+                        status_namespace="lifecycle_status",
+                    )
+                    committed_ids.append(pid)
+            self.conn.commit()
+        return committed_ids
+
     @SQLiteStorageBase.handle_exceptions
     def delete_all_profiles_by_status(self, status: Status) -> int:
-        # Clean up FTS for profiles being deleted
-        pids = [
-            r["profile_id"]
-            for r in self._fetchall(
-                "SELECT profile_id FROM profiles WHERE status = ?", (status.value,)
+        # Atomic: fts + vec + row + lineage in ONE lock/commit — rowid reuse race
+        # prevention (see delete_user_profile comment, #196).
+        batch_request_id = uuid.uuid4().hex
+        with self._lock:
+            rows = self.conn.execute(
+                "SELECT rowid, profile_id FROM profiles WHERE status = ?",
+                (status.value,),
+            ).fetchall()
+            if not rows:
+                return 0
+            pids = [r["profile_id"] for r in rows]
+            rowids = [r["rowid"] for r in rows]
+            self._delete_in_chunks("profiles_fts", "profile_id", pids)
+            if self._has_sqlite_vec and rowids:
+                self._delete_in_chunks("profiles_vec", "rowid", rowids)
+            ph = ",".join("?" for _ in pids)
+            cur = self.conn.execute(
+                f"DELETE FROM profiles WHERE profile_id IN ({ph})",
+                pids,  # noqa: S608
             )
-        ]
-        for pid in pids:
-            self._fts_delete_profile(pid)
-        cur = self._execute("DELETE FROM profiles WHERE status = ?", (status.value,))
+            for pid in pids:
+                _emit_hard_delete_profile(
+                    self.conn,
+                    org_id=self.org_id,
+                    entity_id=str(pid),
+                    request_id=batch_request_id,
+                )
+            self.conn.commit()
         return cur.rowcount
 
     @SQLiteStorageBase.handle_exceptions
@@ -331,15 +817,41 @@ class ProfileMixin:
         return [r["user_id"] for r in rows]
 
     @SQLiteStorageBase.handle_exceptions
-    def delete_profiles_by_ids(self, profile_ids: list[str]) -> int:
+    def delete_profiles_by_ids(
+        self, profile_ids: list[str], *, emit_hard_delete: bool = True
+    ) -> int:
         if not profile_ids:
             return 0
-        for pid in profile_ids:
-            self._fts_delete_profile(pid)
+        # Atomic: fts + vec + row + lineage in ONE lock/commit — rowid reuse race
+        # prevention (see delete_user_profile comment, #196).
         ph = ",".join("?" for _ in profile_ids)
-        cur = self._execute(
-            f"DELETE FROM profiles WHERE profile_id IN ({ph})", profile_ids
-        )
+        batch_request_id = uuid.uuid4().hex
+        with self._lock:
+            pre_rows = self.conn.execute(
+                f"SELECT rowid, profile_id FROM profiles WHERE profile_id IN ({ph})",
+                profile_ids,
+            ).fetchall()
+            if not pre_rows:
+                return 0
+            existing = [r["profile_id"] for r in pre_rows]
+            rowids = [r["rowid"] for r in pre_rows]
+            self._delete_in_chunks("profiles_fts", "profile_id", existing)
+            if self._has_sqlite_vec and rowids:
+                self._delete_in_chunks("profiles_vec", "rowid", rowids)
+            cur = self.conn.execute(
+                f"DELETE FROM profiles WHERE profile_id IN ({ph})",
+                profile_ids,  # noqa: S608
+            )
+            if emit_hard_delete:
+                for pid in existing:
+                    _emit_hard_delete_profile(
+                        self.conn,
+                        org_id=self.org_id,
+                        entity_id=str(pid),
+                        request_id=batch_request_id,
+                        actor="system",
+                    )
+            self.conn.commit()
         return cur.rowcount
 
     # ------------------------------------------------------------------
@@ -370,60 +882,79 @@ class ProfileMixin:
 
     def _insert_interaction(self, interaction: Interaction) -> int:
         created_at_iso = _epoch_to_iso(interaction.created_at)
+        subject_ref = self._subject_ref_for_user_id(interaction.user_id)
         with self._lock:
-            if interaction.interaction_id:
-                self.conn.execute(
-                    """INSERT OR REPLACE INTO interactions
-                       (interaction_id, user_id, content, request_id, created_at,
-                        role, user_action, user_action_description,
-                        interacted_image_url, shadow_content, expert_content,
-                        tools_used, citations, embedding)
-                       VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
-                    (
-                        interaction.interaction_id,
-                        interaction.user_id,
-                        interaction.content,
-                        interaction.request_id,
-                        created_at_iso,
-                        interaction.role,
-                        interaction.user_action.value,
-                        interaction.user_action_description,
-                        interaction.interacted_image_url,
-                        interaction.shadow_content,
-                        interaction.expert_content,
-                        _json_dumps([t.model_dump() for t in interaction.tools_used]),
-                        _json_dumps([c.model_dump() for c in interaction.citations]),
-                        _json_dumps(interaction.embedding),
-                    ),
-                )
-                iid = interaction.interaction_id
-            else:
-                cur = self.conn.execute(
-                    """INSERT INTO interactions
-                       (user_id, content, request_id, created_at,
-                        role, user_action, user_action_description,
-                        interacted_image_url, shadow_content, expert_content,
-                        tools_used, citations, embedding)
-                       VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
-                    (
-                        interaction.user_id,
-                        interaction.content,
-                        interaction.request_id,
-                        created_at_iso,
-                        interaction.role,
-                        interaction.user_action.value,
-                        interaction.user_action_description,
-                        interaction.interacted_image_url,
-                        interaction.shadow_content,
-                        interaction.expert_content,
-                        _json_dumps([t.model_dump() for t in interaction.tools_used]),
-                        _json_dumps([c.model_dump() for c in interaction.citations]),
-                        _json_dumps(interaction.embedding),
-                    ),
-                )
-                iid = cur.lastrowid or 0
-                interaction.interaction_id = iid
-            self.conn.commit()
+            try:
+                self.conn.execute("BEGIN IMMEDIATE")
+                self._assert_subject_writable_locked(subject_ref)
+                if interaction.interaction_id:
+                    self.conn.execute(
+                        """INSERT OR REPLACE INTO interactions
+                           (interaction_id, user_id, content, request_id, created_at,
+                            role, user_action, user_action_description,
+                            interacted_image_url, image_encoding, shadow_content,
+                            expert_content, tools_used, citations, embedding, governance_subject_ref)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                        (
+                            interaction.interaction_id,
+                            interaction.user_id,
+                            interaction.content,
+                            interaction.request_id,
+                            created_at_iso,
+                            interaction.role,
+                            interaction.user_action.value,
+                            interaction.user_action_description,
+                            interaction.interacted_image_url,
+                            interaction.image_encoding,
+                            interaction.shadow_content,
+                            interaction.expert_content,
+                            _json_dumps(
+                                [t.model_dump() for t in interaction.tools_used]
+                            ),
+                            _json_dumps(
+                                [c.model_dump() for c in interaction.citations]
+                            ),
+                            _json_dumps(interaction.embedding),
+                            subject_ref,
+                        ),
+                    )
+                    iid = interaction.interaction_id
+                else:
+                    cur = self.conn.execute(
+                        """INSERT INTO interactions
+                           (user_id, content, request_id, created_at,
+                            role, user_action, user_action_description,
+                            interacted_image_url, image_encoding, shadow_content,
+                            expert_content, tools_used, citations, embedding, governance_subject_ref)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                        (
+                            interaction.user_id,
+                            interaction.content,
+                            interaction.request_id,
+                            created_at_iso,
+                            interaction.role,
+                            interaction.user_action.value,
+                            interaction.user_action_description,
+                            interaction.interacted_image_url,
+                            interaction.image_encoding,
+                            interaction.shadow_content,
+                            interaction.expert_content,
+                            _json_dumps(
+                                [t.model_dump() for t in interaction.tools_used]
+                            ),
+                            _json_dumps(
+                                [c.model_dump() for c in interaction.citations]
+                            ),
+                            _json_dumps(interaction.embedding),
+                            subject_ref,
+                        ),
+                    )
+                    iid = cur.lastrowid or 0
+                    interaction.interaction_id = iid
+                self.conn.commit()
+            except Exception:
+                self.conn.rollback()
+                raise
         # Update FTS and vec
         self._fts_upsert(
             "interactions_fts",
@@ -464,34 +995,49 @@ class ProfileMixin:
 
     @SQLiteStorageBase.handle_exceptions
     def delete_user_interaction(self, request: DeleteUserInteractionRequest) -> None:
-        self._fts_delete("interactions_fts", request.interaction_id)
-        self._execute(
-            "DELETE FROM interactions WHERE user_id = ? AND interaction_id = ?",
-            (request.user_id, request.interaction_id),
-        )
+        with self._lock:
+            row = self.conn.execute(
+                "SELECT interaction_id FROM interactions WHERE user_id = ? AND interaction_id = ?",
+                (request.user_id, request.interaction_id),
+            ).fetchone()
+            if row is None:
+                return
+            self.conn.execute(
+                "DELETE FROM interactions_fts WHERE rowid = ?",
+                (request.interaction_id,),
+            )
+            if self._has_sqlite_vec:
+                self.conn.execute(
+                    "DELETE FROM interactions_vec WHERE rowid = ?",
+                    (request.interaction_id,),
+                )
+            self.conn.execute(
+                "DELETE FROM interactions WHERE user_id = ? AND interaction_id = ?",
+                (request.user_id, request.interaction_id),
+            )
+            self.conn.commit()
 
     @SQLiteStorageBase.handle_exceptions
     def delete_all_interactions_for_user(self, user_id: str) -> None:
-        # Delete FTS entries for this user's interactions
-        ids = [
-            r["interaction_id"]
-            for r in self._fetchall(
+        with self._lock:
+            rows = self.conn.execute(
                 "SELECT interaction_id FROM interactions WHERE user_id = ?", (user_id,)
-            )
-        ]
-        if ids:
-            placeholders = ",".join("?" for _ in ids)
-            with self._lock:
-                self.conn.execute(
-                    f"DELETE FROM interactions_fts WHERE rowid IN ({placeholders})", ids
-                )
-                self.conn.commit()
-        self._execute("DELETE FROM interactions WHERE user_id = ?", (user_id,))
+            ).fetchall()
+            if not rows:
+                return
+            ids = [r["interaction_id"] for r in rows]
+            self._delete_in_chunks("interactions_fts", "rowid", ids)
+            if self._has_sqlite_vec:
+                self._delete_in_chunks("interactions_vec", "rowid", ids)
+            self.conn.execute("DELETE FROM interactions WHERE user_id = ?", (user_id,))
+            self.conn.commit()
 
     @SQLiteStorageBase.handle_exceptions
     def delete_all_interactions(self) -> None:
         with self._lock:
             self.conn.execute("DELETE FROM interactions_fts")
+            if self._has_sqlite_vec:
+                self.conn.execute("DELETE FROM interactions_vec")
             self.conn.execute("DELETE FROM interactions")
             self.conn.commit()
 
@@ -504,22 +1050,18 @@ class ProfileMixin:
     def delete_oldest_interactions(self, count: int) -> int:
         if count <= 0:
             return 0
-        rows = self._fetchall(
-            "SELECT interaction_id FROM interactions ORDER BY created_at ASC LIMIT ?",
-            (count,),
-        )
-        if not rows:
-            return 0
-        ids = [r["interaction_id"] for r in rows]
-        placeholders = ",".join("?" for _ in ids)
         with self._lock:
-            self.conn.execute(
-                f"DELETE FROM interactions_fts WHERE rowid IN ({placeholders})", ids
-            )
-            self.conn.execute(
-                f"DELETE FROM interactions WHERE interaction_id IN ({placeholders})",
-                ids,
-            )
+            rows = self.conn.execute(
+                "SELECT interaction_id FROM interactions ORDER BY created_at ASC LIMIT ?",
+                (count,),
+            ).fetchall()
+            if not rows:
+                return 0
+            ids = [r["interaction_id"] for r in rows]
+            self._delete_in_chunks("interactions_fts", "rowid", ids)
+            if self._has_sqlite_vec:
+                self._delete_in_chunks("interactions_vec", "rowid", ids)
+            self._delete_in_chunks("interactions", "interaction_id", ids)
             self.conn.commit()
         return len(ids)
 
@@ -657,6 +1199,10 @@ class ProfileMixin:
             frag, sparams = _build_status_sql(status_filter)
             conditions.append(frag)
             params.extend(sparams)
+        tag_frag, tag_params = _build_tags_sql("p", req.tags)
+        if tag_frag:
+            conditions.append(tag_frag)
+            params.extend(tag_params)
 
         where_clause = " AND ".join(conditions)
         overfetch = match_count * 5 if mode != SearchMode.FTS else match_count