claude-smart 0.2.47 → 0.2.49

package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py

@@ -7,6 +7,7 @@ are available.
 
 """
 
+import contextlib
 import functools
 import json
 import logging
@@ -14,7 +15,7 @@ import math
 import re
 import sqlite3
 import threading
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Generator, Sequence
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, ClassVar, Literal
@@ -53,12 +54,7 @@ from reflexio.server.services.storage.error import (
     StorageError,
     require_non_empty_session_id,
 )
-from reflexio.server.services.storage.retention import RetentionTarget
-from reflexio.server.services.storage.retention_mixin import (
-    RETENTION_DELETE_CHUNK,
-    RetentionMixin,
-    chunked,
-)
+from reflexio.server.services.storage.retention_mixin import RetentionMixin
 from reflexio.server.services.storage.storage_base import BaseStorage
 from reflexio.server.site_var.site_var_manager import SiteVarManager
 
@@ -288,6 +284,42 @@ def _true_rrf_merge(
 # explicit status_filter on list/count methods.
 _TOMBSTONE_STATUS_VALUES = (Status.MERGED.value, Status.SUPERSEDED.value)
 
+# Profile reads also treat EXPIRED (TTL-expired tombstone) as non-current. Kept
+# separate from _TOMBSTONE_STATUS_VALUES because that tuple is shared with playbook
+# queries (which never carry EXPIRED) via hardcoded placeholders.
+_PROFILE_TOMBSTONE_STATUS_VALUES = (
+    Status.MERGED.value,
+    Status.SUPERSEDED.value,
+    Status.EXPIRED.value,
+)
+
+
+def parse_status(value: str | None) -> Status | None:
+    """Parse a stored status string into a Status, tolerating unknown values.
+
+    Unknown non-null values (e.g. a status written by a newer build after a
+    rollback) map to a non-current tombstone sentinel and log an anomaly, rather
+    than raising ValueError. Never maps to None (which models CURRENT/live).
+
+    Args:
+        value: Raw status string from the database, or None/empty for CURRENT.
+
+    Returns:
+        The matching Status enum member, Status.SUPERSEDED for unknown values,
+        or None for falsy input (representing the CURRENT/live state).
+    """
+    if not value:
+        return None
+    try:
+        return Status(value)
+    except ValueError:
+        from reflexio.server.tracing import capture_anomaly
+
+        capture_anomaly(
+            "storage.status.unknown_value", level="warning", status_value=value
+        )
+        return Status.SUPERSEDED
+
 
 def _status_value(status: Status | None) -> str | None:
     """Convert a Status enum (or None) to its DB string value."""
@@ -363,14 +395,19 @@ def _iso_to_epoch(iso_str: str | None) -> int:
 # stored row) with a valid value.
 _MAX_SAFE_EPOCH_TS = 253_402_300_799  # 9999-12-31T23:59:59Z
 _MIN_SAFE_EPOCH_TS = 0  # 1970-01-01T00:00:00Z
+_MIN_CONTEMPORARY_MILLISECOND_EPOCH_TS = 1_500_000_000_000
 
 
 def _epoch_to_iso(ts: int) -> str:
     """Convert a Unix timestamp (seconds) to an ISO 8601 string.
 
-    Out-of-range sentinel bounds are clamped to the representable range so that
-    callers passing "open" window bounds never trigger a ``ValueError``.
+    Plausible contemporary millisecond timestamps are normalized to seconds.
+    Lower values stay in seconds-space so documented sentinel bounds such as
+    ``to_ts=10**12`` still clamp to the representable range instead of being
+    treated as a 2001 millisecond timestamp.
     """
+    if _MIN_CONTEMPORARY_MILLISECOND_EPOCH_TS <= ts <= _MAX_SAFE_EPOCH_TS * 1000:
+        ts = ts // 1000
     clamped = max(_MIN_SAFE_EPOCH_TS, min(ts, _MAX_SAFE_EPOCH_TS))
     return datetime.fromtimestamp(clamped, tz=UTC).isoformat()
 
@@ -392,7 +429,7 @@ def _row_to_profile(row: sqlite3.Row) -> UserProfile:
         expiration_timestamp=d["expiration_timestamp"],
         custom_features=_json_loads(d.get("custom_features")),
         source=d.get("source") or "",
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         extractor_names=_json_loads(d.get("extractor_names")),
         expanded_terms=d.get("expanded_terms"),
         source_span=d.get("source_span"),
@@ -472,7 +509,7 @@ def _row_to_user_playbook(
         blocking_issue=BlockingIssue(**json.loads(d["blocking_issue"]))
         if d.get("blocking_issue")
         else None,
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         source=d.get("source"),
         source_interaction_ids=_json_loads(d.get("source_interaction_ids")) or [],
         tags=_json_loads(d.get("tags")),
@@ -505,7 +542,7 @@ def _row_to_agent_playbook(row: sqlite3.Row) -> AgentPlaybook:
         playbook_metadata=d.get("playbook_metadata") or "",
         tags=_json_loads(d.get("tags")),
         embedding=[],
-        status=Status(d["status"]) if d.get("status") else None,
+        status=parse_status(d.get("status")),
         expanded_terms=d.get("expanded_terms"),
         merged_into=d.get("merged_into"),
         superseded_by=d.get("superseded_by"),
@@ -546,6 +583,16 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
 
     supports_embedding: ClassVar[bool] = True
 
+    # Chunked bulk-delete helpers provided by SQLiteDeletionMixin via the
+    # composed SQLiteStorage MRO; declared here for clear_user_data's benefit.
+    _delete_in_chunks: Any
+    _delete_source_windows_for_user_playbook_ids: Any
+
+    # FTS/vec index helpers provided by SQLiteFtsVecMixin via the composed
+    # SQLiteStorage MRO; declared here for _migrate_vec_tables's benefit.
+    _vec_upsert: Any
+    _flush_index_op: Any
+
     @staticmethod
     def handle_exceptions(func: Callable[..., Any]) -> Callable[..., Any]:
         @functools.wraps(func)
@@ -588,6 +635,10 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
 
         self.db_path = db_path
         self._lock = threading.RLock()
+        self._scope_depth = 0
+        self._deferred_index_ops: list[
+            tuple[str, Any]
+        ] = []  # (kind, args) flushed post-commit
 
         logger.info("SQLite Storage for org %s using db_path: %s", org_id, db_path)
 
@@ -625,6 +676,45 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         # Create tables
         self.migrate()
 
+    # ------------------------------------------------------------------
+    # Transaction scope
+    # ------------------------------------------------------------------
+
+    def _own_transaction(self) -> bool:
+        """True when the caller is NOT inside a commit_scope (owns its BEGIN/commit)."""
+        return self._scope_depth == 0
+
+    @contextlib.contextmanager
+    def commit_scope(self) -> Generator[None, None, None]:
+        """Group writes into one atomic commit; defer FTS/vec index ops.
+
+        Nested scopes join the outermost: only the outer scope commits. On
+        exception the whole transaction rolls back and deferred index ops
+        are discarded.
+        """
+        with self._lock:
+            if self._scope_depth > 0:
+                self._scope_depth += 1
+                try:
+                    yield
+                finally:
+                    self._scope_depth -= 1
+                return
+            self.conn.execute("BEGIN IMMEDIATE")
+            self._scope_depth = 1
+            try:
+                yield
+                self.conn.commit()
+                ops, self._deferred_index_ops = self._deferred_index_ops, []
+                for kind, args in ops:
+                    self._flush_index_op(kind, args)  # self-commits — fine post-commit
+            except Exception:
+                self.conn.rollback()
+                self._deferred_index_ops = []
+                raise
+            finally:
+                self._scope_depth = 0
+
     # ------------------------------------------------------------------
     # DDL / migration
     # ------------------------------------------------------------------
@@ -669,241 +759,9 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
         self._migrate_retire_profile_change_logs()
         self._migrate_retire_playbook_aggregation_change_logs()
         init_stall_state_table(self.conn)
+        self._migrate_learning_jobs()
         return True
 
-    # -- Retention hooks (see RetentionMixin) --
-
-    @handle_exceptions
-    def _retention_table_exists(self, table_name: str) -> bool:
-        row = self._fetchone(
-            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
-            (table_name,),
-        )
-        return row is not None
-
-    @handle_exceptions
-    def _retention_count_rows(self, target: RetentionTarget) -> int:
-        row = self._fetchone(f"SELECT COUNT(*) as cnt FROM {target.table_name}")  # noqa: S608
-        return int(row["cnt"]) if row else 0
-
-    @handle_exceptions
-    def _retention_select_oldest_keys(
-        self, target: RetentionTarget, count: int
-    ) -> list[tuple[Any, ...]]:
-        id_sql = ", ".join(target.id_columns)
-        tiebreak_sql = id_sql
-        rows = self._fetchall(
-            f"SELECT {id_sql} FROM {target.table_name} "  # noqa: S608
-            f"ORDER BY {target.order_column} ASC, {tiebreak_sql} ASC LIMIT ?",
-            (count,),
-        )
-        return [tuple(row[col] for col in target.id_columns) for row in rows]
-
-    @handle_exceptions
-    def _retention_perform_delete(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        # Wrap dependency + target deletes in a single critical section so
-        # concurrent writers see either both or neither.
-        with self._lock:
-            try:
-                self._retention_delete_dependencies(target, keys)
-                self._retention_delete_target_rows(target, keys)
-                self.conn.commit()
-            except Exception:
-                self.conn.rollback()
-                raise
-
-    def _retention_delete_dependencies(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        ids = [key[0] for key in keys]
-        target_name = target.name
-        if target_name == "requests":
-            self._delete_interactions_for_request_ids([str(v) for v in ids])
-        elif target_name == "interactions":
-            self._delete_interaction_search_rows([int(v) for v in ids])
-        elif target_name == "profiles":
-            self._delete_profile_search_rows([str(v) for v in ids])
-        elif target_name == "user_playbooks":
-            self._delete_source_windows_for_user_playbook_ids([int(v) for v in ids])
-            self._delete_playbook_search_rows(
-                "user", [int(v) for v in ids], commit=False
-            )
-        elif target_name == "agent_playbooks":
-            self._delete_source_windows_for_agent_playbook_ids([int(v) for v in ids])
-            self._delete_playbook_search_rows(
-                "agent", [int(v) for v in ids], commit=False
-            )
-        elif target_name == "playbook_optimization_jobs":
-            self._delete_optimizer_rows_for_job_ids([int(v) for v in ids])
-        elif target_name == "playbook_optimization_candidates":
-            self._delete_optimizer_evaluations_for_candidate_ids([int(v) for v in ids])
-
-    def _retention_delete_target_rows(
-        self, target: RetentionTarget, keys: list[tuple[Any, ...]]
-    ) -> None:
-        if len(target.id_columns) == 1:
-            self._delete_in_chunks(
-                target.table_name,
-                target.id_columns[0],
-                [key[0] for key in keys],
-            )
-            return
-        # Composite-key delete: chunk by row to bound parameter count.
-        params_per_key = len(target.id_columns)
-        rows_per_chunk = max(1, RETENTION_DELETE_CHUNK // params_per_key)
-        for chunk in chunked(keys, rows_per_chunk):
-            where = " OR ".join(
-                "("
-                + " AND ".join(f"{column} = ?" for column in target.id_columns)
-                + ")"
-                for _ in chunk
-            )
-            params = [value for key in chunk for value in key]
-            self.conn.execute(
-                f"DELETE FROM {target.table_name} WHERE {where}",  # noqa: S608
-                params,
-            )
-
-    # -- Chunked-delete primitives shared by the cascade helpers --
-
-    def _delete_in_chunks(
-        self, table_name: str, column_name: str, values: list[Any]
-    ) -> None:
-        """Chunked ``DELETE FROM table WHERE col IN (...)``.
-
-        Chunking keeps parameter count under ``SQLITE_MAX_VARIABLE_NUMBER``
-        on older sqlite builds (default 999) and avoids degenerate plans
-        on very large IN lists.
-        """
-        if not values:
-            return
-        for chunk in chunked(values):
-            placeholders = ",".join("?" for _ in chunk)
-            self.conn.execute(
-                f"DELETE FROM {table_name} WHERE {column_name} IN ({placeholders})",  # noqa: S608
-                chunk,
-            )
-
-    def _select_in_chunks(self, sql_template: str, values: list[Any]) -> list[Any]:
-        """Run ``sql_template`` (containing ``{placeholders}``) over chunks of
-        ``values`` and aggregate the result rows."""
-        results: list[Any] = []
-        for chunk in chunked(values):
-            placeholders = ",".join("?" for _ in chunk)
-            stmt = sql_template.format(placeholders=placeholders)
-            results.extend(self.conn.execute(stmt, chunk).fetchall())
-        return results
-
-    def _delete_interactions_for_request_ids(self, request_ids: list[str]) -> None:
-        if not request_ids:
-            return
-        rows = self._select_in_chunks(
-            "SELECT interaction_id FROM interactions WHERE request_id IN ({placeholders})",
-            request_ids,
-        )
-        self._delete_interaction_search_rows(
-            [int(row["interaction_id"]) for row in rows]
-        )
-        self._delete_in_chunks("interactions", "request_id", request_ids)
-
-    def _delete_interaction_search_rows(self, interaction_ids: list[int]) -> None:
-        """Remove fts + vec index rows for the given interaction IDs.
-
-        Non-committing: participates in the caller's transaction.  Only called
-        from inside the retention atomic block (_retention_perform_delete).
-        """
-        if not interaction_ids:
-            return
-        self._delete_in_chunks("interactions_fts", "rowid", interaction_ids)
-        if self._has_sqlite_vec:
-            self._delete_in_chunks("interactions_vec", "rowid", interaction_ids)
-
-    def _delete_profile_search_rows(self, profile_ids: list[str]) -> None:
-        """Remove fts + vec index rows for the given profile IDs.
-
-        Non-committing: participates in the caller's transaction.  Only called
-        from inside the retention atomic block (_retention_perform_delete).
-        profiles_fts is keyed by profile_id (TEXT); profiles_vec by rowid (INT).
-        """
-        if not profile_ids:
-            return
-        self._delete_in_chunks("profiles_fts", "profile_id", profile_ids)
-        if self._has_sqlite_vec:
-            rows = self._select_in_chunks(
-                "SELECT rowid FROM profiles WHERE profile_id IN ({placeholders})",
-                profile_ids,
-            )
-            rowids = [row["rowid"] for row in rows]
-            if rowids:
-                self._delete_in_chunks("profiles_vec", "rowid", rowids)
-
-    def _delete_playbook_search_rows(
-        self, kind: str, ids: list[int], *, commit: bool = True
-    ) -> None:
-        """Remove fts + vec index rows for the given playbook IDs.
-
-        Args:
-            kind: ``"user"`` or ``"agent"``.
-            ids: Playbook row IDs to remove from the search indexes.
-            commit: When ``True`` (default) commits after the deletes so the
-                after-commit callers in ``_playbook.py`` get a clean, durable
-                cleanup.  Pass ``commit=False`` from inside the retention atomic
-                block so the deletes participate in the single block-level commit
-                (``_retention_perform_delete``).
-
-        Note: callers may already hold ``self._lock`` when calling this (the
-        ``commit=False`` retention/atomic-delete call sites do). The internal
-        ``with self._lock:`` re-acquire is safe ONLY because ``self._lock`` is a
-        reentrant ``threading.RLock``; a non-reentrant lock would deadlock here.
-        """
-        if not ids:
-            return
-        with self._lock:
-            self._delete_in_chunks(f"{kind}_playbooks_fts", "rowid", ids)
-            if self._has_sqlite_vec:
-                self._delete_in_chunks(f"{kind}_playbooks_vec", "rowid", ids)
-            if commit:
-                self.conn.commit()
-
-    def _delete_source_windows_for_agent_playbook_ids(
-        self, agent_playbook_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "agent_playbook_source_user_playbooks",
-            "agent_playbook_id",
-            agent_playbook_ids,
-        )
-
-    def _delete_source_windows_for_user_playbook_ids(
-        self, user_playbook_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "agent_playbook_source_user_playbooks",
-            "user_playbook_id",
-            user_playbook_ids,
-        )
-
-    def _delete_optimizer_rows_for_job_ids(self, job_ids: list[int]) -> None:
-        if not job_ids:
-            return
-        for table in (
-            "playbook_optimization_evaluations",
-            "playbook_optimization_events",
-            "playbook_optimization_candidates",
-        ):
-            self._delete_in_chunks(table, "job_id", job_ids)
-
-    def _delete_optimizer_evaluations_for_candidate_ids(
-        self, candidate_ids: list[int]
-    ) -> None:
-        self._delete_in_chunks(
-            "playbook_optimization_evaluations",
-            "candidate_id",
-            candidate_ids,
-        )
-
     def _try_load_sqlite_vec(self) -> bool:
         """Attempt to load the sqlite-vec extension for native KNN search.
 
@@ -1449,6 +1307,70 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
             )
             self.conn.commit()
 
+    def _migrate_learning_jobs(self) -> None:
+        """Create the learning_jobs table + partial indexes if missing (idempotent).
+
+        Runs ``CREATE TABLE IF NOT EXISTS`` and ``CREATE INDEX IF NOT EXISTS`` only —
+        both are no-ops when the table / indexes already exist.  New columns added in
+        subsequent tasks are backfilled via ``PRAGMA table_info`` + ``ALTER TABLE … ADD
+        COLUMN`` (mirroring ``_migrate_lineage_event_table``), because
+        ``CREATE TABLE IF NOT EXISTS`` silently skips the DDL on existing databases.
+
+        Called at the end of migrate() so the table is always present on startup.
+        """
+        with self._lock:
+            self.conn.executescript("""
+                CREATE TABLE IF NOT EXISTS learning_jobs (
+                    job_id TEXT PRIMARY KEY,
+                    org_id TEXT NOT NULL,
+                    user_id TEXT NOT NULL,
+                    job_type TEXT NOT NULL DEFAULT 'learning',
+                    latest_request_id TEXT,
+                    status TEXT NOT NULL DEFAULT 'pending',
+                    attempts INTEGER NOT NULL DEFAULT 0,
+                    max_attempts INTEGER NOT NULL DEFAULT 3,
+                    claimed_by TEXT,
+                    claim_token TEXT,
+                    claim_expires_at TEXT,
+                    covers_through TEXT,
+                    force_extraction INTEGER NOT NULL DEFAULT 0,
+                    skip_aggregation INTEGER NOT NULL DEFAULT 0,
+                    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+                    updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+                );
+                CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
+                    ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
+                -- Recreate the poll index with 'failed' in the predicate. CREATE
+                -- INDEX IF NOT EXISTS is a no-op on an existing index with the old
+                -- ('pending','claimed') predicate, so DROP first to migrate it.
+                DROP INDEX IF EXISTS learning_jobs_poll;
+                CREATE INDEX IF NOT EXISTS learning_jobs_poll
+                    ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
+            """)
+            # Backfill columns added after the initial release (existing DBs skip
+            # CREATE TABLE IF NOT EXISTS so these must be applied separately).
+            existing_cols = {
+                row["name"]
+                for row in self.conn.execute(
+                    "PRAGMA table_info(learning_jobs)"
+                ).fetchall()
+            }
+            if "force_extraction" not in existing_cols:
+                self.conn.execute(
+                    "ALTER TABLE learning_jobs ADD COLUMN force_extraction INTEGER NOT NULL DEFAULT 0"
+                )
+            if "skip_aggregation" not in existing_cols:
+                self.conn.execute(
+                    "ALTER TABLE learning_jobs ADD COLUMN skip_aggregation INTEGER NOT NULL DEFAULT 0"
+                )
+            self.conn.commit()
+
+    def learning_jobs_columns(self) -> list[str]:
+        """Return the column names of the learning_jobs table."""
+        with self._lock:
+            rows = self.conn.execute("PRAGMA table_info(learning_jobs)").fetchall()
+        return [row["name"] for row in rows]
+
     def _migrate_agent_playbook_source_windows(self) -> None:
         """Add source window snapshots to existing agent source mappings."""
         cols = {
@@ -1639,7 +1561,8 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
     ) -> sqlite3.Cursor:
         with self._lock:
             cur = self.conn.execute(sql, params)
-            self.conn.commit()
+            if self._own_transaction():
+                self.conn.commit()
             return cur
 
     def _fetchone(
@@ -1717,113 +1640,6 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
     def _current_timestamp(self) -> str:
         return datetime.now(UTC).isoformat()
 
-    # FTS helpers
-    def _fts_upsert(self, table: str, rowid: int, **text_fields: str | None) -> None:
-        """Insert or update an FTS row.  Deletes old entry first to avoid duplicates."""
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            cols = list(text_fields.keys())
-            vals = [text_fields[c] or "" for c in cols]
-            placeholders = ",".join("?" for _ in cols)
-            col_str = ",".join(cols)
-            self.conn.execute(
-                f"INSERT INTO {table}(rowid, {col_str}) VALUES (?, {placeholders})",
-                [rowid, *vals],
-            )
-            self.conn.commit()
-
-    def _fts_delete(self, table: str, rowid: int) -> None:
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.commit()
-
-    def _fts_upsert_profile(self, profile_id: str, content: str) -> None:
-        """FTS for profiles uses profile_id TEXT as key column."""
-        with self._lock:
-            self.conn.execute(
-                "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
-            )
-            self.conn.execute(
-                "INSERT INTO profiles_fts(profile_id, content) VALUES (?, ?)",
-                (profile_id, content),
-            )
-            self.conn.commit()
-
-    def _fts_delete_profile(self, profile_id: str) -> None:
-        with self._lock:
-            self.conn.execute(
-                "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
-            )
-            self.conn.commit()
-
-    # Vec helpers (sqlite-vec)
-    def _vec_upsert(self, table: str, rowid: int, embedding: list[float]) -> None:
-        """Insert or update a vec table row. No-op when sqlite-vec is unavailable."""
-        if not self._has_sqlite_vec:
-            return
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.execute(
-                f"INSERT INTO {table}(rowid, embedding) VALUES (?, ?)",
-                (rowid, json.dumps(embedding)),
-            )
-            self.conn.commit()
-
-    def _vec_delete(self, table: str, rowid: int) -> None:
-        """Delete a vec table row. No-op when sqlite-vec is unavailable."""
-        if not self._has_sqlite_vec:
-            return
-        with self._lock:
-            self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,))
-            self.conn.commit()
-
-    def _vec_knn_search(
-        self,
-        vec_table: str,
-        main_table: str,
-        query_embedding: list[float],
-        match_count: int,
-        conditions: list[str] | None = None,
-        params: list[Any] | None = None,
-    ) -> list[sqlite3.Row]:
-        """Run a native KNN search via sqlite-vec and join back to the main table.
-
-        Over-fetches from the KNN index (5x ``match_count``) so that post-filter
-        WHERE conditions (org, user, status, etc.) don't silently reduce the
-        result set below the requested count.
-
-        Args:
-            vec_table: Name of the vec0 virtual table.
-            main_table: Name of the main data table.
-            query_embedding: Query embedding vector.
-            match_count: Number of results to return.
-            conditions: Optional WHERE conditions for the main table.
-            params: Parameters for the conditions.
-
-        Returns:
-            Up to ``match_count`` rows from the main table, ordered by vector
-            distance (ascending).
-        """
-        knn_overfetch = match_count * 5
-        where_clause = " AND ".join(conditions) if conditions else "1=1"
-        sql = f"""SELECT m.* FROM {main_table} m
-                  JOIN (
-                      SELECT rowid, distance FROM {vec_table}
-                      WHERE embedding MATCH ?
-                      ORDER BY distance
-                      LIMIT ?
-                  ) v ON m.rowid = v.rowid
-                  WHERE {where_clause}
-                  ORDER BY v.distance
-                  LIMIT ?"""
-        all_params = [
-            json.dumps(query_embedding),
-            knn_overfetch,
-            *(params or []),
-            match_count,
-        ]
-        return self._fetchall(sql, all_params)
-
     # ------------------------------------------------------------------
     # Per-user data clear
     # ------------------------------------------------------------------
@@ -2389,4 +2205,31 @@ CREATE TABLE IF NOT EXISTS lineage_event (
 );
 CREATE INDEX IF NOT EXISTS idx_lineage_entity ON lineage_event (entity_type, entity_id);
 
+-- ============================================================================
+-- Durable learning pipeline — cross-org job queue
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS learning_jobs (
+    job_id TEXT PRIMARY KEY,
+    org_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    job_type TEXT NOT NULL DEFAULT 'learning',
+    latest_request_id TEXT,
+    status TEXT NOT NULL DEFAULT 'pending',
+    attempts INTEGER NOT NULL DEFAULT 0,
+    max_attempts INTEGER NOT NULL DEFAULT 3,
+    claimed_by TEXT,
+    claim_token TEXT,
+    claim_expires_at TEXT,
+    covers_through TEXT,
+    force_extraction INTEGER NOT NULL DEFAULT 0,
+    skip_aggregation INTEGER NOT NULL DEFAULT 0,
+    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+    updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+);
+CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
+    ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
+CREATE INDEX IF NOT EXISTS learning_jobs_poll
+    ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
+
 """