claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py

@@ -0,0 +1,193 @@
+"""Process-local scheduler for lineage tombstone garbage collection.
+
+The scheduler is per-org and config-gated: each tick it discovers every org,
+reads that org's ``lineage_gc`` config, and — if enabled — hard-deletes
+tombstone rows older than the configured grace window.  One org's failure
+never stalls the loop; errors are captured as Sentry anomalies and the
+scheduler continues to the next org.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.tracing import capture_anomaly
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_POLL_INTERVAL_SECONDS = 86400
+_MIN_POLL_SECONDS = 1
+
+# Window-misconfiguration tripwire: if a single tick deletes more than this
+# many tombstones for one org, something is likely wrong with the grace window.
+_HIGH_VOLUME_THRESHOLD = 1000
+
+_ENTITY_TYPES = ("user_playbook", "agent_playbook", "profile")
+
+
+class LineageGCScheduler:
+    """Polling daemon that garbage-collects expired lineage tombstones per org."""
+
+    def __init__(
+        self,
+        *,
+        request_context_factory: Callable[[str], RequestContext],
+        bootstrap_org_id: str,
+    ) -> None:
+        self.request_context_factory = request_context_factory
+        self.bootstrap_org_id = bootstrap_org_id
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        """Start the daemon thread (idempotent if already running)."""
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._run_loop,
+            name="reflexio-lineage-gc-scheduler",
+            daemon=True,
+        )
+        self._thread.start()
+        logger.info("event=lineage_gc_scheduler_started")
+
+    def stop(self, *, timeout_seconds: float = 5.0) -> None:
+        """Signal the daemon to stop and wait for it to finish."""
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join(timeout=timeout_seconds)
+        self._thread = None
+        logger.info("event=lineage_gc_scheduler_stopped")
+
+    def _discover_org_ids(self, bootstrap_ctx: RequestContext) -> list[str]:
+        """Return every known org, always including the bootstrap org."""
+        storage = getattr(bootstrap_ctx, "storage", None)
+        org_ids: list[str] = []
+        if storage is not None:
+            try:
+                org_ids = storage.list_org_ids()
+            except NotImplementedError:
+                logger.warning(
+                    "event=lineage_gc_list_org_ids_not_implemented "
+                    "backend=%s bootstrap_org_id=%s — GC will only process bootstrap org",
+                    type(storage).__name__,
+                    bootstrap_ctx.org_id,
+                )
+                org_ids = []
+        if bootstrap_ctx.org_id not in org_ids:
+            org_ids = [bootstrap_ctx.org_id, *org_ids]
+        return org_ids
+
+    def _gc_tick(self, org_ids: list[str]) -> None:
+        """Run one GC pass across the given org IDs.
+
+        Factored out of ``_run_loop`` so tests can exercise it without threads.
+
+        Args:
+            org_ids (list[str]): Org IDs to process in this tick.
+        """
+        for org_id in org_ids:
+            if self._stop_event.is_set():
+                break
+            try:
+                ctx = self.request_context_factory(org_id)
+                cfg = ctx.configurator.get_config()
+                if not cfg.lineage_gc.enabled:
+                    continue
+                if ctx.storage is None:
+                    continue
+                older_than_epoch = (
+                    int(time.time())
+                    - cfg.lineage_gc.tombstone_grace_window_days * 86400
+                )
+                total_deleted = 0
+                for entity_type in _ENTITY_TYPES:
+                    count = ctx.storage.gc_expired_tombstones(
+                        entity_type=entity_type,
+                        older_than_epoch=older_than_epoch,
+                    )
+                    total_deleted += count
+                if total_deleted:
+                    logger.info(
+                        "event=lineage_gc_tick org_id=%s deleted=%d",
+                        org_id,
+                        total_deleted,
+                    )
+                if total_deleted > _HIGH_VOLUME_THRESHOLD:
+                    capture_anomaly(
+                        "lineage.gc.high_volume",
+                        org_id=org_id,
+                        count=total_deleted,
+                    )
+            except Exception:
+                capture_anomaly("lineage.gc.run_failed", org_id=org_id)
+                logger.exception("event=lineage_gc_org_failed org_id=%s", org_id)
+
+    def _run_loop(self) -> None:
+        while not self._stop_event.is_set():
+            poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
+            try:
+                bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
+                cfg = bootstrap_ctx.configurator.get_config()
+                poll_interval = cfg.lineage_gc.poll_interval_seconds
+                org_ids = self._discover_org_ids(bootstrap_ctx)
+                self._gc_tick(org_ids)
+            except Exception:
+                logger.exception("event=lineage_gc_scheduler_tick_failed")
+            self._stop_event.wait(max(poll_interval, _MIN_POLL_SECONDS))
+
+
+def maybe_start_lineage_gc(
+    request_context_factory: Callable[[str], RequestContext],
+    *,
+    bootstrap_org_id: str,
+) -> LineageGCScheduler | None:
+    """Start the GC scheduler only when the bootstrap-org config enables it.
+
+    Off by default. Enablement criteria (must ALL hold before enabling for a
+    production org):
+
+    1. **Mechanism**: GC ages tombstones by ``retired_at`` (the INTEGER epoch
+       written at every tombstone write-path).  Rows with ``retired_at = NULL``
+       (created before the column was added) are never eligible and are retained.
+    2. **Grace window**: 90 days is the vetted default (``tombstone_grace_window_days``).
+       This is a per-deployment policy knob — do not shorten without reviewing
+       PII-lifetime obligations (GDPR Art. 5(1)(e)) and audit-depth requirements.
+    3. **B2↔B3 timing gate**: enable per-org only once the grace window is ≥ the
+       reconstruction read-back horizon used by B3 changelog replay, OR once B3 is
+       fully shipped and the horizon is confirmed.  Enabling before this point risks
+       GC'ing tombstones the B3 replay still needs.
+    4. **DPO sign-off**: obtain sign-off on the PII-lifetime and audit-depth
+       implications before enabling in any deployment that processes personal data.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+        bootstrap_org_id: Org used to read config and seed cross-org discovery.
+
+    Returns:
+        LineageGCScheduler: The started scheduler, or ``None`` if not enabled.
+    """
+    try:
+        ctx = request_context_factory(bootstrap_org_id)
+        cfg = ctx.configurator.get_config()
+        if not cfg.lineage_gc.enabled:
+            return None
+    except Exception as exc:
+        logger.warning(
+            "event=lineage_gc_scheduler_start_skipped error_type=%s error=%s",
+            type(exc).__name__,
+            exc,
+        )
+        return None
+
+    scheduler = LineageGCScheduler(
+        request_context_factory=request_context_factory,
+        bootstrap_org_id=bootstrap_org_id,
+    )
+    scheduler.start()
+    return scheduler

package/plugin/vendor/reflexio/reflexio/server/services/lineage/resolve.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from reflexio.models.api_schema.domain.entities import RecordRef
+from reflexio.server.tracing import capture_anomaly
+
+EntityType = Literal["user_playbook", "agent_playbook", "profile"]
+
+_GETTER = {
+    "user_playbook": lambda s, i: s.get_user_playbook_by_id(
+        int(i), include_tombstones=True
+    ),
+    "agent_playbook": lambda s, i: s.get_agent_playbook_by_id(
+        int(i), include_tombstones=True
+    ),
+    "profile": lambda s, i: s.get_profile_by_id(str(i), include_tombstones=True),
+}
+_MAX_HOPS = 8  # Phase A: chains are 1-hop; cap guards malformed pointers
+
+
+def resolve_current(
+    storage: Any, entity_type: EntityType, record_id: Any
+) -> RecordRef | None:
+    """Follow merged_into/superseded_by pointers to the live survivor.
+
+    **Consumer contract:** if the returned ``RecordRef.is_purged`` is ``True``,
+    the survivor's content body has been blanked by ``purge_content`` (GDPR/erasure).
+    Any consumer that dereferences the resolved record's content MUST skip or treat
+    the record as absent when ``is_purged=True`` — reading blank content as if it
+    were valid data is a silent bug.
+
+    Args:
+        storage: Any storage backend exposing get_*_by_id with include_tombstones.
+        entity_type: One of "user_playbook", "agent_playbook", "profile".
+        record_id: The primary key of the record to resolve (int for playbooks, str for profiles).
+
+    Returns:
+        RecordRef pointing to the live survivor (is_purged=True if its body is blank),
+        or None if the record doesn't exist, there's a cycle, or the chain exceeds _MAX_HOPS.
+
+    Raises:
+        ValueError: If ``entity_type`` is not a recognized entity type.
+    """
+    get = _GETTER.get(entity_type)
+    if get is None:
+        raise ValueError(f"unknown entity_type: {entity_type!r}")
+    visited: set[str] = set()
+    cur = get(storage, record_id)
+    if cur is None:
+        return None
+    while True:
+        cur_id = str(_pk(cur, entity_type))
+        if cur_id in visited or len(visited) >= _MAX_HOPS:
+            # A cycle or an over-long chain means a malformed pointer graph;
+            # we return None (callers treat as unresolvable) but surface it to
+            # Sentry so the otherwise-silent breakage is observable.
+            reason = "cycle" if cur_id in visited else "max_hops_exceeded"
+            capture_anomaly(
+                f"lineage.resolve_current.{reason}",
+                entity_type=entity_type,
+                entity_id=str(record_id),
+                hops=len(visited),
+            )
+            return None  # cycle or runaway chain
+        visited.add(cur_id)
+        nxt = cur.merged_into if cur.merged_into is not None else cur.superseded_by
+        if nxt is None:
+            return RecordRef(id=cur_id, is_purged=(not cur.content))
+        nxt_row = get(storage, nxt)
+        if nxt_row is None:
+            return RecordRef(
+                id=cur_id, is_purged=(not cur.content)
+            )  # dangling pointer — stop here
+        cur = nxt_row
+
+
+def _pk(row: Any, entity_type: EntityType) -> Any:
+    """Extract the primary key from a row based on its entity type.
+
+    Args:
+        row: The entity row object.
+        entity_type: One of "user_playbook", "agent_playbook", "profile".
+
+    Returns:
+        The primary key value.
+
+    Raises:
+        ValueError: If ``entity_type`` is not a recognized entity type.
+    """
+    pk = {
+        "user_playbook": "user_playbook_id",
+        "agent_playbook": "agent_playbook_id",
+        "profile": "profile_id",
+    }.get(entity_type)
+    if pk is None:
+        raise ValueError(f"unknown entity_type: {entity_type!r}")
+    return getattr(row, pk)

package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py

@@ -535,6 +535,32 @@ class OperationStateManager:
             state_key,
         )
 
+    def clear_lock_if_owner(
+        self,
+        request_id: str,
+        scope_id: str | None = None,
+    ) -> bool:
+        """Atomically clear a lock only if ``request_id`` still owns it."""
+        state_key = self._lock_key(scope_id)
+        cleared = self.storage.clear_in_progress_lock_if_owner(
+            state_key,
+            request_id,
+            {
+                "in_progress": False,
+                "current_request_id": None,
+                "pending_request_id": None,
+                "pending_request_queue": [],
+            },
+        )
+        if cleared:
+            logger.debug(
+                "Cleared in-progress lock for %s: state_key=%s, request_id=%s",
+                self.service_name,
+                state_key,
+                request_id,
+            )
+        return cleared
+
     def release_lock_pop_queue(
         self,
         request_id: str,

package/plugin/vendor/reflexio/reflexio/server/services/playbook/README.md

@@ -5,10 +5,10 @@ Description: Playbook extraction, aggregation, and consolidation pipeline
 
 ## Main Entry Points
 
-- **Service Orchestrator**: `playbook_generation_service.py` - Manages playbook extraction lifecycle (regular, rerun, manual modes)
-- **Playbook Extractor**: `playbook_extractor.py` - Extracts user playbooks from interactions via LLM
-- **Playbook Aggregator**: `playbook_aggregator.py` - Clusters similar user playbooks and generates aggregated insights
-- **Playbook Consolidator**: `playbook_consolidator.py` - Reconciles newly extracted playbooks against existing storage via LLM. Decides per pair to merge as duplicates, prefer the new entry, prefer the existing entry, differentiate (split with refined triggers), or keep both as independent
+- **Service Orchestrator**: `service.py` - Manages playbook extraction lifecycle (regular, rerun, manual modes)
+- **Playbook Extractor**: `components/extractor.py` - Extracts user playbooks from interactions via LLM
+- **Playbook Aggregator**: `components/aggregator.py` - Clusters similar user playbooks and generates aggregated insights
+- **Playbook Consolidator**: `components/consolidator.py` - Reconciles newly extracted playbooks against existing storage via LLM. Decides per pair to merge as duplicates, prefer the new entry, prefer the existing entry, differentiate (split with refined triggers), or keep both as independent
 
 ## Supporting Files
 
@@ -16,6 +16,7 @@ Description: Playbook extraction, aggregation, and consolidation pipeline
 |------|---------|
 | `playbook_service_constants.py` | Prompt IDs for all playbook operations |
 | `playbook_service_utils.py` | Request dataclasses, Pydantic output schemas, message construction utilities |
+| `user_detail_stripping.py` | Optional aggregation-boundary interfaces and helpers for prompt-only user-detail normalization |
 
 ## Architecture
 
@@ -30,7 +31,7 @@ Interactions
           -> AgentPlaybook (aggregated insights) -> Storage
 ```
 
-### Playbook Extraction (`playbook_extractor.py`)
+### Playbook Extraction (`components/extractor.py`)
 
 Extends `BaseGenerationService` extractor pattern. Each extractor:
 1. Checks stride_size threshold before running
@@ -41,20 +42,18 @@ Extends `BaseGenerationService` extractor pattern. Each extractor:
 
 **Tool Analysis**: Reads `tool_can_use` from root `Config` for tool usage analysis and blocking issue detection.
 
-### Playbook Aggregation (`playbook_aggregator.py`)
+### Playbook Aggregation (`components/aggregator.py`)
 
 Triggered manually via `/api/run_playbook_aggregation`. Clusters user playbooks and generates consolidated insights.
 
 **Key Methods**:
 - `get_clusters(user_playbooks, config)` - HDBSCAN/Agglomerative clustering on embeddings
 - `aggregate()` - Full aggregation pipeline with LLM-based consolidation
-- `_build_change_log()` - Builds `PlaybookAggregationChangeLog` with before/after snapshots (added/removed/updated playbooks)
-
-**Change Log**: After each aggregation, saves a `PlaybookAggregationChangeLog` to storage. In full_archive mode, all old playbooks are "removed" and new ones "added". In incremental mode, maps old->new via fingerprints to detect updates. Saving is best-effort (failures logged, don't block aggregation).
+**Change Log**: The legacy `playbook_aggregation_change_logs` table is retired (Track B, 2026-06-24) — the aggregator no longer writes it. The change-log view is reconstructed on demand from `lineage_event` via `reconstruct_playbook_aggregation_change_log` (`lib/_agent_playbook.py`): each run emits `op=aggregate` events (the "added" side) and `status_change→superseded` events from the supersede calls (the "removed" side), grouped by the run's `request_id`. Per-row `updated` pairing is not reconstructed (`updated_agent_playbooks=[]`, a tolerated parity delta).
 
 **Clustering**: Embeds user playbooks -> HDBSCAN clustering -> falls back to Agglomerative if too few clusters
 
-### Playbook Consolidation (`playbook_consolidator.py`)
+### Playbook Consolidation (`components/consolidator.py`)
 
 Consolidates newly extracted playbooks against existing playbooks in the database via LLM semantic matching. For each NEW vs EXISTING pair the LLM returns one of five decision kinds, and the consolidator applies the chosen kind:
 
@@ -70,10 +69,10 @@ A safety fallback inserts any new candidate that no decision consumed, so extrac
 
 | Constant | Prompt ID | Used By |
 |----------|-----------|---------|
-| `PLAYBOOK_EXTRACTION_SHOULD_GENERATE_PROMPT_ID` | `playbook_should_generate` | PlaybookExtractor |
+| `PLAYBOOK_SHOULD_GENERATE_PROMPT_ID` | `playbook_should_generate` | PlaybookExtractor |
 | `PLAYBOOK_EXTRACTION_CONTEXT_PROMPT_ID` | `playbook_extraction_context` | PlaybookExtractor |
 | `PLAYBOOK_EXTRACTION_PROMPT_ID` | `playbook_extraction_main` | PlaybookExtractor |
-| `PLAYBOOK_GENERATION_PROMPT_ID` | `playbook_generation` | PlaybookAggregator |
+| `PLAYBOOK_AGGREGATION_PROMPT_ID` | `playbook_aggregation` | PlaybookAggregator |
 
 ## Key Output Schemas (in `playbook_service_utils.py`)
 

package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/__init__.py

@@ -0,0 +1,5 @@
+"""Playbook service components.
+
+Import component classes from their concrete modules. Keeping this package
+import-free preserves the lazy consolidator import in the generation service.
+"""