claude-smart 0.2.45 → 0.2.47

package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+from contextlib import suppress
+from typing import Any, Literal, TypedDict
+
+from reflexio.models.api_schema.domain.governance import (
+    AuditEvent,
+    PurgeOperationTarget,
+    SubjectWriteBarrier,
+    UserEraseResult,
+    UserExportResult,
+)
+from reflexio.server.services.governance.config import (
+    get_governance_ref_secret,
+    governance_request_ref,
+    governance_subject_ref,
+)
+from reflexio.server.services.governance.subject_refs import stable_id
+
+_DELETE_TARGET_NAME_TO_RESULT_KEY = {
+    "interaction": "interactions",
+    "user_playbook": "user_playbooks",
+    "profile": "profiles",
+    "request": "requests",
+    "agent_success_evaluation_result": "agent_success_evaluation_results",
+    "profile_purge": "purged_profiles",
+    "user_playbook_purge": "purged_user_playbooks",
+}
+_REQUIRED_DELETE_TARGET_NAMES = tuple(_DELETE_TARGET_NAME_TO_RESULT_KEY)
+_USER_PLAYBOOK_PAGE_SIZE = 1000
+
+
+class GovernanceActorContext(TypedDict):
+    actor_type: Literal["api_token", "jwt", "system"]
+    actor_ref: str | None
+
+
+class GovernanceService:
+    def __init__(self, *, storage: Any, org_id: str, ref_secret: str) -> None:
+        self.storage = storage
+        self.org_id = org_id
+        self.ref_secret = ref_secret
+
+    def export_user(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        actor_context: GovernanceActorContext | None = None,
+    ) -> UserExportResult:
+        self._assert_storage_ref_secret_matches()
+        subref = governance_subject_ref(self.org_id, user_id, self.ref_secret)
+        reqref = governance_request_ref(self.org_id, request_id, self.ref_secret)
+        export_id = stable_id("export", f"{self.org_id}:export:{subref}:{reqref}")
+        actor_type = actor_context["actor_type"] if actor_context else "system"
+        actor_ref = actor_context["actor_ref"] if actor_context else None
+        requests, sessions = self._load_user_requests_and_sessions(user_id)
+        bundle: dict[str, Any] = {
+            "profiles": [
+                profile.model_dump()
+                for profile in self.storage.get_user_profile(user_id)
+            ],
+            "interactions": [
+                interaction.model_dump()
+                for interaction in self.storage.get_user_interaction(user_id)
+            ],
+            "requests": [request.model_dump() for request in requests],
+            "sessions": sessions,
+            "user_playbooks": [
+                playbook.model_dump() for playbook in self._iter_user_playbooks(user_id)
+            ],
+        }
+        self.storage.append_audit_event(
+            AuditEvent(
+                org_id=self.org_id,
+                actor_type=actor_type,
+                actor_ref=actor_ref,
+                operation="EXPORT",
+                entity_type="request",
+                subject_ref=subref,
+                request_ref=reqref,
+                idempotency_key=export_id,
+                detail={"count": sum(len(items) for items in bundle.values())},
+            )
+        )
+        return UserExportResult(subject_ref=subref, export_id=export_id, bundle=bundle)
+
+    def erase_user(
+        self,
+        *,
+        user_id: str,
+        request_id: str,
+        actor_context: GovernanceActorContext | None = None,
+    ) -> UserEraseResult:
+        self._assert_storage_ref_secret_matches()
+        subref = governance_subject_ref(self.org_id, user_id, self.ref_secret)
+        reqref = governance_request_ref(self.org_id, request_id, self.ref_secret)
+        actor_type = actor_context["actor_type"] if actor_context else "system"
+        actor_ref = actor_context["actor_ref"] if actor_context else None
+        idempotency_key = stable_id(
+            "idem",
+            f"{self.org_id}:user_erasure:{subref}:{reqref}",
+        )
+        purge_id = stable_id("purge", idempotency_key)
+        purge = self.storage.begin_purge_operation(
+            purge_id=purge_id,
+            idempotency_key=idempotency_key,
+            operation_type="user_erasure",
+            scope_type="user",
+            subject_ref=subref,
+            request_ref=reqref,
+        )
+        if purge.status == "complete":
+            barrier = self._completed_barrier_for_retry(
+                subject_ref=subref, purge_id=purge_id
+            )
+            if barrier.status != "erased":
+                raise ValueError(
+                    "Completed purge retry requires an erased subject barrier"
+                )
+            return UserEraseResult(
+                subject_ref=subref,
+                purge_id=purge_id,
+                status="complete",
+                deleted_counts=self._deleted_counts_from_targets(purge_id),
+                rebuilt_agent_playbook_ids=(
+                    self._rebuilt_agent_playbook_ids_from_targets(purge_id)
+                ),
+            )
+        try:
+            self.storage.begin_subject_erasure_barrier(subref, purge_id)
+            if not self.storage.purge_targets_prepared(purge_id):
+                self.storage.prepare_governance_erase_targets(
+                    purge_id,
+                    user_id,
+                )
+
+            if not self._delete_targets_complete(purge_id):
+                self.storage.apply_governance_user_data_delete(purge_id, user_id)
+            deleted_counts = self._deleted_counts_from_targets(purge_id)
+
+            rebuilt_agent_playbook_ids: list[int] = []
+            completed = self.storage.complete_subject_erasure_barrier_after_empty_check(
+                purge_id,
+                AuditEvent(
+                    org_id=self.org_id,
+                    actor_type=actor_type,
+                    actor_ref=actor_ref,
+                    operation="ERASE",
+                    entity_type="request",
+                    subject_ref=subref,
+                    request_ref=reqref,
+                    idempotency_key=purge_id,
+                    detail={
+                        "deleted_counts": deleted_counts,
+                        "rebuilt_agent_playbook_ids": rebuilt_agent_playbook_ids,
+                    },
+                ),
+            )
+        except Exception as exc:
+            with suppress(Exception):
+                self.storage.fail_subject_erasure_barrier(
+                    subref,
+                    purge_id,
+                    error_code="governance_erase_failed",
+                    error_detail=type(exc).__name__,
+                )
+            with suppress(Exception):
+                self.storage.fail_purge_operation(
+                    purge_id,
+                    error_code="governance_erase_failed",
+                    error_detail=type(exc).__name__,
+                )
+            raise
+        return UserEraseResult(
+            subject_ref=subref,
+            purge_id=purge_id,
+            status=completed.status,
+            deleted_counts=deleted_counts,
+            rebuilt_agent_playbook_ids=rebuilt_agent_playbook_ids,
+        )
+
+    def _assert_storage_ref_secret_matches(self) -> None:
+        storage_secret = get_governance_ref_secret()
+        if storage_secret != self.ref_secret:
+            raise RuntimeError(
+                "GovernanceService ref_secret must match REFLEXIO_GOVERNANCE_REF_SECRET "
+                "for governance operations"
+            )
+
+    def _completed_barrier_for_retry(
+        self, *, subject_ref: str, purge_id: str
+    ) -> SubjectWriteBarrier:
+        barrier = self.storage.get_subject_write_barrier(subject_ref)
+        if barrier is None or barrier.purge_id != purge_id:
+            raise ValueError(
+                "Completed purge retry requires the matching subject barrier"
+            )
+        return barrier
+
+    def _load_user_requests_and_sessions(
+        self, user_id: str
+    ) -> tuple[list[Any], list[dict[str, Any]]]:
+        requests: list[Any] = []
+        sessions_by_id: dict[str, list[str]] = {}
+        offset = 0
+        page_size = 1000
+
+        while True:
+            grouped_sessions = self.storage.get_sessions(
+                user_id=user_id,
+                top_k=page_size,
+                offset=offset,
+            )
+            returned_rows = 0
+            for session_id, rows in grouped_sessions.items():
+                returned_rows += len(rows)
+                request_ids = sessions_by_id.setdefault(session_id, [])
+                for row in rows:
+                    if row.request is None:
+                        continue
+                    requests.append(row.request)
+                    request_ids.append(row.request.request_id)
+            if returned_rows < page_size:
+                break
+            offset += page_size
+
+        sessions = [
+            {"session_id": session_id, "request_ids": request_ids}
+            for session_id, request_ids in sessions_by_id.items()
+        ]
+        return requests, sessions
+
+    def _iter_user_playbooks(self, user_id: str) -> list[Any]:
+        playbooks: list[Any] = []
+        offset = 0
+        while True:
+            page = self.storage.get_user_playbooks(
+                user_id=user_id,
+                limit=_USER_PLAYBOOK_PAGE_SIZE,
+                offset=offset,
+            )
+            playbooks.extend(page)
+            if len(page) < _USER_PLAYBOOK_PAGE_SIZE:
+                break
+            offset += _USER_PLAYBOOK_PAGE_SIZE
+        return playbooks
+
+    def _delete_targets_complete(self, purge_id: str) -> bool:
+        delete_targets = {
+            target.target_name: target
+            for target in self.storage.list_purge_targets(purge_id, phase="delete")
+        }
+        return all(
+            delete_targets.get(target_name) is not None
+            and delete_targets[target_name].status == "complete"
+            for target_name in _REQUIRED_DELETE_TARGET_NAMES
+        )
+
+    def _deleted_counts_from_targets(self, purge_id: str) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for target in self.storage.list_purge_targets(purge_id, phase="delete"):
+            result_key = _DELETE_TARGET_NAME_TO_RESULT_KEY.get(target.target_name)
+            if result_key is None:
+                continue
+            counts[result_key] = int(target.deleted_count)
+        return counts
+
+    def _rebuilt_agent_playbook_ids_from_targets(self, purge_id: str) -> list[int]:
+        return [
+            int(target.target_ref)
+            for target in self.storage.list_purge_targets(
+                purge_id,
+                phase="rebuild_without_erased_sources",
+            )
+            if (
+                target.target_name == "agent_playbook"
+                and target.target_ref
+                and target.status == "complete"
+            )
+        ]
+
+    def _rebuild_agent_playbooks(self, purge_id: str) -> list[int]:
+        rebuilt_ids: list[int] = []
+        for target in self.storage.list_purge_targets(
+            purge_id,
+            phase="rebuild_without_erased_sources",
+        ):
+            if target.target_name != "agent_playbook" or not target.target_ref:
+                continue
+            agent_playbook_id = int(target.target_ref)
+            if target.status == "complete":
+                rebuilt_ids.append(agent_playbook_id)
+                continue
+            remaining_source_windows = self._remaining_source_windows(target)
+            rebuild_fields = self._build_rebuilt_agent_playbook_fields(
+                remaining_source_windows
+            )
+            self.storage.apply_governance_agent_playbook_rebuild(
+                purge_id=purge_id,
+                agent_playbook_id=agent_playbook_id,
+                remaining_source_windows=remaining_source_windows,
+                content=rebuild_fields["content"],
+                trigger=rebuild_fields["trigger"],
+                rationale=rebuild_fields["rationale"],
+                blocking_issue=rebuild_fields["blocking_issue"],
+                expanded_terms=rebuild_fields["expanded_terms"],
+                tags=rebuild_fields["tags"],
+            )
+            rebuilt_ids.append(agent_playbook_id)
+        return rebuilt_ids
+
+    def _remaining_source_windows(
+        self,
+        target: PurgeOperationTarget,
+    ) -> list[dict[str, object]]:
+        detail = target.detail or {}
+        remaining = detail.get("remaining_source_windows", [])
+        if not isinstance(remaining, list):
+            raise ValueError("remaining_source_windows must be a list")
+        return remaining
+
+    def _build_rebuilt_agent_playbook_fields(
+        self,
+        remaining_source_windows: list[dict[str, object]],
+    ) -> dict[str, Any]:
+        user_playbook_ids: list[int] = []
+        for window in remaining_source_windows:
+            raw_user_playbook_id = window.get("user_playbook_id")
+            if isinstance(raw_user_playbook_id, int):
+                user_playbook_ids.append(raw_user_playbook_id)
+        playbooks_by_id = {
+            playbook.user_playbook_id: playbook
+            for playbook in self.storage.get_user_playbooks_by_ids_any_user(
+                user_playbook_ids
+            )
+            if playbook.user_playbook_id
+        }
+        remaining_playbooks = [
+            playbooks_by_id[user_playbook_id]
+            for user_playbook_id in user_playbook_ids
+            if user_playbook_id in playbooks_by_id
+        ]
+        return {
+            "content": self._join_non_empty_strings(
+                playbook.content for playbook in remaining_playbooks
+            ),
+            "trigger": self._join_non_empty_strings(
+                playbook.trigger for playbook in remaining_playbooks
+            ),
+            "rationale": self._join_non_empty_strings(
+                playbook.rationale for playbook in remaining_playbooks
+            ),
+            "blocking_issue": next(
+                (
+                    playbook.blocking_issue.model_dump()
+                    for playbook in remaining_playbooks
+                    if playbook.blocking_issue is not None
+                ),
+                None,
+            ),
+            "expanded_terms": self._join_non_empty_strings(
+                playbook.expanded_terms for playbook in remaining_playbooks
+            ),
+            "tags": self._merge_tags(remaining_playbooks),
+        }
+
+    def _join_non_empty_strings(self, values: Any) -> str | None:
+        joined = "\n".join(value for value in values if value)
+        return joined or None
+
+    def _merge_tags(self, playbooks: list[Any]) -> list[str] | None:
+        merged_tags: list[str] = []
+        for playbook in playbooks:
+            for tag in playbook.tags or []:
+                if tag not in merged_tags:
+                    merged_tags.append(tag)
+        return merged_tags or None

package/plugin/vendor/reflexio/reflexio/server/services/governance/subject_refs.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+
+
+def _hmac_ref(prefix: str, raw: str, secret: str) -> str:
+    if not raw:
+        raise ValueError("raw value must be non-empty")
+    if not secret:
+        raise ValueError("secret must be non-empty")
+    digest = hmac.new(secret.encode(), raw.encode(), hashlib.sha256).hexdigest()[:32]
+    return f"{prefix}_v1_{digest}"
+
+
+def subject_ref(raw: str, secret: str) -> str:
+    return _hmac_ref("subref", raw, secret)
+
+
+def actor_ref(raw: str, secret: str) -> str:
+    return _hmac_ref("actref", raw, secret)
+
+
+def request_ref(raw: str, secret: str) -> str:
+    return _hmac_ref("reqref", raw, secret)
+
+
+def stable_id(prefix: str, material: str) -> str:
+    if not prefix:
+        raise ValueError("prefix must be non-empty")
+    if not material:
+        raise ValueError("material must be non-empty")
+    digest = hashlib.sha256(material.encode()).hexdigest()[:32]
+    return f"{prefix}_{digest}"

package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py

@@ -0,0 +1,219 @@
+"""Process-local scheduler for lineage tombstone garbage collection.
+
+Startup is gated on ``lineage_gc.enabled`` (default on). Each tick evaluates
+every org independently and hard-deletes expired tombstones per that org's
+``lineage_gc`` config. One org's failure never stalls the loop; errors are
+captured as Sentry anomalies and the loop continues to the next org.
+
+Governance-retention reclamation is a premium concern handled by the enterprise
+GovernanceRetentionCapability (reflexio_ext), not here.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.tracing import capture_anomaly
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_POLL_INTERVAL_SECONDS = 86400
+_MIN_POLL_SECONDS = 1
+
+# Window-misconfiguration tripwire: if a single tick deletes more than this
+# many tombstones for one org, something is likely wrong with the grace window.
+_HIGH_VOLUME_THRESHOLD = 1000
+
+_ENTITY_TYPES = ("user_playbook", "agent_playbook", "profile")
+
+
+class LineageGCScheduler:
+    """Polling daemon that hard-deletes expired tombstones per org."""
+
+    def __init__(
+        self,
+        *,
+        request_context_factory: Callable[[str], RequestContext],
+        bootstrap_org_id: str,
+    ) -> None:
+        self.request_context_factory = request_context_factory
+        self.bootstrap_org_id = bootstrap_org_id
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self) -> None:
+        """Start the daemon thread (idempotent if already running)."""
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._run_loop,
+            name="reflexio-lineage-gc-scheduler",
+            daemon=True,
+        )
+        self._thread.start()
+        logger.info("event=lineage_gc_scheduler_started")
+
+    def stop(self, *, timeout_seconds: float = 5.0) -> None:
+        """Signal the daemon to stop and wait for it to finish."""
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join(timeout=timeout_seconds)
+        self._thread = None
+        logger.info("event=lineage_gc_scheduler_stopped")
+
+    def _discover_org_ids(self, bootstrap_ctx: RequestContext) -> list[str]:
+        """Return every known org, always including the bootstrap org."""
+        storage = getattr(bootstrap_ctx, "storage", None)
+        org_ids: list[str] = []
+        if storage is not None:
+            try:
+                org_ids = storage.list_org_ids()
+            except NotImplementedError:
+                logger.warning(
+                    "event=lineage_gc_list_org_ids_not_implemented "
+                    "backend=%s bootstrap_org_id=%s — GC will only process bootstrap org",
+                    type(storage).__name__,
+                    bootstrap_ctx.org_id,
+                )
+                org_ids = []
+        if bootstrap_ctx.org_id not in org_ids:
+            org_ids = [bootstrap_ctx.org_id, *org_ids]
+        return org_ids
+
+    def _gc_tick(self, org_ids: list[str]) -> None:
+        """Run one GC pass across the given org IDs.
+
+        Factored out of ``_run_loop`` so tests can exercise it without threads.
+
+        Args:
+            org_ids (list[str]): Org IDs to process in this tick.
+        """
+        for org_id in org_ids:
+            if self._stop_event.is_set():
+                break
+            try:
+                ctx = self.request_context_factory(org_id)
+                if ctx.storage is None:
+                    continue
+                cfg = ctx.configurator.get_config()
+                if not cfg.lineage_gc.enabled:
+                    continue
+                older_than_epoch = (
+                    int(time.time())
+                    - cfg.lineage_gc.tombstone_grace_window_days * 86400
+                )
+                tombstone_deleted = 0
+                for entity_type in _ENTITY_TYPES:
+                    tombstone_deleted += ctx.storage.gc_expired_tombstones(
+                        entity_type=entity_type,
+                        older_than_epoch=older_than_epoch,
+                    )
+                if tombstone_deleted:
+                    logger.info(
+                        "event=lineage_gc_tick org_id=%s tombstone_deleted=%d",
+                        org_id,
+                        tombstone_deleted,
+                    )
+                if tombstone_deleted > _HIGH_VOLUME_THRESHOLD:
+                    capture_anomaly(
+                        "lineage.gc.high_volume",
+                        org_id=org_id,
+                        count=tombstone_deleted,
+                    )
+            except Exception:
+                capture_anomaly("lineage.gc.run_failed", org_id=org_id)
+                logger.exception("event=lineage_gc_org_failed org_id=%s", org_id)
+
+    def _run_loop(self) -> None:
+        while not self._stop_event.is_set():
+            poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
+            try:
+                bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
+                cfg = bootstrap_ctx.configurator.get_config()
+                poll_interval = cfg.lineage_gc.poll_interval_seconds
+                org_ids = self._discover_org_ids(bootstrap_ctx)
+                self._gc_tick(org_ids)
+            except Exception:
+                logger.exception("event=lineage_gc_scheduler_tick_failed")
+            self._stop_event.wait(max(poll_interval, _MIN_POLL_SECONDS))
+
+
+def maybe_start_lineage_gc(
+    request_context_factory: Callable[[str], RequestContext],
+    *,
+    bootstrap_org_id: str,
+) -> LineageGCScheduler | None:
+    """Start the scheduler only when bootstrap config enables tombstone GC.
+
+    Startup requires bootstrap-org config to enable tombstone GC via
+    ``lineage_gc.enabled``. Tombstone-GC enablement criteria (must
+    ALL hold before enabling for a production org):
+
+    1. **Mechanism**: GC ages tombstones by ``retired_at`` (the INTEGER epoch
+       written at every tombstone write-path).  Rows with ``retired_at = NULL``
+       (created before the column was added) are never eligible and are retained.
+    2. **Grace window**: 90 days is the vetted default (``tombstone_grace_window_days``).
+       This is a per-deployment policy knob — do not shorten without reviewing
+       PII-lifetime obligations (GDPR Art. 5(1)(e)) and audit-depth requirements.
+    3. **B2↔B3 timing gate**: enable per-org only once the grace window is ≥ the
+       reconstruction read-back horizon used by B3 changelog replay, OR once B3 is
+       fully shipped and the horizon is confirmed.  Enabling before this point risks
+       GC'ing tombstones the B3 replay still needs.
+    4. **DPO sign-off**: obtain sign-off on the PII-lifetime and audit-depth
+       implications before enabling in any deployment that processes personal data.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+        bootstrap_org_id: Org used to read config and seed cross-org discovery.
+
+    Returns:
+        LineageGCScheduler: The started scheduler, or ``None`` if not enabled.
+    """
+    try:
+        ctx = request_context_factory(bootstrap_org_id)
+        cfg = ctx.configurator.get_config()
+
+        # Dead-knob warning: audit-event retention is an ENTERPRISE-only feature
+        # (handled by reflexio_ext GovernanceRetentionCapability). In an OSS-only
+        # deployment the knob is accepted but does nothing — warn loudly. Detected
+        # via the configurator class: enterprise swaps in EnterpriseConfigurator at
+        # construction, so the OSS DefaultConfigurator means "no enterprise here".
+        from reflexio.server.services.configurator.configurator import (  # noqa: PLC0415
+            DefaultConfigurator,
+            get_configurator_class,
+        )
+
+        gr = getattr(cfg, "governance_retention", None)
+        if (
+            gr is not None
+            and getattr(gr, "audit_events_retention_enabled", False)
+            and get_configurator_class() is DefaultConfigurator
+        ):
+            logger.warning(
+                "event=governance_retention_knob_ignored "
+                "audit_events_retention_enabled=True but this is an OSS-only "
+                "deployment — audit-event retention is an enterprise-only feature "
+                "and will NOT run here."
+            )
+
+        if not cfg.lineage_gc.enabled:
+            return None
+    except Exception as exc:
+        logger.warning(
+            "event=lineage_gc_scheduler_start_skipped error_type=%s error=%s",
+            type(exc).__name__,
+            exc,
+        )
+        return None
+
+    scheduler = LineageGCScheduler(
+        request_context_factory=request_context_factory,
+        bootstrap_org_id=bootstrap_org_id,
+    )
+    scheduler.start()
+    return scheduler