claude-smart 0.2.46 → 0.2.48

package/plugin/vendor/reflexio/reflexio/server/services/publish_learning_worker.py

@@ -0,0 +1,288 @@
+"""Process-local queue for learning work after async publish persistence."""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import time
+from dataclasses import dataclass, field
+
+from reflexio.server.cache.reflexio_cache import get_reflexio
+from reflexio.server.env_utils import env_str
+from reflexio.server.operation_limiter import operation_limit, operation_limit_value
+from reflexio.server.services.generation_service import GenerationService
+from reflexio.server.usage_metrics import record_usage_event
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_QUEUE_WARN_SIZE = 1000
+
+
+def _env_int(name: str, default: int, *, minimum: int = 1) -> int:
+    raw = env_str(name, str(default))
+    try:
+        return max(minimum, int(raw))
+    except ValueError:
+        logger.warning("Invalid %s=%r; using %s", name, raw, default)
+        return max(minimum, default)
+
+
+@dataclass(frozen=True)
+class PublishLearningJob:
+    """Metadata needed to run post-persist publish learning."""
+
+    org_id: str
+    user_id: str
+    request_id: str
+    session_id: str | None
+    source: str | None
+    agent_version: str
+    force_extraction: bool
+    skip_aggregation: bool
+    enqueued_at: float = field(default_factory=time.monotonic)
+
+
+class PublishLearningWorker:
+    """Process-local worker pool for deferred publish learning."""
+
+    def __init__(
+        self,
+        *,
+        queue_warn_size: int | None = None,
+        worker_count: int | None = None,
+    ) -> None:
+        default_workers = operation_limit_value("publish")
+        self.queue_warn_size = (
+            queue_warn_size
+            if queue_warn_size is not None
+            else _env_int(
+                "REFLEXIO_PUBLISH_LEARNING_QUEUE_WARN_SIZE",
+                _DEFAULT_QUEUE_WARN_SIZE,
+            )
+        )
+        self.worker_count = (
+            worker_count
+            if worker_count is not None
+            else _env_int("REFLEXIO_PUBLISH_LEARNING_WORKERS", default_workers)
+        )
+        self._queue: queue.Queue[PublishLearningJob] = queue.Queue()
+        self._stop_event = threading.Event()
+        self._threads: list[threading.Thread] = []
+        self._started = False
+        self._start_lock = threading.Lock()
+
+    def start(self) -> None:
+        with self._start_lock:
+            if self._started:
+                return
+            self._stop_event.clear()
+            self._threads = [
+                threading.Thread(
+                    target=self._worker_loop,
+                    name=f"publish-learning-worker-{idx}",
+                    daemon=True,
+                )
+                for idx in range(self.worker_count)
+            ]
+            for thread in self._threads:
+                thread.start()
+            self._started = True
+            logger.info(
+                "event=publish_learning_worker_started workers=%d queue_warn_size=%d",
+                self.worker_count,
+                self.queue_warn_size,
+            )
+
+    def enqueue(self, job: PublishLearningJob) -> bool:
+        self.start()
+        self._queue.put_nowait(job)
+        queue_depth = self._queue.qsize()
+        if queue_depth >= self.queue_warn_size:
+            record_usage_event(
+                org_id=job.org_id,
+                user_id=job.user_id,
+                request_id=job.request_id,
+                session_id=job.session_id,
+                source=job.source,
+                agent_version=job.agent_version,
+                event_name="learning_queue_pressure",
+                event_category="publish_learning",
+                outcome="queued",
+                metadata={
+                    "queue_depth": queue_depth,
+                    "queue_warn_size": self.queue_warn_size,
+                },
+            )
+            logger.warning(
+                "event=publish_learning_queue_pressure org_id=%s user_id=%s "
+                "request_id=%s queue_depth=%d queue_warn_size=%d",
+                job.org_id,
+                job.user_id,
+                job.request_id,
+                queue_depth,
+                self.queue_warn_size,
+            )
+        logger.info(
+            "event=publish_learning_enqueued org_id=%s user_id=%s request_id=%s "
+            "queue_depth=%d queue_warn_size=%d",
+            job.org_id,
+            job.user_id,
+            job.request_id,
+            queue_depth,
+            self.queue_warn_size,
+        )
+        return True
+
+    def stop(self, timeout: float | None = 5.0) -> None:
+        self._stop_event.set()
+        deadline = None if timeout is None else time.monotonic() + timeout
+        for thread in list(self._threads):
+            remaining = (
+                None if deadline is None else max(0.0, deadline - time.monotonic())
+            )
+            thread.join(timeout=remaining)
+        if all(not thread.is_alive() for thread in self._threads):
+            self._threads = []
+            self._started = False
+        logger.info("event=publish_learning_worker_stopped")
+
+    def _worker_loop(self) -> None:
+        while not self._stop_event.is_set() or not self._queue.empty():
+            try:
+                job = self._queue.get(timeout=0.25)
+            except queue.Empty:
+                continue
+            try:
+                self._process_job(job)
+            finally:
+                self._queue.task_done()
+
+    def _process_job(self, job: PublishLearningJob) -> None:
+        try:
+            with operation_limit(
+                job.org_id,
+                "publish",
+                wait_forever=False,
+                log_timeout=False,
+            ):
+                reflexio = get_reflexio(org_id=job.org_id)
+                GenerationService(
+                    llm_client=reflexio.llm_client,
+                    request_context=reflexio.request_context,
+                ).run_deferred_learning(
+                    user_id=job.user_id,
+                    request_id=job.request_id,
+                    session_id=job.session_id,
+                    source=job.source,
+                    agent_version=job.agent_version,
+                    force_extraction=job.force_extraction,
+                    skip_aggregation=job.skip_aggregation,
+                )
+        except TimeoutError:
+            self._requeue_after_limiter_timeout(job)
+            return
+        except Exception as exc:
+            record_usage_event(
+                org_id=job.org_id,
+                user_id=job.user_id,
+                request_id=job.request_id,
+                session_id=job.session_id,
+                source=job.source,
+                agent_version=job.agent_version,
+                event_name="learning_failed",
+                event_category="publish_learning",
+                outcome="failed",
+                error_kind=type(exc).__name__,
+            )
+            logger.exception(
+                "event=publish_learning_failed org_id=%s user_id=%s request_id=%s",
+                job.org_id,
+                job.user_id,
+                job.request_id,
+            )
+            return
+
+        record_usage_event(
+            org_id=job.org_id,
+            user_id=job.user_id,
+            request_id=job.request_id,
+            session_id=job.session_id,
+            source=job.source,
+            agent_version=job.agent_version,
+            event_name="learning_succeeded",
+            event_category="publish_learning",
+            outcome="success",
+        )
+        logger.info(
+            "event=publish_learning_done org_id=%s user_id=%s request_id=%s "
+            "queue_depth=%d",
+            job.org_id,
+            job.user_id,
+            job.request_id,
+            self._queue.qsize(),
+        )
+
+    def _requeue_after_limiter_timeout(self, job: PublishLearningJob) -> None:
+        self._queue.put_nowait(job)
+
+        record_usage_event(
+            org_id=job.org_id,
+            user_id=job.user_id,
+            request_id=job.request_id,
+            session_id=job.session_id,
+            source=job.source,
+            agent_version=job.agent_version,
+            event_name="learning_requeued_limiter_busy",
+            event_category="publish_learning",
+            outcome="requeued",
+            metadata={
+                "age_seconds": self._job_age_seconds(job),
+                "queue_depth": self._queue.qsize(),
+            },
+        )
+        logger.info(
+            "event=publish_learning_requeued_limiter_busy org_id=%s user_id=%s "
+            "request_id=%s queue_depth=%d age_seconds=%.3f",
+            job.org_id,
+            job.user_id,
+            job.request_id,
+            self._queue.qsize(),
+            self._job_age_seconds(job),
+        )
+
+    @staticmethod
+    def _job_age_seconds(job: PublishLearningJob) -> float:
+        return time.monotonic() - job.enqueued_at
+
+
+_worker: PublishLearningWorker | None = None
+_worker_lock = threading.Lock()
+
+
+def get_publish_learning_worker() -> PublishLearningWorker:
+    global _worker  # noqa: PLW0603
+    if _worker is None:
+        with _worker_lock:
+            if _worker is None:
+                _worker = PublishLearningWorker()
+    return _worker
+
+
+def enqueue_publish_learning(job: PublishLearningJob) -> bool:
+    return get_publish_learning_worker().enqueue(job)
+
+
+def stop_publish_learning_worker(timeout: float | None = 5.0) -> None:
+    global _worker  # noqa: PLW0603
+    worker = _worker
+    if worker is not None:
+        worker.stop(timeout=timeout)
+
+
+def reset_publish_learning_worker_for_tests() -> None:
+    global _worker  # noqa: PLW0603
+    worker = _worker
+    if worker is not None:
+        worker.stop(timeout=0.1)
+    _worker = None

package/plugin/vendor/reflexio/reflexio/server/services/retrieval/recency.py

@@ -0,0 +1,211 @@
+"""Recency adjustment helpers for unified search ranking."""
+
+from __future__ import annotations
+
+import logging
+import math
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from datetime import UTC, datetime
+
+from reflexio.server.env_utils import env_str, env_truthy
+
+logger = logging.getLogger(__name__)
+
+SECONDS_PER_DAY = 24 * 60 * 60
+PLAYBOOK_HALF_LIFE_SECONDS = 90 * SECONDS_PER_DAY
+_PROFILE_TTL_HALF_LIFE_FRACTION = 0.5
+
+
+@dataclass(frozen=True)
+class ScoredItem[T]:
+    item: T
+    score: float | None
+
+
+@dataclass(frozen=True)
+class RecencyConfig:
+    enabled: bool = True
+    max_penalty_frac: float = 0.15
+    max_penalty_logit: float = 0.2
+    pool_size: int = 20
+
+    @classmethod
+    def from_env(cls, *, env: Mapping[str, str] | None = None) -> RecencyConfig:
+        return cls(
+            enabled=env_truthy(
+                env_str("REFLEXIO_SEARCH_RECENCY_ENABLED", "true", env=env)
+            ),
+            max_penalty_frac=_float_env(
+                "REFLEXIO_SEARCH_RECENCY_MAX_PENALTY_FRAC",
+                cls.max_penalty_frac,
+                env=env,
+                minimum=0.0,
+                maximum=1.0,
+            ),
+            max_penalty_logit=_float_env(
+                "REFLEXIO_SEARCH_RECENCY_MAX_PENALTY_LOGIT",
+                cls.max_penalty_logit,
+                env=env,
+                minimum=0.0,
+            ),
+            pool_size=_int_env(
+                "REFLEXIO_SEARCH_RECENCY_POOL_SIZE",
+                cls.pool_size,
+                env=env,
+                minimum=1,
+            ),
+        )
+
+    def with_overrides(self, values: Mapping[str, object] | None) -> RecencyConfig:
+        if not values:
+            return self
+        updates: dict[str, object] = {}
+        if "recency_enabled" in values:
+            updates["enabled"] = _bool_value(values["recency_enabled"], self.enabled)
+        if "recency_max_penalty_frac" in values:
+            updates["max_penalty_frac"] = _float_value(
+                values["recency_max_penalty_frac"],
+                self.max_penalty_frac,
+                minimum=0.0,
+                maximum=1.0,
+            )
+        if "recency_max_penalty_logit" in values:
+            updates["max_penalty_logit"] = _float_value(
+                values["recency_max_penalty_logit"],
+                self.max_penalty_logit,
+                minimum=0.0,
+            )
+        if "recency_pool_size" in values:
+            updates["pool_size"] = _int_value(
+                values["recency_pool_size"], self.pool_size, minimum=1
+            )
+        return replace(self, **updates)
+
+
+def decay_for_item(
+    item: object,
+    *,
+    entity_type: str,
+    now: int | None = None,
+) -> float:
+    """Return a freshness decay in [0, 1], where 1 means no penalty."""
+    now = now if now is not None else int(datetime.now(UTC).timestamp())
+    if entity_type == "profiles":
+        return _profile_decay(item, now=now)
+    return _timestamp_decay(
+        getattr(item, "created_at", None),
+        half_life_seconds=PLAYBOOK_HALF_LIFE_SECONDS,
+        now=now,
+    )
+
+
+def decay(age_seconds: float, half_life_seconds: float) -> float:
+    if half_life_seconds <= 0:
+        return 1.0
+    age_seconds = max(0.0, age_seconds)
+    return math.exp(-age_seconds / (half_life_seconds / math.log(2)))
+
+
+def multiplicative_factor(decay_value: float, max_penalty_frac: float) -> float:
+    return 1.0 - _clamp(max_penalty_frac, 0.0, 1.0) * (1.0 - _decay(decay_value))
+
+
+def additive_penalty(decay_value: float, max_penalty_logit: float) -> float:
+    return max(0.0, max_penalty_logit) * (1.0 - _decay(decay_value))
+
+
+def _profile_decay(item: object, *, now: int) -> float:
+    modified_at = _int_or_none(getattr(item, "last_modified_timestamp", None))
+    expires_at = _int_or_none(getattr(item, "expiration_timestamp", None))
+    if modified_at is None or expires_at is None or expires_at <= modified_at:
+        return 1.0
+    half_life = (expires_at - modified_at) * _PROFILE_TTL_HALF_LIFE_FRACTION
+    return decay(now - modified_at, half_life)
+
+
+def _timestamp_decay(value: object, *, half_life_seconds: int, now: int) -> float:
+    timestamp = _int_or_none(value)
+    if timestamp is None or timestamp <= 0:
+        return 1.0
+    return decay(now - timestamp, half_life_seconds)
+
+
+def _float_env(
+    name: str,
+    default: float,
+    *,
+    env: Mapping[str, str] | None,
+    minimum: float,
+    maximum: float | None = None,
+) -> float:
+    return _float_value(
+        env_str(name, str(default), env=env),
+        default,
+        minimum=minimum,
+        maximum=maximum,
+    )
+
+
+def _int_env(
+    name: str,
+    default: int,
+    *,
+    env: Mapping[str, str] | None,
+    minimum: int,
+) -> int:
+    return _int_value(env_str(name, str(default), env=env), default, minimum=minimum)
+
+
+def _bool_value(value: object, default: bool) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.strip():
+        return env_truthy(value)
+    return default
+
+
+def _float_value(
+    value: object,
+    default: float,
+    *,
+    minimum: float,
+    maximum: float | None = None,
+) -> float:
+    try:
+        if not isinstance(value, str | int | float):
+            raise TypeError
+        parsed = float(value)
+    except (TypeError, ValueError):
+        logger.warning("Invalid recency float override %r; using %.3f", value, default)
+        return default
+    parsed = max(minimum, parsed)
+    if maximum is not None:
+        parsed = min(maximum, parsed)
+    return parsed
+
+
+def _int_value(value: object, default: int, *, minimum: int) -> int:
+    try:
+        if not isinstance(value, str | int):
+            raise TypeError
+        parsed = int(value)
+    except (TypeError, ValueError):
+        logger.warning("Invalid recency integer override %r; using %d", value, default)
+        return default
+    return max(minimum, parsed)
+
+
+def _int_or_none(value: object) -> int | None:
+    try:
+        return int(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return None
+
+
+def _decay(value: float) -> float:
+    return _clamp(value, 0.0, 1.0)
+
+
+def _clamp(value: float, low: float, high: float) -> float:
+    return min(high, max(low, value))

package/plugin/vendor/reflexio/reflexio/server/services/retrieval/relevance_floor.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import logging
 from collections.abc import Callable, Sequence
+from dataclasses import dataclass
 from typing import Any
 
 from reflexio.server.llm.rerank import score_pairs
@@ -21,12 +22,20 @@ from reflexio.server.tracing import profile_step
 logger = logging.getLogger(__name__)
 
 
-def _floor_and_sort[T](items: list[T], scores: list[float], floor: float) -> list[T]:
-    """Return items scoring >= ``floor``, sorted by score descending."""
+@dataclass(frozen=True)
+class RelevanceFloorResult:
+    items: list[Any]
+    scores: list[float] | None
+
+
+def _floor_and_sort[T](
+    items: list[T], scores: list[float], floor: float
+) -> list[tuple[T, float]]:
+    """Return ``(item, score)`` pairs scoring >= ``floor``, sorted descending."""
     ranked = sorted(
         zip(items, scores, strict=True), key=lambda pair: pair[1], reverse=True
     )
-    return [item for item, score in ranked if score >= floor]
+    return [(item, score) for item, score in ranked if score >= floor]
 
 
 def apply_relevance_floor[T](
@@ -71,7 +80,7 @@ def apply_relevance_floor[T](
             return items[:top_k]
         span.set_data("available", True)
 
-        survivors = _floor_and_sort(items, scores, floor)
+        survivors = [item for item, _score in _floor_and_sort(items, scores, floor)]
         dropped = len(items) - len(survivors)
         span.set_data("kept", len(survivors))
         span.set_data("dropped", dropped)
@@ -92,7 +101,7 @@ def apply_relevance_floors(
     top_k: int,
     *,
     content_of: Callable[[Any], str] = lambda item: item.content,
-) -> list[list[Any]]:
+) -> list[RelevanceFloorResult]:
     """Floor every arm with a single cross-encoder batch.
 
     CPU cross-encoder inference does not parallelize across threads, so
@@ -108,12 +117,13 @@ def apply_relevance_floors(
         content_of: Extracts the text to score for an item.
 
     Returns:
-        One survivor list per arm, in input order — each sorted by score
-        descending and capped at ``top_k``. On reranker unavailability,
-        every arm returns ``items[:top_k]`` unchanged (logged).
+        One result per arm, in input order. Available reranker results are
+        sorted by score descending and left uncapped with the paired raw logits.
+        On reranker unavailability, every arm returns the original full item
+        pool with a ``None`` score sentinel (logged).
     """
     if not any(items for _, items, _ in arms):
-        return [[] for _ in arms]
+        return [RelevanceFloorResult([], []) for _ in arms]
     arm_names = [name for name, _, _ in arms]
     contents: list[str] = []
     for _, items, _ in arms:
@@ -123,20 +133,21 @@ def apply_relevance_floors(
         arm="all",
         arms=arm_names,
         items=len(contents),
+        top_k=top_k,
     ) as span:
         try:
             scores = score_pairs(query, contents)
         except CrossEncoderUnavailableError:
             span.set_data("available", False)
             logger.warning(
-                "event=relevance_floor_unavailable arms=%s items=%d (returning unfiltered top_k)",
+                "event=relevance_floor_unavailable arms=%s items=%d (returning unfiltered pool)",
                 arm_names,
                 len(contents),
             )
-            return [items[:top_k] for _, items, _ in arms]
+            return [RelevanceFloorResult(list(items), None) for _, items, _ in arms]
         span.set_data("available", True)
 
-        results: list[list[Any]] = []
+        results: list[RelevanceFloorResult] = []
         offset = 0
         for name, items, floor in arms:
             arm_scores = scores[offset : offset + len(items)]
@@ -153,5 +164,10 @@ def apply_relevance_floors(
                     dropped,
                     floor,
                 )
-            results.append(survivors[:top_k])
+            results.append(
+                RelevanceFloorResult(
+                    [item for item, _score in survivors],
+                    [score for _item, score in survivors],
+                )
+            )
         return results

package/plugin/vendor/reflexio/reflexio/server/services/storage/error.py

@@ -14,6 +14,10 @@ class StorageError(Exception):
         return f"StorageError: {self.message}"
 
 
+class SubjectWriteBarrierError(StorageError):
+    """Raised when a write targets a subject with an active erasure barrier."""
+
+
 def require_non_empty_session_id(value: Any) -> str:
     """Return a stripped, non-empty request ``session_id`` or raise ``StorageError``.