claude-smart 0.2.47 → 0.2.49

package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py

@@ -48,6 +48,10 @@ def parse_item_id(item_id: str) -> tuple[str, int] | None:
     """
     Parse a prompt-format item ID like 'NEW-0' or 'EXISTING-1' into its prefix and index.
 
+    Weak models sometimes echo the rendered display label with its brackets
+    (``[NEW-0]``); strip a single surrounding pair so those outputs parse
+    instead of being silently dropped.
+
     Args:
         item_id (str): Item ID string in the format 'PREFIX-N' (e.g., 'NEW-0', 'EXISTING-1')
 
@@ -55,7 +59,10 @@ def parse_item_id(item_id: str) -> tuple[str, int] | None:
         Optional[tuple[str, int]]: A tuple of (prefix, index) where prefix is 'NEW' or 'EXISTING',
             or None if the item ID is invalid
     """
-    parts = item_id.rsplit("-", 1)
+    stripped = item_id.strip()
+    if stripped.startswith("[") and stripped.endswith("]"):
+        stripped = stripped[1:-1].strip()
+    parts = stripped.rsplit("-", 1)
     if len(parts) != 2:
         logger.warning("Invalid item ID format: %s", item_id)
         return None

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py

@@ -0,0 +1,13 @@
+"""Durable learning worker: claim → process in commit_scope → fenced completion."""
+
+from reflexio.server.services.durable_learning.scheduler import (
+    DurableLearningScheduler,
+    maybe_start_durable_learning,
+)
+from reflexio.server.services.durable_learning.worker import DurableLearningWorker
+
+__all__ = [
+    "DurableLearningScheduler",
+    "DurableLearningWorker",
+    "maybe_start_durable_learning",
+]

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py

@@ -0,0 +1,142 @@
+"""Process-local scheduler for the durable learning-job queue (Task 6).
+
+Each tick discovers the orgs with actionable learning jobs (via an injected
+``org_ids_provider``) and drains each through a :class:`DurableLearningWorker`.
+The worker's claims are org-scoped and the fenced complete/fail transitions make
+the drain exactly-once, so racing scheduler instances across processes are safe.
+
+Gating: startup is gated on ``REFLEXIO_DURABLE_LEARNING_QUEUE`` (a rollout flag,
+scalability design 2026-07-04 §8 stage 2). When off, :meth:`start` is a no-op.
+
+Multi-ref capability (design §3.1, per-ref polling): the mechanism supports
+per-data-ref fan-out — inject an ``org_ids_provider`` that yields orgs across
+refs plus a ``request_context_factory`` that routes each org to its ref's
+storage. The default provider (single-ref: bootstrap-storage discovery) is wired
+in :func:`maybe_start_durable_learning`; the enterprise cross-ref enumerator is a
+deferred follow-up (tracked with the pre-Task-8 gates).
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Iterable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.env_utils import env_str, env_truthy
+from reflexio.server.scheduling import ThreadedScheduler
+from reflexio.server.services.durable_learning.worker import DurableLearningWorker
+
+logger = logging.getLogger(__name__)
+
+
+class DurableLearningScheduler(ThreadedScheduler):
+    """Polling daemon that drains the durable learning queue per org.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+            Passed straight to the worker; a multi-ref deployment routes each org
+            to its data ref's storage here.
+        org_ids_provider: Called once per tick; yields the org_ids to drain. The
+            single-ref default (see :func:`maybe_start_durable_learning`) reads
+            them from the bootstrap storage's cross-org discovery query.
+        instance_id: Stable label written to ``claimed_by`` on each claim.
+    """
+
+    def __init__(
+        self,
+        *,
+        request_context_factory: Callable[[str], RequestContext],
+        org_ids_provider: Callable[[], Iterable[str]],
+        instance_id: str | None = None,
+    ) -> None:
+        super().__init__(thread_name="reflexio-durable-learning-scheduler")
+        self._worker = DurableLearningWorker(
+            request_context_factory, instance_id=instance_id
+        )
+        self._org_ids_provider = org_ids_provider
+        # Read env INSIDE __init__ so tests can set the values before construction.
+        self._poll = float(env_str("REFLEXIO_DURABLE_LEARNING_POLL_SECONDS", "2.0"))
+        self._batch = int(env_str("REFLEXIO_DURABLE_LEARNING_BATCH", "16"))
+        self._lease = int(env_str("REFLEXIO_DURABLE_LEARNING_LEASE_SECONDS", "300"))
+
+    def _should_start(self) -> bool:
+        if env_truthy(env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")):
+            return True
+        logger.info(
+            "event=durable_learning_scheduler_disabled "
+            "REFLEXIO_DURABLE_LEARNING_QUEUE not set — not starting"
+        )
+        return False
+
+    def _on_started(self) -> None:
+        logger.info("event=durable_learning_scheduler_started")
+
+    def _on_stopped(self) -> None:
+        logger.info("event=durable_learning_scheduler_stopped")
+
+    def _run_once(self) -> float:
+        """Drain every discovered org once; per-org error isolation.
+
+        A single org raising never aborts the tick, and a failure to enumerate
+        orgs never kills the daemon thread. Returns the poll interval.
+        """
+        try:
+            for org_id in self._org_ids_provider():
+                if self._stop_event.is_set():
+                    break
+                try:
+                    self._worker.drain_org(org_id, self._batch, self._lease)
+                except Exception:
+                    logger.exception(
+                        "event=durable_learning_drain_failed org_id=%s", org_id
+                    )
+        except Exception:
+            logger.exception("event=durable_learning_tick_failed")
+        return self._poll
+
+
+def maybe_start_durable_learning(
+    request_context_factory: Callable[[str], RequestContext],
+    *,
+    bootstrap_org_id: str,
+    org_ids_provider: Callable[[], Iterable[str]] | None = None,
+) -> DurableLearningScheduler | None:
+    """Start the durable-learning scheduler when the rollout flag is on.
+
+    Returns ``None`` when ``REFLEXIO_DURABLE_LEARNING_QUEUE`` is off (mirrors
+    ``maybe_start_resume_scheduler``). The default ``org_ids_provider`` performs
+    single-ref discovery: it queries the bootstrap org's storage for every org
+    with actionable work (covers OSS/local and the shared managed DATA ref). The
+    enterprise cross-ref enumerator is a deferred follow-up.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+        bootstrap_org_id: Org used to reach a storage for cross-org discovery.
+        org_ids_provider: Optional explicit provider (e.g. a future enterprise
+            cross-ref enumerator); when ``None`` the single-ref default is used.
+    """
+    if not env_truthy(env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")):
+        return None
+
+    def _default_provider() -> list[str]:
+        try:
+            ctx = request_context_factory(bootstrap_org_id)
+            storage = getattr(ctx, "storage", None)
+            if storage is None:
+                return []
+            return list(storage.list_org_ids_with_pending_learning_jobs())
+        except NotImplementedError:
+            return []
+        except Exception:
+            logger.exception(
+                "event=durable_learning_discovery_failed bootstrap_org_id=%s",
+                bootstrap_org_id,
+            )
+            return []
+
+    scheduler = DurableLearningScheduler(
+        request_context_factory=request_context_factory,
+        org_ids_provider=org_ids_provider or _default_provider,
+    )
+    scheduler.start()
+    return scheduler

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py

@@ -0,0 +1,193 @@
+"""DurableLearningWorker: claim jobs from the durable queue, process in a fenced
+commit_scope so exactly one of N racing workers commits its outputs.
+
+Design constraints (v1 — do not remove):
+- LLM calls are inside the commit_scope (compute/persist separation is deferred).
+- No heartbeat thread (deferred; would deadlock a SQLite scope anyway).
+- Exactly-once is guaranteed by the fenced complete_learning_job:
+    rowcount == 0  -> lease was stolen -> _SupersededError -> rollback.
+    rowcount == 1  -> we own the lease -> commit succeeds.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from collections.abc import Callable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.cache.reflexio_cache import get_reflexio
+from reflexio.server.services.generation_service import GenerationService
+from reflexio.server.services.storage.storage_base._learning_jobs import LearningJob
+
+logger = logging.getLogger(__name__)
+
+
+class _SupersededError(Exception):
+    """Raised inside commit_scope when complete_learning_job returns 0.
+
+    Propagating this exception causes commit_scope to roll back all writes
+    (profiles, playbooks) made by run_deferred_learning for the superseded
+    worker — guaranteeing exactly-once side effects.
+    """
+
+    def __init__(self, job_id: str) -> None:
+        super().__init__(f"learning job {job_id} superseded (lease stolen)")
+
+
+class DurableLearningWorker:
+    """Claim durable learning jobs and process each in a fenced commit_scope.
+
+    Two instances racing the same job produce profile/playbook side effects
+    identical to a single run: the loser's complete_learning_job returns 0,
+    which raises _SupersededError inside the scope, rolling back its writes.
+
+    Args:
+        request_context_factory: Callable that returns a RequestContext for a
+            given org_id.  Called once per drain_org invocation.
+        instance_id: Stable label written to claimed_by on each claim.
+            Defaults to a random short UUID suffix.
+    """
+
+    def __init__(
+        self,
+        request_context_factory: Callable[[str], RequestContext],
+        *,
+        instance_id: str | None = None,
+    ) -> None:
+        self._factory = request_context_factory
+        self._instance_id = instance_id or str(uuid.uuid4())[:8]
+
+    def drain_org(self, org_id: str, batch_size: int, lease_seconds: int) -> int:
+        """Claim up to batch_size of this org's jobs and process each.
+
+        Args:
+            org_id: Organisation to drain.
+            batch_size: Maximum number of jobs to claim in one call.
+            lease_seconds: Lease duration passed to claim_learning_jobs.
+
+        Returns:
+            Number of jobs successfully completed (not counting superseded or failed).
+        """
+        ctx = self._factory(org_id)
+        storage = ctx.storage
+        if storage is None:
+            logger.error("event=learning_worker_no_storage org_id=%s", org_id)
+            return 0
+        jobs = storage.claim_learning_jobs(
+            claimed_by=self._instance_id,
+            limit=batch_size,
+            lease_seconds=lease_seconds,
+        )
+        processed = 0
+        for job in jobs:
+            if self._process_job(ctx, job):
+                processed += 1
+        return processed
+
+    def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool:
+        """Process a single claimed job.
+
+        Args:
+            ctx: RequestContext for the job's org.
+            job: The claimed LearningJob (claim_token set).
+
+        Returns:
+            True if the job was successfully completed; False if superseded or failed.
+        """
+        storage = ctx.storage
+        if storage is None:
+            logger.error(
+                "event=learning_job_no_storage job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        # claim_learning_jobs always sets claim_token on a returned job.
+        claim_token = job.claim_token
+        if claim_token is None:
+            logger.error(
+                "event=learning_job_no_claim_token job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        dead = job.attempts >= job.max_attempts
+
+        # The job does not carry agent_version / session_id / source.
+        # These must be read from the durably-persisted Request.
+        if job.latest_request_id is None:
+            logger.error(
+                "event=learning_job_no_request_id job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        request = storage.get_request(job.latest_request_id)
+        if request is None:
+            logger.error(
+                "event=learning_job_missing_request job_id=%s request_id=%s",
+                job.job_id,
+                job.latest_request_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        try:
+            reflexio = get_reflexio(
+                org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir
+            )
+            gen = GenerationService(llm_client=reflexio.llm_client, request_context=ctx)
+
+            with storage.commit_scope():
+                gen.run_deferred_learning(
+                    user_id=job.user_id,
+                    request_id=job.latest_request_id,
+                    session_id=request.session_id,
+                    source=request.source,
+                    agent_version=request.agent_version,
+                    force_extraction=job.force_extraction,
+                    skip_aggregation=job.skip_aggregation,
+                    sequential=True,  # prevent ThreadPoolExecutor deadlock on commit_scope RLock
+                )
+                rows = storage.complete_learning_job(
+                    job_id=job.job_id, claim_token=claim_token
+                )
+                if rows == 0:
+                    raise _SupersededError(job.job_id)
+            logger.info(
+                "event=learning_job_done job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            return True
+        except _SupersededError:
+            logger.info(
+                "event=learning_job_superseded job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+        except Exception:
+            logger.exception(
+                "event=learning_job_failed job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            # attempts was incremented by claim_learning_jobs; go dead when we've
+            # exhausted max_attempts total claim attempts.
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py

@@ -11,11 +11,11 @@ with — never another tenant's runs.
 from __future__ import annotations
 
 import logging
-import threading
 from collections.abc import Callable
 from datetime import UTC, datetime
 
 from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.scheduling import ThreadedScheduler
 from reflexio.server.services.extraction.resumable_agent import (
     pending_tool_calls_enabled,
 )
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 _DEFAULT_POLL_INTERVAL_SECONDS = 5.0
 
 
-class ExtractionResumeScheduler:
+class ExtractionResumeScheduler(ThreadedScheduler):
     """Small polling wrapper that drives :class:`ExtractionResumeWorker` per org."""
 
     def __init__(
@@ -37,29 +37,15 @@ class ExtractionResumeScheduler:
         bootstrap_org_id: str,
         max_runs_per_tick: int = 10,
     ) -> None:
+        super().__init__(thread_name="reflexio-extraction-resume-scheduler")
         self.request_context_factory = request_context_factory
         self.bootstrap_org_id = bootstrap_org_id
         self.max_runs_per_tick = max_runs_per_tick
-        self._stop_event = threading.Event()
-        self._thread: threading.Thread | None = None
 
-    def start(self) -> None:
-        if self._thread is not None and self._thread.is_alive():
-            return
-        self._stop_event.clear()
-        self._thread = threading.Thread(
-            target=self._run_loop,
-            name="reflexio-extraction-resume-scheduler",
-            daemon=True,
-        )
-        self._thread.start()
+    def _on_started(self) -> None:
         logger.info("event=extraction_resume_scheduler_started")
 
-    def stop(self, *, timeout_seconds: float = 5.0) -> None:
-        self._stop_event.set()
-        if self._thread is not None:
-            self._thread.join(timeout=timeout_seconds)
-        self._thread = None
+    def _on_stopped(self) -> None:
         logger.info("event=extraction_resume_scheduler_stopped")
 
     def _discover_org_ids(self, bootstrap_ctx: RequestContext) -> list[str]:
@@ -116,28 +102,25 @@ class ExtractionResumeScheduler:
                     org_id,
                 )
 
-    def _run_loop(self) -> None:
-        while not self._stop_event.is_set():
-            poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
-            try:
-                bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
-                config = bootstrap_ctx.configurator.get_config()
-                poll_interval = (
-                    config.pending_tool_call_config.resume_poll_interval_seconds
-                )
-                self._expire_pending_tool_calls(bootstrap_ctx)
-                for org_id in self._discover_org_ids(bootstrap_ctx):
-                    if self._stop_event.is_set():
-                        break
-                    self._drain_org(org_id)
-            except Exception as exc:
-                with sentry_tags(
-                    subsystem="extraction",
-                    op="scheduler_tick",
-                    error_type=type(exc).__name__,
-                ):
-                    logger.exception("event=extraction_resume_scheduler_tick_failed")
-            self._stop_event.wait(poll_interval)
+    def _run_once(self) -> float:
+        poll_interval = _DEFAULT_POLL_INTERVAL_SECONDS
+        try:
+            bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
+            config = bootstrap_ctx.configurator.get_config()
+            poll_interval = config.pending_tool_call_config.resume_poll_interval_seconds
+            self._expire_pending_tool_calls(bootstrap_ctx)
+            for org_id in self._discover_org_ids(bootstrap_ctx):
+                if self._stop_event.is_set():
+                    break
+                self._drain_org(org_id)
+        except Exception as exc:
+            with sentry_tags(
+                subsystem="extraction",
+                op="scheduler_tick",
+                error_type=type(exc).__name__,
+            ):
+                logger.exception("event=extraction_resume_scheduler_tick_failed")
+        return poll_interval
 
 
 def maybe_start_resume_scheduler(