claude-smart 0.2.48 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py

@@ -0,0 +1,270 @@
+"""Write-plan dataclasses for the durable-learning compute/persist split (gate b).
+
+Compute resolves these in-memory plans issuing **no** learning DB write; persist
+applies them inside one short fenced ``commit_scope``. This module starts with the
+extractor stride-bookmark advance and grows additional write-plan dataclasses in the
+later gate-(b) tasks (profile/playbook/reflection/generation/deferred plans).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from reflexio.models.api_schema.domain.entities import (
+        UserPlaybook,
+        UserProfile,
+    )
+    from reflexio.models.api_schema.service_schemas import Interaction
+    from reflexio.server.llm.token_accounting import RunTokenTotals
+    from reflexio.server.services.base_generation_service import (
+        BaseGenerationService,
+        PreparedGenerationRun,
+    )
+    from reflexio.server.services.reflection.reflection_service_utils import (
+        ReflectionDecision,
+        ReflectionResult,
+        ReflectionServiceRequest,
+    )
+    from reflexio.server.services.reflection.service import ReflectionService
+
+
+@dataclass(frozen=True)
+class ExtractorBookmarkAdvance:
+    """The extractor stride-bookmark advance, deferred out of the extractor (F1).
+
+    The extractor no longer self-advances its bookmark inside ``run()`` (a DB
+    write). It emits this on the ``ExtractionOutcome`` instead, so the advance is
+    applied later — inside the persist fence on the durable path, or right after
+    result-processing on the synchronous ``.run()`` path — keeping the bookmark
+    advance atomic with the row writes it corresponds to.
+    """
+
+    extractor_name: str
+    processed_interactions: list[Interaction]
+    # str | None (not str): the playbook extractor runs org-level with
+    # user_id=None (PlaybookGenerationServiceConfig.user_id is Optional), which
+    # maps to an unscoped bookmark key. update_extractor_bookmark accepts None.
+    user_id: str | None
+
+
+@dataclass
+class ProfileWritePlan:
+    """Resolved profile write-plan for the compute/persist split (gate b).
+
+    ``ProfileGenerationService._resolve_write_plan`` produces this in compute
+    (dedup 2nd-LLM call + existing-row reads + source/status assignment +
+    precomputed ``.embedding`` via ``precompute_profile_embeddings``), issuing
+    **no** learning DB write. ``_persist_write_plan`` applies it inside the
+    fence: ``add_user_profile(..., skip_embedding=True)`` (embeddings already
+    set) then ``supersede_profiles_by_ids``.
+
+    Attributes:
+        user_id: Owning user id for the row writes.
+        request_id: Generation request id — the lineage key
+            ``supersede_profiles_by_ids`` records. Compute drops any supersede
+            ids when this is empty (unreconstructable), so persist only ever
+            supersedes with a non-empty request_id.
+        new_profiles: New profile rows with ``source``/``status`` set and
+            ``.embedding`` precomputed in compute.
+        superseded_ids: Existing profile ids to soft-supersede on the dedup
+            path.
+    """
+
+    user_id: str
+    request_id: str
+    new_profiles: list[UserProfile]
+    superseded_ids: list[str]
+
+
+@dataclass
+class PlaybookWritePlan:
+    """Resolved playbook write-plan for the compute/persist split (gate b).
+
+    ``PlaybookGenerationService._resolve_write_plan`` produces this in compute
+    (``dedupe_and_drop_empty`` + the deduplicator's 2nd-LLM call + existing-row
+    reads + source/status assignment + precomputed ``.embedding`` via
+    ``precompute_user_playbook_embeddings``), issuing **no** learning DB write.
+    ``_persist_write_plan`` applies it inside the fence:
+    ``save_user_playbooks(..., skip_embedding=True)`` (which assigns survivor
+    ids) then ``_apply_consolidation_lineage`` (which MUST see those ids, so it
+    runs AFTER the save).
+
+    The off-thread schedulers (``_enqueue_user_playbook_optimization`` +
+    ``_trigger_playbook_aggregation``) are NOT part of persist — they fire
+    post-commit in ``emit_generation_side_effects`` (durable / ``.run()`` path)
+    or right after persist in the permanent ``_finalize_extracted_items``
+    wrapper (synchronous resume/manual path). ``output_pending_status`` /
+    ``skip_aggregation`` are snapshotted here so the scheduler dispatch reads the
+    plan rather than the reused service instance.
+
+    Attributes:
+        request_id: Generation request id — the lineage key
+            ``_apply_consolidation_lineage`` records on merges/supersedes.
+        output_pending_status: Whether the run emits PENDING rows (rerun mode);
+            when True the aggregation trigger is suppressed.
+        skip_aggregation: Whether aggregation is skipped (extract-only); when
+            True the aggregation trigger is suppressed.
+        new_playbooks: New playbook rows with ``source``/``status`` set and
+            ``.embedding`` precomputed in compute. Survivor ids are assigned by
+            ``save_user_playbooks`` in persist, before lineage reads them.
+        superseded_ids: ALL archived existing ids (merge sources + leftovers)
+            routed through ``_apply_consolidation_lineage``.
+        merge_groups: ``(survivor_index_into_new_playbooks, source_existing_ids)``
+            per dedup merge group.
+    """
+
+    request_id: str
+    output_pending_status: bool
+    skip_aggregation: bool
+    new_playbooks: list[UserPlaybook]
+    superseded_ids: list[int]
+    merge_groups: list[tuple[int, list[int]]]
+
+
+@dataclass
+class ReflectionWritePlan:
+    """Resolved reflection write-plan for the compute/persist split (F5, V2).
+
+    ``ReflectionService.compute`` produces this issuing **no** learning DB write;
+    ``persist`` applies the whole cap/validate/CAS apply loop verbatim inside the
+    fence, then advances the reflection stride-bookmark; ``emit_side_effects``
+    fires the post-commit billing + aggregation triggers. The apply loop stays a
+    single monolith (no per-revision ``ResolvedReflectionRevision`` dataclass).
+
+    Attributes:
+        request: The originating reflection request (user_id / request_id /
+            agent_version).
+        result: The reflection result. Compute seeds it with the gate/citation
+            counters resolved so far; persist's apply loop mutates it in place
+            (revised/failed/skipped/... counters) as it applies each decision.
+        decisions: ``ReflectionOutput.decisions`` to apply — ``[]`` on the four
+            early bookmark-advance paths (nothing to apply, only advance).
+        profiles_by_id: Cited profile rows resolved in compute, keyed by
+            ``profile_id`` (the apply loop's ``_apply_revision`` input).
+        playbooks_by_id: Cited playbook rows resolved in compute, keyed by
+            ``user_playbook_id``.
+        max_revisions_per_pass: Per-pass revision cap applied inside the loop.
+        bookmark_interactions: The window the reflection stride-bookmark advances
+            over in persist (``[]`` on the empty-window advance path).
+        advance_bookmark: ``True`` on all five bookmark-advance paths (the four
+            early-advance returns + the normal end). Persist advances the
+            reflection bookmark iff this is ``True``.
+        record_learnings: ``True`` only on the normal-end path — gates the
+            post-commit ``_record_learnings_generated`` billing event in emit.
+        replacement_profiles: Replacement ``UserProfile`` rows whose embeddings
+            were precomputed in compute (V2), keyed by the **cited** profile_id
+            they replace. Persist inserts these with ``skip_embedding=True`` so
+            no embedding runs inside the fence. Empty on non-compute (direct)
+            ``_replace_*`` calls, which then build+embed the row themselves.
+        replacement_playbooks: Same, keyed by the cited ``user_playbook_id``.
+        aggregation_successor_ids: Playbook successor ids collected during the
+            persist apply loop; their aggregation trigger is dispatched in
+            ``emit_side_effects`` (post-commit) so it never fires on a
+            rolled-back job.
+    """
+
+    request: ReflectionServiceRequest
+    result: ReflectionResult
+    decisions: list[ReflectionDecision]
+    profiles_by_id: dict[str, UserProfile]
+    playbooks_by_id: dict[int, UserPlaybook]
+    max_revisions_per_pass: int
+    bookmark_interactions: list[Interaction]
+    advance_bookmark: bool
+    record_learnings: bool
+    replacement_profiles: dict[str, UserProfile] = field(default_factory=dict)
+    replacement_playbooks: dict[int, UserPlaybook] = field(default_factory=dict)
+    aggregation_successor_ids: list[int] = field(default_factory=list)
+
+
+@dataclass
+class GenerationComputePlan:
+    """Resolved compute output of one ``BaseGenerationService`` run (gate b).
+
+    ``compute_generation`` runs the prepare gate + extractor + dedup/embedding
+    resolution (``_resolve_write_plan``) and drives the ``agent_run`` rows to
+    their terminal state (``_finalize_extraction_runs`` — agent_run only, §4.3),
+    issuing **no** learning DB write. ``persist_generation`` applies
+    ``write_plan`` + the extractor bookmark advance inside the fence;
+    ``emit_generation_side_effects`` fires the post-commit telemetry + billing.
+
+    The billing inputs (``extraction_run_ids`` / ``token_totals`` /
+    ``generated_count`` / ``prepared``) are **snapshotted at compute time** so
+    the fence-crossing emit reads this plan rather than the reused service
+    instance's mutable ``_last_*`` accumulators (purity contract, plan §File
+    Structure). See ``emit_generation_side_effects`` for the single-use-instance
+    invariant that also keeps the money helper's ``self._last_*`` reads safe.
+
+    Attributes:
+        prepared: The prepared generation run (identifier / extractor_name /
+            extractor_config), reused by emit for telemetry + billing input.
+        generated_count: Learnings produced by this extraction run.
+        write_plan: The resolved write-plan (``ProfileWritePlan`` /
+            ``PlaybookWritePlan`` in Tasks 6-7, a ``_LegacyItems`` shim marker
+            until then) or ``None`` when the extractor produced nothing.
+        bookmark_advance: The deferred extractor stride-bookmark advance (F1),
+            applied inside the persist scope. ``None`` when the extractor
+            produced no output or the service has no stride bookmark.
+        generation_start: ``perf_counter`` captured at compute start; emit reads
+            it for the ``generation_succeeded`` ``duration_ms`` (parity).
+        extraction_run_ids: Snapshot of the run's ``agent_run`` ids.
+        token_totals: Snapshot of the run's LLM token totals (billing cost
+            facet), or ``None`` when the extractor reported none.
+    """
+
+    prepared: PreparedGenerationRun[Any]
+    generated_count: int
+    write_plan: Any
+    bookmark_advance: ExtractorBookmarkAdvance | None
+    generation_start: float
+    extraction_run_ids: list[str]
+    token_totals: RunTokenTotals | None
+
+
+@dataclass
+class DeferredLearningPlan:
+    """One durable-learning job's resolved compute output (gate b, Task 8).
+
+    ``GenerationService.compute_deferred_learning`` acquires the same-user guard
+    (F4), then runs reflection + profile + playbook compute holding **no**
+    ``commit_scope``, and assembles this plan. It carries each present half as a
+    ``(held_service_instance, its_compute_plan)`` pair so
+    ``persist_deferred_learning`` can apply the fence-critical writes and
+    ``emit_deferred_learning_side_effects`` can fire the post-commit telemetry on
+    the very instances that produced the plans (the single-use-instance invariant
+    that keeps the money helpers' ``self._last_*`` reads valid — see
+    ``BaseGenerationService.emit_generation_side_effects``).
+
+    Attributes:
+        request_id: The job's generation request id (lineage key).
+        user_id: The user the job learns for; also the per-user F4 lock scope.
+        agent_version: Resolved agent version — carried so the post-commit
+            ``schedule_tagging`` dispatch in emit has it (the plan is all emit
+            receives).
+        lock_acquired: ``False`` when the F4 same-user guard denied this job
+            (another durable job for the same user is mid-flight). On ``False``
+            the three plan halves are ``None`` and NO LLM/compute ran — the
+            worker (Task 9) must leave the job reclaimable and must NOT
+            ``complete_learning_job`` it.
+        reflection: ``(ReflectionService, ReflectionWritePlan)`` when reflection
+            resolved something to persist, else ``None`` (gate closed / disabled
+            / compute error captured into ``warnings``).
+        profile: ``(ProfileGenerationService, GenerationComputePlan)`` when the
+            profile extractor produced a plan, else ``None``.
+        playbook: ``(PlaybookGenerationService, GenerationComputePlan)`` when the
+            playbook extractor produced a plan, else ``None``.
+        warnings: Best-effort per-half compute failures (mirrors
+            ``GenerationServiceResult.warnings``) — a failed half is dropped from
+            the plan, the others still persist.
+    """
+
+    request_id: str
+    user_id: str
+    agent_version: str
+    lock_acquired: bool
+    reflection: tuple[ReflectionService, ReflectionWritePlan] | None
+    profile: tuple[BaseGenerationService, GenerationComputePlan] | None
+    playbook: tuple[BaseGenerationService, GenerationComputePlan] | None
+    warnings: list[str] = field(default_factory=list)

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py

@@ -0,0 +1,13 @@
+"""Durable learning worker: claim → process in commit_scope → fenced completion."""
+
+from reflexio.server.services.durable_learning.scheduler import (
+    DurableLearningScheduler,
+    maybe_start_durable_learning,
+)
+from reflexio.server.services.durable_learning.worker import DurableLearningWorker
+
+__all__ = [
+    "DurableLearningScheduler",
+    "DurableLearningWorker",
+    "maybe_start_durable_learning",
+]

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py

@@ -0,0 +1,142 @@
+"""Process-local scheduler for the durable learning-job queue (Task 6).
+
+Each tick discovers the orgs with actionable learning jobs (via an injected
+``org_ids_provider``) and drains each through a :class:`DurableLearningWorker`.
+The worker's claims are org-scoped and the fenced complete/fail transitions make
+the drain exactly-once, so racing scheduler instances across processes are safe.
+
+Gating: startup is gated on ``REFLEXIO_DURABLE_LEARNING_QUEUE`` (a rollout flag,
+scalability design 2026-07-04 §8 stage 2). When off, :meth:`start` is a no-op.
+
+Multi-ref capability (design §3.1, per-ref polling): the mechanism supports
+per-data-ref fan-out — inject an ``org_ids_provider`` that yields orgs across
+refs plus a ``request_context_factory`` that routes each org to its ref's
+storage. The default provider (single-ref: bootstrap-storage discovery) is wired
+in :func:`maybe_start_durable_learning`; the enterprise cross-ref enumerator is a
+deferred follow-up (tracked with the pre-Task-8 gates).
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Iterable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.env_utils import env_str, env_truthy
+from reflexio.server.scheduling import ThreadedScheduler
+from reflexio.server.services.durable_learning.worker import DurableLearningWorker
+
+logger = logging.getLogger(__name__)
+
+
+class DurableLearningScheduler(ThreadedScheduler):
+    """Polling daemon that drains the durable learning queue per org.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+            Passed straight to the worker; a multi-ref deployment routes each org
+            to its data ref's storage here.
+        org_ids_provider: Called once per tick; yields the org_ids to drain. The
+            single-ref default (see :func:`maybe_start_durable_learning`) reads
+            them from the bootstrap storage's cross-org discovery query.
+        instance_id: Stable label written to ``claimed_by`` on each claim.
+    """
+
+    def __init__(
+        self,
+        *,
+        request_context_factory: Callable[[str], RequestContext],
+        org_ids_provider: Callable[[], Iterable[str]],
+        instance_id: str | None = None,
+    ) -> None:
+        super().__init__(thread_name="reflexio-durable-learning-scheduler")
+        self._worker = DurableLearningWorker(
+            request_context_factory, instance_id=instance_id
+        )
+        self._org_ids_provider = org_ids_provider
+        # Read env INSIDE __init__ so tests can set the values before construction.
+        self._poll = float(env_str("REFLEXIO_DURABLE_LEARNING_POLL_SECONDS", "2.0"))
+        self._batch = int(env_str("REFLEXIO_DURABLE_LEARNING_BATCH", "16"))
+        self._lease = int(env_str("REFLEXIO_DURABLE_LEARNING_LEASE_SECONDS", "300"))
+
+    def _should_start(self) -> bool:
+        if env_truthy(env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")):
+            return True
+        logger.info(
+            "event=durable_learning_scheduler_disabled "
+            "REFLEXIO_DURABLE_LEARNING_QUEUE not set — not starting"
+        )
+        return False
+
+    def _on_started(self) -> None:
+        logger.info("event=durable_learning_scheduler_started")
+
+    def _on_stopped(self) -> None:
+        logger.info("event=durable_learning_scheduler_stopped")
+
+    def _run_once(self) -> float:
+        """Drain every discovered org once; per-org error isolation.
+
+        A single org raising never aborts the tick, and a failure to enumerate
+        orgs never kills the daemon thread. Returns the poll interval.
+        """
+        try:
+            for org_id in self._org_ids_provider():
+                if self._stop_event.is_set():
+                    break
+                try:
+                    self._worker.drain_org(org_id, self._batch, self._lease)
+                except Exception:
+                    logger.exception(
+                        "event=durable_learning_drain_failed org_id=%s", org_id
+                    )
+        except Exception:
+            logger.exception("event=durable_learning_tick_failed")
+        return self._poll
+
+
+def maybe_start_durable_learning(
+    request_context_factory: Callable[[str], RequestContext],
+    *,
+    bootstrap_org_id: str,
+    org_ids_provider: Callable[[], Iterable[str]] | None = None,
+) -> DurableLearningScheduler | None:
+    """Start the durable-learning scheduler when the rollout flag is on.
+
+    Returns ``None`` when ``REFLEXIO_DURABLE_LEARNING_QUEUE`` is off (mirrors
+    ``maybe_start_resume_scheduler``). The default ``org_ids_provider`` performs
+    single-ref discovery: it queries the bootstrap org's storage for every org
+    with actionable work (covers OSS/local and the shared managed DATA ref). The
+    enterprise cross-ref enumerator is a deferred follow-up.
+
+    Args:
+        request_context_factory: Builds an org-scoped :class:`RequestContext`.
+        bootstrap_org_id: Org used to reach a storage for cross-org discovery.
+        org_ids_provider: Optional explicit provider (e.g. a future enterprise
+            cross-ref enumerator); when ``None`` the single-ref default is used.
+    """
+    if not env_truthy(env_str("REFLEXIO_DURABLE_LEARNING_QUEUE", "false")):
+        return None
+
+    def _default_provider() -> list[str]:
+        try:
+            ctx = request_context_factory(bootstrap_org_id)
+            storage = getattr(ctx, "storage", None)
+            if storage is None:
+                return []
+            return list(storage.list_org_ids_with_pending_learning_jobs())
+        except NotImplementedError:
+            return []
+        except Exception:
+            logger.exception(
+                "event=durable_learning_discovery_failed bootstrap_org_id=%s",
+                bootstrap_org_id,
+            )
+            return []
+
+    scheduler = DurableLearningScheduler(
+        request_context_factory=request_context_factory,
+        org_ids_provider=org_ids_provider or _default_provider,
+    )
+    scheduler.start()
+    return scheduler

package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py

@@ -0,0 +1,262 @@
+"""DurableLearningWorker: claim jobs from the durable queue, split each into
+compute (no writer transaction) -> persist (short fenced commit_scope) ->
+post-commit side-effects so exactly one of N racing workers commits its outputs.
+
+Design constraints (gate b — do not remove):
+- LLM extraction + dedup + embeddings run in compute_deferred_learning, OUTSIDE
+  the commit_scope. Only the fence-critical writes (profile/playbook rows +
+  bookmark advances + complete_learning_job) run inside the scope; billing /
+  telemetry / tagging / off-thread schedulers + the per-user lock release run
+  AFTER the scope commits (emit_deferred_learning_side_effects). Holding the
+  writer transaction across the ~30-120s LLM window is exactly what this split
+  removes.
+- No heartbeat thread (deferred; would deadlock a SQLite scope anyway).
+- Exactly-once is guaranteed by the fenced complete_learning_job:
+    rowcount == 0  -> lease was stolen -> _SupersededError -> rollback.
+    rowcount == 1  -> we own the lease -> commit succeeds.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from collections.abc import Callable
+
+from reflexio.server.api_endpoints.request_context import RequestContext
+from reflexio.server.cache.reflexio_cache import get_reflexio
+from reflexio.server.services.generation_service import GenerationService
+from reflexio.server.services.storage.storage_base._learning_jobs import LearningJob
+
+logger = logging.getLogger(__name__)
+
+
+class _SupersededError(Exception):
+    """Raised inside commit_scope when complete_learning_job returns 0.
+
+    Propagating this exception causes commit_scope to roll back all writes
+    (profiles, playbooks, bookmark advances) made by persist_deferred_learning
+    for the superseded worker — guaranteeing exactly-once side effects. The
+    post-commit emit is skipped, so a superseded job phantom-bills nothing.
+    """
+
+    def __init__(self, job_id: str) -> None:
+        super().__init__(f"learning job {job_id} superseded (lease stolen)")
+
+
+class DurableLearningWorker:
+    """Claim durable learning jobs and process each in a fenced commit_scope.
+
+    Two instances racing the same job produce profile/playbook side effects
+    identical to a single run: the loser's complete_learning_job returns 0,
+    which raises _SupersededError inside the scope, rolling back its writes.
+
+    Args:
+        request_context_factory: Callable that returns a RequestContext for a
+            given org_id.  Called once per drain_org invocation.
+        instance_id: Stable label written to claimed_by on each claim.
+            Defaults to a random short UUID suffix.
+    """
+
+    def __init__(
+        self,
+        request_context_factory: Callable[[str], RequestContext],
+        *,
+        instance_id: str | None = None,
+    ) -> None:
+        self._factory = request_context_factory
+        self._instance_id = instance_id or str(uuid.uuid4())[:8]
+
+    def drain_org(self, org_id: str, batch_size: int, lease_seconds: int) -> int:
+        """Claim up to batch_size of this org's jobs and process each.
+
+        Args:
+            org_id: Organisation to drain.
+            batch_size: Maximum number of jobs to claim in one call.
+            lease_seconds: Lease duration passed to claim_learning_jobs.
+
+        Returns:
+            Number of jobs successfully completed (not counting superseded or failed).
+        """
+        ctx = self._factory(org_id)
+        storage = ctx.storage
+        if storage is None:
+            logger.error("event=learning_worker_no_storage org_id=%s", org_id)
+            return 0
+        jobs = storage.claim_learning_jobs(
+            claimed_by=self._instance_id,
+            limit=batch_size,
+            lease_seconds=lease_seconds,
+        )
+        processed = 0
+        for job in jobs:
+            if self._process_job(ctx, job):
+                processed += 1
+        return processed
+
+    def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool:
+        """Process a single claimed job.
+
+        Args:
+            ctx: RequestContext for the job's org.
+            job: The claimed LearningJob (claim_token set).
+
+        Returns:
+            True if the job was successfully completed; False if superseded or failed.
+        """
+        storage = ctx.storage
+        if storage is None:
+            logger.error(
+                "event=learning_job_no_storage job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        # claim_learning_jobs always sets claim_token on a returned job.
+        claim_token = job.claim_token
+        if claim_token is None:
+            logger.error(
+                "event=learning_job_no_claim_token job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            return False
+
+        dead = job.attempts >= job.max_attempts
+
+        # The job does not carry agent_version / session_id / source.
+        # These must be read from the durably-persisted Request.
+        if job.latest_request_id is None:
+            logger.error(
+                "event=learning_job_no_request_id job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        request = storage.get_request(job.latest_request_id)
+        if request is None:
+            logger.error(
+                "event=learning_job_missing_request job_id=%s request_id=%s",
+                job.job_id,
+                job.latest_request_id,
+            )
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+        gen: GenerationService | None = None
+        try:
+            reflexio = get_reflexio(
+                org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir
+            )
+            gen = GenerationService(llm_client=reflexio.llm_client, request_context=ctx)
+
+            # COMPUTE — LLM extraction + dedup + embeddings, NO writer transaction
+            # held. Acquires the per-user F4 lock; issues no learning DB write.
+            plan = gen.compute_deferred_learning(
+                user_id=job.user_id,
+                request_id=job.latest_request_id,
+                session_id=request.session_id,
+                source=request.source,
+                agent_version=request.agent_version,
+                force_extraction=job.force_extraction,
+                skip_aggregation=job.skip_aggregation,
+            )
+
+            # Same-user contention (F4): another same-user durable job holds the
+            # per-user lock. Leave THIS job reclaimable (dead=False) — do NOT
+            # complete it — so the queue re-claims it once the holder finishes.
+            # REFUND the attempt (refund_attempt=True): claim_learning_jobs did
+            # attempts += 1 on this claim, but no real work ran, and the ~2s poll
+            # would otherwise re-claim (and re-increment) every couple of seconds
+            # while the holder is in its ~60s compute — inflating attempts past
+            # max_attempts in seconds so the eventual winner dead-letters on its
+            # first transient error with zero real retries. The refund nets each
+            # contention cycle (claim +1, release -1) to zero. No lock to release
+            # (compute never acquired it).
+            if not plan.lock_acquired:
+                storage.fail_learning_job(
+                    job_id=job.job_id,
+                    claim_token=claim_token,
+                    dead=False,
+                    refund_attempt=True,
+                )
+                return False
+
+            # PERSIST — short fenced scope: fence-critical writes + bookmark
+            # advances only, then the claim-token fence. rows == 0 -> lease
+            # stolen -> _SupersededError -> the scope rolls persist back.
+            with storage.commit_scope():
+                gen.persist_deferred_learning(plan)
+                rows = storage.complete_learning_job(
+                    job_id=job.job_id, claim_token=claim_token
+                )
+                if rows == 0:
+                    raise _SupersededError(job.job_id)
+
+            # POST-COMMIT — billing / telemetry / tagging / off-thread schedulers
+            # + the per-user lock release, only for the winning worker.
+            gen.emit_deferred_learning_side_effects(plan)
+            logger.info(
+                "event=learning_job_done job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            return True
+        except _SupersededError:
+            logger.info(
+                "event=learning_job_superseded job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )
+            # The persist rolled back and emit never ran, so the per-user lock is
+            # still held by this compute — release it so the reclaim isn't blocked.
+            self._release_user_lock(gen, job)
+            return False
+        except Exception:
+            logger.exception(
+                "event=learning_job_failed job_id=%s org_id=%s user_id=%s",
+                job.job_id,
+                job.org_id,
+                job.user_id,
+            )
+            # emit (which releases the lock) never ran on this path — release the
+            # per-user lock so a failed job doesn't strand it.
+            self._release_user_lock(gen, job)
+            # attempts was incremented by claim_learning_jobs; go dead when we've
+            # exhausted max_attempts total claim attempts.
+            storage.fail_learning_job(
+                job_id=job.job_id, claim_token=claim_token, dead=dead
+            )
+            return False
+
+    def _release_user_lock(
+        self, gen: GenerationService | None, job: LearningJob
+    ) -> None:
+        """Best-effort release of the per-user F4 lock after a rolled-back/failed
+        job, so a job whose emit (the normal release site) never ran does not
+        strand the lock and block the same user's re-claim.
+
+        Safe to call even when compute never acquired the lock: the release is a
+        CAS on the holder (``clear_in_progress_lock_if_owner``) — a no-op unless
+        this compute's ``request_id`` still owns it. Never raises.
+        """
+        if gen is None or job.latest_request_id is None:
+            return
+        try:
+            gen._release_durable_learning_lock(
+                user_id=job.user_id, request_id=job.latest_request_id
+            )
+        except Exception:
+            logger.exception(
+                "event=learning_job_lock_release_failed job_id=%s org_id=%s",
+                job.job_id,
+                job.org_id,
+            )