mdbxmou 0.1.26

package/deps/libmdbx/src/meta.h

@@ -0,0 +1,168 @@
+/// \copyright SPDX-License-Identifier: Apache-2.0
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+
+#pragma once
+
+#include "essentials.h"
+
+static inline uint64_t meta_sign_calculate(const meta_t *meta) {
+  uint64_t sign = DATASIGN_NONE;
+#if 0 /* TODO */
+  sign = hippeus_hash64(...);
+#else
+  (void)meta;
+#endif
+  /* LY: newer returns DATASIGN_NONE or DATASIGN_WEAK */
+  return (sign > DATASIGN_WEAK) ? sign : ~sign;
+}
+
+static inline uint64_t meta_sign_get(const volatile meta_t *meta) { return unaligned_peek_u64_volatile(4, meta->sign); }
+
+static inline void meta_sign_as_steady(meta_t *meta) { unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); }
+
+static inline bool meta_is_steady(const volatile meta_t *meta) { return SIGN_IS_STEADY(meta_sign_get(meta)); }
+
+MDBX_INTERNAL troika_t meta_tap(const MDBX_env *env);
+MDBX_INTERNAL unsigned meta_eq_mask(const troika_t *troika);
+MDBX_INTERNAL bool meta_should_retry(const MDBX_env *env, troika_t *troika);
+MDBX_MAYBE_UNUSED MDBX_INTERNAL bool troika_verify_fsm(void);
+
+struct meta_ptr {
+  txnid_t txnid;
+  union {
+    const volatile meta_t *ptr_v;
+    const meta_t *ptr_c;
+  };
+  size_t is_steady;
+};
+
+MDBX_INTERNAL meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n);
+MDBX_INTERNAL txnid_t meta_txnid(const volatile meta_t *meta);
+MDBX_INTERNAL txnid_t recent_committed_txnid(const MDBX_env *env);
+MDBX_INTERNAL int meta_sync(const MDBX_env *env, const meta_ptr_t head);
+
+MDBX_INTERNAL const char *durable_caption(const meta_t *const meta);
+MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, const troika_t *troika);
+
+#define METAPAGE(env, n) page_meta(pgno2page(env, n))
+#define METAPAGE_END(env) METAPAGE(env, NUM_METAS)
+
+static inline meta_ptr_t meta_recent(const MDBX_env *env, const troika_t *troika) {
+  meta_ptr_t r;
+  r.txnid = troika->txnid[troika->recent];
+  r.ptr_v = METAPAGE(env, troika->recent);
+  r.is_steady = (troika->fsm >> troika->recent) & 1;
+  return r;
+}
+
+static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, const troika_t *troika) {
+  meta_ptr_t r;
+  r.txnid = troika->txnid[troika->prefer_steady];
+  r.ptr_v = METAPAGE(env, troika->prefer_steady);
+  r.is_steady = (troika->fsm >> troika->prefer_steady) & 1;
+  return r;
+}
+
+static inline meta_ptr_t meta_tail(const MDBX_env *env, const troika_t *troika) {
+  const uint8_t tail = troika->tail_and_flags & 3;
+  MDBX_ANALYSIS_ASSUME(tail < NUM_METAS);
+  meta_ptr_t r;
+  r.txnid = troika->txnid[tail];
+  r.ptr_v = METAPAGE(env, tail);
+  r.is_steady = (troika->fsm >> tail) & 1;
+  return r;
+}
+
+static inline bool meta_is_used(const troika_t *troika, unsigned n) {
+  return n == troika->recent || n == troika->prefer_steady;
+}
+
+static inline bool meta_bootid_match(const meta_t *meta) {
+
+  return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && (globals.bootid.x | globals.bootid.y) != 0;
+}
+
+static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, const int lck_exclusive) {
+  return lck_exclusive
+             ? /* exclusive lock */ meta_bootid_match(meta)
+             : /* db already opened */ env->lck_mmap.lck && (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t constmeta_txnid(const meta_t *meta) {
+  const txnid_t a = unaligned_peek_u64(4, &meta->txnid_a);
+  const txnid_t b = unaligned_peek_u64(4, &meta->txnid_b);
+  return likely(a == b) ? a : 0;
+}
+
+static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
+  eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
+  eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && unaligned_peek_u64(4, meta->txnid_b) < txnid);
+  (void)env;
+#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
+  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, 0, mo_AcquireRelease);
+  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, mo_AcquireRelease);
+#else
+  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
+  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease);
+  atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
+  atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
+#endif
+}
+
+static inline void meta_update_end(const MDBX_env *env, meta_t *meta, txnid_t txnid) {
+  eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env));
+  eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) == txnid);
+  eASSERT(env, unaligned_peek_u64(4, meta->txnid_b) < txnid);
+  (void)env;
+  jitter4testing(true);
+  memcpy(&meta->bootid, &globals.bootid, 16);
+#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8
+  atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, mo_AcquireRelease);
+#else
+  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease);
+  atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease);
+#endif
+}
+
+static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, const txnid_t txnid) {
+  eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env));
+  (void)env;
+  /* update inconsistently since this function used ONLY for filling meta-image
+   * for writing, but not the actual meta-page */
+  memcpy(&meta->bootid, &globals.bootid, 16);
+  unaligned_poke_u64(4, meta->txnid_a, txnid);
+  unaligned_poke_u64(4, meta->txnid_b, txnid);
+}
+
+static inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) {
+  return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s;
+}
+
+static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
+  assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
+  return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady);
+}
+
+static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, bool b_steady) {
+  assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */);
+  return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1);
+}
+
+static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
+  return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
+}
+
+static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) {
+  return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady);
+}
+
+MDBX_INTERNAL meta_t *meta_init_triplet(const MDBX_env *env, void *buffer);
+
+MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number,
+                                unsigned *guess_pagesize);
+
+MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest);
+
+MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape);
+
+MDBX_INTERNAL int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto);

package/deps/libmdbx/src/mvcc-readers.c

@@ -0,0 +1,414 @@
+/// \copyright SPDX-License-Identifier: Apache-2.0
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+
+#include "internals.h"
+
+bsr_t mvcc_bind_slot(MDBX_env *env) {
+  eASSERT(env, env->lck_mmap.lck);
+  eASSERT(env, env->lck->magic_and_version == MDBX_LOCK_MAGIC);
+  eASSERT(env, env->lck->os_and_format == MDBX_LOCK_FORMAT);
+
+  bsr_t result = {lck_rdt_lock(env), nullptr};
+  if (unlikely(MDBX_IS_ERROR(result.err)))
+    return result;
+  if (unlikely(env->flags & ENV_FATAL_ERROR)) {
+    lck_rdt_unlock(env);
+    result.err = MDBX_PANIC;
+    return result;
+  }
+  if (unlikely(!env->dxb_mmap.base)) {
+    lck_rdt_unlock(env);
+    result.err = MDBX_EPERM;
+    return result;
+  }
+
+  if (unlikely(env->registered_reader_pid != env->pid)) {
+    result.err = lck_rpid_set(env);
+    if (unlikely(result.err != MDBX_SUCCESS)) {
+      lck_rdt_unlock(env);
+      return result;
+    }
+    env->registered_reader_pid = env->pid;
+  }
+
+  result.err = MDBX_SUCCESS;
+  size_t slot, nreaders;
+  while (1) {
+    nreaders = env->lck->rdt_length.weak;
+    for (slot = 0; slot < nreaders; slot++)
+      if (!atomic_load32(&env->lck->rdt[slot].pid, mo_AcquireRelease))
+        break;
+
+    if (likely(slot < env->max_readers))
+      break;
+
+    result.err = mvcc_cleanup_dead(env, true, nullptr);
+    if (result.err != MDBX_RESULT_TRUE) {
+      lck_rdt_unlock(env);
+      result.err = (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err;
+      return result;
+    }
+  }
+
+  result.rslot = &env->lck->rdt[slot];
+  /* Claim the reader slot, carefully since other code
+   * uses the reader table un-mutexed: First reset the
+   * slot, next publish it in lck->rdt_length.  After
+   * that, it is safe for mdbx_env_close() to touch it.
+   * When it will be closed, we can finally claim it. */
+  atomic_store32(&result.rslot->pid, 0, mo_AcquireRelease);
+  safe64_reset(&result.rslot->txnid, true);
+  if (slot == nreaders)
+    env->lck->rdt_length.weak = (uint32_t)++nreaders;
+  result.rslot->tid.weak = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self();
+  atomic_store32(&result.rslot->pid, env->pid, mo_AcquireRelease);
+  lck_rdt_unlock(env);
+
+  if (likely(env->flags & ENV_TXKEY)) {
+    eASSERT(env, env->registered_reader_pid == env->pid);
+    thread_rthc_set(env->me_txkey, result.rslot);
+  }
+  return result;
+}
+
+__hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) {
+  const uint32_t nothing_changed = MDBX_STRING_TETRAD("None");
+  eASSERT(env, steady <= env->basal_txn->txnid);
+
+  lck_t *const lck = env->lck_mmap.lck;
+  if (unlikely(lck == nullptr /* exclusive without-lck mode */)) {
+    eASSERT(env, env->lck == lckless_stub(env));
+    env->lck->rdt_refresh_flag.weak = nothing_changed;
+    return env->lck->cached_oldest.weak = steady;
+  }
+
+  const txnid_t prev_oldest = atomic_load64(&lck->cached_oldest, mo_AcquireRelease);
+  eASSERT(env, steady >= prev_oldest);
+
+  txnid_t new_oldest = prev_oldest;
+  while (nothing_changed != atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) {
+    lck->rdt_refresh_flag.weak = nothing_changed;
+    jitter4testing(false);
+    const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
+    new_oldest = steady;
+
+    for (size_t i = 0; i < snap_nreaders; ++i) {
+      const uint32_t pid = atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease);
+      if (!pid)
+        continue;
+      jitter4testing(true);
+
+      const txnid_t rtxn = safe64_read(&lck->rdt[i].txnid);
+      if (unlikely(rtxn < prev_oldest)) {
+        if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) &&
+            safe64_reset_compare(&lck->rdt[i].txnid, rtxn)) {
+          NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN,
+                 i, snap_nreaders, pid, rtxn, prev_oldest, steady);
+        }
+        continue;
+      }
+
+      if (rtxn < new_oldest) {
+        new_oldest = rtxn;
+        if (!MDBX_DEBUG && !MDBX_FORCE_ASSERTIONS && new_oldest == prev_oldest)
+          break;
+      }
+    }
+  }
+
+  if (new_oldest != prev_oldest) {
+    VERBOSE("update oldest %" PRIaTXN " -> %" PRIaTXN, prev_oldest, new_oldest);
+    eASSERT(env, new_oldest >= lck->cached_oldest.weak);
+    atomic_store64(&lck->cached_oldest, new_oldest, mo_Relaxed);
+  }
+  return new_oldest;
+}
+
+pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) {
+  lck_t *const lck = env->lck_mmap.lck;
+  if (likely(lck != nullptr /* check for exclusive without-lck mode */)) {
+  retry:;
+    const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
+    for (size_t i = 0; i < snap_nreaders; ++i) {
+      if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) {
+        /* jitter4testing(true); */
+        const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
+        const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
+        if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
+                     snap_txnid != safe64_read(&lck->rdt[i].txnid)))
+          goto retry;
+        if (last_used_page < snap_pages && snap_txnid <= env->basal_txn->txnid)
+          last_used_page = snap_pages;
+      }
+    }
+  }
+
+  return last_used_page;
+}
+
+/* Find largest mvcc-snapshot still referenced by this process. */
+pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest) {
+  lck_t *const lck = env->lck_mmap.lck;
+  if (likely(lck != nullptr /* exclusive mode */)) {
+    const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
+    for (size_t i = 0; i < snap_nreaders; ++i) {
+    retry:
+      if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease) == env->pid) {
+        /* jitter4testing(true); */
+        const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed);
+        const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid);
+        if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) ||
+                     snap_txnid != safe64_read(&lck->rdt[i].txnid)))
+          goto retry;
+        if (largest < snap_pages &&
+            atomic_load64(&lck->cached_oldest, mo_AcquireRelease) <=
+                /* ignore pending updates */ snap_txnid &&
+            snap_txnid <= MAX_TXNID)
+          largest = snap_pages;
+      }
+    }
+  }
+  return largest;
+}
+
+static bool pid_insert(uint32_t *list, uint32_t pid) {
+  /* binary search of pid in list */
+  size_t base = 0;
+  size_t cursor = 1;
+  int32_t val = 0;
+  size_t n = /* length */ list[0];
+
+  while (n > 0) {
+    size_t pivot = n >> 1;
+    cursor = base + pivot + 1;
+    val = pid - list[cursor];
+
+    if (val < 0) {
+      n = pivot;
+    } else if (val > 0) {
+      base = cursor;
+      n -= pivot + 1;
+    } else {
+      /* found, so it's a duplicate */
+      return false;
+    }
+  }
+
+  if (val > 0)
+    ++cursor;
+
+  list[0]++;
+  for (n = list[0]; n > cursor; n--)
+    list[n] = list[n - 1];
+  list[n] = pid;
+  return true;
+}
+
+__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int *dead) {
+  int rc = check_env(env, true);
+  if (unlikely(rc != MDBX_SUCCESS))
+    return rc;
+
+  eASSERT(env, rdt_locked >= 0);
+  lck_t *const lck = env->lck_mmap.lck;
+  if (unlikely(lck == nullptr)) {
+    /* exclusive mode */
+    if (dead)
+      *dead = 0;
+    return MDBX_SUCCESS;
+  }
+
+  const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease);
+  uint32_t pidsbuf_onstask[142];
+  uint32_t *const pids = (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask))
+                             ? pidsbuf_onstask
+                             : osal_malloc((snap_nreaders + 1) * sizeof(uint32_t));
+  if (unlikely(!pids))
+    return MDBX_ENOMEM;
+
+  pids[0] = 0;
+  int count = 0;
+  for (size_t i = 0; i < snap_nreaders; i++) {
+    const uint32_t pid = atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease);
+    if (pid == 0)
+      continue /* skip empty */;
+    if (pid == env->pid)
+      continue /* skip self */;
+    if (!pid_insert(pids, pid))
+      continue /* such pid already processed */;
+
+    int err = lck_rpid_check(env, pid);
+    if (err == MDBX_RESULT_TRUE)
+      continue /* reader is live */;
+
+    if (err != MDBX_SUCCESS) {
+      rc = err;
+      break /* lck_rpid_check() failed */;
+    }
+
+    /* stale reader found */
+    if (!rdt_locked) {
+      err = lck_rdt_lock(env);
+      if (MDBX_IS_ERROR(err)) {
+        rc = err;
+        break;
+      }
+
+      rdt_locked = -1;
+      if (err == MDBX_RESULT_TRUE) {
+        /* mutex recovered, the mdbx_ipclock_failed() checked all readers */
+        rc = MDBX_RESULT_TRUE;
+        break;
+      }
+
+      /* a other process may have clean and reused slot, recheck */
+      if (lck->rdt[i].pid.weak != pid)
+        continue;
+
+      err = lck_rpid_check(env, pid);
+      if (MDBX_IS_ERROR(err)) {
+        rc = err;
+        break;
+      }
+
+      if (err != MDBX_SUCCESS)
+        continue /* the race with other process, slot reused */;
+    }
+
+    /* clean it */
+    for (size_t ii = i; ii < snap_nreaders; ii++) {
+      if (lck->rdt[ii].pid.weak == pid) {
+        DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->rdt[ii].txnid.weak);
+        atomic_store32(&lck->rdt[ii].pid, 0, mo_Relaxed);
+        atomic_store32(&lck->rdt_refresh_flag, true, mo_AcquireRelease);
+        count++;
+      }
+    }
+  }
+
+  if (likely(!MDBX_IS_ERROR(rc)))
+    atomic_store64(&lck->readers_check_timestamp, osal_monotime(), mo_Relaxed);
+
+  if (rdt_locked < 0)
+    lck_rdt_unlock(env);
+
+  if (pids != pidsbuf_onstask)
+    osal_free(pids);
+
+  if (dead)
+    *dead = count;
+  return rc;
+}
+
+__cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) {
+  DEBUG("DB size maxed out by reading #%" PRIaTXN, straggler);
+  osal_memory_fence(mo_AcquireRelease, false);
+  MDBX_hsr_func *const callback = env->hsr_callback;
+  txnid_t oldest = 0;
+  bool notify_eof_of_loop = false;
+  int retry = 0;
+  do {
+    const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady];
+    env->lck->rdt_refresh_flag.weak = /* force refresh */ true;
+    oldest = mvcc_shapshot_oldest(env, steady);
+    eASSERT(env, oldest < env->basal_txn->txnid);
+    eASSERT(env, oldest >= straggler);
+    eASSERT(env, oldest >= env->lck->cached_oldest.weak);
+
+    lck_t *const lck = env->lck_mmap.lck;
+    if (oldest == steady || oldest > straggler || /* without-LCK mode */ !lck)
+      break;
+
+    if (MDBX_IS_ERROR(mvcc_cleanup_dead(env, false, nullptr)))
+      break;
+
+    reader_slot_t *stucked = nullptr;
+    uint64_t hold_retired = 0;
+    for (size_t i = 0; i < lck->rdt_length.weak; ++i) {
+      uint32_t pid;
+      reader_slot_t *const rslot = &lck->rdt[i];
+      txnid_t rtxn = safe64_read(&rslot->txnid);
+    retry:
+      if (rtxn == straggler && (pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) {
+        const uint64_t tid = safe64_read(&rslot->tid);
+        if (tid == MDBX_TID_TXN_PARKED) {
+          /* Читающая транзакция была помечена владельцем как "припаркованная",
+           * т.е. подлежащая асинхронному прерыванию, либо восстановлению
+           * по активности читателя.
+           *
+           * Если первый CAS(slot->tid) будет успешным, то
+           * safe64_reset_compare() безопасно очистит txnid, либо откажется
+           * из-за того что читатель сбросил и/или перезапустил транзакцию.
+           * При этом читатеть может не заметить вытестения, если приступит
+           * к завершению транзакции. Все эти исходы нас устраивют.
+           *
+           * Если первый CAS(slot->tid) будет НЕ успешным, то значит читатеть
+           * восстановил транзакцию, либо завершил её, либо даже освободил слот.
+           */
+          bool ousted =
+#if MDBX_64BIT_CAS
+              atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, MDBX_TID_TXN_OUSTED);
+#else
+              atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)MDBX_TID_TXN_OUSTED);
+#endif
+          if (likely(ousted)) {
+            ousted = safe64_reset_compare(&rslot->txnid, rtxn);
+            NOTICE("ousted-%s parked read-txn %" PRIaTXN ", pid %u, tid 0x%" PRIx64, ousted ? "complete" : "half", rtxn,
+                   pid, tid);
+            eASSERT(env, ousted || safe64_read(&rslot->txnid) > straggler);
+            continue;
+          }
+          rtxn = safe64_read(&rslot->txnid);
+          goto retry;
+        }
+        hold_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed);
+        stucked = rslot;
+      }
+    }
+
+    if (!callback || !stucked)
+      break;
+
+    uint32_t pid = atomic_load32(&stucked->pid, mo_AcquireRelease);
+    uint64_t tid = safe64_read(&stucked->tid);
+    if (safe64_read(&stucked->txnid) != straggler || !pid)
+      continue;
+
+    const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika);
+    const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP;
+    const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired);
+    const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0;
+    int rc = callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler,
+                      (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry);
+    if (rc < 0)
+      /* hsr returned error and/or agree MDBX_MAP_FULL error */
+      break;
+
+    if (rc > 0) {
+      if (rc == 1) {
+        /* hsr reported transaction (will be) aborted asynchronous */
+        safe64_reset_compare(&stucked->txnid, straggler);
+      } else {
+        /* hsr reported reader process was killed and slot should be cleared */
+        safe64_reset(&stucked->txnid, true);
+        atomic_store64(&stucked->tid, 0, mo_Relaxed);
+        atomic_store32(&stucked->pid, 0, mo_AcquireRelease);
+      }
+    } else if (!notify_eof_of_loop) {
+#if MDBX_ENABLE_PROFGC
+      env->lck->pgops.gc_prof.kicks += 1;
+#endif /* MDBX_ENABLE_PROFGC */
+      notify_eof_of_loop = true;
+    }
+
+  } while (++retry < INT_MAX);
+
+  if (notify_eof_of_loop) {
+    /* notify end of hsr-loop */
+    const txnid_t turn = oldest - straggler;
+    if (turn)
+      NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, straggler, oldest, turn);
+    callback(env, env->txn, 0, 0, straggler, (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry);
+  }
+  return oldest;
+}