mdbxmou 0.1.26

package/deps/libmdbx/src/cogs.c

@@ -0,0 +1,309 @@
+/// \copyright SPDX-License-Identifier: Apache-2.0
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+
+#include "internals.h"
+
+/*------------------------------------------------------------------------------
+ * Pack/Unpack 16-bit values for Grow step & Shrink threshold */
+
+MDBX_NOTHROW_CONST_FUNCTION static inline pgno_t me2v(size_t m, size_t e) {
+  assert(m < 2048 && e < 8);
+  return (pgno_t)(32768 + ((m + 1) << (e + 8)));
+}
+
+MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t v2me(size_t v, size_t e) {
+  assert(v > (e ? me2v(2047, e - 1) : 32768));
+  assert(v <= me2v(2047, e));
+  size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8);
+  m -= m > 0;
+  assert(m < 2048 && e < 8);
+  // f e d c b a 9 8 7 6 5 4 3 2 1 0
+  // 1 e e e m m m m m m m m m m m 1
+  const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1));
+  assert(pv != 65535);
+  return pv;
+}
+
+/* Convert 16-bit packed (exponential quantized) value to number of pages */
+pgno_t pv2pages(uint16_t pv) {
+  if ((pv & 0x8001) != 0x8001)
+    return pv;
+  if (pv == 65535)
+    return 65536;
+  // f e d c b a 9 8 7 6 5 4 3 2 1 0
+  // 1 e e e m m m m m m m m m m m 1
+  return me2v((pv >> 1) & 2047, (pv >> 12) & 7);
+}
+
+/* Convert number of pages to 16-bit packed (exponential quantized) value */
+uint16_t pages2pv(size_t pages) {
+  if (pages < 32769 || (pages < 65536 && (pages & 1) == 0))
+    return (uint16_t)pages;
+  if (pages <= me2v(2047, 0))
+    return v2me(pages, 0);
+  if (pages <= me2v(2047, 1))
+    return v2me(pages, 1);
+  if (pages <= me2v(2047, 2))
+    return v2me(pages, 2);
+  if (pages <= me2v(2047, 3))
+    return v2me(pages, 3);
+  if (pages <= me2v(2047, 4))
+    return v2me(pages, 4);
+  if (pages <= me2v(2047, 5))
+    return v2me(pages, 5);
+  if (pages <= me2v(2047, 6))
+    return v2me(pages, 6);
+  return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533;
+}
+
+__cold bool pv2pages_verify(void) {
+  bool ok = true, dump_translation = false;
+  for (size_t i = 0; i < 65536; ++i) {
+    size_t pages = pv2pages(i);
+    size_t x = pages2pv(pages);
+    size_t xp = pv2pages(x);
+    if (pages != xp) {
+      ERROR("%zu => %zu => %zu => %zu\n", i, pages, x, xp);
+      ok = false;
+    } else if (dump_translation && !(x == i || (x % 2 == 0 && x < 65536))) {
+      DEBUG("%zu => %zu => %zu => %zu\n", i, pages, x, xp);
+    }
+  }
+  return ok;
+}
+
+/*----------------------------------------------------------------------------*/
+
+MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes) {
+  return ceil_powerof2(bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno) {
+  return ceil_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno) {
+  return bytes2pgno(env, pgno_align2os_bytes(env, pgno));
+}
+
+/*----------------------------------------------------------------------------*/
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_int_inline(const size_t expected_alignment, const MDBX_val *a,
+                                                                     const MDBX_val *b) {
+  if (likely(a->iov_len == b->iov_len)) {
+    if (sizeof(size_t) > 7 && likely(a->iov_len == 8))
+      return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base),
+                     unaligned_peek_u64(expected_alignment, b->iov_base));
+    if (likely(a->iov_len == 4))
+      return CMP2INT(unaligned_peek_u32(expected_alignment, a->iov_base),
+                     unaligned_peek_u32(expected_alignment, b->iov_base));
+    if (sizeof(size_t) < 8 && likely(a->iov_len == 8))
+      return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base),
+                     unaligned_peek_u64(expected_alignment, b->iov_base));
+  }
+  ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base,
+        b->iov_len);
+  return 0;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) {
+  return cmp_int_inline(1, a, b);
+}
+
+#ifndef cmp_int_align2
+/* Compare two items pointing at 2-byte aligned unsigned int's. */
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a, const MDBX_val *b) {
+  return cmp_int_inline(2, a, b);
+}
+#endif /* cmp_int_align2 */
+
+#ifndef cmp_int_align4
+/* Compare two items pointing at 4-byte aligned unsigned int's. */
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a, const MDBX_val *b) {
+  return cmp_int_inline(4, a, b);
+}
+#endif /* cmp_int_align4 */
+
+/* Compare two items lexically */
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, const MDBX_val *b) {
+  if (a->iov_len == b->iov_len)
+    return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0;
+
+  const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1;
+  const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
+  int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0;
+  return likely(diff_data) ? diff_data : diff_len;
+}
+
+MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned tail3le(const uint8_t *p, size_t l) {
+  STATIC_ASSERT(sizeof(unsigned) > 2);
+  // 1: 0 0 0
+  // 2: 0 1 1
+  // 3: 0 1 2
+  return p[0] | p[l >> 1] << 8 | p[l - 1] << 16;
+}
+
+/* Compare two items in reverse byte order */
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, const MDBX_val *b) {
+  size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len;
+  if (likely(left)) {
+    const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len);
+    const uint8_t *pb = ptr_disp(b->iov_base, b->iov_len);
+    while (left >= sizeof(size_t)) {
+      pa -= sizeof(size_t);
+      pb -= sizeof(size_t);
+      left -= sizeof(size_t);
+      STATIC_ASSERT(sizeof(size_t) == 4 || sizeof(size_t) == 8);
+      if (sizeof(size_t) == 4) {
+        uint32_t xa = unaligned_peek_u32(1, pa);
+        uint32_t xb = unaligned_peek_u32(1, pb);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+        xa = osal_bswap32(xa);
+        xb = osal_bswap32(xb);
+#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
+        if (xa != xb)
+          return (xa < xb) ? -1 : 1;
+      } else {
+        uint64_t xa = unaligned_peek_u64(1, pa);
+        uint64_t xb = unaligned_peek_u64(1, pb);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+        xa = osal_bswap64(xa);
+        xb = osal_bswap64(xb);
+#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
+        if (xa != xb)
+          return (xa < xb) ? -1 : 1;
+      }
+    }
+    if (sizeof(size_t) == 8 && left >= 4) {
+      pa -= 4;
+      pb -= 4;
+      left -= 4;
+      uint32_t xa = unaligned_peek_u32(1, pa);
+      uint32_t xb = unaligned_peek_u32(1, pb);
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
+      xa = osal_bswap32(xa);
+      xb = osal_bswap32(xb);
+#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */
+      if (xa != xb)
+        return (xa < xb) ? -1 : 1;
+    }
+    if (left) {
+      unsigned xa = tail3le(pa - left, left);
+      unsigned xb = tail3le(pb - left, left);
+      if (xa != xb)
+        return (xa < xb) ? -1 : 1;
+    }
+  }
+  return CMP2INT(a->iov_len, b->iov_len);
+}
+
+/* Fast non-lexically comparator */
+MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) {
+  int diff = CMP2INT(a->iov_len, b->iov_len);
+  return (likely(diff) || a->iov_len == 0) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len);
+}
+
+MDBX_NOTHROW_PURE_FUNCTION __hot bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) {
+  if (likely(l > 3)) {
+    if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9))
+      return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) |
+              (unaligned_peek_u32(1, a + l - 4) - unaligned_peek_u32(1, b + l - 4))) == 0;
+    if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17))
+      return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) |
+              (unaligned_peek_u64(1, a + l - 8) - unaligned_peek_u64(1, b + l - 8))) == 0;
+    return memcmp(a, b, l) == 0;
+  }
+  if (likely(l))
+    return tail3le(a, l) == tail3le(b, l);
+  return true;
+}
+
+int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : 1; }
+
+int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : -1; }
+
+/*----------------------------------------------------------------------------*/
+
+__cold void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release) {
+  for (;;) {
+    const pgno_t mlock_pgno_before = atomic_load32(&env->mlocked_pgno, mo_AcquireRelease);
+    eASSERT(env, pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before);
+    eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == new_aligned_mlocked_pgno);
+    if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno)
+                         : (mlock_pgno_before <= new_aligned_mlocked_pgno))
+      break;
+    if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, new_aligned_mlocked_pgno)))
+      for (;;) {
+        mdbx_atomic_uint32_t *const mlcnt = env->lck->mlcnt;
+        const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed);
+        const int32_t snap_unlocked = atomic_load32(mlcnt + 1, mo_Relaxed);
+        if (mlock_pgno_before == 0 && (snap_locked - snap_unlocked) < INT_MAX) {
+          eASSERT(env, lock_not_release);
+          if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1)))
+            continue;
+        }
+        if (new_aligned_mlocked_pgno == 0 && (snap_locked - snap_unlocked) > 0) {
+          eASSERT(env, !lock_not_release);
+          if (unlikely(!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1)))
+            continue;
+        }
+        NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", lock_not_release ? "lock" : "unlock",
+               lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno,
+               lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, snap_locked - snap_unlocked,
+               atomic_load32(mlcnt + 0, mo_Relaxed) - atomic_load32(mlcnt + 1, mo_Relaxed));
+        return;
+      }
+  }
+}
+
+__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes) {
+  if (atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) > aligned_pgno) {
+    int err = MDBX_ENOSYS;
+    const size_t munlock_begin = pgno2bytes(env, aligned_pgno);
+    const size_t munlock_size = end_bytes - munlock_begin;
+    eASSERT(env, end_bytes % globals.sys_pagesize == 0 && munlock_begin % globals.sys_pagesize == 0 &&
+                     munlock_size % globals.sys_pagesize == 0);
+#if defined(_WIN32) || defined(_WIN64)
+    err = VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? MDBX_SUCCESS : (int)GetLastError();
+    if (err == ERROR_NOT_LOCKED)
+      err = MDBX_SUCCESS;
+#elif defined(_POSIX_MEMLOCK_RANGE)
+    err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? errno : MDBX_SUCCESS;
+#endif
+    if (likely(err == MDBX_SUCCESS))
+      update_mlcnt(env, aligned_pgno, false);
+    else {
+#if defined(_WIN32) || defined(_WIN64)
+      WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
+#else
+      WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err);
+#endif
+    }
+  }
+}
+
+__cold void munlock_all(const MDBX_env *env) {
+  munlock_after(env, 0, bytes_align2os_bytes(env, env->dxb_mmap.current));
+}
+
+/*----------------------------------------------------------------------------*/
+
+uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) {
+  uint32_t r = a | b;
+
+  /* avoid false MDBX_UTTERLY_NOSYNC */
+  if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))
+    r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC;
+
+  /* convert DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */
+  if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) &&
+      !F_ISSET(r, MDBX_UTTERLY_NOSYNC))
+    r = (r - DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC;
+
+  /* force MDBX_NOMETASYNC if NOSYNC enabled */
+  if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC))
+    r |= MDBX_NOMETASYNC;
+
+  assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)));
+  return r;
+}