mdbxmou 0.1.26

package/deps/libmdbx/test/test.h++

@@ -0,0 +1,298 @@
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+/// \copyright SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "base.h++"
+#include "chrono.h++"
+#include "config.h++"
+#include "keygen.h++"
+#include "log.h++"
+#include "osal.h++"
+#include "utils.h++"
+
+#include <deque>
+#include <set>
+#include <stack>
+#include <tuple>
+
+#ifndef HAVE_cxx17_std_string_view
+#if __cplusplus >= 201703L && __has_include(<string_view>)
+#include <string_view>
+#define HAVE_cxx17_std_string_view 1
+#else
+#define HAVE_cxx17_std_string_view 0
+#endif
+#endif /* HAVE_cxx17_std_string_view */
+
+#if HAVE_cxx17_std_string_view
+#include <string_view>
+#endif
+
+bool test_execute(const actor_config &config);
+std::string thunk_param(const actor_config &config);
+void testcase_setup(const char *casename, const actor_params &params, unsigned &last_space_id);
+void configure_actor(unsigned &last_space_id, const actor_testcase testcase, const char *space_id_cstr,
+                     actor_params params);
+void keycase_setup(const char *casename, actor_params &params);
+
+namespace global {
+
+extern const char thunk_param_prefix[];
+extern std::vector<actor_config> actors;
+extern std::unordered_map<unsigned, actor_config *> events;
+extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
+extern std::set<std::string> databases;
+extern unsigned nactors;
+extern chrono::time start_monotonic;
+extern chrono::time deadline_monotonic;
+extern bool singlemode;
+
+namespace config {
+extern unsigned timeout_duration_seconds;
+extern bool dump_config;
+extern bool cleanup_before;
+extern bool cleanup_after;
+extern bool failfast;
+extern bool progress_indicator;
+extern bool console_mode;
+extern bool geometry_jitter;
+} /* namespace config */
+
+} /* namespace global */
+
+//-----------------------------------------------------------------------------
+
+struct db_deleter /* : public std::unary_function<void, MDBX_env *> */ {
+  void operator()(MDBX_env *env) const { mdbx_env_close(env); }
+};
+
+struct txn_deleter /* : public std::unary_function<void, MDBX_txn *> */ {
+  void operator()(MDBX_txn *txn) const {
+    int rc = mdbx_txn_abort(txn);
+    if (rc)
+      log_trouble(__func__, "mdbx_txn_abort()", rc);
+  }
+};
+
+struct cursor_deleter /* : public std::unary_function<void, MDBX_cursor *> */ {
+  void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); }
+};
+
+using scoped_db_guard = std::unique_ptr<MDBX_env, db_deleter>;
+using scoped_txn_guard = std::unique_ptr<MDBX_txn, txn_deleter>;
+using scoped_cursor_guard = std::unique_ptr<MDBX_cursor, cursor_deleter>;
+
+//-----------------------------------------------------------------------------
+
+class testcase;
+
+class registry {
+  struct record {
+    actor_testcase id = ac_none;
+    std::string name;
+    bool (*review_params)(actor_params &, unsigned space_id) = nullptr;
+    testcase *(*constructor)(const actor_config &, const mdbx_pid_t) = nullptr;
+  };
+  std::unordered_map<std::string, const record *> name2id;
+  std::unordered_map<int, const record *> id2record;
+  static bool add(const record *item);
+  static registry *instance();
+
+public:
+  template <class TESTCASE> struct factory : public record {
+    factory(const actor_testcase id, const char *name) {
+      this->id = id;
+      this->name = name;
+      review_params = TESTCASE::review_params;
+      constructor = [](const actor_config &config, const mdbx_pid_t pid) -> testcase * {
+        return new TESTCASE(config, pid);
+      };
+      add(this);
+    }
+  };
+  static bool review_actor_params(const actor_testcase id, actor_params &params, const unsigned space_id);
+  static testcase *create_actor(const actor_config &config, const mdbx_pid_t pid);
+};
+
+#define REGISTER_TESTCASE(NAME)                                                                                        \
+  static registry::factory<testcase_##NAME> gRegister_##NAME(ac_##NAME, MDBX_STRINGIFY(NAME))
+
+class testcase {
+protected:
+  using data_view = mdbx::slice;
+  static inline data_view iov2dataview(const MDBX_val &v) {
+    return (v.iov_base && v.iov_len) ? data_view(static_cast<const char *>(v.iov_base), v.iov_len) : data_view();
+  }
+  static inline data_view iov2dataview(const keygen::buffer &b) { return iov2dataview(b->value); }
+
+  using Item = std::pair<::mdbx::buffer<>, ::mdbx::buffer<>>;
+
+  static MDBX_val dataview2iov(const data_view &v) {
+    MDBX_val r;
+    r.iov_base = (void *)v.data();
+    r.iov_len = v.size();
+    return r;
+  }
+  struct ItemCompare {
+    const testcase *context;
+    ItemCompare(const testcase *owner) : context(owner) { /* The context->txn_guard may be empty/null here */ }
+
+    bool operator()(const Item &a, const Item &b) const {
+      MDBX_val va = dataview2iov(a.first), vb = dataview2iov(b.first);
+      assert(context->txn_guard.get() != nullptr);
+      int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
+      if (cmp == 0 && (context->config.params.table_flags & MDBX_DUPSORT) != 0) {
+        va = dataview2iov(a.second);
+        vb = dataview2iov(b.second);
+        cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
+      }
+      return cmp < 0;
+    }
+  };
+
+  // for simplify the set<pair<key,value>>
+  // is used instead of multimap<key,value>
+  using SET = std::set<Item, ItemCompare>;
+
+  const actor_config &config;
+  const mdbx_pid_t pid;
+
+  MDBX_dbi dbi{0};
+  scoped_db_guard db_guard;
+  scoped_txn_guard txn_guard;
+  scoped_cursor_guard cursor_guard;
+  bool signalled{false};
+  bool need_speculum_assign{false};
+
+  uint64_t nops_completed{0};
+  chrono::time start_timestamp;
+  keygen::buffer key;
+  keygen::buffer data;
+  keygen::maker keyvalue_maker;
+
+  struct {
+    MDBX_canary canary;
+  } last;
+
+  SET speculum{ItemCompare(this)}, speculum_committed{ItemCompare(this)};
+#ifndef SPECULUM_CURSORS
+#define SPECULUM_CURSORS 1
+#endif /* SPECULUM_CURSORS */
+#if SPECULUM_CURSORS
+  scoped_cursor_guard speculum_cursors[5 + 1];
+  void speculum_prepare_cursors(const Item &item);
+  bool speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it,
+                             int cursor_err, const MDBX_val &cursor_key, const MDBX_val &cursor_data,
+                             MDBX_cursor *cursor) const;
+  bool speculum_check_cursor(const char *where, const char *stage, const testcase::SET::const_iterator &it,
+                             MDBX_cursor *cursor, const MDBX_cursor_op op) const;
+  void speculum_render(const testcase::SET::const_iterator &it, const MDBX_cursor *ref) const;
+#endif /* SPECULUM_CURSORS */
+  bool speculum_check_iterator(const char *where, const char *stage, const testcase::SET::const_iterator &it,
+                               const MDBX_val &k, const MDBX_val &v, MDBX_cursor *cursor) const;
+
+  void verbose(const char *where, const char *stage, const testcase::SET::const_iterator &it) const;
+  void verbose(const char *where, const char *stage, const MDBX_val &k, const MDBX_val &v,
+               int err = MDBX_SUCCESS) const;
+
+  bool is_same(const Item &a, const Item &b) const;
+  bool is_same(const SET::const_iterator &it, const MDBX_val &k, const MDBX_val &v) const;
+
+  bool speculum_verify();
+  bool check_batch_get();
+  int insert(const keygen::buffer &akey, const keygen::buffer &adata, MDBX_put_flags_t flags);
+  int replace(const keygen::buffer &akey, const keygen::buffer &new_value, const keygen::buffer &old_value,
+              MDBX_put_flags_t flags, bool hush_keygen_mistakes = true);
+  int remove(const keygen::buffer &akey, const keygen::buffer &adata);
+
+  static int hsr_callback(const MDBX_env *env, const MDBX_txn *txn, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard,
+                          unsigned gap, size_t space, int retry) MDBX_CXX17_NOEXCEPT;
+
+  MDBX_env_flags_t actual_env_mode{MDBX_ENV_DEFAULTS};
+  bool is_nested_txn_available() const { return (actual_env_mode & MDBX_WRITEMAP) == 0; }
+  void kick_progress(bool active) const;
+  void db_prepare();
+  void db_open();
+  void db_close();
+  virtual void txn_begin(bool readonly, MDBX_txn_flags_t flags = MDBX_TXN_READWRITE);
+  int breakable_commit();
+  virtual void txn_end(bool abort);
+  int breakable_restart();
+  void txn_restart(bool abort, bool readonly, MDBX_txn_flags_t flags = MDBX_TXN_READWRITE);
+  void cursor_open(MDBX_dbi handle);
+  void cursor_close();
+  void cursor_renew();
+  void txn_inject_writefault(void);
+  void txn_inject_writefault(MDBX_txn *txn);
+  bool txn_probe_parking();
+
+  void fetch_canary();
+  void update_canary(uint64_t increment);
+  bool checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, MDBX_val expected_valued);
+  unsigned txn_underutilization_x256(MDBX_txn *txn) const;
+
+  using tablename_buf = char[32];
+  const char *db_tablename(tablename_buf &buffer, const char *suffix = "") const;
+  MDBX_dbi db_table_open(bool create, bool expect_failure = false);
+  void db_table_drop(MDBX_dbi handle);
+  void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr);
+  void db_table_close(MDBX_dbi handle);
+  int db_open__begin__table_create_open_clean(MDBX_dbi &handle);
+  bool is_handle_created_in_current_txn(const MDBX_dbi handle, MDBX_txn *txn);
+
+  bool wait4start();
+  void report(size_t nops_done);
+  void signal();
+  bool should_continue(bool check_timeout_only = false) const;
+
+  bool MDBX_PRINTF_ARGS(2, 3) failure(const char *fmt, ...) const;
+  void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, keygen::buffer &out_value,
+                     keygen::serial_t data_age) {
+    keyvalue_maker.pair(serial, out_key, out_value, data_age, false);
+  }
+
+  void generate_pair(const keygen::serial_t serial) { keyvalue_maker.pair(serial, key, data, 0, true); }
+
+  bool mode_readonly() const { return (config.params.mode_flags & MDBX_RDONLY) ? true : false; }
+
+public:
+  testcase(const actor_config &config, const mdbx_pid_t pid) : config(config), pid(pid) {
+    start_timestamp.reset();
+    memset(&last, 0, sizeof(last));
+  }
+
+  static bool review_params(actor_params &params, unsigned space_id) {
+    // silently fix key/data length for fixed-length modes
+    params.prng_seed += bleach32(space_id);
+    if ((params.table_flags & MDBX_INTEGERKEY) && params.keylen_min != params.keylen_max)
+      params.keylen_min = params.keylen_max;
+    if ((params.table_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) && params.datalen_min != params.datalen_max)
+      params.datalen_min = params.datalen_max;
+    return true;
+  }
+
+  virtual bool setup();
+  virtual bool run() { return true; }
+  virtual bool teardown();
+  virtual ~testcase() {}
+};
+
+//-----------------------------------------------------------------------------
+
+class testcase_ttl : public testcase {
+  using inherited = testcase;
+
+protected:
+  struct {
+    unsigned max_window_size{0};
+    unsigned max_step_size{0};
+  } sliding;
+  unsigned edge2window(uint64_t edge);
+  unsigned edge2count(uint64_t edge);
+
+public:
+  testcase_ttl(const actor_config &config, const mdbx_pid_t pid) : inherited(config, pid) {}
+  bool setup() override;
+  bool run() override;
+};

package/deps/libmdbx/test/tmux.conf

@@ -0,0 +1,3 @@
+setw -g aggressive-resize on
+set-option -g remain-on-exit on
+# set-option -g remain-on-exit-format FAILED

package/deps/libmdbx/test/try.c++

@@ -0,0 +1,30 @@
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+/// \copyright SPDX-License-Identifier: Apache-2.0
+
+#include "test.h++"
+
+class testcase_try : public testcase {
+public:
+  testcase_try(const actor_config &config, const mdbx_pid_t pid) : testcase(config, pid) {}
+  bool run() override;
+};
+REGISTER_TESTCASE(try);
+
+bool testcase_try::run() {
+  db_open();
+  assert(!txn_guard);
+
+  MDBX_txn *txn = nullptr;
+  MDBX_txn *txn2 = nullptr;
+  int rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TXN_READWRITE, &txn);
+  if (unlikely(rc != MDBX_SUCCESS))
+    failure_perror("mdbx_txn_begin(MDBX_TXN_TRY)", rc);
+  else {
+    rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TXN_TRY, &txn2);
+    if (unlikely(rc != MDBX_BUSY))
+      failure_perror("mdbx_txn_begin(MDBX_TXN_TRY)", rc);
+  }
+
+  txn_guard.reset(txn);
+  return true;
+}

package/deps/libmdbx/test/ttl.c++

@@ -0,0 +1,240 @@
+/// \author Леонид Юрьев aka Leonid Yuriev <leo@yuriev.ru> \date 2015-2025
+/// \copyright SPDX-License-Identifier: Apache-2.0
+
+#include "test.h++"
+#include <cmath>
+#include <deque>
+
+/* LY: тест "эмуляцией time-to-live":
+ *  - организуется "скользящее окно", которое двигается вперед вдоль
+ *    числовой оси каждую транзакцию.
+ *  - по переднему краю "скользящего окна" записи добавляются в таблицу,
+ *    а по заднему удаляются.
+ *  - количество добавляемых/удаляемых записей псевдослучайно зависит
+ *    от номера транзакции, но с экспоненциальным распределением.
+ *  - размер "скользящего окна" также псевдослучайно зависит от номера
+ *    транзакции с "отрицательным" экспоненциальным распределением
+ *    MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний
+ *    край и удаляются записи позади него.
+ *
+ *  Таким образом имитируется поведение таблицы с TTL: записи стохастически
+ *  добавляются и удаляются, но изредка происходит массивное удаление.
+ */
+REGISTER_TESTCASE(ttl);
+
+unsigned testcase_ttl::edge2count(uint64_t edge) {
+  const double rnd = u64_to_double1(prng64_map1_white(edge));
+  const unsigned count = unsigned(std::lrint(std::pow(sliding.max_step_size, rnd)));
+  // average value: (X - 1) / ln(X), where X = sliding.max_step_size
+  return count;
+}
+
+unsigned testcase_ttl::edge2window(uint64_t edge) {
+  const double rnd = u64_to_double1(bleach64(edge));
+  const unsigned size = sliding.max_window_size - unsigned(std::lrint(std::pow(sliding.max_window_size, rnd)));
+  // average value: Y - (Y - 1) / ln(Y), where Y = sliding.max_window_size
+  return size;
+}
+
+static inline double estimate(const double x, const double y) {
+  /* среднее кол-во операций N = X' * Y', где X' и Y' средние значения
+   * размера окна и кол-ва добавляемых за один шаг записей:
+   *  X' = (X - 1) / ln(X), где X = sliding.max_step_size
+   *  Y' = Y - (Y - 1) / ln(Y), где Y = sliding.max_window_size */
+  return (x - 1) / std::log(x) * (y - (y - 1) / std::log(y));
+}
+
+bool testcase_ttl::setup() {
+  const unsigned window_top_lower = 7 /* нижний предел для верхней границы диапазона, в котором будет
+                                         стохастически колебаться размер окна */
+      ;
+  const unsigned count_top_lower = 7 /* нижний предел для верхней границы диапазона, в котором будет
+                                        стохастически колебаться кол-во записей добавляемых на одном шаге */
+      ;
+
+  /* для параметризации используем подходящие параметры,
+   * которые не имеют здесь смысла в первоначальном значении. */
+  const double ratio = double(config.params.batch_read ? config.params.batch_read : 1) /
+                       double(config.params.batch_write ? config.params.batch_write : 1);
+
+  /* проще найти двоичным поиском (вариация метода Ньютона) */
+  double hi = config.params.test_nops, lo = 1;
+  double x = std::sqrt(hi + lo) / ratio;
+  while (hi > lo) {
+    const double n = estimate(x, x * ratio);
+    if (n > config.params.test_nops)
+      hi = x - 1;
+    else
+      lo = x + 1;
+    x = (hi + lo) / 2;
+  }
+
+  sliding.max_step_size = unsigned(std::lrint(x));
+  if (sliding.max_step_size < count_top_lower)
+    sliding.max_step_size = count_top_lower;
+  sliding.max_window_size = unsigned(std::lrint(x * ratio));
+  if (sliding.max_window_size < window_top_lower)
+    sliding.max_window_size = window_top_lower;
+
+  while (estimate(sliding.max_step_size, sliding.max_window_size) > config.params.test_nops * 2.0) {
+    if (ratio * sliding.max_step_size > sliding.max_window_size) {
+      if (sliding.max_step_size < count_top_lower)
+        break;
+      sliding.max_step_size = sliding.max_step_size * 7 / 8;
+    } else {
+      if (sliding.max_window_size < window_top_lower)
+        break;
+      sliding.max_window_size = sliding.max_window_size * 7 / 8;
+    }
+  }
+
+  log_verbose("come up window_max %u from `batch_read`", sliding.max_window_size);
+  log_verbose("come up step_max %u from `batch_write`", sliding.max_step_size);
+  return inherited::setup();
+}
+
+bool testcase_ttl::run() {
+  int err = db_open__begin__table_create_open_clean(dbi);
+  if (unlikely(err != MDBX_SUCCESS)) {
+    log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));
+    return false;
+  }
+
+  uint64_t seed = prng64_map2_white(prng_state) + config.space_id;
+  keyvalue_maker.setup(config.params, 0 /* thread_number */);
+  key = keygen::alloc(config.params.keylen_max);
+  data = keygen::alloc(config.params.datalen_max);
+  const MDBX_put_flags_t insert_flags =
+      (config.params.table_flags & MDBX_DUPSORT) ? MDBX_NODUPDATA : MDBX_NODUPDATA | MDBX_NOOVERWRITE;
+
+  std::deque<std::pair<uint64_t, unsigned>> fifo;
+  uint64_t serial = 0;
+  bool rc = false;
+  unsigned clear_wholetable_passed = 0;
+  unsigned clear_stepbystep_passed = 0;
+  unsigned dbfull_passed = 0;
+  unsigned loops = 0;
+  bool keyspace_overflow = false;
+  while (true) {
+    const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
+
+    const unsigned window_width = (!should_continue() || flipcoin_x4()) ? 0 : edge2window(salt);
+    unsigned head_count = edge2count(salt);
+    log_debug("ttl: step #%" PRIu64 " (serial %" PRIu64 ", window %u, count %u) salt %" PRIu64, nops_completed, serial,
+              window_width, head_count, salt);
+
+    if (window_width || flipcoin()) {
+      clear_stepbystep_passed += window_width == 0;
+      while (fifo.size() > window_width) {
+        uint64_t tail_serial = fifo.back().first;
+        const unsigned tail_count = fifo.back().second;
+        log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, tail_count);
+        fifo.pop_back();
+        for (unsigned n = 0; n < tail_count; ++n) {
+          log_trace("ttl: remove-tail %" PRIu64, tail_serial);
+          generate_pair(tail_serial);
+          err = remove(key, data);
+          if (unlikely(err != MDBX_SUCCESS)) {
+            if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
+              log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err));
+              goto bailout;
+            }
+            failure_perror("mdbx_del(tail)", err);
+          }
+          if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
+            failure("ttl: unexpected key-space overflow on the tail");
+        }
+        report(tail_count);
+      }
+    } else {
+      log_trace("ttl: purge state");
+      db_table_clear(dbi);
+      fifo.clear();
+      clear_wholetable_passed += 1;
+      report(1);
+    }
+
+    err = breakable_restart();
+    if (unlikely(err != MDBX_SUCCESS)) {
+      log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err));
+      break;
+    }
+    if (!speculum_verify()) {
+      log_notice("ttl: bailout after tail-trim");
+      return false;
+    }
+
+    if (!keyspace_overflow && (should_continue() || !clear_wholetable_passed || !clear_stepbystep_passed)) {
+      unsigned underutilization_x256 = txn_underutilization_x256(txn_guard.get());
+      if (dbfull_passed > underutilization_x256) {
+        log_notice("ttl: skip head-grow to avoid one more dbfull (was %u, "
+                   "underutilization %.2f%%)",
+                   dbfull_passed, underutilization_x256 / 2.560);
+        continue;
+      }
+      fifo.push_front(std::make_pair(serial, head_count));
+    retry:
+      for (unsigned n = 0; n < head_count; ++n) {
+        log_trace("ttl: insert-head %" PRIu64, serial);
+        generate_pair(serial);
+        err = insert(key, data, insert_flags);
+        if (unlikely(err != MDBX_SUCCESS)) {
+          if ((err == MDBX_TXN_FULL || err == MDBX_MAP_FULL) && config.params.ignore_dbfull) {
+            log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
+            txn_restart(true, false);
+            serial = fifo.front().first;
+            fifo.front().second = head_count = n;
+            dbfull_passed += 1;
+            goto retry;
+          }
+          failure_perror("mdbx_put(head)", err);
+        }
+
+        if (unlikely(!keyvalue_maker.increment(serial, 1))) {
+          log_notice("ttl: unexpected key-space overflow");
+          keyspace_overflow = true;
+          txn_restart(true, false);
+          serial = fifo.front().first;
+          fifo.front().second = head_count = n;
+          goto retry;
+        }
+      }
+      err = breakable_restart();
+      if (unlikely(err != MDBX_SUCCESS)) {
+        log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
+        serial = fifo.front().first;
+        fifo.pop_front();
+      }
+      if (!speculum_verify()) {
+        log_notice("ttl: bailout after head-grow");
+        return false;
+      }
+      loops += 1;
+    } else if (fifo.empty()) {
+      log_notice("ttl: done %u whole loops, %" PRIu64 " ops, %" PRIu64 " items", loops, nops_completed, serial);
+      rc = true;
+      break;
+    } else {
+      log_notice("ttl: done, wait for empty, skip head-grow");
+    }
+  }
+
+bailout:
+  if (!rc && err == MDBX_MAP_FULL && config.params.ignore_dbfull)
+    rc = true;
+  txn_end(true);
+  if (dbi) {
+    if (config.params.drop_table && !mode_readonly()) {
+      txn_begin(false);
+      db_table_drop(dbi);
+      err = breakable_commit();
+      if (unlikely(err != MDBX_SUCCESS)) {
+        log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err));
+        if (err != MDBX_MAP_FULL || !config.params.ignore_dbfull)
+          rc = false;
+      }
+    } else
+      db_table_close(dbi);
+  }
+  return rc;
+}