@nxtedition/rocksdb 7.0.38 → 7.0.41

package/binding.cc

@@ -24,6 +24,8 @@
 #include <thread>
 #include <vector>
 
+#include "max_rev_operator.h"
+
 class NullLogger : public rocksdb::Logger {
  public:
   using rocksdb::Logger::Logv;
@@ -130,17 +132,6 @@ static std::optional<int> Int32Property(napi_env env, napi_value obj, const std:
   return {};
 }
 
-static std::optional<int64_t> Int64Property(napi_env env, napi_value obj, const std::string_view& key) {
-  if (HasProperty(env, obj, key.data())) {
-    const auto value = GetProperty(env, obj, key.data());
-    int64_t result;
-    napi_get_value_int64(env, value, &result);
-    return result;
-  }
-
-  return {};
-}
-
 static napi_status ToString(napi_env env, napi_value from, std::string& to) {
   napi_valuetype type;
   NAPI_STATUS_RETURN(napi_typeof(env, from, &type));
@@ -167,6 +158,37 @@ static napi_status ToString(napi_env env, napi_value from, std::string& to) {
   return napi_ok;
 }
 
+void noop(void* arg1, void* arg2) {}
+
+// TODO (fix): Should use rocksdb::Slice since "to" cannot outlive "from".
+static napi_status ToString(napi_env env, napi_value from, rocksdb::PinnableSlice& to) {
+  napi_valuetype type;
+  NAPI_STATUS_RETURN(napi_typeof(env, from, &type));
+
+  if (type == napi_string) {
+    size_t length = 0;
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, nullptr, 0, &length));
+    to.GetSelf()->resize(length, '\0');
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, &(*to.GetSelf())[0], length + 1, &length));
+    to.PinSelf();
+  } else {
+    bool isBuffer;
+    NAPI_STATUS_RETURN(napi_is_buffer(env, from, &isBuffer));
+
+    if (isBuffer) {
+      char* buf = nullptr;
+      size_t length = 0;
+      NAPI_STATUS_RETURN(napi_get_buffer_info(env, from, reinterpret_cast<void**>(&buf), &length));
+
+      to.PinSlice(rocksdb::Slice(buf, length), noop, nullptr, nullptr);
+    } else {
+      return napi_invalid_arg;
+    }
+  }
+
+  return napi_ok;
+}
+
 static std::optional<std::string> StringProperty(napi_env env, napi_value opts, const std::string_view& name) {
   if (HasProperty(env, opts, name)) {
     const auto property = GetProperty(env, opts, name);
@@ -226,7 +248,9 @@ napi_status Convert(napi_env env, T&& s, bool asBuffer, napi_value& result) {
   if (!s) {
     return napi_get_null(env, &result);
   } else if (asBuffer) {
-    return napi_create_buffer_copy(env, s->size(), s->data(), NULL, &result);
+    using Y = typename std::remove_pointer<typename std::decay<decltype(*s)>::type>::type;
+    auto ptr = new Y(std::move(*s));
+    return napi_create_external_buffer(env, ptr->size(), const_cast<char*>(ptr->data()), Finalize<Y>, ptr, &result);
   } else {
     return napi_create_string_utf8(env, s->data(), s->size(), &result);
   }
@@ -760,8 +784,12 @@ rocksdb::Status InitOptions(napi_env env, T& columnOptions, const U& options) {
 
   const auto mergeOperatorOpt = StringProperty(env, options, "mergeOperator");
   if (mergeOperatorOpt) {
-    ROCKS_STATUS_RETURN(
-        rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
+    if (*mergeOperatorOpt == "maxRev") {
+      columnOptions.merge_operator = std::make_shared<MaxRevOperator>();
+    } else {
+      ROCKS_STATUS_RETURN(
+          rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
+    }
   }
 
   const auto cacheSize = Uint32Property(env, options, "cacheSize").value_or(8 << 20);
@@ -1180,10 +1208,10 @@ NAPI_METHOD(db_put) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-  std::string key;
+  rocksdb::PinnableSlice key;
   NAPI_STATUS_THROWS(ToString(env, argv[1], key));
 
-  std::string val;
+  rocksdb::PinnableSlice val;
   NAPI_STATUS_THROWS(ToString(env, argv[2], val));
 
   rocksdb::ColumnFamilyHandle* column;
@@ -1398,7 +1426,7 @@ NAPI_METHOD(db_del) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-  std::string key;
+  rocksdb::PinnableSlice key;
   NAPI_STATUS_THROWS(ToString(env, argv[1], key));
 
   rocksdb::ColumnFamilyHandle* column;
@@ -1507,7 +1535,7 @@ NAPI_METHOD(db_get_property) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-  std::string property;
+  rocksdb::PinnableSlice property;
   NAPI_STATUS_THROWS(ToString(env, argv[1], property));
 
   std::string value;
@@ -1528,7 +1556,7 @@ NAPI_METHOD(db_get_latest_sequence) {
   const auto seq = database->db_->GetLatestSequenceNumber();
 
   napi_value result;
-  NAPI_STATUS_THROWS(napi_create_bigint_int64(env, seq, &result));
+  NAPI_STATUS_THROWS(napi_create_int64(env, seq, &result));
 
   return result;
 }
@@ -1579,7 +1607,7 @@ NAPI_METHOD(iterator_seek) {
   Iterator* iterator;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  std::string target;
+  rocksdb::PinnableSlice target;
   NAPI_STATUS_THROWS(ToString(env, argv[1], target));
 
   iterator->first_ = true;
@@ -1609,7 +1637,7 @@ NAPI_METHOD(iterator_get_sequence) {
   const auto seq = iterator->snapshot_->GetSequenceNumber();
 
   napi_value result;
-  NAPI_STATUS_THROWS(napi_create_bigint_int64(env, seq, &result));
+  NAPI_STATUS_THROWS(napi_create_int64(env, seq, &result));
 
   return result;
 }
@@ -1716,14 +1744,14 @@ NAPI_METHOD(batch_do) {
 
   rocksdb::WriteBatch batch;
 
-  std::string type;
-  std::string key;
-  std::string value;
-
   uint32_t length;
   NAPI_STATUS_THROWS(napi_get_array_length(env, argv[1], &length));
 
   for (uint32_t i = 0; i < length; i++) {
+    rocksdb::PinnableSlice type;
+    rocksdb::PinnableSlice key;
+    rocksdb::PinnableSlice value;
+
     napi_value element;
     NAPI_STATUS_THROWS(napi_get_element(env, argv[1], i, &element));
 
@@ -1800,10 +1828,10 @@ NAPI_METHOD(batch_put) {
   rocksdb::WriteBatch* batch;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[1], (void**)(&batch)));
 
-  std::string key;
+  rocksdb::PinnableSlice key;
   NAPI_STATUS_THROWS(ToString(env, argv[2], key));
 
-  std::string val;
+  rocksdb::PinnableSlice val;
   NAPI_STATUS_THROWS(ToString(env, argv[3], val));
 
   rocksdb::ColumnFamilyHandle* column;
@@ -1823,7 +1851,7 @@ NAPI_METHOD(batch_del) {
   rocksdb::WriteBatch* batch;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[1], reinterpret_cast<void**>(&batch)));
 
-  std::string key;
+  rocksdb::PinnableSlice key;
   NAPI_STATUS_THROWS(ToString(env, argv[2], key));
 
   rocksdb::ColumnFamilyHandle* column;
@@ -1869,7 +1897,7 @@ NAPI_METHOD(batch_put_log_data) {
   rocksdb::WriteBatch* batch;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[1], reinterpret_cast<void**>(&batch)));
 
-  std::string logData;
+  rocksdb::PinnableSlice logData;
   NAPI_STATUS_THROWS(ToString(env, argv[2], logData));
 
   ROCKS_STATUS_THROWS(batch->PutLogData(logData));
@@ -1886,10 +1914,10 @@ NAPI_METHOD(batch_merge) {
   rocksdb::WriteBatch* batch;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[1], (void**)(&batch)));
 
-  std::string key;
+  rocksdb::PinnableSlice key;
   NAPI_STATUS_THROWS(ToString(env, argv[2], key));
 
-  std::string val;
+  rocksdb::PinnableSlice val;
   NAPI_STATUS_THROWS(ToString(env, argv[3], val));
 
   rocksdb::ColumnFamilyHandle* column;
@@ -1907,7 +1935,7 @@ NAPI_METHOD(batch_count) {
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[1], reinterpret_cast<void**>(&batch)));
 
   napi_value result;
-  NAPI_STATUS_THROWS(napi_create_bigint_int64(env, batch->Count(), &result));
+  NAPI_STATUS_THROWS(napi_create_int64(env, batch->Count(), &result));
 
   return result;
 }
@@ -1925,7 +1953,8 @@ NAPI_METHOD(db_flush_wal) {
   return 0;
 }
 
-napi_status FromLogFile(napi_env env, const auto& file, napi_value* obj) {
+template <typename T>
+napi_status FromLogFile(napi_env env, const T& file, napi_value* obj) {
   NAPI_STATUS_RETURN(napi_create_object(env, obj));
 
   napi_value pathName;

package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc

@@ -114,6 +114,8 @@ DEFINE_uint32(
     "(-stress_cache_key) Simulated file size in MiB, for accounting purposes");
 DEFINE_uint32(sck_reopen_nfiles, 100,
               "(-stress_cache_key) Simulate DB re-open average every n files");
+DEFINE_uint32(sck_newdb_nreopen, 1000,
+              "(-stress_cache_key) Simulate new DB average every n re-opens");
 DEFINE_uint32(sck_restarts_per_day, 24,
               "(-stress_cache_key) Average simulated process restarts per day "
               "(across DBs)");
@@ -780,7 +782,7 @@ class StressCacheKey {
 
   void RunOnce() {
     // Re-initialized simulated state
-    const size_t db_count = FLAGS_sck_db_count;
+    const size_t db_count = std::max(size_t{FLAGS_sck_db_count}, size_t{1});
     dbs_.reset(new TableProperties[db_count]{});
     const size_t table_mask = (size_t{1} << FLAGS_sck_table_bits) - 1;
     table_.reset(new uint64_t[table_mask + 1]{});
@@ -797,7 +799,8 @@ class StressCacheKey {
 
     process_count_ = 0;
     session_count_ = 0;
-    ResetProcess();
+    newdb_count_ = 0;
+    ResetProcess(/*newdbs*/ true);
 
     Random64 r{std::random_device{}()};
 
@@ -816,9 +819,9 @@ class StressCacheKey {
       }
       // Any other periodic actions before simulating next file
       if (!FLAGS_sck_footer_unique_id && r.OneIn(FLAGS_sck_reopen_nfiles)) {
-        ResetSession(db_i);
+        ResetSession(db_i, /*newdb*/ r.OneIn(FLAGS_sck_newdb_nreopen));
       } else if (r.OneIn(restart_nfiles_)) {
-        ResetProcess();
+        ResetProcess(/*newdbs*/ false);
       }
       // Simulate next file
       OffsetableCacheKey ock;
@@ -870,7 +873,7 @@ class StressCacheKey {
         // Our goal is to predict probability of no collisions, not expected
         // number of collisions. To make the distinction, we have to get rid
         // of observing correlated collisions, which this takes care of:
-        ResetProcess();
+        ResetProcess(/*newdbs*/ false);
       } else {
         // Replace (end of lifetime for file that was in this slot)
         table_[pos] = reduced_key;
@@ -888,10 +891,11 @@ class StressCacheKey {
         }
         // Report
         printf(
-            "%" PRIu64 " days, %" PRIu64 " proc, %" PRIu64
-            " sess, %u coll, occ %g%%, ejected %g%%   \r",
+            "%" PRIu64 " days, %" PRIu64 " proc, %" PRIu64 " sess, %" PRIu64
+            " newdb, %u coll, occ %g%%, ejected %g%%      \r",
             file_count / FLAGS_sck_files_per_day, process_count_,
-            session_count_, collisions_this_run, 100.0 * sampled_count / 1000.0,
+            session_count_, newdb_count_ - FLAGS_sck_db_count,
+            collisions_this_run, 100.0 * sampled_count / 1000.0,
             100.0 * (1.0 - sampled_count / 1000.0 * table_mask / file_count));
         fflush(stdout);
       }
@@ -899,16 +903,27 @@ class StressCacheKey {
     collisions_ += collisions_this_run;
   }
 
-  void ResetSession(size_t i) {
+  void ResetSession(size_t i, bool newdb) {
     dbs_[i].db_session_id = DBImpl::GenerateDbSessionId(nullptr);
+    if (newdb) {
+      ++newdb_count_;
+      if (FLAGS_sck_footer_unique_id) {
+        // Simulate how footer id would behave
+        dbs_[i].db_id = "none";
+      } else {
+        // db_id might be ignored, depending on the implementation details
+        dbs_[i].db_id = std::to_string(newdb_count_);
+        dbs_[i].orig_file_number = 0;
+      }
+    }
     session_count_++;
   }
 
-  void ResetProcess() {
+  void ResetProcess(bool newdbs) {
     process_count_++;
     DBImpl::TEST_ResetDbSessionIdGen();
     for (size_t i = 0; i < FLAGS_sck_db_count; ++i) {
-      ResetSession(i);
+      ResetSession(i, newdbs);
     }
     if (FLAGS_sck_footer_unique_id) {
       // For footer unique ID, this tracks process-wide generated SST file
@@ -923,6 +938,7 @@ class StressCacheKey {
   std::unique_ptr<uint64_t[]> table_;
   uint64_t process_count_ = 0;
   uint64_t session_count_ = 0;
+  uint64_t newdb_count_ = 0;
   uint64_t collisions_ = 0;
   uint32_t restart_nfiles_ = 0;
   double multiplier_ = 0.0;