@nxtedition/rocksdb 7.1.34 → 8.0.1

package/BUILDING.md

@@ -10,9 +10,9 @@
   - Copy `libfolly.a` to `/usr/lib/x86_64-linux-gnu`.
   - Copy headers to `/usr/lib/x86_64-linux-gnu/include`.
   - Copy boost headers from folly scratchpad to `/usr/lib/x86_64-linux-gnu/include`.
-- `npx prebuildify -t -t 17.8.0 --napi --strip --arch x64`
+- `JOBS=8 npx prebuildify -t 18.11.0 --napi --strip --arch x64`
 
 # OSX
 
 - `brew install zstd`
-- `npx prebuildify -t -t 17.8.0 --napi --strip --arch x64`
+- `JOBS=8 npx prebuildify -t 18.11.0 --napi --strip --arch arm64`

package/binding.cc

@@ -26,7 +26,6 @@
 #include <thread>
 #include <vector>
 
-#include "max_rev_operator.h"
 #include "util.h"
 
 class NullLogger : public rocksdb::Logger {
@@ -38,7 +37,6 @@ class NullLogger : public rocksdb::Logger {
 
 struct Database;
 struct Iterator;
-struct Updates;
 
 struct ColumnFamily {
   napi_ref ref;
@@ -254,48 +252,6 @@ struct BatchIterator : public rocksdb::WriteBatch::Handler {
   std::vector<BatchEntry> cache_;
 };
 
-struct Updates : public BatchIterator, public Closable {
-  Updates(Database* database,
-          const int64_t seqNumber,
-          const bool keys,
-          const bool values,
-          const bool data,
-          const rocksdb::ColumnFamilyHandle* column,
-          const Encoding keyEncoding,
-          const Encoding valueEncoding)
-      : BatchIterator(database, keys, values, data, column, keyEncoding, valueEncoding),
-        database_(database),
-        start_(seqNumber) {}
-
-  virtual ~Updates() { assert(!iterator_); }
-
-  rocksdb::Status Close() override {
-    iterator_.reset();
-    return rocksdb::Status::OK();
-  }
-
-  napi_status Attach(napi_env env, napi_value context) {
-    NAPI_STATUS_RETURN(napi_create_reference(env, context, 1, &ref_));
-    database_->closables.insert(this);
-    return napi_ok;
-  }
-
-  napi_status Detach(napi_env env) {
-    database_->closables.erase(this);
-    if (ref_) {
-      NAPI_STATUS_RETURN(napi_delete_reference(env, ref_));
-    }
-    return napi_ok;
-  }
-
-  Database* database_;
-  int64_t start_;
-  std::unique_ptr<rocksdb::TransactionLogIterator> iterator_;
-
- private:
-  napi_ref ref_ = nullptr;
-};
-
 struct BaseIterator : public Closable {
   BaseIterator(Database* database,
                rocksdb::ColumnFamilyHandle* column,
@@ -600,12 +556,8 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   std::optional<std::string> mergeOperatorOpt;
   NAPI_STATUS_RETURN(GetProperty(env, options, "mergeOperator", mergeOperatorOpt));
   if (mergeOperatorOpt) {
-    if (*mergeOperatorOpt == "maxRev") {
-      columnOptions.merge_operator = std::make_shared<MaxRevOperator>();
-    } else {
-      ROCKS_STATUS_RETURN_NAPI(
-          rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
-    }
+		ROCKS_STATUS_RETURN_NAPI(
+				rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
   }
 
   std::optional<std::string> compactionPriority;
@@ -850,106 +802,6 @@ NAPI_METHOD(db_close) {
   return 0;
 }
 
-NAPI_METHOD(updates_init) {
-  NAPI_ARGV(2);
-
-  Database* database;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
-
-  const auto options = argv[1];
-
-  int64_t since = 0;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "since", since));
-
-  bool keys = true;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "keys", keys));
-
-  bool values = true;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "values", values));
-
-  bool data = true;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "data", data));
-
-  Encoding keyEncoding = Encoding::String;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "keyEncoding", keyEncoding));
-
-  Encoding valueEncoding = Encoding::String;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
-
-  rocksdb::ColumnFamilyHandle* column = nullptr;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
-
-  auto updates =
-      std::unique_ptr<Updates>(new Updates(database, since, keys, values, data, column, keyEncoding, valueEncoding));
-
-  napi_value result;
-  NAPI_STATUS_THROWS(napi_create_external(env, updates.get(), Finalize<Updates>, updates.get(), &result));
-
-  // Prevent GC of JS object before the iterator is closed (explicitly or on
-  // db close) and keep track of non-closed iterators to end them on db close.
-  NAPI_STATUS_THROWS(updates.release()->Attach(env, result));
-
-  return result;
-}
-
-NAPI_METHOD(updates_next) {
-  NAPI_ARGV(2);
-
-  Updates* updates;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&updates)));
-
-  auto callback = argv[1];
-
-  runAsync<rocksdb::BatchResult>(
-      "leveldown.updates.next", env, callback,
-      [=](auto& batchResult) {
-        if (!updates->iterator_) {
-          rocksdb::TransactionLogIterator::ReadOptions options;
-          const auto status = updates->database_->db->GetUpdatesSince(updates->start_, &updates->iterator_, options);
-          if (!status.ok()) {
-            return status;
-          }
-        } else if (updates->iterator_->Valid()) {
-          updates->iterator_->Next();
-        }
-
-        if (!updates->iterator_->Valid() || !updates->iterator_->status().ok()) {
-          return updates->iterator_->status();
-        }
-
-        batchResult = updates->iterator_->GetBatch();
-
-        return rocksdb::Status::OK();
-      },
-      [=](auto& batchResult, auto env, auto& argv) {
-        if (!batchResult.writeBatchPtr) {
-          return napi_ok;
-        }
-
-        argv.resize(5);
-        NAPI_STATUS_RETURN(updates->Iterate(env, *batchResult.writeBatchPtr, &argv[1]));
-        NAPI_STATUS_RETURN(napi_create_int64(env, batchResult.sequence, &argv[2]));
-        NAPI_STATUS_RETURN(napi_create_int64(env, batchResult.writeBatchPtr->Count(), &argv[3]));
-        NAPI_STATUS_RETURN(napi_create_int64(env, updates->start_, &argv[4]));
-
-        return napi_ok;
-      });
-
-  return 0;
-}
-
-NAPI_METHOD(updates_close) {
-  NAPI_ARGV(1);
-
-  Updates* updates;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&updates)));
-
-  ROCKS_STATUS_THROWS_NAPI(updates->Close());
-  NAPI_STATUS_THROWS(updates->Detach(env));
-
-  return 0;
-}
-
 NAPI_METHOD(db_get_many) {
   NAPI_ARGV(4);
 
@@ -1731,10 +1583,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(iterator_nextv);
   NAPI_EXPORT_FUNCTION(iterator_get_sequence);
 
-  NAPI_EXPORT_FUNCTION(updates_init);
-  NAPI_EXPORT_FUNCTION(updates_close);
-  NAPI_EXPORT_FUNCTION(updates_next);
-
   NAPI_EXPORT_FUNCTION(batch_do);
   NAPI_EXPORT_FUNCTION(batch_init);
   NAPI_EXPORT_FUNCTION(batch_put);

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -250,6 +250,8 @@ option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
 option(FORCE_AVX2 "force building with AVX2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
+  add_definitions(-DROCKSDB_PORTABLE)
+
   # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
   # is available, it is available by default.
   if(FORCE_SSE42 AND NOT MSVC)
@@ -292,8 +294,7 @@ if(NOT MSVC)
   set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
 endif()
 
-if (NOT PORTABLE OR FORCE_SSE42)
-  CHECK_CXX_SOURCE_COMPILES("
+CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
 #include <wmmintrin.h>
@@ -305,12 +306,11 @@ int main() {
   auto d = _mm_cvtsi128_si64(c);
 }
 " HAVE_SSE42)
-  if(HAVE_SSE42)
-    add_definitions(-DHAVE_SSE42)
-    add_definitions(-DHAVE_PCLMUL)
-  elseif(FORCE_SSE42)
-    message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
-  endif()
+if(HAVE_SSE42)
+  add_definitions(-DHAVE_SSE42)
+  add_definitions(-DHAVE_PCLMUL)
+elseif(FORCE_SSE42)
+  message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
 endif()
 
 # Check if -latomic is required or not

package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc

@@ -1888,6 +1888,43 @@ TEST_F(DBSecondaryCacheTest, SecondaryCacheFailureTest) {
   Destroy(options);
 }
 
+TEST_F(DBSecondaryCacheTest, TestSecondaryWithCompressedCache) {
+  if (!Snappy_Supported()) {
+    ROCKSDB_GTEST_SKIP("Compressed cache test requires snappy support");
+    return;
+  }
+  LRUCacheOptions opts(2000 /* capacity */, 0 /* num_shard_bits */,
+                       false /* strict_capacity_limit */,
+                       0.5 /* high_pri_pool_ratio */,
+                       nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+                       kDontChargeCacheMetadata);
+  std::shared_ptr<TestSecondaryCache> secondary_cache(
+      new TestSecondaryCache(2048 * 1024));
+  opts.secondary_cache = secondary_cache;
+  std::shared_ptr<Cache> cache = NewLRUCache(opts);
+  BlockBasedTableOptions table_options;
+  table_options.block_cache_compressed = cache;
+  table_options.no_block_cache = true;
+  table_options.block_size = 1234;
+  Options options = GetDefaultOptions();
+  options.compression = kSnappyCompression;
+  options.create_if_missing = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+  Random rnd(301);
+  const int N = 6;
+  for (int i = 0; i < N; i++) {
+    // Partly compressible
+    std::string p_v = rnd.RandomString(507) + std::string(500, ' ');
+    ASSERT_OK(Put(Key(i), p_v));
+  }
+  ASSERT_OK(Flush());
+  for (int i = 0; i < 2 * N; i++) {
+    std::string v = Get(Key(i % N));
+    ASSERT_EQ(1007, v.size());
+  }
+}
+
 TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
   LRUCacheOptions opts(1024 /* capacity */, 2 /* num_shard_bits */,
                        false /* strict_capacity_limit */,

package/deps/rocksdb/rocksdb/db/db_iter.cc

@@ -158,6 +158,8 @@ void DBIter::Next() {
 
   local_stats_.next_count_++;
   if (ok && iter_.Valid()) {
+    ClearSavedValue();
+
     if (prefix_same_as_start_) {
       assert(prefix_extractor_ != nullptr);
       const Slice prefix = prefix_.GetUserKey();
@@ -544,8 +546,7 @@ bool DBIter::MergeValuesNewToOld() {
       // hit a put, merge the put value with operands and store the
       // final result in saved_value_. We are done!
       const Slice val = iter_.value();
-      Status s = Merge(&val, ikey.user_key);
-      if (!s.ok()) {
+      if (!Merge(&val, ikey.user_key)) {
         return false;
       }
       // iter_ is positioned after put
@@ -574,8 +575,7 @@ bool DBIter::MergeValuesNewToOld() {
         return false;
       }
       valid_ = true;
-      Status s = Merge(&blob_value_, ikey.user_key);
-      if (!s.ok()) {
+      if (!Merge(&blob_value_, ikey.user_key)) {
         return false;
       }
 
@@ -589,11 +589,18 @@ bool DBIter::MergeValuesNewToOld() {
       }
       return true;
     } else if (kTypeWideColumnEntity == ikey.type) {
-      // TODO: support wide-column entities
-      status_ = Status::NotSupported(
-          "Merge currently not supported for wide-column entities");
-      valid_ = false;
-      return false;
+      if (!MergeEntity(iter_.value(), ikey.user_key)) {
+        return false;
+      }
+
+      // iter_ is positioned after put
+      iter_.Next();
+      if (!iter_.status().ok()) {
+        valid_ = false;
+        return false;
+      }
+
+      return true;
     } else {
       valid_ = false;
       status_ = Status::Corruption(
@@ -612,8 +619,7 @@ bool DBIter::MergeValuesNewToOld() {
   // a deletion marker.
   // feed null as the existing value to the merge operator, such that
   // client can differentiate this scenario and do things accordingly.
-  Status s = Merge(nullptr, saved_key_.GetUserKey());
-  if (!s.ok()) {
+  if (!Merge(nullptr, saved_key_.GetUserKey())) {
     return false;
   }
   assert(status_.ok());
@@ -636,6 +642,8 @@ void DBIter::Prev() {
     }
   }
   if (ok) {
+    ClearSavedValue();
+
     Slice prefix;
     if (prefix_same_as_start_) {
       assert(prefix_extractor_ != nullptr);
@@ -960,8 +968,7 @@ bool DBIter::FindValueForCurrentKey() {
       if (last_not_merge_type == kTypeDeletion ||
           last_not_merge_type == kTypeSingleDeletion ||
           last_not_merge_type == kTypeDeletionWithTimestamp) {
-        s = Merge(nullptr, saved_key_.GetUserKey());
-        if (!s.ok()) {
+        if (!Merge(nullptr, saved_key_.GetUserKey())) {
           return false;
         }
         return true;
@@ -976,8 +983,7 @@ bool DBIter::FindValueForCurrentKey() {
           return false;
         }
         valid_ = true;
-        s = Merge(&blob_value_, saved_key_.GetUserKey());
-        if (!s.ok()) {
+        if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
           return false;
         }
 
@@ -985,15 +991,14 @@ bool DBIter::FindValueForCurrentKey() {
 
         return true;
       } else if (last_not_merge_type == kTypeWideColumnEntity) {
-        // TODO: support wide-column entities
-        status_ = Status::NotSupported(
-            "Merge currently not supported for wide-column entities");
-        valid_ = false;
-        return false;
+        if (!MergeEntity(pinned_value_, saved_key_.GetUserKey())) {
+          return false;
+        }
+
+        return true;
       } else {
         assert(last_not_merge_type == kTypeValue);
-        s = Merge(&pinned_value_, saved_key_.GetUserKey());
-        if (!s.ok()) {
+        if (!Merge(&pinned_value_, saved_key_.GetUserKey())) {
           return false;
         }
         return true;
@@ -1177,8 +1182,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 
     if (ikey.type == kTypeValue) {
       const Slice val = iter_.value();
-      Status s = Merge(&val, saved_key_.GetUserKey());
-      if (!s.ok()) {
+      if (!Merge(&val, saved_key_.GetUserKey())) {
         return false;
       }
       return true;
@@ -1197,8 +1201,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
         return false;
       }
       valid_ = true;
-      Status s = Merge(&blob_value_, saved_key_.GetUserKey());
-      if (!s.ok()) {
+      if (!Merge(&blob_value_, saved_key_.GetUserKey())) {
         return false;
       }
 
@@ -1206,11 +1209,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 
       return true;
     } else if (ikey.type == kTypeWideColumnEntity) {
-      // TODO: support wide-column entities
-      status_ = Status::NotSupported(
-          "Merge currently not supported for wide-column entities");
-      valid_ = false;
-      return false;
+      if (!MergeEntity(iter_.value(), saved_key_.GetUserKey())) {
+        return false;
+      }
+
+      return true;
     } else {
       valid_ = false;
       status_ = Status::Corruption(
@@ -1220,8 +1223,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     }
   }
 
-  Status s = Merge(nullptr, saved_key_.GetUserKey());
-  if (!s.ok()) {
+  if (!Merge(nullptr, saved_key_.GetUserKey())) {
     return false;
   }
 
@@ -1244,7 +1246,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   return true;
 }
 
-Status DBIter::Merge(const Slice* val, const Slice& user_key) {
+bool DBIter::Merge(const Slice* val, const Slice& user_key) {
   Status s = MergeHelper::TimedFullMerge(
       merge_operator_, user_key, val, merge_context_.GetOperands(),
       &saved_value_, logger_, statistics_, clock_, &pinned_value_,
@@ -1252,14 +1254,33 @@ Status DBIter::Merge(const Slice* val, const Slice& user_key) {
   if (!s.ok()) {
     valid_ = false;
     status_ = s;
-    return s;
+    return false;
   }
 
   SetValueAndColumnsFromPlain(pinned_value_.data() ? pinned_value_
                                                    : saved_value_);
 
   valid_ = true;
-  return s;
+  return true;
+}
+
+bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) {
+  Status s = MergeHelper::TimedFullMergeWithEntity(
+      merge_operator_, user_key, entity, merge_context_.GetOperands(),
+      &saved_value_, logger_, statistics_, clock_,
+      /* update_num_ops_stats */ true);
+  if (!s.ok()) {
+    valid_ = false;
+    status_ = s;
+    return false;
+  }
+
+  if (!SetValueAndColumnsFromEntity(saved_value_)) {
+    return false;
+  }
+
+  valid_ = true;
+  return true;
 }
 
 // Move backwards until the key smaller than saved_key_.

package/deps/rocksdb/rocksdb/db/db_iter.h

@@ -318,7 +318,8 @@ class DBIter final : public Iterator {
   }
 
   // If user-defined timestamp is enabled, `user_key` includes timestamp.
-  Status Merge(const Slice* val, const Slice& user_key);
+  bool Merge(const Slice* val, const Slice& user_key);
+  bool MergeEntity(const Slice& entity, const Slice& user_key);
 
   const SliceTransform* prefix_extractor_;
   Env* const env_;

package/deps/rocksdb/rocksdb/db/memtable.cc

@@ -1066,11 +1066,21 @@ static bool SaveValue(void* arg, const char* entry) {
           assert(s->do_merge);
 
           if (s->value || s->columns) {
+            std::string result;
             *(s->status) = MergeHelper::TimedFullMerge(
                 merge_operator, s->key->user_key(), &v,
-                merge_context->GetOperands(), s->value, s->columns, s->logger,
-                s->statistics, s->clock, /* result_operand */ nullptr,
+                merge_context->GetOperands(), &result, s->logger, s->statistics,
+                s->clock, /* result_operand */ nullptr,
                 /* update_num_ops_stats */ true);
+
+            if (s->status->ok()) {
+              if (s->value) {
+                *(s->value) = std::move(result);
+              } else {
+                assert(s->columns);
+                s->columns->SetPlainValue(result);
+              }
+            }
           }
         } else if (s->value) {
           s->value->assign(v.data(), v.size());
@@ -1115,11 +1125,27 @@ static bool SaveValue(void* arg, const char* entry) {
         } else if (*(s->merge_in_progress)) {
           assert(s->do_merge);
 
-          if (s->value || s->columns) {
+          if (s->value) {
+            Slice value_of_default;
+            *(s->status) = WideColumnSerialization::GetValueOfDefaultColumn(
+                v, value_of_default);
+            if (s->status->ok()) {
+              *(s->status) = MergeHelper::TimedFullMerge(
+                  merge_operator, s->key->user_key(), &value_of_default,
+                  merge_context->GetOperands(), s->value, s->logger,
+                  s->statistics, s->clock, /* result_operand */ nullptr,
+                  /* update_num_ops_stats */ true);
+            }
+          } else if (s->columns) {
+            std::string result;
             *(s->status) = MergeHelper::TimedFullMergeWithEntity(
                 merge_operator, s->key->user_key(), v,
-                merge_context->GetOperands(), s->value, s->columns, s->logger,
-                s->statistics, s->clock, /* update_num_ops_stats */ true);
+                merge_context->GetOperands(), &result, s->logger, s->statistics,
+                s->clock, /* update_num_ops_stats */ true);
+
+            if (s->status->ok()) {
+              *(s->status) = s->columns->SetWideColumnValue(result);
+            }
           }
         } else if (s->value) {
           Slice value_of_default;
@@ -1150,11 +1176,21 @@ static bool SaveValue(void* arg, const char* entry) {
       case kTypeRangeDeletion: {
         if (*(s->merge_in_progress)) {
           if (s->value || s->columns) {
+            std::string result;
             *(s->status) = MergeHelper::TimedFullMerge(
                 merge_operator, s->key->user_key(), nullptr,
-                merge_context->GetOperands(), s->value, s->columns, s->logger,
-                s->statistics, s->clock, /* result_operand */ nullptr,
+                merge_context->GetOperands(), &result, s->logger, s->statistics,
+                s->clock, /* result_operand */ nullptr,
                 /* update_num_ops_stats */ true);
+
+            if (s->status->ok()) {
+              if (s->value) {
+                *(s->value) = std::move(result);
+              } else {
+                assert(s->columns);
+                s->columns->SetPlainValue(result);
+              }
+            }
           }
         } else {
           *(s->status) = Status::NotFound();
@@ -1180,11 +1216,21 @@ static bool SaveValue(void* arg, const char* entry) {
         if (s->do_merge && merge_operator->ShouldMerge(
                                merge_context->GetOperandsDirectionBackward())) {
           if (s->value || s->columns) {
+            std::string result;
             *(s->status) = MergeHelper::TimedFullMerge(
                 merge_operator, s->key->user_key(), nullptr,
-                merge_context->GetOperands(), s->value, s->columns, s->logger,
-                s->statistics, s->clock, /* result_operand */ nullptr,
+                merge_context->GetOperands(), &result, s->logger, s->statistics,
+                s->clock, /* result_operand */ nullptr,
                 /* update_num_ops_stats */ true);
+
+            if (s->status->ok()) {
+              if (s->value) {
+                *(s->value) = std::move(result);
+              } else {
+                assert(s->columns);
+                s->columns->SetPlainValue(result);
+              }
+            }
           }
 
           *(s->found_final_value) = true;