@nxtedition/rocksdb 8.0.4 → 8.1.0

package/BUILDING.md

@@ -10,9 +10,9 @@
   - Copy `libfolly.a` to `/usr/lib/x86_64-linux-gnu`.
   - Copy headers to `/usr/lib/x86_64-linux-gnu/include`.
   - Copy boost headers from folly scratchpad to `/usr/lib/x86_64-linux-gnu/include`.
-- `JOBS=8 npx prebuildify -t 18.11.0 --napi --strip --arch x64`
+- `JONS=16 npx prebuildify -t 18.11.0 --napi --strip --arch x64`
 
 # OSX
 
 - `brew install zstd`
-- `JOBS=8 npx prebuildify -t 18.11.0 --napi --strip --arch arm64`
+- `JONS=16 npx prebuildify -t 18.11.0 --napi --strip --arch arm64`

package/binding.cc

@@ -26,6 +26,7 @@
 #include <thread>
 #include <vector>
 
+#include "max_rev_operator.h"
 #include "util.h"
 
 class NullLogger : public rocksdb::Logger {
@@ -556,8 +557,12 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   std::optional<std::string> mergeOperatorOpt;
   NAPI_STATUS_RETURN(GetProperty(env, options, "mergeOperator", mergeOperatorOpt));
   if (mergeOperatorOpt) {
-		ROCKS_STATUS_RETURN_NAPI(
-				rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
+    if (*mergeOperatorOpt == "maxRev") {
+      columnOptions.merge_operator = std::make_shared<MaxRevOperator>();
+    } else {
+      ROCKS_STATUS_RETURN_NAPI(
+          rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
+    }
   }
 
   std::optional<std::string> compactionPriority;
@@ -631,6 +636,70 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   return napi_ok;
 }
 
+NAPI_METHOD(db_get_merge_operands) {
+  NAPI_ARGV(4);
+
+  Database* database;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
+
+	std::string key;
+	NAPI_STATUS_THROWS(GetValue(env, argv[1], key));
+
+  const auto options = argv[2];
+
+  Encoding valueEncoding = Encoding::String;
+  NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
+
+  rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
+  NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
+
+  auto callback = argv[3];
+
+  runAsync<std::vector<rocksdb::PinnableSlice>>(
+      "leveldown.get.mergeOperands", env, callback,
+      [=, key = std::move(key)](auto& values) {
+        rocksdb::ReadOptions readOptions;
+
+				values.resize(16); // TODO (fix): Make option
+
+				rocksdb::GetMergeOperandsOptions mergeOperandsOptions;
+				mergeOperandsOptions.expected_max_number_of_operands = values.size();
+
+  			int size = 0;
+				const auto status = database->db->GetMergeOperands(
+					readOptions,
+					column,
+					key,
+					values.data(),
+					&mergeOperandsOptions,
+					&size
+				);
+
+				values.resize(size);
+
+        return status;
+      },
+      [=](auto& values, auto env, auto& argv) {
+        argv.resize(2);
+
+        NAPI_STATUS_RETURN(napi_create_array_with_length(env, values.size(), &argv[1]));
+
+        for (size_t idx = 0; idx < values.size(); idx++) {
+          napi_value element;
+          if (values[idx].GetSelf()) {
+            NAPI_STATUS_RETURN(Convert(env, &values[idx], valueEncoding, element));
+          } else {
+            NAPI_STATUS_RETURN(napi_get_undefined(env, &element));
+          }
+          NAPI_STATUS_RETURN(napi_set_element(env, argv[1], static_cast<uint32_t>(idx), element));
+        }
+
+        return napi_ok;
+      });
+
+  return 0;
+}
+
 NAPI_METHOD(db_get_identity) {
   NAPI_ARGV(1);
 
@@ -1576,6 +1645,7 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_get_latest_sequence);
   NAPI_EXPORT_FUNCTION(db_get_sorted_wal_files);
   NAPI_EXPORT_FUNCTION(db_flush_wal);
+  NAPI_EXPORT_FUNCTION(db_get_merge_operands);
 
   NAPI_EXPORT_FUNCTION(iterator_init);
   NAPI_EXPORT_FUNCTION(iterator_seek);

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -980,6 +980,12 @@ if ( ROCKSDB_PLUGINS )
         plugin/${plugin}/${src}
         PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
     endforeach()
+    foreach (test ${${plugin}_TESTS})
+      list(APPEND PLUGIN_TESTS plugin/${plugin}/${test})
+      set_source_files_properties(
+        plugin/${plugin}/${test}
+        PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
+    endforeach()
     foreach (path ${${plugin}_INCLUDE_PATHS})
       include_directories(${path})
     endforeach()
@@ -1471,6 +1477,7 @@ if(WITH_TESTS)
         utilities/ttl/ttl_test.cc
         utilities/util_merge_operators_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
+	${PLUGIN_TESTS}
     )
   endif()
 

package/deps/rocksdb/rocksdb/Makefile

@@ -266,6 +266,7 @@ ROCKSDB_PLUGIN_EXTERNS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), int $($(p)_FUNC
 ROCKSDB_PLUGIN_BUILTINS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), {\"$(p)\"\, $($(p)_FUNC)}\,)
 ROCKSDB_PLUGIN_LDFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS))
 ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_PKGCONFIG_REQUIRES))
+ROCKSDB_PLUGIN_TESTS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach test, $($(p)_TESTS), plugin/$(p)/$(test)))
 
 CXXFLAGS += $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_CXXFLAGS))
 PLATFORM_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
@@ -647,10 +648,12 @@ STRESS_OBJECTS =  $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES))
 ALL_SOURCES  = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
 ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)
-ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES)
+ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES) $(ROCKSDB_PLUGIN_TESTS)
 
+PLUGIN_TESTS = $(patsubst %.cc, %, $(notdir $(ROCKSDB_PLUGIN_TESTS)))
 TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES)))
 TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C)))
+TESTS += $(PLUGIN_TESTS)
 
 # `make check-headers` to very that each header file includes its own
 # dependencies
@@ -702,6 +705,7 @@ NON_PARALLEL_TEST = \
 	env_test \
 	deletefile_test \
 	db_bloom_filter_test \
+	$(PLUGIN_TESTS) \
 
 PARALLEL_TEST = $(filter-out $(NON_PARALLEL_TEST), $(TESTS))
 
@@ -1355,6 +1359,14 @@ db_sanity_test: $(OBJ_DIR)/tools/db_sanity_test.o $(LIBRARY)
 db_repl_stress: $(OBJ_DIR)/tools/db_repl_stress.o $(LIBRARY)
 	$(AM_LINK)
 
+define MakeTestRule
+$(notdir $(1:%.cc=%)): $(1:%.cc=$$(OBJ_DIR)/%.o) $$(TEST_LIBRARY) $$(LIBRARY)
+	$$(AM_LINK)
+endef
+
+# For each PLUGIN test, create a rule to generate the test executable
+$(foreach test, $(ROCKSDB_PLUGIN_TESTS), $(eval $(call MakeTestRule, $(test))))
+
 arena_test: $(OBJ_DIR)/memory/arena_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 

package/deps/rocksdb/rocksdb/db/builder.cc

@@ -71,8 +71,9 @@ Status BuildTable(
     int job_id, const Env::IOPriority io_priority,
     TableProperties* table_properties, Env::WriteLifeTimeHint write_hint,
     const std::string* full_history_ts_low,
-    BlobFileCompletionCallback* blob_callback, uint64_t* num_input_entries,
-    uint64_t* memtable_payload_bytes, uint64_t* memtable_garbage_bytes) {
+    BlobFileCompletionCallback* blob_callback, Version* version,
+    uint64_t* num_input_entries, uint64_t* memtable_payload_bytes,
+    uint64_t* memtable_garbage_bytes) {
   assert((tboptions.column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          tboptions.column_family_name.empty());
@@ -246,9 +247,17 @@ Status BuildTable(
         auto tombstone = range_del_it->Tombstone();
         auto kv = tombstone.Serialize();
         builder->Add(kv.first.Encode(), kv.second);
-        meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
-                                       tombstone.seq_,
+        InternalKey tombstone_end = tombstone.SerializeEndKey();
+        meta->UpdateBoundariesForRange(kv.first, tombstone_end, tombstone.seq_,
                                        tboptions.internal_comparator);
+        if (version) {
+          SizeApproximationOptions approx_opts;
+          approx_opts.files_size_error_margin = 0.1;
+          meta->compensated_range_deletion_size += versions->ApproximateSize(
+              approx_opts, version, kv.first.Encode(), tombstone_end.Encode(),
+              0 /* start_level */, -1 /* end_level */,
+              TableReaderCaller::kFlush);
+        }
       }
     }
 

package/deps/rocksdb/rocksdb/db/builder.h

@@ -13,6 +13,7 @@
 #include "db/range_tombstone_fragmenter.h"
 #include "db/seqno_to_time_mapping.h"
 #include "db/table_properties_collector.h"
+#include "db/version_set.h"
 #include "logging/event_logger.h"
 #include "options/cf_options.h"
 #include "rocksdb/comparator.h"
@@ -70,7 +71,7 @@ extern Status BuildTable(
     Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
     const std::string* full_history_ts_low = nullptr,
     BlobFileCompletionCallback* blob_callback = nullptr,
-    uint64_t* num_input_entries = nullptr,
+    Version* version = nullptr, uint64_t* num_input_entries = nullptr,
     uint64_t* memtable_payload_bytes = nullptr,
     uint64_t* memtable_garbage_bytes = nullptr);
 

package/deps/rocksdb/rocksdb/db/c.cc

@@ -2588,6 +2588,12 @@ void rocksdb_block_based_options_set_partition_filters(
   options->rep.partition_filters = partition_filters;
 }
 
+void rocksdb_block_based_options_set_optimize_filters_for_memory(
+    rocksdb_block_based_table_options_t* options,
+    unsigned char optimize_filters_for_memory) {
+  options->rep.optimize_filters_for_memory = optimize_filters_for_memory;
+}
+
 void rocksdb_block_based_options_set_use_delta_encoding(
     rocksdb_block_based_table_options_t* options,
     unsigned char use_delta_encoding) {

package/deps/rocksdb/rocksdb/db/column_family.cc

@@ -1218,6 +1218,7 @@ Compaction* ColumnFamilyData::CompactRange(
   if (result != nullptr) {
     result->SetInputVersion(current_);
   }
+  TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return");
   return result;
 }
 

package/deps/rocksdb/rocksdb/db/compaction/compaction.cc

@@ -235,12 +235,19 @@ Compaction::Compaction(
       inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
       grandparents_(std::move(_grandparents)),
       score_(_score),
-      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
+      bottommost_level_(
+          // For simplicity, we don't support the concept of "bottommost level"
+          // with
+          // `CompactionReason::kExternalSstIngestion` and
+          // `CompactionReason::kRefitLevel`
+          (_compaction_reason == CompactionReason::kExternalSstIngestion ||
+           _compaction_reason == CompactionReason::kRefitLevel)
+              ? false
+              : IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
       is_manual_compaction_(_manual_compaction),
       trim_ts_(_trim_ts),
       is_trivial_move_(false),
-
       compaction_reason_(_compaction_reason),
       notify_on_compaction_completion_(false),
       enable_blob_garbage_collection_(
@@ -255,8 +262,15 @@ Compaction::Compaction(
                   _blob_garbage_collection_age_cutoff > 1
               ? mutable_cf_options()->blob_garbage_collection_age_cutoff
               : _blob_garbage_collection_age_cutoff),
-      penultimate_level_(EvaluatePenultimateLevel(
-          vstorage, immutable_options_, start_level_, output_level_)) {
+      penultimate_level_(
+          // For simplicity, we don't support the concept of "penultimate level"
+          // with `CompactionReason::kExternalSstIngestion` and
+          // `CompactionReason::kRefitLevel`
+          _compaction_reason == CompactionReason::kExternalSstIngestion ||
+                  _compaction_reason == CompactionReason::kRefitLevel
+              ? Compaction::kInvalidLevel
+              : EvaluatePenultimateLevel(vstorage, immutable_options_,
+                                         start_level_, output_level_)) {
   MarkFilesBeingCompacted(true);
   if (is_manual_compaction_) {
     compaction_reason_ = CompactionReason::kManualCompaction;

package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc

@@ -99,6 +99,8 @@ const char* GetCompactionReasonString(CompactionReason compaction_reason) {
       return "ForcedBlobGC";
     case CompactionReason::kRoundRobinTtl:
       return "RoundRobinTtl";
+    case CompactionReason::kRefitLevel:
+      return "RefitLevel";
     case CompactionReason::kNumOfReasons:
       // fall through
     default:

package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc

@@ -386,7 +386,8 @@ class CompactionJobTestBase : public testing::Test {
         oldest_blob_file_number, kUnknownOldestAncesterTime,
         kUnknownFileCreationTime,
         versions_->GetColumnFamilySet()->GetDefault()->NewEpochNumber(),
-        kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2,
+        0);
 
     mutex_.Lock();
     EXPECT_OK(

package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc

@@ -525,7 +525,8 @@ Status CompactionOutputs::AddRangeDels(
            ucmp->CompareWithoutTimestamp(*lower_bound, kv.second) < 0);
     // Range tombstone is not supported by output validator yet.
     builder_->Add(kv.first.Encode(), kv.second);
-    InternalKey smallest_candidate = std::move(kv.first);
+    InternalKey tombstone_start = std::move(kv.first);
+    InternalKey smallest_candidate{tombstone_start};
     if (lower_bound != nullptr &&
         ucmp->CompareWithoutTimestamp(smallest_candidate.user_key(),
                                       *lower_bound) <= 0) {
@@ -594,7 +595,8 @@ Status CompactionOutputs::AddRangeDels(
         smallest_candidate = InternalKey(*lower_bound, 0, kTypeRangeDeletion);
       }
     }
-    InternalKey largest_candidate = tombstone.SerializeEndKey();
+    InternalKey tombstone_end = tombstone.SerializeEndKey();
+    InternalKey largest_candidate{tombstone_end};
     if (upper_bound != nullptr &&
         ucmp->CompareWithoutTimestamp(*upper_bound,
                                       largest_candidate.user_key()) <= 0) {
@@ -636,6 +638,24 @@ Status CompactionOutputs::AddRangeDels(
 #endif
     meta.UpdateBoundariesForRange(smallest_candidate, largest_candidate,
                                   tombstone.seq_, icmp);
+    if (!bottommost_level) {
+      // Range tombstones are truncated at file boundaries
+      if (icmp.Compare(tombstone_start, meta.smallest) < 0) {
+        tombstone_start = meta.smallest;
+      }
+      if (icmp.Compare(tombstone_end, meta.largest) > 0) {
+        tombstone_end = meta.largest;
+      }
+      SizeApproximationOptions approx_opts;
+      approx_opts.files_size_error_margin = 0.1;
+      auto approximate_covered_size =
+          compaction_->input_version()->version_set()->ApproximateSize(
+              approx_opts, compaction_->input_version(),
+              tombstone_start.Encode(), tombstone_end.Encode(),
+              compaction_->output_level() + 1 /* start_level */,
+              -1 /* end_level */, kCompaction);
+      meta.compensated_range_deletion_size += approximate_covered_size;
+    }
     // The smallest key in a file is used for range tombstone truncation, so
     // it cannot have a seqnum of 0 (unless the smallest data key in a file
     // has a seqnum of 0). Otherwise, the truncated tombstone may expose

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc

@@ -1126,7 +1126,11 @@ void CompactionPicker::RegisterCompaction(Compaction* c) {
          c->output_level() == 0 ||
          !FilesRangeOverlapWithCompaction(*c->inputs(), c->output_level(),
                                           c->GetPenultimateLevel()));
-  if (c->start_level() == 0 ||
+  // CompactionReason::kExternalSstIngestion's start level is just a placeholder
+  // number without actual meaning as file ingestion technically does not have
+  // an input level like other compactions
+  if ((c->start_level() == 0 &&
+       c->compaction_reason() != CompactionReason::kExternalSstIngestion) ||
       ioptions_.compaction_style == kCompactionStyleUniversal) {
     level0_compactions_in_progress_.insert(c);
   }

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc

@@ -447,21 +447,21 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
       compaction_inputs_.push_back(output_level_inputs_);
     }
 
+    // In some edge cases we could pick a compaction that will be compacting
+    // a key range that overlap with another running compaction, and both
+    // of them have the same output level. This could happen if
+    // (1) we are running a non-exclusive manual compaction
+    // (2) AddFile ingest a new file into the LSM tree
+    // We need to disallow this from happening.
+    if (compaction_picker_->FilesRangeOverlapWithCompaction(
+            compaction_inputs_, output_level_,
+            Compaction::EvaluatePenultimateLevel(
+                vstorage_, ioptions_, start_level_, output_level_))) {
+      // This compaction output could potentially conflict with the output
+      // of a currently running compaction, we cannot run it.
+      return false;
+    }
     if (!is_l0_trivial_move_) {
-      // In some edge cases we could pick a compaction that will be compacting
-      // a key range that overlap with another running compaction, and both
-      // of them have the same output level. This could happen if
-      // (1) we are running a non-exclusive manual compaction
-      // (2) AddFile ingest a new file into the LSM tree
-      // We need to disallow this from happening.
-      if (compaction_picker_->FilesRangeOverlapWithCompaction(
-              compaction_inputs_, output_level_,
-              Compaction::EvaluatePenultimateLevel(
-                  vstorage_, ioptions_, start_level_, output_level_))) {
-        // This compaction output could potentially conflict with the output
-        // of a currently running compaction, we cannot run it.
-        return false;
-      }
       compaction_picker_->GetGrandparents(vstorage_, start_level_inputs_,
                                           output_level_inputs_, &grandparents_);
     }

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc

@@ -148,7 +148,7 @@ class CompactionPickerTestBase : public testing::Test {
         smallest_seq, largest_seq, marked_for_compact, temperature,
         kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
         kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum,
-        kUnknownFileChecksumFuncName, kNullUniqueId64x2);
+        kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
     f->compensated_file_size =
         (compensated_file_size != 0) ? compensated_file_size : file_size;
     f->oldest_ancester_time = oldest_ancestor_time;
@@ -2873,7 +2873,6 @@ TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) {
   ASSERT_EQ(0, compaction->output_level());
 }
 
-
 #ifndef ROCKSDB_LITE
 TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   const uint64_t kFileSize = 100000;

package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc

@@ -1229,7 +1229,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
      *
      *  The test is designed in a way such that the reservation for (p1 - b')
      *  will trigger at least another dummy entry insertion
-     *  (or equivelantly to saying, creating another peak).
+     *  (or equivalently to saying, creating another peak).
      *
      * kStandard128Ribbon + FullFilter +
      * detect_filter_construct_corruption
@@ -2618,8 +2618,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
       BottommostLevelCompaction::kSkip;
   compact_options.change_level = true;
   compact_options.target_level = 7;
-  ASSERT_TRUE(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)
-                  .IsNotSupported());
+  ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
 
   ASSERT_EQ(trivial_move, 1);
   ASSERT_EQ(non_trivial_move, 0);