@nxtedition/rocksdb 8.1.3 → 8.1.4

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc

@@ -77,9 +77,8 @@ class CompactionPickerTestBase : public testing::Test {
   void NewVersionStorage(int num_levels, CompactionStyle style) {
     DeleteVersionStorage();
     options_.num_levels = num_levels;
-    vstorage_.reset(new VersionStorageInfo(
-        &icmp_, ucmp_, options_.num_levels, style, nullptr, false,
-        EpochNumberRequirement::kMustPresent));
+    vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
+                                           style, nullptr, false));
     vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
   }
 
@@ -88,7 +87,7 @@ class CompactionPickerTestBase : public testing::Test {
   void AddVersionStorage() {
     temp_vstorage_.reset(new VersionStorageInfo(
         &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style,
-        vstorage_.get(), false, EpochNumberRequirement::kMustPresent));
+        vstorage_.get(), false));
   }
 
   void DeleteVersionStorage() {
@@ -106,8 +105,7 @@ class CompactionPickerTestBase : public testing::Test {
            size_t compensated_file_size = 0, bool marked_for_compact = false,
            Temperature temperature = Temperature::kUnknown,
            uint64_t oldest_ancestor_time = kUnknownOldestAncesterTime,
-           Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice(),
-           uint64_t epoch_number = kUnknownEpochNumber) {
+           Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice()) {
     assert(ts_of_smallest.size() == ucmp_->timestamp_size());
     assert(ts_of_largest.size() == ucmp_->timestamp_size());
 
@@ -147,8 +145,8 @@ class CompactionPickerTestBase : public testing::Test {
         file_number, path_id, file_size, smallest_ikey, largest_ikey,
         smallest_seq, largest_seq, marked_for_compact, temperature,
         kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-        kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum,
-        kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+        kUnknownFileCreationTime, kUnknownFileChecksum,
+        kUnknownFileChecksumFuncName, kNullUniqueId64x2);
     f->compensated_file_size =
         (compensated_file_size != 0) ? compensated_file_size : file_size;
     f->oldest_ancester_time = oldest_ancestor_time;
@@ -2873,6 +2871,39 @@ TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) {
   ASSERT_EQ(0, compaction->output_level());
 }
 
+TEST_F(CompactionPickerTest, IntraL0ForEarliestSeqno) {
+  // Intra L0 compaction triggers only if there are at least
+  // level0_file_num_compaction_trigger + 2 L0 files.
+  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+  mutable_cf_options_.max_compaction_bytes = 999999u;
+  NewVersionStorage(6, kCompactionStyleLevel);
+
+  // 4 out of 6 L0 files will be picked for intra L0 compaction due to
+  // being_compact limit. And the latest one L0 will be skipped due to earliest
+  // seqno. The one L1 file spans entire L0 key range and is marked as being
+  // compacted to avoid L0->L1 compaction.
+  Add(1, 1U, "100", "350", 200000U, 0, 110, 111);
+  Add(0, 2U, "301", "350", 1U, 0, 108, 109);
+  Add(0, 3U, "251", "300", 1U, 0, 106, 107);
+  Add(0, 4U, "201", "250", 1U, 0, 104, 105);
+  Add(0, 5U, "151", "200", 1U, 0, 102, 103);
+  Add(0, 6U, "100", "150", 1U, 0, 100, 101);
+  Add(0, 7U, "100", "100", 1U, 0, 99, 100);
+  vstorage_->LevelFiles(0)[5]->being_compacted = true;
+  vstorage_->LevelFiles(1)[0]->being_compacted = true;
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+      &log_buffer_, 107));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(1U, compaction->num_input_levels());
+  ASSERT_EQ(4U, compaction->num_input_files(0));
+  ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
+            compaction->compaction_reason());
+  ASSERT_EQ(0, compaction->output_level());
+}
+
 #ifndef ROCKSDB_LITE
 TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   const uint64_t kFileSize = 100000;
@@ -2885,23 +2916,9 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   // should fail
   NewVersionStorage(5, kCompactionStyleUniversal);
 
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
-      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 3);
-  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 2);
-  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 1);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
+  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450);
+  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300);
   Add(3, 5U, "010", "080", 8 * kFileSize, 0, 200, 251);
   Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150);
   Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150);
@@ -2923,11 +2940,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
 
   AddVersionStorage();
   // Simulate a flush and mark the file for compaction
-  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 4);
+  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(
@@ -2949,11 +2962,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
   NewVersionStorage(5, kCompactionStyleUniversal);
 
   // Mark file number 4 for compaction
-  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 1);
+  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true);
   Add(3, 5U, "240", "290", 8 * kFileSize, 0, 201, 250);
   Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150);
   Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150);
@@ -2974,17 +2983,8 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
   ASSERT_EQ(1U, compaction->num_input_files(1));
 
   AddVersionStorage();
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
-      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 3);
-  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 2);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
+  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(
@@ -3150,29 +3150,10 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
   NewVersionStorage(1, kCompactionStyleUniversal);
 
   // Mark file number 5 for compaction
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 4);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 3);
-  Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 2);
-  Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100,
-      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 1);
+  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300);
+  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true);
+  Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150);
+  Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(
@@ -3192,18 +3173,8 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
   ASSERT_TRUE(file_map_[6].first->being_compacted);
 
   AddVersionStorage();
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
-      /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 6);
-  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450, /*compensated_file_size*/ 0,
-      /*marked_for_compact*/ false,
-      /* temperature*/ Temperature::kUnknown,
-      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
-      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
-      /*epoch_number*/ 5);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
+  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc

@@ -293,7 +293,7 @@ bool UniversalCompactionPicker::NeedsCompaction(
 Compaction* UniversalCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-    LogBuffer* log_buffer) {
+    LogBuffer* log_buffer, SequenceNumber /* earliest_memtable_seqno */) {
   UniversalCompactionBuilder builder(ioptions_, icmp_, cf_name,
                                      mutable_cf_options, mutable_db_options,
                                      vstorage, this, log_buffer);
@@ -400,7 +400,6 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
   if (!vstorage_->FilesMarkedForPeriodicCompaction().empty()) {
     // Always need to do a full compaction for periodic compaction.
     c = PickPeriodicCompaction();
-    TEST_SYNC_POINT_CALLBACK("PostPickPeriodicCompaction", c);
   }
 
   // Check for size amplification.
@@ -409,7 +408,6 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
           static_cast<size_t>(
               mutable_cf_options_.level0_file_num_compaction_trigger)) {
     if ((c = PickCompactionToReduceSizeAmp()) != nullptr) {
-      TEST_SYNC_POINT("PickCompactionToReduceSizeAmpReturnNonnullptr");
       ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: compacting for size amp\n",
                        cf_name_.c_str());
     } else {
@@ -419,7 +417,6 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
           mutable_cf_options_.compaction_options_universal.size_ratio;
 
       if ((c = PickCompactionToReduceSortedRuns(ratio, UINT_MAX)) != nullptr) {
-        TEST_SYNC_POINT("PickCompactionToReduceSortedRunsReturnNonnullptr");
         ROCKS_LOG_BUFFER(log_buffer_,
                          "[%s] Universal: compacting for size ratio\n",
                          cf_name_.c_str());
@@ -460,7 +457,6 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
 
   if (c == nullptr) {
     if ((c = PickDeleteTriggeredCompaction()) != nullptr) {
-      TEST_SYNC_POINT("PickDeleteTriggeredCompactionReturnNonnullptr");
       ROCKS_LOG_BUFFER(log_buffer_,
                        "[%s] Universal: delete triggered compaction\n",
                        cf_name_.c_str());

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h

@@ -18,11 +18,11 @@ class UniversalCompactionPicker : public CompactionPicker {
   UniversalCompactionPicker(const ImmutableOptions& ioptions,
                             const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     const MutableDBOptions& mutable_db_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
+  virtual Compaction* PickCompaction(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
+      LogBuffer* log_buffer,
+      SequenceNumber earliest_memtable_seqno = kMaxSequenceNumber) override;
   virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
 
   virtual bool NeedsCompaction(

package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc

@@ -190,7 +190,6 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
     meta.largest.DecodeFrom(file.largest_internal_key);
     meta.oldest_ancester_time = file.oldest_ancester_time;
     meta.file_creation_time = file.file_creation_time;
-    meta.epoch_number = file.epoch_number;
     meta.marked_for_compaction = file.marked_for_compaction;
     meta.unique_id = file.unique_id;
 
@@ -334,9 +333,8 @@ Status CompactionServiceCompactionJob::Run() {
         MakeTableFileName(meta.fd.GetNumber()), meta.fd.smallest_seqno,
         meta.fd.largest_seqno, meta.smallest.Encode().ToString(),
         meta.largest.Encode().ToString(), meta.oldest_ancester_time,
-        meta.file_creation_time, meta.epoch_number,
-        output_file.validator.GetHash(), meta.marked_for_compaction,
-        meta.unique_id);
+        meta.file_creation_time, output_file.validator.GetHash(),
+        meta.marked_for_compaction, meta.unique_id);
   }
   InternalStats::CompactionStatsFull compaction_stats;
   sub_compact->AggregateCompactionStats(compaction_stats);
@@ -491,10 +489,6 @@ static std::unordered_map<std::string, OptionTypeInfo>
          {offsetof(struct CompactionServiceOutputFile, file_creation_time),
           OptionType::kUInt64T, OptionVerificationType::kNormal,
           OptionTypeFlags::kNone}},
-        {"epoch_number",
-         {offsetof(struct CompactionServiceOutputFile, epoch_number),
-          OptionType::kUInt64T, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
         {"paranoid_hash",
          {offsetof(struct CompactionServiceOutputFile, paranoid_hash),
           OptionType::kUInt64T, OptionVerificationType::kNormal,

package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h

@@ -84,11 +84,6 @@ class SubcompactionState {
   // Assign range dels aggregator, for each range_del, it can only be assigned
   // to one output level, for per_key_placement, it's going to be the
   // penultimate level.
-  // TODO: This does not work for per_key_placement + user-defined timestamp +
-  //  DeleteRange() combo. If user-defined timestamp is enabled,
-  //  it is possible for a range tombstone to belong to bottommost level (
-  //  seqno < earliest snapshot) without being dropped (garbage collection
-  //  for user-defined timestamp).
   void AssignRangeDelAggregator(
       std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
     if (compaction->SupportsPerKeyPlacement()) {
@@ -201,11 +196,8 @@ class SubcompactionState {
                               const CompactionFileCloseFunc& close_file_func) {
     // Call FinishCompactionOutputFile() even if status is not ok: it needs to
     // close the output file.
-    // CloseOutput() may open new compaction output files.
-    is_current_penultimate_level_ = true;
     Status s = penultimate_level_outputs_.CloseOutput(
         curr_status, open_file_func, close_file_func);
-    is_current_penultimate_level_ = false;
     s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func);
     return s;
   }