@nxtedition/rocksdb 13.1.4 → 13.2.0

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "db/compaction/compaction.h"
+#include "db/snapshot_checker.h"
 #include "db/version_set.h"
 #include "options/cf_options.h"
 #include "rocksdb/env.h"
@@ -55,17 +56,17 @@ class CompactionPicker {
   // Returns nullptr if there is no compaction to be done.
   // Otherwise returns a pointer to a heap-allocated object that
   // describes the compaction.  Caller should delete the result.
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     const MutableDBOptions& mutable_db_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) = 0;
-
-  // Return a compaction object for compacting the range [begin,end] in
-  // the specified level.  Returns nullptr if there is nothing in that
-  // level that overlaps the specified range.  Caller should delete
-  // the result.
-  //
+  // Currently, only universal compaction will query existing snapshots and
+  // pass it to aid compaction picking. And it's only passed when user-defined
+  // timestamps is not enabled. The other compaction styles do not pass or use
+  // `existing_snapshots` or `snapshot_checker`.
+  virtual Compaction* PickCompaction(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options,
+      const std::vector<SequenceNumber>& existing_snapshots,
+      const SnapshotChecker* snapshot_checker, VersionStorageInfo* vstorage,
+      LogBuffer* log_buffer) = 0;
+
   // The returned Compaction might not include the whole requested range.
   // In that case, compaction_end will be set to the next key that needs
   // compacting. In case the compaction will compact the whole range,
@@ -96,9 +97,8 @@ class CompactionPicker {
   // non-ok status with specific reason.
   //
   Status SanitizeAndConvertCompactionInputFiles(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta, const int output_level,
-      const VersionStorageInfo* vstorage,
+      std::unordered_set<uint64_t>* input_files, const int output_level,
+      Version* version,
       std::vector<CompactionInputFiles>* converted_input_files) const;
 
   // Free up the files that participated in a compaction
@@ -203,10 +203,11 @@ class CompactionPicker {
                        const CompactionInputFiles& output_level_inputs,
                        std::vector<FileMetaData*>* grandparents);
 
-  void PickFilesMarkedForCompaction(const std::string& cf_name,
-                                    VersionStorageInfo* vstorage,
-                                    int* start_level, int* output_level,
-                                    CompactionInputFiles* start_level_inputs);
+  void PickFilesMarkedForCompaction(
+      const std::string& cf_name, VersionStorageInfo* vstorage,
+      int* start_level, int* output_level,
+      CompactionInputFiles* start_level_inputs,
+      std::function<bool(const FileMetaData*)> skip_marked_file);
 
   bool GetOverlappingL0Files(VersionStorageInfo* vstorage,
                              CompactionInputFiles* start_level_inputs,
@@ -257,11 +258,13 @@ class NullCompactionPicker : public CompactionPicker {
   virtual ~NullCompactionPicker() {}
 
   // Always return "nullptr"
-  Compaction* PickCompaction(const std::string& /*cf_name*/,
-                             const MutableCFOptions& /*mutable_cf_options*/,
-                             const MutableDBOptions& /*mutable_db_options*/,
-                             VersionStorageInfo* /*vstorage*/,
-                             LogBuffer* /* log_buffer */) override {
+  Compaction* PickCompaction(
+      const std::string& /*cf_name*/,
+      const MutableCFOptions& /*mutable_cf_options*/,
+      const MutableDBOptions& /*mutable_db_options*/,
+      const std::vector<SequenceNumber>& /*existing_snapshots*/,
+      const SnapshotChecker* /*snapshot_checker*/,
+      VersionStorageInfo* /*vstorage*/, LogBuffer* /* log_buffer */) override {
     return nullptr;
   }
 

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc

@@ -79,10 +79,14 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
       FileMetaData* f = *ritr;
       assert(f);
       if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
+        uint64_t newest_key_time = f->TryGetNewestKeyTime();
         uint64_t creation_time =
             f->fd.table_reader->GetTableProperties()->creation_time;
-        if (creation_time == 0 ||
-            creation_time >= (current_time - mutable_cf_options.ttl)) {
+        uint64_t est_newest_key_time = newest_key_time == kUnknownNewestKeyTime
+                                           ? creation_time
+                                           : newest_key_time;
+        if (est_newest_key_time == kUnknownNewestKeyTime ||
+            est_newest_key_time >= (current_time - mutable_cf_options.ttl)) {
           break;
         }
       }
@@ -102,15 +106,19 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   }
 
   for (const auto& f : inputs[0].files) {
-    uint64_t creation_time = 0;
     assert(f);
+    uint64_t newest_key_time = f->TryGetNewestKeyTime();
+    uint64_t creation_time = 0;
     if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
       creation_time = f->fd.table_reader->GetTableProperties()->creation_time;
     }
+    uint64_t est_newest_key_time = newest_key_time == kUnknownNewestKeyTime
+                                       ? creation_time
+                                       : newest_key_time;
     ROCKS_LOG_BUFFER(log_buffer,
                      "[%s] FIFO compaction: picking file %" PRIu64
-                     " with creation time %" PRIu64 " for deletion",
-                     cf_name.c_str(), f->fd.GetNumber(), creation_time);
+                     " with estimated newest key time %" PRIu64 " for deletion",
+                     cf_name.c_str(), f->fd.GetNumber(), est_newest_key_time);
   }
 
   Compaction* c = new Compaction(
@@ -118,7 +126,9 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
       std::move(inputs), 0, 0, 0, 0, kNoCompression,
       mutable_cf_options.compression_opts,
       mutable_cf_options.default_write_temperature,
-      /* max_subcompactions */ 0, {}, /* is manual */ false,
+      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
       /* is deletion compaction */ true, /* l0_files_might_overlap */ true,
       CompactionReason::kFIFOTtl);
@@ -188,7 +198,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             0 /* output path ID */, mutable_cf_options.compression,
             mutable_cf_options.compression_opts,
             mutable_cf_options.default_write_temperature,
-            0 /* max_subcompactions */, {}, /* is manual */ false,
+            0 /* max_subcompactions */, {},
+            /* earliest_snapshot */ std::nullopt,
+            /* snapshot_checker */ nullptr, /* is manual */ false,
             /* trim_ts */ "", vstorage->CompactionScore(0),
             /* is deletion compaction */ false,
             /* l0_files_might_overlap */ true,
@@ -284,7 +296,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
       /* output_path_id */ 0, kNoCompression,
       mutable_cf_options.compression_opts,
       mutable_cf_options.default_write_temperature,
-      /* max_subcompactions */ 0, {}, /* is manual */ false,
+      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
       /* is deletion compaction */ true,
       /* l0_files_might_overlap */ true, CompactionReason::kFIFOMaxSize);
@@ -344,38 +358,29 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
   Temperature compaction_target_temp = Temperature::kLastTemperature;
   if (current_time > min_age) {
     uint64_t create_time_threshold = current_time - min_age;
-    // We will ideally identify a file qualifying for temperature change by
-    // knowing the timestamp for the youngest entry in the file. However, right
-    // now we don't have the information. We infer it by looking at timestamp of
-    // the previous file's (which is just younger) oldest entry's timestamp.
-    // avoid index underflow
     assert(level_files.size() >= 1);
-    for (size_t index = level_files.size() - 1; index >= 1; --index) {
+    for (size_t index = level_files.size(); index >= 1; --index) {
       // Try to add cur_file to compaction inputs.
-      FileMetaData* cur_file = level_files[index];
-      // prev_file is just younger than cur_file
-      FileMetaData* prev_file = level_files[index - 1];
+      FileMetaData* cur_file = level_files[index - 1];
+      FileMetaData* prev_file = index < 2 ? nullptr : level_files[index - 2];
       if (cur_file->being_compacted) {
         // Should not happen since we check for
         // `level0_compactions_in_progress_` above. Here we simply just don't
         // schedule anything.
         return nullptr;
       }
-      uint64_t oldest_ancestor_time = prev_file->TryGetOldestAncesterTime();
-      if (oldest_ancestor_time == kUnknownOldestAncesterTime) {
-        // Older files might not have enough information. It is possible to
-        // handle these files by looking at newer files, but maintaining the
-        // logic isn't worth it.
-        break;
+      uint64_t est_newest_key_time = cur_file->TryGetNewestKeyTime(prev_file);
+      // Newer file could have newest_key_time populated
+      if (est_newest_key_time == kUnknownNewestKeyTime) {
+        continue;
       }
-      if (oldest_ancestor_time > create_time_threshold) {
-        // cur_file is too fresh
+      if (est_newest_key_time > create_time_threshold) {
         break;
       }
       Temperature cur_target_temp = ages[0].temperature;
       for (size_t i = 1; i < ages.size(); ++i) {
         if (current_time >= ages[i].age &&
-            oldest_ancestor_time <= current_time - ages[i].age) {
+            est_newest_key_time <= current_time - ages[i].age) {
           cur_target_temp = ages[i].temperature;
         }
       }
@@ -390,8 +395,8 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
       ROCKS_LOG_BUFFER(
           log_buffer,
           "[%s] FIFO compaction: picking file %" PRIu64
-          " with next file's oldest time %" PRIu64 " for temperature %s.",
-          cf_name.c_str(), cur_file->fd.GetNumber(), oldest_ancestor_time,
+          " with estimated newest key time %" PRIu64 " for temperature %s.",
+          cf_name.c_str(), cur_file->fd.GetNumber(), est_newest_key_time,
           temperature_to_string[cur_target_temp].c_str());
       break;
     }
@@ -410,8 +415,9 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
       0 /* max compaction bytes, not applicable */, 0 /* output path ID */,
       mutable_cf_options.compression, mutable_cf_options.compression_opts,
       compaction_target_temp,
-      /* max_subcompactions */ 0, {}, /* is manual */ false, /* trim_ts */ "",
-      vstorage->CompactionScore(0),
+      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false, /* trim_ts */ "", vstorage->CompactionScore(0),
       /* is deletion compaction */ false, /* l0_files_might_overlap */ true,
       CompactionReason::kChangeTemperature);
   return c;
@@ -419,7 +425,9 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
 
 Compaction* FIFOCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
+    const MutableDBOptions& mutable_db_options,
+    const std::vector<SequenceNumber>& /* existing_snapshots */,
+    const SnapshotChecker* /* snapshot_checker */, VersionStorageInfo* vstorage,
     LogBuffer* log_buffer) {
   Compaction* c = nullptr;
   if (mutable_cf_options.ttl > 0) {
@@ -454,8 +462,10 @@ Compaction* FIFOCompactionPicker::CompactRange(
   assert(output_level == 0);
   *compaction_end = nullptr;
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.logger);
-  Compaction* c = PickCompaction(cf_name, mutable_cf_options,
-                                 mutable_db_options, vstorage, &log_buffer);
+  Compaction* c =
+      PickCompaction(cf_name, mutable_cf_options, mutable_db_options,
+                     /*existing_snapshots*/ {}, /*snapshot_checker*/ nullptr,
+                     vstorage, &log_buffer);
   log_buffer.FlushBufferToLog();
   return c;
 }

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h

@@ -18,11 +18,12 @@ class FIFOCompactionPicker : public CompactionPicker {
                        const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
 
-  Compaction* PickCompaction(const std::string& cf_name,
-                             const MutableCFOptions& mutable_cf_options,
-                             const MutableDBOptions& mutable_db_options,
-                             VersionStorageInfo* version,
-                             LogBuffer* log_buffer) override;
+  Compaction* PickCompaction(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options,
+      const std::vector<SequenceNumber>& /* existing_snapshots */,
+      const SnapshotChecker* /* snapshot_checker */,
+      VersionStorageInfo* version, LogBuffer* log_buffer) override;
 
   Compaction* CompactRange(const std::string& cf_name,
                            const MutableCFOptions& mutable_cf_options,

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc

@@ -262,7 +262,10 @@ void LevelCompactionBuilder::SetupInitialFiles() {
   parent_index_ = base_index_ = -1;
 
   compaction_picker_->PickFilesMarkedForCompaction(
-      cf_name_, vstorage_, &start_level_, &output_level_, &start_level_inputs_);
+      cf_name_, vstorage_, &start_level_, &output_level_, &start_level_inputs_,
+      /*skip_marked_file*/ [](const FileMetaData* /* file */) {
+        return false;
+      });
   if (!start_level_inputs_.empty()) {
     compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
     return;
@@ -411,8 +414,9 @@ void LevelCompactionBuilder::SetupOtherFilesWithRoundRobinExpansion() {
                                                     &tmp_start_level_inputs) ||
         compaction_picker_->FilesRangeOverlapWithCompaction(
             {tmp_start_level_inputs}, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
       // Constraint 1a
       tmp_start_level_inputs.clear();
       return;
@@ -486,8 +490,9 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
     // We need to disallow this from happening.
     if (compaction_picker_->FilesRangeOverlapWithCompaction(
             compaction_inputs_, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
       // This compaction output could potentially conflict with the output
       // of a currently running compaction, we cannot run it.
       return false;
@@ -554,7 +559,9 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
                          vstorage_->base_level()),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_),
       mutable_cf_options_.default_write_temperature,
-      /* max_subcompactions */ 0, std::move(grandparents_), is_manual_,
+      /* max_subcompactions */ 0, std::move(grandparents_),
+      /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
+      is_manual_,
       /* trim_ts */ "", start_level_score_, false /* deletion_compaction */,
       l0_files_might_overlap, compaction_reason_);
 
@@ -839,8 +846,9 @@ bool LevelCompactionBuilder::PickFileToCompact() {
                                                     &start_level_inputs_) ||
         compaction_picker_->FilesRangeOverlapWithCompaction(
             {start_level_inputs_}, output_level_,
-            Compaction::EvaluatePenultimateLevel(
-                vstorage_, ioptions_, start_level_, output_level_))) {
+            Compaction::EvaluatePenultimateLevel(vstorage_, mutable_cf_options_,
+                                                 ioptions_, start_level_,
+                                                 output_level_))) {
       // A locked (pending compaction) input-level file was pulled in due to
       // user-key overlap.
       start_level_inputs_.clear();
@@ -967,7 +975,9 @@ bool LevelCompactionBuilder::PickSizeBasedIntraL0Compaction() {
 
 Compaction* LevelCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
+    const MutableDBOptions& mutable_db_options,
+    const std::vector<SequenceNumber>& /*existing_snapshots */,
+    const SnapshotChecker* /*snapshot_checker*/, VersionStorageInfo* vstorage,
     LogBuffer* log_buffer) {
   LevelCompactionBuilder builder(cf_name, vstorage, this, log_buffer,
                                  mutable_cf_options, ioptions_,

package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h

@@ -20,11 +20,12 @@ class LevelCompactionPicker : public CompactionPicker {
   LevelCompactionPicker(const ImmutableOptions& ioptions,
                         const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
-  Compaction* PickCompaction(const std::string& cf_name,
-                             const MutableCFOptions& mutable_cf_options,
-                             const MutableDBOptions& mutable_db_options,
-                             VersionStorageInfo* vstorage,
-                             LogBuffer* log_buffer) override;
+  Compaction* PickCompaction(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options,
+      const std::vector<SequenceNumber>& /* existing_snapshots */,
+      const SnapshotChecker* /* snapshot_checker */,
+      VersionStorageInfo* vstorage, LogBuffer* log_buffer) override;
 
   bool NeedsCompaction(const VersionStorageInfo* vstorage) const override;
 };