@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc

@@ -6,6 +6,9 @@
 
 #include "db/periodic_task_scheduler.h"
 
+#include <set>
+
+#include "db/column_family.h"
 #include "db/db_test_util.h"
 #include "env/composite_env_wrapper.h"
 #include "test_util/mock_time_env.h"
@@ -124,15 +127,289 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
   ASSERT_EQ(3, pst_st_counter);
   ASSERT_EQ(5, flush_info_log_counter);
 
-  ASSERT_EQ(0, trigger_compaction_counter);
-  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
-    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
-  });
-  ASSERT_EQ(1, trigger_compaction_counter);
-  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
-    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
-  });
-  ASSERT_EQ(2, trigger_compaction_counter);
+  // With the new ComputeTriggerCompactionPeriod() logic, since
+  // stats_dump_period_sec = stats_persist_period_sec = kPeriodSec, the trigger
+  // compaction period is also kPeriodSec. Unlike other periodic tasks,
+  // kTriggerCompaction uses run_immediately=false, so it doesn't fire on
+  // initial registration. It fires after each period elapses.
+  // By now we've had: initial 9s + 10s + 10s + 10s + 10s = 49s elapsed.
+  // kTriggerCompaction fires at t=10, 20, 30, 40 = 4 times.
+  ASSERT_EQ(4, trigger_compaction_counter);
+
+  // Sleep one more period and verify it increments
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
+  ASSERT_EQ(5, trigger_compaction_counter);
+
+  Close();
+}
+
+TEST_F(PeriodicTaskSchedulerTest, TriggerCompactionPeriodComputation) {
+  // Test that ComputeTriggerCompactionPeriod() returns the expected values
+  // for different configurations.
+  Close();
+
+  // Helper to capture the trigger compaction period on Reopen
+  uint64_t trigger_compaction_period = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "PeriodicTaskScheduler::Register:TaskRegistered", [&](void* arg) {
+        auto* task_info =
+            static_cast<std::pair<PeriodicTaskType, uint64_t>*>(arg);
+        if (task_info->first == PeriodicTaskType::kTriggerCompaction) {
+          trigger_compaction_period = task_info->second;
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Helper to test the trigger compaction period with optional second CF
+  auto test_period = [&](Options& options, uint64_t expected_period,
+                         const ColumnFamilyOptions* cf2_options = nullptr) {
+    options.create_if_missing = true;
+    options.env = mock_env_.get();
+    trigger_compaction_period = 0;
+
+    if (cf2_options == nullptr) {
+      Reopen(options);
+    } else {
+      // Open with two column families
+      // First destroy DB to start fresh
+      ASSERT_OK(DestroyDB(dbname_, options));
+      // Open and create cf2
+      ASSERT_OK(TryReopen(options));
+      CreateColumnFamilies({"cf2"}, options);
+      Close();
+      // Now reopen with both CFs
+      std::vector<ColumnFamilyDescriptor> cf_descs;
+      cf_descs.emplace_back(kDefaultColumnFamilyName, options);
+      cf_descs.emplace_back("cf2", *cf2_options);
+      std::vector<ColumnFamilyHandle*> handles;
+      ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_));
+      for (auto* h : handles) {
+        delete h;
+      }
+    }
+    ASSERT_EQ(expected_period, trigger_compaction_period);
+    Close();
+    // Destroy DB to clean up for next test case
+    ASSERT_OK(DestroyDB(dbname_, options));
+  };
+
+  // Case 1: 12-hour cap when stats_dump_period is longer
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;  // 24 hours
+    options.stats_persist_period_sec = 0;
+    test_period(options, 12 * 60 * 60);  // Expect 12 hours (cap)
+  }
+
+  // Case 2: stats_dump_period_sec sets the period (when smaller than 12 hours)
+  {
+    Options options;
+    options.stats_dump_period_sec = 3600;  // 1 hour
+    options.stats_persist_period_sec = 0;
+    test_period(options, 3600);
+  }
+
+  // Case 3: stats_persist_period_sec sets the period
+  {
+    Options options;
+    options.stats_dump_period_sec = 0;
+    options.stats_persist_period_sec = 1800;  // 30 minutes
+    test_period(options, 1800);
+  }
+
+  // Case 4: periodic_compaction_seconds / kTriggerDivisor
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.periodic_compaction_seconds = 500;  // 500 / 5 = 100
+    test_period(options, 100);
+  }
+
+  // Case 5: ttl / kTriggerDivisor
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.ttl = 600;  // 600 / 5 = 120
+    test_period(options, 120);
+  }
+
+  // Case 6: bottommost_file_compaction_delay / kTriggerDivisor
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.bottommost_file_compaction_delay = 250;  // 250 / 5 = 50
+    test_period(options, 50);
+  }
+
+  // Case 7: file_temperature_age_thresholds / kTriggerDivisor (FIFO)
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.compaction_style = kCompactionStyleFIFO;
+    options.num_levels = 1;  // Required for file_temperature_age_thresholds
+    options.compaction_options_fifo.file_temperature_age_thresholds = {
+        {Temperature::kWarm, 1000}};  // 1000 / 5 = 200
+    // FIFO requires max_table_files_size to be set
+    options.compaction_options_fifo.max_table_files_size = 1024 * 1024 * 1024;
+    test_period(options, 200);
+  }
+
+  // Case 8: Minimum of multiple CF options is used
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.periodic_compaction_seconds = 1000;  // 1000 / 5 = 200
+
+    ColumnFamilyOptions cf2_opts;
+    cf2_opts.ttl = 500;  // 500 / 5 = 100 (smaller, should be used)
+
+    test_period(options, 100, &cf2_opts);
+  }
+
+  // Case 9: Second CF has larger value, first CF's value is used
+  {
+    Options options;
+    options.stats_dump_period_sec = 24 * 60 * 60;
+    options.stats_persist_period_sec = 0;
+    options.ttl = 300;  // 300 / 5 = 60
+
+    ColumnFamilyOptions cf2_opts;
+    cf2_opts.periodic_compaction_seconds = 1000;  // 1000 / 5 = 200 (larger)
+
+    test_period(options, 60, &cf2_opts);
+  }
+
+  // Case 10: Minimum is 1 second (never 0)
+  {
+    Options options;
+    options.stats_dump_period_sec = 0;
+    options.stats_persist_period_sec = 0;
+    options.ttl = 3;  // 3 / 5 = 0, but clamped to 1
+    test_period(options, 1);
+  }
+}
+
+// Test that TriggerPeriodicCompaction() properly considers CFs for compaction
+// based on all time-based compaction options (not just
+// periodic_compaction_seconds). This is a regression test for the bug where
+// only periodic_compaction_seconds was checked, causing CFs with ttl,
+// bottommost_file_compaction_delay, or file_temperature_age_thresholds to not
+// be considered.
+//
+// NOTE: This test uses a separate test class without mock time because the
+// PeriodicTaskSchedulerTest fixture's mock clock setup does not integrate
+// properly with opening the DB via DB::Open() with multiple column families
+// having different options.
+class TriggerCompactionTest : public DBTestBase {
+ public:
+  TriggerCompactionTest()
+      : DBTestBase("trigger_compaction_test", /*env_do_fsync=*/true) {}
+};
+
+TEST_F(TriggerCompactionTest, QueuesAllTimeBasedOptions) {
+  Close();
+
+  // Track which CFs get their compaction score computed during
+  // TriggerPeriodicCompaction
+  std::set<std::string> cfs_with_score_computed;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::TriggerPeriodicCompaction:BeforeComputeCompactionScore",
+      [&](void* arg) {
+        auto* cfd = static_cast<ColumnFamilyData*>(arg);
+        cfs_with_score_computed.insert(cfd->GetName());
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Use small period so the test completes quickly with real time
+  constexpr unsigned int kPeriodSec = 1;
+
+  // Default CF: no time-based compaction options
+  Options options;
+  options.create_if_missing = true;
+  options.stats_dump_period_sec = kPeriodSec;
+  options.stats_persist_period_sec = 0;
+  options.periodic_compaction_seconds = 0;
+  options.ttl = 0;
+  options.bottommost_file_compaction_delay = 0;
+
+  // Create the DB with default CF first
+  DestroyAndReopen(options);
+
+  // Create the additional CFs (using same options, will set specific options
+  // on reopen)
+  CreateColumnFamilies(
+      {"cf_periodic", "cf_ttl", "cf_bottommost", "cf_fifo", "cf_none"},
+      options);
+  Close();
+
+  // Now reopen with specific options for each CF
+  std::vector<std::string> cf_names = {
+      kDefaultColumnFamilyName, "cf_periodic", "cf_ttl",
+      "cf_bottommost",          "cf_fifo",     "cf_none"};
+  std::vector<Options> cf_options;
+
+  // default: no time-based options
+  cf_options.push_back(options);
+
+  // cf_periodic: periodic_compaction_seconds
+  Options opt_periodic = options;
+  opt_periodic.periodic_compaction_seconds = 100;
+  cf_options.push_back(opt_periodic);
+
+  // cf_ttl: ttl
+  Options opt_ttl = options;
+  opt_ttl.ttl = 100;
+  cf_options.push_back(opt_ttl);
+
+  // cf_bottommost: bottommost_file_compaction_delay
+  Options opt_bottommost = options;
+  opt_bottommost.bottommost_file_compaction_delay = 100;
+  cf_options.push_back(opt_bottommost);
+
+  // cf_fifo: file_temperature_age_thresholds (requires FIFO compaction)
+  Options opt_fifo = options;
+  opt_fifo.compaction_style = kCompactionStyleFIFO;
+  opt_fifo.num_levels = 1;
+  opt_fifo.compaction_options_fifo.max_table_files_size = 1024 * 1024 * 1024;
+  opt_fifo.compaction_options_fifo.file_temperature_age_thresholds = {
+      {Temperature::kWarm, 100}};
+  cf_options.push_back(opt_fifo);
+
+  // cf_none: explicitly no time-based options
+  Options opt_none = options;
+  opt_none.periodic_compaction_seconds = 0;
+  opt_none.ttl = 0;
+  opt_none.bottommost_file_compaction_delay = 0;
+  cf_options.push_back(opt_none);
+
+  ReopenWithColumnFamilies(cf_names, cf_options);
+
+  // Wait for TriggerPeriodicCompaction to run using TEST_WaitForPeriodicTaskRun
+  // with a real sleep callback to advance real time
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { Env::Default()->SleepForMicroseconds(kPeriodSec * 1000000); });
+
+  // Verify CFs with time-based options had their compaction score computed
+  EXPECT_GT(cfs_with_score_computed.count("cf_periodic"), 0u)
+      << "Expected cf_periodic to have compaction score computed";
+  EXPECT_GT(cfs_with_score_computed.count("cf_ttl"), 0u)
+      << "Expected cf_ttl to have compaction score computed";
+  EXPECT_GT(cfs_with_score_computed.count("cf_bottommost"), 0u)
+      << "Expected cf_bottommost to have compaction score computed";
+  EXPECT_GT(cfs_with_score_computed.count("cf_fifo"), 0u)
+      << "Expected cf_fifo to have compaction score computed";
+
+  // CFs without time-based options should NOT have score computed
+  EXPECT_EQ(cfs_with_score_computed.count("default"), 0u)
+      << "Expected default CF to NOT have compaction score computed";
+  EXPECT_EQ(cfs_with_score_computed.count("cf_none"), 0u)
+      << "Expected cf_none to NOT have compaction score computed";
 
   Close();
 }
@@ -157,13 +434,13 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
                                         [&](void*) { pst_st_counter++; });
   SyncPoint::GetInstance()->EnableProcessing();
 
-  auto dbs = std::vector<DB*>(kInstanceNum);
+  auto dbs = std::vector<std::unique_ptr<DB>>(kInstanceNum);
   for (int i = 0; i < kInstanceNum; i++) {
     ASSERT_OK(
         DB::Open(options, test::PerThreadDBPath(std::to_string(i)), &(dbs[i])));
   }
 
-  auto dbi = static_cast_with_check<DBImpl>(dbs[kInstanceNum - 1]);
+  auto dbi = static_cast_with_check<DBImpl>(dbs[kInstanceNum - 1].get());
 
   const PeriodicTaskScheduler& scheduler = dbi->TEST_GetPeriodicTaskScheduler();
   // kRecordSeqnoTime is not registered since the feature is not enabled
@@ -190,7 +467,7 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
 
   int half = kInstanceNum / 2;
   for (int i = 0; i < half; i++) {
-    delete dbs[i];
+    dbs[i].reset();
   }
 
   expected_run += (kInstanceNum - half) * 2;
@@ -204,7 +481,7 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
 
   for (int i = half; i < kInstanceNum; i++) {
     ASSERT_OK(dbs[i]->Close());
-    delete dbs[i];
+    dbs[i].reset();
   }
 }
 
@@ -229,11 +506,11 @@ TEST_F(PeriodicTaskSchedulerTest, MultiEnv) {
   options1.env = mock_env2.get();
 
   std::string dbname = test::PerThreadDBPath("multi_env_test");
-  DB* db;
+  std::unique_ptr<DB> db;
   ASSERT_OK(DB::Open(options2, dbname, &db));
 
   ASSERT_OK(db->Close());
-  delete db;
+  db.reset();
   Close();
 }
 

package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc

@@ -98,7 +98,7 @@ class PlainTableDBTest : public testing::Test,
  private:
   std::string dbname_;
   Env* env_;
-  DB* db_;
+  std::unique_ptr<DB> db_;
 
   bool mmap_mode_;
   Options last_options_;
@@ -107,7 +107,7 @@ class PlainTableDBTest : public testing::Test,
   PlainTableDBTest() : env_(Env::Default()) {}
 
   ~PlainTableDBTest() override {
-    delete db_;
+    db_.reset();
     EXPECT_OK(DestroyDB(dbname_, Options()));
   }
 
@@ -115,7 +115,7 @@ class PlainTableDBTest : public testing::Test,
     mmap_mode_ = GetParam();
     dbname_ = test::PerThreadDBPath("plain_table_db_test");
     EXPECT_OK(DestroyDB(dbname_, Options()));
-    db_ = nullptr;
+    db_.reset();
     Reopen();
   }
 
@@ -144,14 +144,11 @@ class PlainTableDBTest : public testing::Test,
     return options;
   }
 
-  DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
+  DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_.get()); }
 
   void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
 
-  void Close() {
-    delete db_;
-    db_ = nullptr;
-  }
+  void Close() { db_.reset(); }
 
   bool mmap_mode() const { return mmap_mode_; }
 
@@ -162,24 +159,21 @@ class PlainTableDBTest : public testing::Test,
   }
 
   void Destroy(Options* options) {
-    delete db_;
-    db_ = nullptr;
+    db_.reset();
     ASSERT_OK(DestroyDB(dbname_, *options));
   }
 
-  Status PureReopen(Options* options, DB** db) {
+  Status PureReopen(Options* options, std::unique_ptr<DB>* db) {
     return DB::Open(*options, dbname_, db);
   }
 
   Status ReopenForReadOnly(Options* options) {
-    delete db_;
-    db_ = nullptr;
+    db_.reset();
     return DB::OpenForReadOnly(*options, dbname_, &db_);
   }
 
   Status TryReopen(Options* options = nullptr) {
-    delete db_;
-    db_ = nullptr;
+    db_.reset();
     Options opts;
     if (options != nullptr) {
       opts = *options;
@@ -495,8 +489,7 @@ TEST_P(PlainTableDBTest, Flush) {
             ASSERT_GT(int_num, 0U);
 
             TablePropertiesCollection ptc;
-            ASSERT_OK(
-                static_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc));
+            ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&ptc));
             ASSERT_EQ(1U, ptc.size());
             auto row = ptc.begin();
             auto tp = row->second;

package/deps/rocksdb/rocksdb/db/prefix_test.cc

@@ -220,8 +220,6 @@ class SamePrefixTransform : public SliceTransform {
     return false;
   }
 
-  bool InRange(const Slice& dst) const override { return dst == prefix_; }
-
   bool FullLengthEnabled(size_t* /*len*/) const override { return false; }
 };
 
@@ -229,8 +227,8 @@ class SamePrefixTransform : public SliceTransform {
 
 class PrefixTest : public testing::Test {
  public:
-  std::shared_ptr<DB> OpenDb() {
-    DB* db;
+  std::unique_ptr<DB> OpenDb() {
+    std::unique_ptr<DB> db;
 
     options.create_if_missing = true;
     options.write_buffer_size = FLAGS_write_buffer_size;
@@ -251,7 +249,7 @@ class PrefixTest : public testing::Test {
 
     Status s = DB::Open(options, kDbName, &db);
     EXPECT_OK(s);
-    return std::shared_ptr<DB>(db);
+    return db;
   }
 
   void FirstOption() { option_config_ = kBegin; }
@@ -304,7 +302,7 @@ class PrefixTest : public testing::Test {
 };
 
 TEST(SamePrefixTest, InDomainTest) {
-  DB* db;
+  std::unique_ptr<DB> db;
   Options options;
   options.create_if_missing = true;
   options.prefix_extractor.reset(new SamePrefixTransform("HHKB"));
@@ -331,7 +329,7 @@ TEST(SamePrefixTest, InDomainTest) {
     ASSERT_EQ(db_iter->value(), "idk");
 
     delete db_iter;
-    delete db;
+    db.reset();
     ASSERT_OK(DestroyDB(kDbName, Options()));
   }
 
@@ -348,7 +346,7 @@ TEST(SamePrefixTest, InDomainTest) {
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_OK(db_iter->status());
     delete db_iter;
-    delete db;
+    db.reset();
     ASSERT_OK(DestroyDB(kDbName, Options()));
   }
 }

package/deps/rocksdb/rocksdb/db/repair.cc

@@ -704,17 +704,17 @@ class Repairer {
       VersionEdit dummy_edit;
       for (const auto* table : cf_id_and_tables.second) {
         // TODO(opt): separate out into multiple levels
+        const auto& meta = table->meta;
         dummy_edit.AddFile(
-            0, table->meta.fd.GetNumber(), table->meta.fd.GetPathId(),
-            table->meta.fd.GetFileSize(), table->meta.smallest,
-            table->meta.largest, table->meta.fd.smallest_seqno,
-            table->meta.fd.largest_seqno, table->meta.marked_for_compaction,
-            table->meta.temperature, table->meta.oldest_blob_file_number,
-            table->meta.oldest_ancester_time, table->meta.file_creation_time,
-            table->meta.epoch_number, table->meta.file_checksum,
-            table->meta.file_checksum_func_name, table->meta.unique_id,
-            table->meta.compensated_range_deletion_size, table->meta.tail_size,
-            table->meta.user_defined_timestamps_persisted);
+            0, meta.fd.GetNumber(), meta.fd.GetPathId(), meta.fd.GetFileSize(),
+            meta.smallest, meta.largest, meta.fd.smallest_seqno,
+            meta.fd.largest_seqno, meta.marked_for_compaction, meta.temperature,
+            meta.oldest_blob_file_number, meta.oldest_ancester_time,
+            meta.file_creation_time, meta.epoch_number, meta.file_checksum,
+            meta.file_checksum_func_name, meta.unique_id,
+            meta.compensated_range_deletion_size, meta.tail_size,
+            meta.user_defined_timestamps_persisted, meta.min_timestamp,
+            meta.max_timestamp);
       }
       s = dummy_version_builder.Apply(&dummy_edit);
       if (s.ok()) {

package/deps/rocksdb/rocksdb/db/seqno_time_test.cc

@@ -661,14 +661,14 @@ TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) {
   options.stats_dump_period_sec = 0;
   options.stats_persist_period_sec = 0;
 
-  auto dbs = std::vector<DB*>(kInstanceNum);
+  auto dbs = std::vector<std::unique_ptr<DB>>(kInstanceNum);
   for (int i = 0; i < kInstanceNum; i++) {
     ASSERT_OK(
         DB::Open(options, test::PerThreadDBPath(std::to_string(i)), &(dbs[i])));
   }
 
   // Make sure the second instance has the worker enabled
-  auto dbi = static_cast_with_check<DBImpl>(dbs[1]);
+  auto dbi = static_cast_with_check<DBImpl>(dbs[1].get());
   WriteOptions wo;
   for (int i = 0; i < 200; i++) {
     ASSERT_OK(dbi->Put(wo, Key(i), "value"));
@@ -680,7 +680,7 @@ TEST_P(SeqnoTimeTablePropTest, MultiInstancesBasic) {
 
   for (int i = 0; i < kInstanceNum; i++) {
     ASSERT_OK(dbs[i]->Close());
-    delete dbs[i];
+    dbs[i].reset();
   }
 }
 
@@ -792,8 +792,8 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
   }
   ASSERT_GT(num_seqno_zeroing, 0);
   std::vector<KeyVersion> key_versions;
-  ASSERT_OK(GetAllKeyVersions(db_, {}, {}, std::numeric_limits<size_t>::max(),
-                              &key_versions));
+  ASSERT_OK(GetAllKeyVersions(
+      db_.get(), {}, {}, std::numeric_limits<size_t>::max(), &key_versions));
   // make sure there're more than 300 keys and first 100 keys are having seqno
   // zeroed out, the last 100 key seqno not zeroed out
   ASSERT_GT(key_versions.size(), 300);