@nxtedition/rocksdb 13.5.7 → 13.5.9

package/deps/rocksdb/rocksdb/db/db_test_util.cc

@@ -11,6 +11,7 @@
 
 #include "cache/cache_reservation_manager.h"
 #include "db/forward_iterator.h"
+#include "env/fs_readonly.h"
 #include "env/mock_env.h"
 #include "port/lang.h"
 #include "rocksdb/cache.h"
@@ -122,7 +123,6 @@ DBTestBase::~DBTestBase() {
 }
 
 bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
-
   if ((skip_mask & kSkipUniversalCompaction) &&
       (option_config == kUniversalCompaction ||
        option_config == kUniversalCompactionMultiLevel ||
@@ -592,7 +592,6 @@ Options DBTestBase::GetOptions(
       options_override.level_compaction_dynamic_level_bytes;
   options.env = env_;
   options.create_if_missing = true;
-  options.fail_if_options_file_error = true;
   return options;
 }
 
@@ -717,6 +716,17 @@ Status DBTestBase::ReadOnlyReopen(const Options& options) {
   return DB::OpenForReadOnly(options, dbname_, &db_);
 }
 
+Status DBTestBase::EnforcedReadOnlyReopen(const Options& options) {
+  Close();
+  Options options_copy = options;
+  MaybeInstallTimeElapseOnlySleep(options_copy);
+  auto fs_read_only =
+      std::make_shared<ReadOnlyFileSystem>(env_->GetFileSystem());
+  env_read_only_ = std::make_shared<CompositeEnvWrapper>(env_, fs_read_only);
+  options_copy.env = env_read_only_.get();
+  return DB::OpenForReadOnly(options_copy, dbname_, &db_);
+}
+
 Status DBTestBase::TryReopen(const Options& options) {
   Close();
   last_options_.table_factory.reset();
@@ -1208,7 +1218,6 @@ std::string DBTestBase::FilesPerLevel(int cf) {
   return result;
 }
 
-
 std::vector<uint64_t> DBTestBase::GetBlobFileNumbers() {
   VersionSet* const versions = dbfull()->GetVersionSet();
   assert(versions);
@@ -1667,7 +1676,7 @@ void DBTestBase::VerifyDBFromMap(
     data_iter = true_data.begin();
     for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++data_iter) {
       ASSERT_EQ(iter->key().ToString(), data_iter->first);
-        ASSERT_EQ(iter->value().ToString(), data_iter->second);
+      ASSERT_EQ(iter->value().ToString(), data_iter->second);
       iter_cnt++;
       total_reads++;
     }
@@ -1683,7 +1692,7 @@ void DBTestBase::VerifyDBFromMap(
     auto data_rev = true_data.rbegin();
     for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
       ASSERT_EQ(iter->key().ToString(), data_rev->first);
-        ASSERT_EQ(iter->value().ToString(), data_rev->second);
+      ASSERT_EQ(iter->value().ToString(), data_rev->second);
       iter_cnt++;
       total_reads++;
     }
@@ -1765,7 +1774,6 @@ void DBTestBase::VerifyDBInternal(
   iter->~InternalIterator();
 }
 
-
 uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
     DB* db, std::string column_family_name) {
   std::vector<LiveFileMetaData> metadata;

package/deps/rocksdb/rocksdb/db/db_test_util.h

@@ -1062,6 +1062,7 @@ class DBTestBase : public testing::Test {
   MockEnv* mem_env_;
   Env* encrypted_env_;
   SpecialEnv* env_;
+  std::shared_ptr<Env> env_read_only_;
   std::shared_ptr<Env> env_guard_;
   DB* db_;
   std::vector<ColumnFamilyHandle*> handles_;
@@ -1106,6 +1107,11 @@ class DBTestBase : public testing::Test {
     return std::string(buf);
   }
 
+  // Expects valid key created by Key().
+  static int IdFromKey(const std::string& key) {
+    return std::stoi(key.substr(3));
+  }
+
   static bool ShouldSkipOptions(int option_config, int skip_mask = kNoSkip);
 
   // Switch to a fresh database with the next option configuration to
@@ -1173,6 +1179,9 @@ class DBTestBase : public testing::Test {
 
   Status ReadOnlyReopen(const Options& options);
 
+  // With a filesystem wrapper that fails on attempted write
+  Status EnforcedReadOnlyReopen(const Options& options);
+
   Status TryReopen(const Options& options);
 
   bool IsDirectIOSupported();

package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc

@@ -1672,55 +1672,75 @@ TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
   }
   const int kNumFilesTrigger = 3;
   Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.max_background_compactions = 2;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
-  options.target_file_size_base = 32 << 10;  // 32KB
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  // Trigger compaction if size amplification exceeds 110%
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  DestroyAndReopen(options);
-
-  // Need to get a token to enable compaction parallelism up to
-  // `max_background_compactions` jobs.
-  auto pressure_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
-      {// wait for the full compaction to be picked before adding files intended
-       // for the second one.
-       {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
-        "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
-       // the full (bottom-pri) compaction waits until a partial (low-pri)
-       // compaction has started to verify they can run in parallel.
-       {"DBImpl::BackgroundCompaction:NonTrivial",
-        "DBImpl::BGWorkBottomCompaction"}});
-  SyncPoint::GetInstance()->EnableProcessing();
 
-  Random rnd(301);
-  for (int i = 0; i < 2; ++i) {
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
-      // use no_wait above because that one waits for flush and compaction. We
-      // don't want to wait for compaction because the full compaction is
-      // intentionally blocked while more files are flushed.
-      ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  for (bool universal_reduce_file_locking : {true, false}) {
+    Options options = CurrentOptions();
+    options.compaction_style = kCompactionStyleUniversal;
+    options.compaction_options_universal.reduce_file_locking =
+        universal_reduce_file_locking;
+    options.max_background_compactions = 2;
+    options.num_levels = num_levels_;
+    options.write_buffer_size = 100 << 10;     // 100KB
+    options.target_file_size_base = 32 << 10;  // 32KB
+    options.level0_file_num_compaction_trigger = kNumFilesTrigger;
+    // Trigger compaction if size amplification exceeds 110%
+    options.compaction_options_universal.max_size_amplification_percent = 110;
+    DestroyAndReopen(options);
+
+    // Need to get a token to enable compaction parallelism up to
+    // `max_background_compactions` jobs.
+    auto pressure_token =
+        dbfull()->TEST_write_controler().GetCompactionPressureToken();
+    if (universal_reduce_file_locking) {
+      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+          {// Wait for the full compaction to be repicked before adding files
+           // intended for the second compaction.
+           {"DBImpl::BackgroundCompaction():AfterPickCompactionBottomPri",
+            "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
+           // Wait for the second compaction to run before running the full
+           // compaction to verify they can run in parallel
+           {"DBImpl::BackgroundCompaction:NonTrivial:BeforeRun",
+            "DBImpl::BackgroundCompaction:NonTrivial:BeforeRunBottomPri"}});
+    } else {
+      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+          {// Wait for the full compaction to be forwarded before adding files
+           // intended for the second compaction.
+           {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
+            "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
+           // Wait for the second compaction to run before running the full
+           // compaction to verify they can run in parallel
+           {"DBImpl::BackgroundCompaction:NonTrivial:BeforeRun",
+            "DBImpl::BackgroundCompaction:NonTrivial:BeforeRunBottomPri"}});
     }
-    if (i == 0) {
-      TEST_SYNC_POINT(
-          "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
+
+    SyncPoint::GetInstance()->EnableProcessing();
+
+    Random rnd(301);
+    for (int i = 0; i < 2; ++i) {
+      for (int num = 0; num < kNumFilesTrigger; num++) {
+        int key_idx = 0;
+        GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
+        // use no_wait above because that one waits for flush and compaction. We
+        // don't want to wait for compaction because the full compaction is
+        // intentionally blocked while more files are flushed.
+        ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+      }
+      if (i == 0) {
+        TEST_SYNC_POINT(
+            "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
+      }
     }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+    // First compaction should output to bottom level. Second should output to
+    // L0 since older L0 files pending compaction prevent it from being placed
+    // lower.
+    ASSERT_EQ(NumSortedRuns(), 2);
+    ASSERT_GT(NumTableFilesAtLevel(0), 0);
+    ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
   }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
-  // First compaction should output to bottom level. Second should output to L0
-  // since older L0 files pending compaction prevent it from being placed lower.
-  ASSERT_EQ(NumSortedRuns(), 2);
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-  ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
   Env::Default()->SetBackgroundThreads(0, Env::Priority::BOTTOM);
 }
 
@@ -2369,7 +2389,6 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompactionOffpeak) {
 
 }  // namespace ROCKSDB_NAMESPACE
 
-
 int main(int argc, char** argv) {
   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);

package/deps/rocksdb/rocksdb/db/db_wal_test.cc

@@ -1153,6 +1153,7 @@ TEST_F(DBWALTest, FullPurgePreservesRecycledLog) {
     dbfull()->FindObsoleteFiles(&job_context, true /* force */);
     dbfull()->TEST_UnlockMutex();
     dbfull()->PurgeObsoleteFiles(job_context);
+    job_context.Clean();
 
     if (i == 0) {
       ASSERT_OK(
@@ -1838,9 +1839,194 @@ class RecoveryTestHelper {
   }
 };
 
-class DBWALTestWithParams : public DBWALTestBase,
-                            public ::testing::WithParamInterface<
-                                std::tuple<bool, int, int, CompressionType>> {
+TEST_F(DBWALTest, TrackAndVerifyWALsRecycleWAL) {
+  Options options = CurrentOptions();
+  options.avoid_flush_during_shutdown = true;
+  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
+  options.recycle_log_file_num = 1;
+  options.track_and_verify_wals = true;
+
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
+  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
+  ASSERT_OK(Flush());
+
+  // Stop background flush to avoid deleting any WAL
+  options.env->SetBackgroundThreads(1, Env::HIGH);
+  test::SleepingBackgroundTask sleeping_task;
+  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                        &sleeping_task, Env::Priority::HIGH);
+
+  // Recycle the first WAL
+  ASSERT_OK(Put("key1", "old_value"));
+  // Recycle the second WAL
+  ASSERT_OK(dbfull()->TEST_SwitchWAL());
+  ASSERT_OK(Put("key1", "new_value"));
+
+  // Create a WAL hole on sequence number by truncating the first WAL to 0 byte
+  VectorWalPtr log_files;
+  ASSERT_OK(db_->GetSortedWalFiles(log_files));
+  ASSERT_EQ(log_files.size(), 2);
+  std::string log_name = LogFileName(dbname_, log_files.front()->LogNumber());
+  Close();
+  // Drop `Put("key1", "old_value")` in the first WAL
+  ASSERT_OK(test::TruncateFile(options.env, log_name, 0 /* new_length */));
+
+  Status s = DB::Open(options, dbname_, &db_);
+
+  ASSERT_OK(s);
+
+  ASSERT_EQ("wal_to_recycle", Get("key_ignore2"));
+  ASSERT_EQ("NOT_FOUND", Get("key1"));
+
+  Close();
+}
+
+class DBWALTrackAndVerifyWALsWithParamsTest
+    : public DBWALTestBase,
+      public ::testing::WithParamInterface<WALRecoveryMode> {
+ public:
+  DBWALTrackAndVerifyWALsWithParamsTest()
+      : DBWALTestBase("/db_wal_track_and_verify_wals_with_params_test") {}
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBWALTrackAndVerifyWALsWithParamsTest,
+    DBWALTrackAndVerifyWALsWithParamsTest,
+    ::testing::Values(WALRecoveryMode::kTolerateCorruptedTailRecords,
+                      WALRecoveryMode::kAbsoluteConsistency,
+                      WALRecoveryMode::kPointInTimeRecovery,
+                      WALRecoveryMode::kSkipAnyCorruptedRecords));
+
+TEST_P(DBWALTrackAndVerifyWALsWithParamsTest, Basic) {
+  Options options = CurrentOptions();
+  options.avoid_flush_during_shutdown = true;
+  options.track_and_verify_wals = true;
+  options.wal_recovery_mode = GetParam();
+
+  // Stop background flush to avoid deleting any WAL
+  options.env->SetBackgroundThreads(1, Env::HIGH);
+  test::SleepingBackgroundTask sleeping_task;
+  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                        &sleeping_task, Env::Priority::HIGH);
+
+  for (int i = 0; i < 5; i++) {
+    DestroyAndReopen(options);
+
+    ASSERT_OK(Put("key1", "old_value"));
+    SequenceNumber last_seqno_recorded_in_fist_wal =
+        dbfull()->GetLatestSequenceNumber();
+
+    ASSERT_OK(dbfull()->TEST_SwitchWAL());
+    ASSERT_OK(Put("key1", "new_value"));
+
+    VectorWalPtr log_files;
+    ASSERT_OK(db_->GetSortedWalFiles(log_files));
+    ASSERT_EQ(log_files.size(), 2);
+    uint64_t first_log_number = log_files.front()->LogNumber();
+    std::string first_log_name = LogFileName(dbname_, first_log_number);
+    std::string second_log_name =
+        LogFileName(dbname_, log_files.back()->LogNumber());
+
+    if (i == 0) {
+      // Delete the obsolete WAL and verify it will not be seen as a WAL hole
+      sleeping_task.WakeUp();
+      sleeping_task.WaitUntilDone();
+      ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+      // Stop background flush to avoid deleting any WAL
+      sleeping_task.Reset();
+      options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                            &sleeping_task, Env::Priority::HIGH);
+      Close();
+    } else if (i == 1) {
+      // Create a WAL hole on WAL number by deleting the first WAL and verify
+      // the hole will be detected
+      Close();
+      ASSERT_OK(options.env->DeleteFile(first_log_name));
+    } else if (i == 2) {
+      // Create a WAL hole on sequence number by truncating the first WAL and
+      // verify the hole will be detected
+      Close();
+      ASSERT_OK(
+          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
+    } else if (i == 3) {
+      // Create a WAL hole on size difference by truncating the first WAL and
+      // mocking a correct sequence number to force triggering corruption based
+      // on size instead of sequence number and verify the hole will be detected
+      Close();
+      ASSERT_OK(
+          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
+      SyncPoint::GetInstance()->SetCallBack(
+          "DBImpl::UpdatePredecessorWALInfo", [&](void* arg) {
+            std::pair<uint64_t, SequenceNumber*>* pair =
+                static_cast<std::pair<uint64_t, SequenceNumber*>*>(arg);
+            if (pair->first == first_log_number) {
+              *(pair->second) = last_seqno_recorded_in_fist_wal;
+            }
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+    } else if (i == 4) {
+      // Delete all wals and verify opening a DB with no WAL will be detected
+      Close();
+      ASSERT_OK(options.env->DeleteFile(first_log_name));
+      ASSERT_OK(options.env->DeleteFile(second_log_name));
+    }
+
+    Status s = DB::Open(options, dbname_, &db_);
+
+    if (i == 0) {
+      ASSERT_OK(s);
+      ASSERT_EQ("new_value", Get("key1"));
+      continue;
+    } else if (i == 3) {
+      SyncPoint::GetInstance()->ClearAllCallBacks();
+      SyncPoint::GetInstance()->DisableProcessing();
+    } else if (i == 4) {
+      ASSERT_TRUE(s.IsCorruption());
+      ASSERT_TRUE(
+          s.ToString().find("Opening an existing DB with no WAL files") !=
+          std::string::npos);
+      Close();
+      continue;
+    }
+
+    if (options.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) {
+      ASSERT_OK(s);
+      ASSERT_EQ("NOT_FOUND", Get("key1"));
+    } else if (options.wal_recovery_mode ==
+                   WALRecoveryMode::kAbsoluteConsistency ||
+               options.wal_recovery_mode ==
+                   WALRecoveryMode::kTolerateCorruptedTailRecords) {
+      ASSERT_TRUE(s.IsCorruption());
+      std::string msg;
+      if (i == 1) {
+        msg = "Missing WAL";
+      } else if (i == 2) {
+        msg = "Mismatched last sequence number recorded in the WAL";
+      } else if (i == 3) {
+        msg = "Mismatched size of the WAL";
+      }
+      ASSERT_TRUE(s.ToString().find(msg) != std::string::npos);
+    } else {
+      ASSERT_OK(s);
+      ASSERT_EQ("new_value", Get("key1"));
+    }
+
+    Close();
+  }
+}
+
+class DBWALTestWithParams
+    : public DBWALTestBase,
+      public ::testing::WithParamInterface<
+          std::tuple<bool, int, int, CompressionType, bool>> {
  public:
   DBWALTestWithParams() : DBWALTestBase("/db_wal_test_with_params") {}
 };
@@ -1853,7 +2039,8 @@ INSTANTIATE_TEST_CASE_P(
                                             RecoveryTestHelper::kWALFilesCount,
                                         1),
                        ::testing::Values(CompressionType::kNoCompression,
-                                         CompressionType::kZSTD)));
+                                         CompressionType::kZSTD),
+                       ::testing::Bool()));
 
 class DBWALTestWithParamsVaryingRecoveryMode
     : public DBWALTestBase,
@@ -1891,6 +2078,7 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {
 
   // Fill data for testing
   Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
   const size_t row_count = RecoveryTestHelper::FillData(this, &options);
   // test checksum failure or parsing
   RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3,
@@ -1914,6 +2102,7 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {
 TEST_P(DBWALTestWithParams, kAbsoluteConsistency) {
   // Verify clean slate behavior
   Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
   const size_t row_count = RecoveryTestHelper::FillData(this, &options);
   options.create_if_missing = false;
   ASSERT_OK(TryReopen(options));
@@ -2124,7 +2313,6 @@ TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) {
         wal_synced = true;
       });
 
-
   SyncPoint::GetInstance()->EnableProcessing();
 
   ASSERT_OK(Flush());
@@ -2165,6 +2353,7 @@ TEST_P(DBWALTestWithParams, kPointInTimeRecovery) {
 
   // Fill data for testing
   Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
   options.wal_compression = compression_type;
   const size_t row_count = RecoveryTestHelper::FillData(this, &options);
 
@@ -2222,6 +2411,7 @@ TEST_P(DBWALTestWithParams, kSkipAnyCorruptedRecords) {
 
   // Fill data for testing
   Options options = CurrentOptions();
+  options.track_and_verify_wals = std::get<4>(GetParam());
   options.wal_compression = compression_type;
   const size_t row_count = RecoveryTestHelper::FillData(this, &options);
 
@@ -2699,12 +2889,12 @@ TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
   // create DB and close with file truncate disabled
   std::atomic_bool enable_truncate{false};
 
-  SyncPoint::GetInstance()->SetCallBack(
-      "PosixWritableFile::Close", [&](void* arg) {
-        if (!enable_truncate) {
-          *(static_cast<size_t*>(arg)) = 0;
-        }
-      });
+  SyncPoint::GetInstance()->SetCallBack("PosixWritableFile::Close",
+                                        [&](void* arg) {
+                                          if (!enable_truncate) {
+                                            *(static_cast<size_t*>(arg)) = 0;
+                                          }
+                                        });
   SyncPoint::GetInstance()->EnableProcessing();
 
   DestroyAndReopen(options);
@@ -2793,7 +2983,6 @@ TEST_F(DBWALTest, WalInManifestButNotInSortedWals) {
   Close();
 }
 
-
 TEST_F(DBWALTest, WalTermTest) {
   Options options = CurrentOptions();
   options.env = env_;
@@ -2835,13 +3024,13 @@ TEST_F(DBWALTest, GetCompressedWalsAfterSync) {
   options.wal_compression = kZSTD;
   DestroyAndReopen(options);
 
-  // Write something to memtable and WAL so that log_empty_ will be false after
+  // Write something to memtable and WAL so that wal_empty_ will be false after
   // next DB::Open().
   ASSERT_OK(Put("a", "v"));
 
   Reopen(options);
 
-  // New WAL is created, thanks to !log_empty_.
+  // New WAL is created, thanks to !wal_empty_.
   ASSERT_OK(dbfull()->TEST_SwitchWAL());
 
   ASSERT_OK(Put("b", "v"));