@nxtedition/rocksdb 10.0.13 → 10.0.15

package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc

@@ -7,12 +7,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include <memory>
+
 #include "db/db_test_util.h"
 #include "file/sst_file_manager_impl.h"
 #include "port/stack_trace.h"
 #include "rocksdb/io_status.h"
 #include "rocksdb/sst_file_manager.h"
 #include "test_util/sync_point.h"
+#include "test_util/testharness.h"
 #include "util/random.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/fault_injection_fs.h"
@@ -27,6 +30,14 @@ class DBErrorHandlingFSTest : public DBTestBase {
     fault_env_.reset(new CompositeEnvWrapper(env_, fault_fs_));
   }
 
+  ~DBErrorHandlingFSTest() {
+    // Before destroying fault_env_
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->LoadDependency({});
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+    Close();
+  }
+
   std::string GetManifestNameFromLiveFiles() {
     std::vector<std::string> live_files;
     uint64_t manifest_size;
@@ -152,9 +163,9 @@ class ErrorHandlerFSListener : public EventListener {
   FaultInjectionTestFS* fault_fs_;
 };
 
-TEST_F(DBErrorHandlingFSTest, FLushWriteError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -197,9 +208,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) {
 // All the NoSpace IOError will be handled as the regular BG Error no matter the
 // retryable flag is set of not. So the auto resume for retryable IO Error will
 // not be triggered. Also, it is mapped as hard error.
-TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteNoSpaceError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -241,9 +252,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteRetryableError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -315,9 +326,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteFileScopeError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -399,9 +410,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWALWriteRetryableError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -443,9 +454,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWALAtomicWriteRetryableError) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -489,9 +500,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) {
 }
 
 // The flush error is injected before we finish the table build
-TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableError1) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -545,9 +556,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
 }
 
 // The retryable IO error is injected before we sync table
-TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError2) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -589,9 +600,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) {
 }
 
 // The retryable IO error is injected before we close the table file
-TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError3) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -633,8 +644,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) {
 }
 
 TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -658,6 +669,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
   fault_fs_->SetFilesystemActive(true);
@@ -666,6 +678,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -674,8 +687,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -701,6 +714,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) {
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
   fault_fs_->SetFilesystemActive(true);
@@ -709,6 +723,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -717,8 +732,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -746,6 +761,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) {
       [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
@@ -755,6 +771,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -763,8 +780,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -792,6 +809,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) {
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
   fault_fs_->SetFilesystemActive(true);
@@ -800,6 +818,7 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -808,8 +827,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -833,11 +852,13 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) {
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   fault_fs_->SetFilesystemActive(true);
 
   // This Resume() will attempt to create a new manifest file and fail again
   s = dbfull()->Resume();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   fault_fs_->SetFilesystemActive(true);
   SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
@@ -848,6 +869,7 @@ TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -860,8 +882,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
     ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
     return;
   }
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -914,6 +936,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
   fault_fs_->SetFilesystemActive(true);
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
   TEST_SYNC_POINT("CompactionManifestWriteError:1");
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   TEST_SYNC_POINT("CompactionManifestWriteError:2");
 
   s = dbfull()->TEST_WaitForCompact();
@@ -922,6 +945,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
   ASSERT_EQ("val", Get(Key(1)));
@@ -930,8 +954,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -980,6 +1004,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) {
   ASSERT_OK(s);
 
   TEST_SYNC_POINT("CompactionManifestWriteError:0");
+  ASSERT_FALSE(dbfull()->TEST_GetFilesToQuarantine().empty());
   TEST_SYNC_POINT("CompactionManifestWriteError:1");
 
   s = dbfull()->TEST_WaitForCompact();
@@ -993,6 +1018,7 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) {
 
   new_manifest = GetManifestNameFromLiveFiles();
   ASSERT_NE(new_manifest, old_manifest);
+  ASSERT_TRUE(dbfull()->TEST_GetFilesToQuarantine().empty());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -1002,8 +1028,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, CompactionWriteError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1044,8 +1070,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1091,8 +1117,8 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteFileScopeError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1183,8 +1209,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) {
     ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
     return;
   }
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1220,6 +1246,7 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) {
                    ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
   ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
                    ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
+  ASSERT_OK(dbfull()->SyncWAL());
 
   Reopen(options);
   ASSERT_EQ("val", Get(Key(0)));
@@ -1228,8 +1255,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1257,8 +1284,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) {
     ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
     return;
   }
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1330,8 +1357,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) {
 }
 
 TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1425,8 +1452,8 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) {
     ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
     return;
   }
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1757,10 +1784,10 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) {
 // to soft error and trigger auto resume. During auto resume, SwitchMemtable
 // is disabled to avoid small SST tables. Write can still be applied before
 // the bg error is cleaned unless the memtable is full.
-TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) {
+TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover1) {
   // Activate the FS before the first resume
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1781,7 +1808,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) {
   ASSERT_OK(Put(Key(1), "val1", wo));
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
       {{"RecoverFromRetryableBGIOError:LoopOut",
-        "FLushWritNoWALRetryableeErrorAutoRecover1:1"}});
+        "FlushWritNoWALRetryableeErrorAutoRecover1:1"}});
   SyncPoint::GetInstance()->SetCallBack(
       "BuildTable:BeforeFinishBuildTable",
       [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
@@ -1790,7 +1817,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) {
   s = Flush();
   ASSERT_EQ("val1", Get(Key(1)));
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
-  TEST_SYNC_POINT("FLushWritNoWALRetryableeErrorAutoRecover1:1");
+  TEST_SYNC_POINT("FlushWritNoWALRetryableeErrorAutoRecover1:1");
   ASSERT_EQ("val1", Get(Key(1)));
   ASSERT_EQ("val1", Get(Key(1)));
   SyncPoint::GetInstance()->DisableProcessing();
@@ -1827,10 +1854,94 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) {
+TEST_F(DBErrorHandlingFSTest, MultipleRecoveryThreads) {
+  // This test creates a scenario where second write's recovery can get started
+  // while mutex is released for a short period during
+  // NotifyOnErrorRecoveryEnd() from the first write's recovery. This is to make
+  // sure RecoverFromRetryableBGIOError() from the second write's recovery
+  // thread does not start with recovery_in_prog_ = false;
+
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
+  Options options = GetDefaultOptions();
+  options.env = fault_env_.get();
+  options.create_if_missing = true;
+  options.listeners.emplace_back(listener);
+  options.max_bgerror_resume_count = 100;
+  options.bgerror_resume_retry_interval = 1000000;  // 1 second
+  options.statistics = CreateDBStatistics();
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+
+  IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
+  error_msg.SetRetryable(true);
+
+  WriteOptions wo = WriteOptions();
+  wo.disableWAL = true;
+  fault_fs_->SetFilesystemActive(false, error_msg);
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"NotifyOnErrorRecoveryEnd:MutexUnlocked:1",
+        "MultipleRecoveryThreads:1"},
+       {"MultipleRecoveryThreads:2",
+        "NotifyOnErrorRecoveryEnd:MutexUnlocked:2"},
+       {"StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread",
+        "MultipleRecoveryThreads:3"},
+       {"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "MultipleRecoveryThreads:4"},
+       {"MultipleRecoveryThreads:4",
+        "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // First write with read fault injected and recovery will start
+  {
+    ASSERT_OK(Put(Key(1), "val1", wo));
+    Status s = Flush();
+    ASSERT_NOK(s);
+  }
+  // Remove read fault injection so that first recovery can go through
+  fault_fs_->SetFilesystemActive(true);
+
+  // At this point, first recovery is now at NotifyOnErrorRecoveryEnd. Mutex is
+  // released.
+  TEST_SYNC_POINT("MultipleRecoveryThreads:1");
+
+  ROCKSDB_NAMESPACE::port::Thread second_write([&] {
+    // Second write with read fault injected
+    fault_fs_->SetFilesystemActive(false, error_msg);
+    ASSERT_OK(Put(Key(2), "val2", wo));
+    Status s = Flush();
+    ASSERT_NOK(s);
+  });
+  // Second bg thread before waiting for the first thread's recovery thread
+  TEST_SYNC_POINT("MultipleRecoveryThreads:3");
+  // First thread's recovery thread continues
+  TEST_SYNC_POINT("MultipleRecoveryThreads:2");
+  // Wait for the first thread's recovery to finish
+  // (this sets recovery_in_prog_ = false)
+  // And second thread continues and starts recovery thread
+  TEST_SYNC_POINT("MultipleRecoveryThreads:4");
+  second_write.join();
+  // Remove error injection so that second thread recovery can go through
+  fault_fs_->SetFilesystemActive(true);
+
+  // Set up sync point so that we can wait for the recovery thread to finish
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"RecoverFromRetryableBGIOError:RecoverSuccess",
+        "MultipleRecoveryThreads:6"}});
+
+  // Wait for the second thread's recovery to be done
+  TEST_SYNC_POINT("MultipleRecoveryThreads:6");
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  Destroy(options);
+}
+
+TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover2) {
   // Activate the FS before the first resume
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1888,10 +1999,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) {
 
 // Auto resume fromt the flush retryable IO error. Activate the FS before the
 // first resume. Resume is successful
-TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) {
+TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover1) {
   // Activate the FS before the first resume
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1930,10 +2041,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) {
 
 // Auto resume fromt the flush retryable IO error and set the retry limit count.
 // Never activate the FS and auto resume should fail at the end
-TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
+TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover2) {
   // Fail all the resume and let user to resume
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -1950,18 +2061,18 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
 
   ASSERT_OK(Put(Key(1), "val1"));
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
-      {{"FLushWritRetryableeErrorAutoRecover2:0",
+      {{"FlushWritRetryableeErrorAutoRecover2:0",
         "RecoverFromRetryableBGIOError:BeforeStart"},
        {"RecoverFromRetryableBGIOError:LoopOut",
-        "FLushWritRetryableeErrorAutoRecover2:1"}});
+        "FlushWritRetryableeErrorAutoRecover2:1"}});
   SyncPoint::GetInstance()->SetCallBack(
       "BuildTable:BeforeFinishBuildTable",
       [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
-  TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:0");
-  TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:1");
+  TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:0");
+  TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:1");
   fault_fs_->SetFilesystemActive(true);
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
   SyncPoint::GetInstance()->DisableProcessing();
@@ -1983,8 +2094,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
 // Fail the first resume and let the second resume be successful.
 TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) {
   // Fail the first resume and let the second resume be successful
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2036,8 +2147,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) {
 
 TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) {
   // Fail the first resume and let the second resume be successful
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2091,8 +2202,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) {
 
 TEST_F(DBErrorHandlingFSTest,
        CompactionManifestWriteRetryableErrorAutoRecover) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2181,8 +2292,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) {
   // compaction, the FS is set to active and compaction is successful, so
   // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync
   // point.
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2235,8 +2346,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) {
 }
 
 TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2338,8 +2449,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) {
 
 TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) {
   // Fail the first recover and try second time.
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2439,10 +2550,10 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) {
 
 // Fail auto resume from a flush retryable error and verify that
 // OnErrorRecoveryEnd listener callback is called
-TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) {
+TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAbortRecovery) {
   // Activate the FS before the first resume
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2473,6 +2584,59 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) {
   Destroy(options);
 }
 
+TEST_F(DBErrorHandlingFSTest, FlushErrorRecoveryRaceWithDBDestruction) {
+  Options options = GetDefaultOptions();
+  options.env = fault_env_.get();
+  options.create_if_missing = true;
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
+  options.listeners.emplace_back(listener);
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("k1", "val"));
+
+  // Inject retryable flush error
+  bool error_set = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BuildTable:BeforeOutputValidation", [&](void*) {
+        if (error_set) {
+          return;
+        }
+        IOStatus st = IOStatus::IOError("Injected");
+        st.SetRetryable(true);
+        fault_fs_->SetFilesystemActive(false, st);
+        error_set = true;
+      });
+
+  port::Thread db_close_thread;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BuildTable:BeforeDeleteFile", [&](void*) {
+        // Clear retryable flush error injection
+        fault_fs_->SetFilesystemActive(true);
+
+        // Coerce race between ending auto recovery in db destruction and flush
+        // error recovery
+        ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+            {{"PostEndAutoRecovery", "FlushJob::WriteLevel0Table"}});
+        db_close_thread = port::Thread([&] { Close(); });
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Status s = Flush();
+  ASSERT_NOK(s);
+
+  int placeholder = 1;
+  listener->WaitForRecovery(placeholder);
+  ASSERT_TRUE(listener->new_bg_error().IsShutdownInProgress());
+
+  // Prior to the fix, the db close will crash due to the recovery thread for
+  // flush error is not joined by the time of destruction.
+  db_close_thread.join();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  Destroy(options);
+}
+
 TEST_F(DBErrorHandlingFSTest, FlushReadError) {
   std::shared_ptr<ErrorHandlerFSListener> listener =
       std::make_shared<ErrorHandlerFSListener>();
@@ -2680,9 +2844,9 @@ TEST_F(DBErrorHandlingFSTest, CompactionReadRetryableErrorAutoRecover) {
 class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest,
                                    public testing::WithParamInterface<bool> {};
 
-TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+TEST_P(DBErrorHandlingFencingTest, FlushWriteFenced) {
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2709,8 +2873,8 @@ TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) {
 }
 
 TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2744,8 +2908,8 @@ TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) {
 }
 
 TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;
@@ -2785,8 +2949,8 @@ TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) {
 }
 
 TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) {
-  std::shared_ptr<ErrorHandlerFSListener> listener(
-      new ErrorHandlerFSListener());
+  std::shared_ptr<ErrorHandlerFSListener> listener =
+      std::make_shared<ErrorHandlerFSListener>();
   Options options = GetDefaultOptions();
   options.env = fault_env_.get();
   options.create_if_missing = true;