@nxtedition/rocksdb 11.0.3 → 11.0.4

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc

@@ -632,7 +632,6 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
   for (auto cfh : column_families_) {
     for (int64_t k = 0; k != number_of_keys; ++k) {
       const std::string key = Key(k);
-
       PendingExpectedValue pending_expected_value =
           shared->PreparePut(cf_idx, k);
       const uint32_t value_base = pending_expected_value.GetFinalValueBase();
@@ -1011,26 +1010,12 @@ void StressTest::OperateDb(ThreadState* thread) {
           thread->rand.OneIn(FLAGS_verify_db_one_in)) {
         //  Temporarily disable error injection for verification
         if (fault_fs_guard) {
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kWrite);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataWrite);
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
         }
         ContinuouslyVerifyDb(thread);
         //  Enable back error injection disabled for verification
         if (fault_fs_guard) {
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kRead);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kWrite);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataRead);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataWrite);
+          fault_fs_guard->EnableAllThreadLocalErrorInjection();
         }
         if (thread->shared->ShouldStopTest()) {
           break;
@@ -1056,14 +1041,7 @@ void StressTest::OperateDb(ThreadState* thread) {
         } else if (s.ok()) {
           //  Temporarily disable error injection for verification
           if (fault_fs_guard) {
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kRead);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataRead);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kWrite);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataWrite);
+            fault_fs_guard->DisableAllThreadLocalErrorInjection();
           }
 
           // Verify no writes during LockWAL
@@ -1118,14 +1096,7 @@ void StressTest::OperateDb(ThreadState* thread) {
 
           //  Enable back error injection disabled for verification
           if (fault_fs_guard) {
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kRead);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataRead);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kWrite);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataWrite);
+            fault_fs_guard->EnableAllThreadLocalErrorInjection();
           }
         }
       }
@@ -1238,27 +1209,13 @@ void StressTest::OperateDb(ThreadState* thread) {
         // failed due to injected error. So we disable fault injection to avoid
         // false positive
         if (fault_fs_guard) {
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataWrite);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kWrite);
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
         }
 
         TestGetProperty(thread);
 
         if (fault_fs_guard) {
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataWrite);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataRead);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kRead);
-          fault_fs_guard->EnableThreadLocalErrorInjection(
-              FaultInjectionIOType::kWrite);
+          fault_fs_guard->EnableAllThreadLocalErrorInjection();
         }
       }
 
@@ -1289,25 +1246,11 @@ void StressTest::OperateDb(ThreadState* thread) {
           // TODO(hx235): enable error injection with
           // backup/restore after fixing the various issues it surfaces
           if (fault_fs_guard) {
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataRead);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataWrite);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kRead);
-            fault_fs_guard->DisableThreadLocalErrorInjection(
-                FaultInjectionIOType::kWrite);
+            fault_fs_guard->DisableAllThreadLocalErrorInjection();
           }
           Status s = TestBackupRestore(thread, rand_column_families, rand_keys);
           if (fault_fs_guard) {
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataWrite);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kMetadataRead);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kRead);
-            fault_fs_guard->EnableThreadLocalErrorInjection(
-                FaultInjectionIOType::kWrite);
+            fault_fs_guard->EnableAllThreadLocalErrorInjection();
           }
           ProcessStatus(shared, "Backup/restore", s);
         }
@@ -1344,7 +1287,13 @@ void StressTest::OperateDb(ThreadState* thread) {
       if (thread->rand.OneInOpt(FLAGS_key_may_exist_one_in)) {
         TestKeyMayExist(thread, read_opts, rand_column_families, rand_keys);
       }
-
+      // Prefix-recoverability relies on tracing successful user writes.
+      // Currently we trace all user writes regardless of whether it later
+      // succeeds or not. To simplify, we disable any fault injection during
+      // user write.
+      // TODO(hx235): support tracing user writes with fault injection.
+      bool disable_fault_injection_during_user_write =
+          fault_fs_guard && MightHaveUnsyncedDataLoss();
       int prob_op = thread->rand.Uniform(100);
       // Reset this in case we pick something other than a read op. We don't
       // want to use a stale value when deciding at the beginning of the loop
@@ -1403,16 +1352,34 @@ void StressTest::OperateDb(ThreadState* thread) {
       } else if (prob_op < write_bound) {
         assert(prefix_bound <= prob_op);
         // OPERATION write
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
+        }
         TestPut(thread, write_opts, read_opts, rand_column_families, rand_keys,
                 value);
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->EnableAllThreadLocalErrorInjection();
+        }
       } else if (prob_op < del_bound) {
         assert(write_bound <= prob_op);
         // OPERATION delete
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
+        }
         TestDelete(thread, write_opts, rand_column_families, rand_keys);
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->EnableAllThreadLocalErrorInjection();
+        }
       } else if (prob_op < delrange_bound) {
         assert(del_bound <= prob_op);
         // OPERATION delete range
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
+        }
         TestDeleteRange(thread, write_opts, rand_column_families, rand_keys);
+        if (disable_fault_injection_during_user_write) {
+          fault_fs_guard->EnableAllThreadLocalErrorInjection();
+        }
       } else if (prob_op < iterate_bound) {
         assert(delrange_bound <= prob_op);
         // OPERATION iterate
@@ -1456,14 +1423,7 @@ void StressTest::OperateDb(ThreadState* thread) {
 
 #ifndef NDEBUG
     if (fault_fs_guard) {
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
+      fault_fs_guard->DisableAllThreadLocalErrorInjection();
     }
 #endif  // NDEBUG
   }
@@ -2344,14 +2304,7 @@ Status StressTest::TestBackupRestore(
 
   // Temporarily disable error injection for clean up
   if (fault_fs_guard) {
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->DisableAllThreadLocalErrorInjection();
   }
 
   if (s.ok() || IsErrorInjectedAndRetryable(s)) {
@@ -2373,14 +2326,7 @@ Status StressTest::TestBackupRestore(
 
   // Enable back error injection disabled for clean up
   if (fault_fs_guard) {
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->EnableAllThreadLocalErrorInjection();
   }
 
   if (!s.ok() && !IsErrorInjectedAndRetryable(s)) {
@@ -2520,26 +2466,12 @@ Status StressTest::TestCheckpoint(ThreadState* thread,
   tmp_opts.sst_file_manager.reset();
   //  Temporarily disable error injection for clean-up
   if (fault_fs_guard) {
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->DisableAllThreadLocalErrorInjection();
   }
   DestroyDB(checkpoint_dir, tmp_opts);
   // Enable back error injection disabled for clean-up
   if (fault_fs_guard) {
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->EnableAllThreadLocalErrorInjection();
   }
   Checkpoint* checkpoint = nullptr;
   Status s = Checkpoint::Create(db_, &checkpoint);
@@ -2647,14 +2579,7 @@ Status StressTest::TestCheckpoint(ThreadState* thread,
 
   //  Temporarily disable error injection for clean-up
   if (fault_fs_guard) {
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->DisableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->DisableAllThreadLocalErrorInjection();
   }
 
   if (!s.ok() && !IsErrorInjectedAndRetryable(s)) {
@@ -2666,14 +2591,7 @@ Status StressTest::TestCheckpoint(ThreadState* thread,
 
   // Enable back error injection disabled for clean-up
   if (fault_fs_guard) {
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kWrite);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataRead);
-    fault_fs_guard->EnableThreadLocalErrorInjection(
-        FaultInjectionIOType::kMetadataWrite);
+    fault_fs_guard->EnableAllThreadLocalErrorInjection();
   }
   return s;
 }
@@ -3048,14 +2966,7 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
   if (thread->rand.OneIn(2)) {
     // Temporarily disable error injection to for validation
     if (fault_fs_guard) {
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
+      fault_fs_guard->DisableAllThreadLocalErrorInjection();
     }
 
     // Declare a snapshot and compare the data before and after the compaction
@@ -3065,14 +2976,7 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
 
     // Enable back error injection disabled for validation
     if (fault_fs_guard) {
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
+      fault_fs_guard->EnableAllThreadLocalErrorInjection();
     }
   }
   std::ostringstream compact_range_opt_oss;
@@ -3110,14 +3014,7 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
   if (pre_snapshot != nullptr) {
     // Temporarily disable error injection for validation
     if (fault_fs_guard) {
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
+      fault_fs_guard->DisableAllThreadLocalErrorInjection();
     }
     uint32_t post_hash =
         GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key);
@@ -3136,14 +3033,7 @@ void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
     db_->ReleaseSnapshot(pre_snapshot);
     if (fault_fs_guard) {
       // Enable back error injection disabled for validation
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
-      fault_fs_guard->EnableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
+      fault_fs_guard->EnableAllThreadLocalErrorInjection();
     }
   }
 }
@@ -3523,14 +3413,7 @@ void StressTest::Open(SharedState* shared, bool reopen) {
     // If this is for DB reopen,  error injection may have been enabled.
     // Disable it here in case there is no open fault injection.
     if (fault_fs_guard) {
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kWrite);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataRead);
-      fault_fs_guard->DisableThreadLocalErrorInjection(
-          FaultInjectionIOType::kMetadataWrite);
+      fault_fs_guard->DisableAllThreadLocalErrorInjection();
     }
     // TODO; test transaction DB Open with fault injection
     if (!FLAGS_use_txn) {
@@ -3611,14 +3494,7 @@ void StressTest::Open(SharedState* shared, bool reopen) {
         if (inject_sync_fault || inject_open_meta_read_error ||
             inject_open_meta_write_error || inject_open_read_error ||
             inject_open_write_error) {
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kWrite);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataRead);
-          fault_fs_guard->DisableThreadLocalErrorInjection(
-              FaultInjectionIOType::kMetadataWrite);
+          fault_fs_guard->DisableAllThreadLocalErrorInjection();
 
           if (s.ok()) {
             // Injected errors might happen in background compactions. We
@@ -3798,7 +3674,7 @@ void StressTest::Reopen(ThreadState* thread) {
   // crash-recovery verification does. Therefore it always expects no data loss
   // and we should ensure no data loss in testing.
   // TODO(hx235): eliminate the FlushWAL(true /* sync */)/SyncWAL() below
-  if (!FLAGS_disable_wal && !FLAGS_avoid_flush_during_shutdown) {
+  if (!FLAGS_disable_wal && FLAGS_avoid_flush_during_shutdown) {
     Status s;
     if (FLAGS_manual_wal_flush_one_in > 0) {
       s = db_->FlushWAL(/*sync=*/true);
@@ -3956,6 +3832,10 @@ void CheckAndSetOptionsForUserTimestamp(Options& options) {
       FLAGS_persist_user_defined_timestamps;
 }
 
+bool ShouldDisableAutoCompactionsBeforeVerifyDb() {
+  return !FLAGS_disable_auto_compactions && FLAGS_enable_compaction_filter;
+}
+
 bool InitializeOptionsFromFile(Options& options) {
   DBOptions db_options;
   ConfigOptions config_options;
@@ -3983,6 +3863,8 @@ void InitializeOptionsFromFlags(
     const std::shared_ptr<const FilterPolicy>& filter_policy,
     Options& options) {
   BlockBasedTableOptions block_based_options;
+  block_based_options.decouple_partitioned_filters =
+      FLAGS_decouple_partitioned_filters;
   block_based_options.block_cache = cache;
   block_based_options.cache_index_and_filter_blocks =
       FLAGS_cache_index_and_filter_blocks;
@@ -4069,7 +3951,11 @@ void InitializeOptionsFromFlags(
         new WriteBufferManager(FLAGS_db_write_buffer_size, block_cache));
   }
   options.memtable_whole_key_filtering = FLAGS_memtable_whole_key_filtering;
-  options.disable_auto_compactions = FLAGS_disable_auto_compactions;
+  if (ShouldDisableAutoCompactionsBeforeVerifyDb()) {
+    options.disable_auto_compactions = true;
+  } else {
+    options.disable_auto_compactions = FLAGS_disable_auto_compactions;
+  }
   options.max_background_compactions = FLAGS_max_background_compactions;
   options.max_background_flushes = FLAGS_max_background_flushes;
   options.compaction_style =
@@ -4169,6 +4055,7 @@ void InitializeOptionsFromFlags(
   options.memtable_protection_bytes_per_key =
       FLAGS_memtable_protection_bytes_per_key;
   options.block_protection_bytes_per_key = FLAGS_block_protection_bytes_per_key;
+  options.paranoid_memory_checks = FLAGS_paranoid_memory_checks;
 
   // Integrated BlobDB
   options.enable_blob_files = FLAGS_enable_blob_files;
@@ -4384,6 +4271,7 @@ void InitializeOptionsGeneral(
     options.disable_auto_compactions = true;
   }
 
+  options.table_properties_collector_factories.clear();
   options.table_properties_collector_factories.emplace_back(
       std::make_shared<DbStressTablePropertiesCollectorFactory>());
 

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h

@@ -48,7 +48,11 @@ class StressTest {
     return FLAGS_sync_fault_injection || FLAGS_disable_wal ||
            FLAGS_manual_wal_flush_one_in > 0;
   }
-
+  Status EnableAutoCompaction() {
+    assert(options_.disable_auto_compactions);
+    Status s = db_->EnableAutoCompaction(column_families_);
+    return s;
+  }
   void CleanUp();
 
  protected:
@@ -63,6 +67,55 @@ class StressTest {
       return 0;
     }
   }
+
+  void UpdateIfInitialWriteFails(Env* db_stress_env, const Status& write_s,
+                                 Status* initial_write_s,
+                                 bool* initial_wal_write_may_succeed,
+                                 uint64_t* wait_for_recover_start_time) {
+    assert(db_stress_env && initial_write_s && initial_wal_write_may_succeed &&
+           wait_for_recover_start_time);
+    // Only update `initial_write_s`, `initial_wal_write_may_succeed` when the
+    // first write fails
+    if (!write_s.ok() && (*initial_write_s).ok()) {
+      *initial_write_s = write_s;
+      *initial_wal_write_may_succeed =
+          !FaultInjectionTestFS::IsFailedToWriteToWALError(*initial_write_s);
+      *wait_for_recover_start_time = db_stress_env->NowMicros();
+    }
+  }
+
+  void PrintWriteRecoveryWaitTimeIfNeeded(Env* db_stress_env,
+                                          const Status& initial_write_s,
+                                          bool initial_wal_write_may_succeed,
+                                          uint64_t wait_for_recover_start_time,
+                                          const std::string& thread_name) {
+    assert(db_stress_env);
+    bool waited_for_recovery = !initial_write_s.ok() &&
+                               IsErrorInjectedAndRetryable(initial_write_s) &&
+                               initial_wal_write_may_succeed;
+    if (waited_for_recovery) {
+      uint64_t elapsed_sec =
+          (db_stress_env->NowMicros() - wait_for_recover_start_time) / 1000000;
+      if (elapsed_sec > 10) {
+        fprintf(stdout,
+                "%s thread slept to wait for write recovery for "
+                "%" PRIu64 " seconds\n",
+                thread_name.c_str(), elapsed_sec);
+      }
+    }
+  }
+  void GetDeleteRangeKeyLocks(
+      ThreadState* thread, int rand_column_family, int64_t rand_key,
+      std::vector<std::unique_ptr<MutexLock>>* range_locks) {
+    for (int j = 0; j < FLAGS_range_deletion_width; ++j) {
+      if (j == 0 ||
+          ((rand_key + j) & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) {
+        range_locks->emplace_back(new MutexLock(
+            thread->shared->GetMutexForKey(rand_column_family, rand_key + j)));
+      }
+    }
+  }
+
   Status AssertSame(DB* db, ColumnFamilyHandle* cf,
                     ThreadState::SnapshotState& snap_state);
 
@@ -283,7 +336,8 @@ class StressTest {
 
   bool IsErrorInjectedAndRetryable(const Status& error_s) const {
     assert(!error_s.ok());
-    return error_s.getState() && std::strstr(error_s.getState(), "inject") &&
+    return error_s.getState() &&
+           FaultInjectionTestFS::IsInjectedError(error_s) &&
            !status_to_io_status(Status(error_s)).GetDataLoss();
   }
 
@@ -397,5 +451,6 @@ void InitializeOptionsGeneral(
 // user-defined timestamp.
 void CheckAndSetOptionsForUserTimestamp(Options& options);
 
+bool ShouldDisableAutoCompactionsBeforeVerifyDb();
 }  // namespace ROCKSDB_NAMESPACE
 #endif  // GFLAGS

package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc

@@ -34,11 +34,19 @@ void ExpectedState::Precommit(int cf, int64_t key, const ExpectedValue& value) {
 
 PendingExpectedValue ExpectedState::PreparePut(int cf, int64_t key) {
   ExpectedValue expected_value = Load(cf, key);
+
+  // Calculate the original expected value
   const ExpectedValue orig_expected_value = expected_value;
+
+  // Calculate the pending expected value
   expected_value.Put(true /* pending */);
   const ExpectedValue pending_expected_value = expected_value;
+
+  // Calculate the final expected value
   expected_value.Put(false /* pending */);
   const ExpectedValue final_expected_value = expected_value;
+
+  // Precommit
   Precommit(cf, key, pending_expected_value);
   return PendingExpectedValue(&Value(cf, key), orig_expected_value,
                               final_expected_value);
@@ -46,21 +54,26 @@ PendingExpectedValue ExpectedState::PreparePut(int cf, int64_t key) {
 
 ExpectedValue ExpectedState::Get(int cf, int64_t key) { return Load(cf, key); }
 
-PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key,
-                                                  bool* prepared) {
+PendingExpectedValue ExpectedState::PrepareDelete(int cf, int64_t key) {
   ExpectedValue expected_value = Load(cf, key);
+
+  // Calculate the original expected value
   const ExpectedValue orig_expected_value = expected_value;
+
+  // Calculate the pending expected value
   bool res = expected_value.Delete(true /* pending */);
-  if (prepared) {
-    *prepared = res;
-  }
   if (!res) {
-    return PendingExpectedValue(&Value(cf, key), orig_expected_value,
-                                orig_expected_value);
+    PendingExpectedValue ret = PendingExpectedValue(
+        &Value(cf, key), orig_expected_value, orig_expected_value);
+    return ret;
   }
   const ExpectedValue pending_expected_value = expected_value;
+
+  // Calculate the final expected value
   expected_value.Delete(false /* pending */);
   const ExpectedValue final_expected_value = expected_value;
+
+  // Precommit
   Precommit(cf, key, pending_expected_value);
   return PendingExpectedValue(&Value(cf, key), orig_expected_value,
                               final_expected_value);
@@ -73,16 +86,11 @@ PendingExpectedValue ExpectedState::PrepareSingleDelete(int cf, int64_t key) {
 std::vector<PendingExpectedValue> ExpectedState::PrepareDeleteRange(
     int cf, int64_t begin_key, int64_t end_key) {
   std::vector<PendingExpectedValue> pending_expected_values;
+
   for (int64_t key = begin_key; key < end_key; ++key) {
-    bool prepared = false;
-    PendingExpectedValue pending_expected_value =
-        PrepareDelete(cf, key, &prepared);
-    if (prepared) {
-      pending_expected_values.push_back(pending_expected_value);
-    } else {
-      pending_expected_value.PermitUnclosedPendingState();
-    }
+    pending_expected_values.push_back(PrepareDelete(cf, key));
   }
+
   return pending_expected_values;
 }
 

package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h

@@ -49,14 +49,13 @@ class ExpectedState {
   // Does not requires external locking.
   ExpectedValue Get(int cf, int64_t key);
 
-  // Prepare a Delete that will be started but not finished yet
+  // Prepare a Delete that will be started but not finished yet.
   // This is useful for crash-recovery testing when the process may crash
   // before updating the corresponding expected value
   //
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.
-  PendingExpectedValue PrepareDelete(int cf, int64_t key,
-                                     bool* prepared = nullptr);
+  PendingExpectedValue PrepareDelete(int cf, int64_t key);
 
   // Requires external locking covering `key` in `cf` to prevent concurrent
   // write or delete to the same `key`.

package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.cc

@@ -21,7 +21,7 @@ void ExpectedValue::Put(bool pending) {
 }
 
 bool ExpectedValue::Delete(bool pending) {
-  if (!Exists()) {
+  if (pending && !Exists()) {
     return false;
   }
   if (pending) {

package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.h

@@ -37,7 +37,10 @@ class ExpectedValue {
   explicit ExpectedValue(uint32_t expected_value)
       : expected_value_(expected_value) {}
 
-  bool Exists() const { return PendingWrite() || !IsDeleted(); }
+  bool Exists() const {
+    assert(!PendingWrite() && !PendingDelete());
+    return !IsDeleted();
+  }
 
   uint32_t Read() const { return expected_value_; }