@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc

@@ -7,6 +7,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#include "db_stress_tool/expected_state.h"
 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
 #include "rocksdb/utilities/transaction_db.h"
@@ -121,8 +122,7 @@ class NonBatchedOpsStressTest : public StressTest {
           }
 
           VerifyOrSyncValue(static_cast<int>(cf), i, options, shared, from_db,
-                            /* msg_prefix */ "Iterator verification", s,
-                            /* strict */ true);
+                            /* msg_prefix */ "Iterator verification", s);
 
           if (!from_db.empty()) {
             PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
@@ -141,8 +141,7 @@ class NonBatchedOpsStressTest : public StressTest {
           Status s = db_->Get(options, column_families_[cf], key, &from_db);
 
           VerifyOrSyncValue(static_cast<int>(cf), i, options, shared, from_db,
-                            /* msg_prefix */ "Get verification", s,
-                            /* strict */ true);
+                            /* msg_prefix */ "Get verification", s);
 
           if (!from_db.empty()) {
             PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
@@ -178,8 +177,7 @@ class NonBatchedOpsStressTest : public StressTest {
           }
 
           VerifyOrSyncValue(static_cast<int>(cf), i, options, shared, from_db,
-                            /* msg_prefix */ "GetEntity verification", s,
-                            /* strict */ true);
+                            /* msg_prefix */ "GetEntity verification", s);
 
           if (!from_db.empty()) {
             PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
@@ -214,7 +212,7 @@ class NonBatchedOpsStressTest : public StressTest {
 
             VerifyOrSyncValue(static_cast<int>(cf), i + j, options, shared,
                               from_db, /* msg_prefix */ "MultiGet verification",
-                              statuses[j], /* strict */ true);
+                              statuses[j]);
 
             if (!from_db.empty()) {
               PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i + j),
@@ -264,10 +262,9 @@ class NonBatchedOpsStressTest : public StressTest {
               }
             }
 
-            VerifyOrSyncValue(static_cast<int>(cf), i + j, options, shared,
-                              from_db,
-                              /* msg_prefix */ "MultiGetEntity verification",
-                              statuses[j], /* strict */ true);
+            VerifyOrSyncValue(
+                static_cast<int>(cf), i + j, options, shared, from_db,
+                /* msg_prefix */ "MultiGetEntity verification", statuses[j]);
 
             if (!from_db.empty()) {
               PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i + j),
@@ -319,8 +316,8 @@ class NonBatchedOpsStressTest : public StressTest {
           }
 
           VerifyOrSyncValue(static_cast<int>(cf), i, options, shared, from_db,
-                            /* msg_prefix */ "GetMergeOperands verification", s,
-                            /* strict */ true);
+                            /* msg_prefix */ "GetMergeOperands verification",
+                            s);
 
           if (!from_db.empty()) {
             PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
@@ -479,9 +476,6 @@ class NonBatchedOpsStressTest : public StressTest {
       SharedState::ignore_read_error = false;
     }
 
-    std::unique_ptr<MutexLock> lock(new MutexLock(
-        thread->shared->GetMutexForKey(rand_column_families[0], rand_keys[0])));
-
     ReadOptions read_opts_copy = read_opts;
     std::string read_ts_str;
     Slice read_ts_slice;
@@ -493,7 +487,11 @@ class NonBatchedOpsStressTest : public StressTest {
     bool read_older_ts = MaybeUseOlderTimestampForPointLookup(
         thread, read_ts_str, read_ts_slice, read_opts_copy);
 
+    const ExpectedValue pre_read_expected_value =
+        thread->shared->Get(rand_column_families[0], rand_keys[0]);
     Status s = db_->Get(read_opts_copy, cfh, key, &from_db);
+    const ExpectedValue post_read_expected_value =
+        thread->shared->Get(rand_column_families[0], rand_keys[0]);
     if (fault_fs_guard) {
       error_count = fault_fs_guard->GetAndResetErrorCount();
     }
@@ -512,23 +510,35 @@ class NonBatchedOpsStressTest : public StressTest {
       // found case
       thread->stats.AddGets(1, 1);
       // we only have the latest expected state
-      if (!FLAGS_skip_verifydb && !read_older_ts &&
-          thread->shared->Get(rand_column_families[0], rand_keys[0]) ==
-              SharedState::DELETION_SENTINEL) {
-        thread->shared->SetVerificationFailure();
-        fprintf(stderr,
-                "error : inconsistent values for key %s: Get returns %s, "
-                "expected state does not have the key.\n",
-                key.ToString(true).c_str(), StringToHex(from_db).c_str());
+      if (!FLAGS_skip_verifydb && !read_older_ts) {
+        if (ExpectedValueHelper::MustHaveNotExisted(pre_read_expected_value,
+                                                    post_read_expected_value)) {
+          thread->shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent values for key %s: Get returns %s, "
+                  "but expected state is \"deleted\".\n",
+                  key.ToString(true).c_str(), StringToHex(from_db).c_str());
+        }
+        Slice from_db_slice(from_db);
+        uint32_t value_base_from_db = GetValueBase(from_db_slice);
+        if (!ExpectedValueHelper::InExpectedValueBaseRange(
+                value_base_from_db, pre_read_expected_value,
+                post_read_expected_value)) {
+          thread->shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent values for key %s: Get returns %s with "
+                  "value base %d that falls out of expected state's value base "
+                  "range.\n",
+                  key.ToString(true).c_str(), StringToHex(from_db).c_str(),
+                  value_base_from_db);
+        }
       }
     } else if (s.IsNotFound()) {
       // not found case
       thread->stats.AddGets(1, 0);
       if (!FLAGS_skip_verifydb && !read_older_ts) {
-        auto expected =
-            thread->shared->Get(rand_column_families[0], rand_keys[0]);
-        if (expected != SharedState::DELETION_SENTINEL &&
-            expected != SharedState::UNKNOWN_SENTINEL) {
+        if (ExpectedValueHelper::MustHaveExisted(pre_read_expected_value,
+                                                 post_read_expected_value)) {
           thread->shared->SetVerificationFailure();
           fprintf(stderr,
                   "error : inconsistent values for key %s: expected state has "
@@ -611,8 +621,7 @@ class NonBatchedOpsStressTest : public StressTest {
           switch (op) {
             case 0:
             case 1: {
-              uint32_t value_base =
-                  thread->rand.Next() % thread->shared->UNKNOWN_SENTINEL;
+              const uint32_t value_base = 0;
               char value[100];
               size_t sz = GenerateValue(value_base, value, sizeof(value));
               Slice v(value, sz);
@@ -803,15 +812,16 @@ class NonBatchedOpsStressTest : public StressTest {
 
       if (!FLAGS_skip_verifydb) {
         const WideColumns& columns = from_db.columns();
-
+        ExpectedValue expected =
+            shared->Get(rand_column_families[0], rand_keys[0]);
         if (!VerifyWideColumns(columns)) {
           shared->SetVerificationFailure();
           fprintf(stderr,
                   "error : inconsistent columns returned by GetEntity for key "
                   "%s: %s\n",
                   StringToHex(key).c_str(), WideColumnsToHex(columns).c_str());
-        } else if (shared->Get(rand_column_families[0], rand_keys[0]) ==
-                   SharedState::DELETION_SENTINEL) {
+        } else if (ExpectedValueHelper::MustHaveNotExisted(expected,
+                                                           expected)) {
           shared->SetVerificationFailure();
           fprintf(
               stderr,
@@ -824,9 +834,9 @@ class NonBatchedOpsStressTest : public StressTest {
       thread->stats.AddGets(1, 0);
 
       if (!FLAGS_skip_verifydb) {
-        auto expected = shared->Get(rand_column_families[0], rand_keys[0]);
-        if (expected != SharedState::DELETION_SENTINEL &&
-            expected != SharedState::UNKNOWN_SENTINEL) {
+        ExpectedValue expected =
+            shared->Get(rand_column_families[0], rand_keys[0]);
+        if (ExpectedValueHelper::MustHaveExisted(expected, expected)) {
           shared->SetVerificationFailure();
           fprintf(stderr,
                   "error : inconsistent values for key %s: expected state has "
@@ -847,6 +857,157 @@ class NonBatchedOpsStressTest : public StressTest {
     }
   }
 
+  void TestMultiGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                          const std::vector<int>& rand_column_families,
+                          const std::vector<int64_t>& rand_keys) override {
+    assert(thread);
+
+    ManagedSnapshot snapshot_guard(db_);
+
+    ReadOptions read_opts_copy(read_opts);
+    read_opts_copy.snapshot = snapshot_guard.snapshot();
+
+    assert(!rand_column_families.empty());
+    assert(rand_column_families[0] >= 0);
+    assert(rand_column_families[0] < static_cast<int>(column_families_.size()));
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    assert(!rand_keys.empty());
+
+    const size_t num_keys = rand_keys.size();
+
+    std::vector<std::string> keys(num_keys);
+    std::vector<Slice> key_slices(num_keys);
+
+    for (size_t i = 0; i < num_keys; ++i) {
+      keys[i] = Key(rand_keys[i]);
+      key_slices[i] = keys[i];
+    }
+
+    std::vector<PinnableWideColumns> results(num_keys);
+    std::vector<Status> statuses(num_keys);
+
+    if (fault_fs_guard) {
+      fault_fs_guard->EnableErrorInjection();
+      SharedState::ignore_read_error = false;
+    }
+
+    db_->MultiGetEntity(read_opts_copy, cfh, num_keys, key_slices.data(),
+                        results.data(), statuses.data());
+
+    int error_count = 0;
+
+    if (fault_fs_guard) {
+      error_count = fault_fs_guard->GetAndResetErrorCount();
+
+      if (error_count && !SharedState::ignore_read_error) {
+        int stat_nok = 0;
+        for (const auto& s : statuses) {
+          if (!s.ok() && !s.IsNotFound()) {
+            stat_nok++;
+          }
+        }
+
+        if (stat_nok < error_count) {
+          // Grab mutex so multiple threads don't try to print the
+          // stack trace at the same time
+          assert(thread->shared);
+          MutexLock l(thread->shared->GetMutex());
+
+          fprintf(stderr, "Didn't get expected error from MultiGetEntity\n");
+          fprintf(stderr, "num_keys %zu Expected %d errors, seen %d\n",
+                  num_keys, error_count, stat_nok);
+          fprintf(stderr, "Call stack that injected the fault\n");
+          fault_fs_guard->PrintFaultBacktrace();
+          std::terminate();
+        }
+      }
+
+      fault_fs_guard->DisableErrorInjection();
+    }
+
+    const bool check_get_entity = !error_count && thread->rand.OneIn(4);
+
+    for (size_t i = 0; i < num_keys; ++i) {
+      const Status& s = statuses[i];
+
+      bool is_consistent = true;
+
+      if (s.ok() && !VerifyWideColumns(results[i].columns())) {
+        fprintf(
+            stderr,
+            "error : inconsistent columns returned by MultiGetEntity for key "
+            "%s: %s\n",
+            StringToHex(keys[i]).c_str(),
+            WideColumnsToHex(results[i].columns()).c_str());
+        is_consistent = false;
+      } else if (check_get_entity && (s.ok() || s.IsNotFound())) {
+        PinnableWideColumns cmp_result;
+
+        const Status cmp_s =
+            db_->GetEntity(read_opts_copy, cfh, key_slices[i], &cmp_result);
+
+        if (!cmp_s.ok() && !cmp_s.IsNotFound()) {
+          fprintf(stderr, "GetEntity error: %s\n", cmp_s.ToString().c_str());
+          is_consistent = false;
+        } else if (cmp_s.IsNotFound()) {
+          if (s.ok()) {
+            fprintf(stderr,
+                    "Inconsistent results for key %s: MultiGetEntity returned "
+                    "ok, GetEntity returned not found\n",
+                    StringToHex(keys[i]).c_str());
+            is_consistent = false;
+          }
+        } else {
+          assert(cmp_s.ok());
+
+          if (s.IsNotFound()) {
+            fprintf(stderr,
+                    "Inconsistent results for key %s: MultiGetEntity returned "
+                    "not found, GetEntity returned ok\n",
+                    StringToHex(keys[i]).c_str());
+            is_consistent = false;
+          } else {
+            assert(s.ok());
+
+            if (results[i] != cmp_result) {
+              fprintf(
+                  stderr,
+                  "Inconsistent results for key %s: MultiGetEntity returned "
+                  "%s, GetEntity returned %s\n",
+                  StringToHex(keys[i]).c_str(),
+                  WideColumnsToHex(results[i].columns()).c_str(),
+                  WideColumnsToHex(cmp_result.columns()).c_str());
+              is_consistent = false;
+            }
+          }
+        }
+      }
+
+      if (!is_consistent) {
+        fprintf(stderr,
+                "TestMultiGetEntity error: results are not consistent\n");
+        thread->stats.AddErrors(1);
+        // Fail fast to preserve the DB state
+        thread->shared->SetVerificationFailure();
+        break;
+      } else if (s.ok()) {
+        thread->stats.AddGets(1, 1);
+      } else if (s.IsNotFound()) {
+        thread->stats.AddGets(1, 0);
+      } else {
+        if (error_count == 0) {
+          fprintf(stderr, "MultiGetEntity error: %s\n", s.ToString().c_str());
+          thread->stats.AddErrors(1);
+        } else {
+          thread->stats.AddVerifiedErrors(1);
+        }
+      }
+    }
+  }
+
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override {
@@ -982,16 +1143,17 @@ class NonBatchedOpsStressTest : public StressTest {
       Status s = db_->Get(read_opts, cfh, k, &from_db);
       if (!VerifyOrSyncValue(rand_column_family, rand_key, read_opts, shared,
                              /* msg_prefix */ "Pre-Put Get verification",
-                             from_db, s, /* strict */ true)) {
+                             from_db, s)) {
         return s;
       }
     }
 
-    const uint32_t value_base = thread->rand.Next() % shared->UNKNOWN_SENTINEL;
+    PendingExpectedValue pending_expected_value =
+        shared->PreparePut(rand_column_family, rand_key);
+    const uint32_t value_base = pending_expected_value.GetFinalValueBase();
     const size_t sz = GenerateValue(value_base, value, sizeof(value));
     const Slice v(value, sz);
 
-    shared->Put(rand_column_family, rand_key, value_base, true /* pending */);
 
     Status s;
 
@@ -1035,7 +1197,7 @@ class NonBatchedOpsStressTest : public StressTest {
       }
     }
 
-    shared->Put(rand_column_family, rand_key, value_base, false /* pending */);
+    pending_expected_value.Commit();
 
     if (!s.ok()) {
       if (FLAGS_injest_error_severity >= 2) {
@@ -1080,7 +1242,8 @@ class NonBatchedOpsStressTest : public StressTest {
     // otherwise.
     Status s;
     if (shared->AllowsOverwrite(rand_key)) {
-      shared->Delete(rand_column_family, rand_key, true /* pending */);
+      PendingExpectedValue pending_expected_value =
+          shared->PrepareDelete(rand_column_family, rand_key);
       if (!FLAGS_use_txn) {
         if (FLAGS_user_timestamp_size == 0) {
           s = db_->Delete(write_opts, cfh, key);
@@ -1097,7 +1260,8 @@ class NonBatchedOpsStressTest : public StressTest {
           }
         }
       }
-      shared->Delete(rand_column_family, rand_key, false /* pending */);
+      pending_expected_value.Commit();
+
       thread->stats.AddDeletes(1);
       if (!s.ok()) {
         if (FLAGS_injest_error_severity >= 2) {
@@ -1115,7 +1279,8 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       }
     } else {
-      shared->SingleDelete(rand_column_family, rand_key, true /* pending */);
+      PendingExpectedValue pending_expected_value =
+          shared->PrepareSingleDelete(rand_column_family, rand_key);
       if (!FLAGS_use_txn) {
         if (FLAGS_user_timestamp_size == 0) {
           s = db_->SingleDelete(write_opts, cfh, key);
@@ -1132,7 +1297,7 @@ class NonBatchedOpsStressTest : public StressTest {
           }
         }
       }
-      shared->SingleDelete(rand_column_family, rand_key, false /* pending */);
+      pending_expected_value.Commit();
       thread->stats.AddSingleDeletes(1);
       if (!s.ok()) {
         if (FLAGS_injest_error_severity >= 2) {
@@ -1177,10 +1342,10 @@ class NonBatchedOpsStressTest : public StressTest {
             shared->GetMutexForKey(rand_column_family, rand_key + j)));
       }
     }
-    shared->DeleteRange(rand_column_family, rand_key,
-                        rand_key + FLAGS_range_deletion_width,
-                        true /* pending */);
-
+    std::vector<PendingExpectedValue> pending_expected_values =
+        shared->PrepareDeleteRange(rand_column_family, rand_key,
+                                   rand_key + FLAGS_range_deletion_width);
+    const int covered = static_cast<int>(pending_expected_values.size());
     std::string keystr = Key(rand_key);
     Slice key = keystr;
     auto cfh = column_families_[rand_column_family];
@@ -1210,9 +1375,10 @@ class NonBatchedOpsStressTest : public StressTest {
         std::terminate();
       }
     }
-    int covered = shared->DeleteRange(rand_column_family, rand_key,
-                                      rand_key + FLAGS_range_deletion_width,
-                                      false /* pending */);
+    for (PendingExpectedValue& pending_expected_value :
+         pending_expected_values) {
+      pending_expected_value.Commit();
+    }
     thread->stats.AddRangeDeletions(1);
     thread->stats.AddCoveredByRangeDeletions(covered);
     return s;
@@ -1242,6 +1408,8 @@ class NonBatchedOpsStressTest : public StressTest {
     keys.reserve(FLAGS_ingest_external_file_width);
     std::vector<uint32_t> values;
     values.reserve(FLAGS_ingest_external_file_width);
+    std::vector<PendingExpectedValue> pending_expected_values;
+    pending_expected_values.reserve(FLAGS_ingest_external_file_width);
     SharedState* shared = thread->shared;
 
     assert(FLAGS_nooverwritepercent < 100);
@@ -1256,15 +1424,16 @@ class NonBatchedOpsStressTest : public StressTest {
             new MutexLock(shared->GetMutexForKey(column_family, key)));
       }
       if (!shared->AllowsOverwrite(key)) {
-        // We could alternatively include `key` on the condition its current
-        // value is `DELETION_SENTINEL`.
+        // We could alternatively include `key` that is deleted.
         continue;
       }
       keys.push_back(key);
 
-      uint32_t value_base = thread->rand.Next() % shared->UNKNOWN_SENTINEL;
+      PendingExpectedValue pending_expected_value =
+          shared->PreparePut(column_family, key);
+      const uint32_t value_base = pending_expected_value.GetFinalValueBase();
       values.push_back(value_base);
-      shared->Put(column_family, key, value_base, true /* pending */);
+      pending_expected_values.push_back(pending_expected_value);
 
       char value[100];
       size_t value_len = GenerateValue(value_base, value, sizeof(value));
@@ -1287,8 +1456,9 @@ class NonBatchedOpsStressTest : public StressTest {
       fprintf(stderr, "file ingestion error: %s\n", s.ToString().c_str());
       std::terminate();
     }
-    for (size_t i = 0; i < keys.size(); ++i) {
-      shared->Put(column_family, keys[i], values[i], false /* pending */);
+
+    for (size_t i = 0; i < pending_expected_values.size(); ++i) {
+      pending_expected_values[i].Commit();
     }
   }
 
@@ -1320,8 +1490,13 @@ class NonBatchedOpsStressTest : public StressTest {
     // Lock the whole range over which we might iterate to ensure it doesn't
     // change under us.
     const int rand_column_family = rand_column_families[0];
-    std::vector<std::unique_ptr<MutexLock>> range_locks =
-        shared->GetLocksForKeyRange(rand_column_family, lb, ub);
+
+    // Testing parallel read and write to the same key with user timestamp
+    // is not currently supported
+    std::vector<std::unique_ptr<MutexLock>> range_locks;
+    if (FLAGS_user_timestamp_size > 0) {
+      range_locks = shared->GetLocksForKeyRange(rand_column_family, lb, ub);
+    }
 
     ReadOptions ro(read_opts);
     ro.total_order_seek = true;
@@ -1348,7 +1523,22 @@ class NonBatchedOpsStressTest : public StressTest {
     ColumnFamilyHandle* const cfh = column_families_[rand_column_family];
     assert(cfh);
 
+    const std::size_t expected_values_size = static_cast<std::size_t>(ub - lb);
+    std::vector<ExpectedValue> pre_read_expected_values;
+    std::vector<ExpectedValue> post_read_expected_values;
+
+    for (int64_t i = 0; i < static_cast<int64_t>(expected_values_size); ++i) {
+      pre_read_expected_values.push_back(
+          shared->Get(rand_column_family, i + lb));
+    }
     std::unique_ptr<Iterator> iter(db_->NewIterator(ro, cfh));
+    for (int64_t i = 0; i < static_cast<int64_t>(expected_values_size); ++i) {
+      post_read_expected_values.push_back(
+          shared->Get(rand_column_family, i + lb));
+    }
+
+    assert(pre_read_expected_values.size() == expected_values_size &&
+           pre_read_expected_values.size() == post_read_expected_values.size());
 
     std::string op_logs;
 
@@ -1378,10 +1568,15 @@ class NonBatchedOpsStressTest : public StressTest {
 
     auto check_no_key_in_range = [&](int64_t start, int64_t end) {
       for (auto j = std::max(start, lb); j < std::min(end, ub); ++j) {
-        auto expected_value =
-            shared->Get(rand_column_family, static_cast<int64_t>(j));
-        if (expected_value != shared->DELETION_SENTINEL &&
-            expected_value != shared->UNKNOWN_SENTINEL) {
+        std::size_t index = static_cast<std::size_t>(j - lb);
+        assert(index < pre_read_expected_values.size() &&
+               index < post_read_expected_values.size());
+        const ExpectedValue pre_read_expected_value =
+            pre_read_expected_values[index];
+        const ExpectedValue post_read_expected_value =
+            post_read_expected_values[index];
+        if (ExpectedValueHelper::MustHaveExisted(pre_read_expected_value,
+                                                 post_read_expected_value)) {
           // Fail fast to preserve the DB state.
           thread->shared->SetVerificationFailure();
           if (iter->Valid()) {
@@ -1495,9 +1690,23 @@ class NonBatchedOpsStressTest : public StressTest {
     }
 
     if (thread->rand.OneIn(2)) {
+      pre_read_expected_values.clear();
+      post_read_expected_values.clear();
       // Refresh after forward/backward scan to allow higher chance of SV
-      // change. It is safe to refresh since the testing key range is locked.
+      // change.
+      for (int64_t i = 0; i < static_cast<int64_t>(expected_values_size); ++i) {
+        pre_read_expected_values.push_back(
+            shared->Get(rand_column_family, i + lb));
+      }
       iter->Refresh();
+      for (int64_t i = 0; i < static_cast<int64_t>(expected_values_size); ++i) {
+        post_read_expected_values.push_back(
+            shared->Get(rand_column_family, i + lb));
+      }
+
+      assert(pre_read_expected_values.size() == expected_values_size &&
+             pre_read_expected_values.size() ==
+                 post_read_expected_values.size());
     }
 
     // start from middle of [lb, ub) otherwise it is easy to iterate out of
@@ -1539,9 +1748,19 @@ class NonBatchedOpsStressTest : public StressTest {
         iter->Prev();
         op_logs += "P";
       } else {
-        const uint32_t expected_value =
-            shared->Get(rand_column_family, static_cast<int64_t>(curr));
-        if (expected_value == shared->DELETION_SENTINEL) {
+        const uint32_t value_base_from_db = GetValueBase(iter->value());
+        std::size_t index = static_cast<std::size_t>(curr - lb);
+        assert(index < pre_read_expected_values.size() &&
+               index < post_read_expected_values.size());
+        const ExpectedValue pre_read_expected_value =
+            pre_read_expected_values[index];
+        const ExpectedValue post_read_expected_value =
+            post_read_expected_values[index];
+        if (ExpectedValueHelper::MustHaveNotExisted(pre_read_expected_value,
+                                                    post_read_expected_value) ||
+            !ExpectedValueHelper::InExpectedValueBaseRange(
+                value_base_from_db, pre_read_expected_value,
+                post_read_expected_value)) {
           // Fail fast to preserve the DB state.
           thread->shared->SetVerificationFailure();
           fprintf(stderr, "Iterator has key %s, but expected state does not.\n",
@@ -1597,59 +1816,70 @@ class NonBatchedOpsStressTest : public StressTest {
 
   bool VerifyOrSyncValue(int cf, int64_t key, const ReadOptions& /*opts*/,
                          SharedState* shared, const std::string& value_from_db,
-                         std::string msg_prefix, const Status& s,
-                         bool strict = false) const {
+                         std::string msg_prefix, const Status& s) const {
     if (shared->HasVerificationFailedYet()) {
       return false;
     }
+    const ExpectedValue expected_value = shared->Get(cf, key);
 
-    // compare value_from_db with the value in the shared state
-    uint32_t value_base = shared->Get(cf, key);
-    if (value_base == SharedState::UNKNOWN_SENTINEL) {
+    if (expected_value.PendingWrite() || expected_value.PendingDelete()) {
       if (s.ok()) {
         // Value exists in db, update state to reflect that
         Slice slice(value_from_db);
-        value_base = GetValueBase(slice);
-        shared->Put(cf, key, value_base, false);
+        uint32_t value_base = GetValueBase(slice);
+        shared->SyncPut(cf, key, value_base);
       } else if (s.IsNotFound()) {
         // Value doesn't exist in db, update state to reflect that
-        shared->SingleDelete(cf, key, false);
+        shared->SyncDelete(cf, key);
       }
       return true;
     }
-    if (value_base == SharedState::DELETION_SENTINEL && !strict) {
-      return true;
-    }
 
+    // compare value_from_db with the value in the shared state
     if (s.ok()) {
-      char value[kValueMaxLen];
-      if (value_base == SharedState::DELETION_SENTINEL) {
+      const Slice slice(value_from_db);
+      const uint32_t value_base_from_db = GetValueBase(slice);
+      if (ExpectedValueHelper::MustHaveNotExisted(expected_value,
+                                                  expected_value) ||
+          !ExpectedValueHelper::InExpectedValueBaseRange(
+              value_base_from_db, expected_value, expected_value)) {
         VerificationAbort(shared, msg_prefix + ": Unexpected value found", cf,
                           key, value_from_db, "");
         return false;
       }
-      size_t sz = GenerateValue(value_base, value, sizeof(value));
-      if (value_from_db.length() != sz) {
+      char expected_value_data[kValueMaxLen];
+      size_t expected_value_data_size =
+          GenerateValue(expected_value.GetValueBase(), expected_value_data,
+                        sizeof(expected_value_data));
+      if (value_from_db.length() != expected_value_data_size) {
         VerificationAbort(shared,
                           msg_prefix + ": Length of value read is not equal",
-                          cf, key, value_from_db, Slice(value, sz));
+                          cf, key, value_from_db,
+                          Slice(expected_value_data, expected_value_data_size));
         return false;
       }
-      if (memcmp(value_from_db.data(), value, sz) != 0) {
+      if (memcmp(value_from_db.data(), expected_value_data,
+                 expected_value_data_size) != 0) {
         VerificationAbort(shared,
                           msg_prefix + ": Contents of value read don't match",
-                          cf, key, value_from_db, Slice(value, sz));
+                          cf, key, value_from_db,
+                          Slice(expected_value_data, expected_value_data_size));
         return false;
       }
-    } else {
-      if (value_base != SharedState::DELETION_SENTINEL) {
-        char value[kValueMaxLen];
-        size_t sz = GenerateValue(value_base, value, sizeof(value));
-        VerificationAbort(shared,
-                          msg_prefix + ": Value not found: " + s.ToString(), cf,
-                          key, "", Slice(value, sz));
+    } else if (s.IsNotFound()) {
+      if (ExpectedValueHelper::MustHaveExisted(expected_value,
+                                               expected_value)) {
+        char expected_value_data[kValueMaxLen];
+        size_t expected_value_data_size =
+            GenerateValue(expected_value.GetValueBase(), expected_value_data,
+                          sizeof(expected_value_data));
+        VerificationAbort(
+            shared, msg_prefix + ": Value not found: " + s.ToString(), cf, key,
+            "", Slice(expected_value_data, expected_value_data_size));
         return false;
       }
+    } else {
+      assert(false);
     }
     return true;
   }