@nxtedition/rocksdb 13.5.13 → 14.0.0

package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.cc

@@ -5,11 +5,49 @@
 
 #include "utilities/transactions/lock/point/point_lock_manager_test.h"
 
+#include "utilities/transactions/lock/point/any_lock_manager_test.h"
+
 namespace ROCKSDB_NAMESPACE {
 
+struct SpotLockManagerTestParam {
+  bool use_per_key_point_lock_manager;
+  int deadlock_timeout_us;
+};
+
+// Define operator<< for SpotLockManagerTestParam to stop valgrind from
+// complaining uinitialized value when printing SpotLockManagerTestParam.
+std::ostream& operator<<(std::ostream& os,
+                         const SpotLockManagerTestParam& param) {
+  os << "use_per_key_point_lock_manager: "
+     << param.use_per_key_point_lock_manager
+     << ", deadlock_timeout_us: " << param.deadlock_timeout_us;
+  return os;
+}
+
+// including test for both PointLockManager and PerKeyPointLockManager
+class SpotLockManagerTest
+    : public PointLockManagerTest,
+      public testing::WithParamInterface<SpotLockManagerTestParam> {
+ public:
+  void SetUp() override {
+    init();
+    // If a custom setup function was provided, use it. Otherwise, use what we
+    // have inherited.
+    auto param = GetParam();
+    if (param.use_per_key_point_lock_manager) {
+      locker_.reset(new PerKeyPointLockManager(
+          static_cast<PessimisticTransactionDB*>(db_), txndb_opt_));
+    } else {
+      locker_.reset(new PointLockManager(
+          static_cast<PessimisticTransactionDB*>(db_), txndb_opt_));
+    }
+    deadlock_timeout_us = param.deadlock_timeout_us;
+  }
+};
+
 // This test is not applicable for Range Lock manager as Range Lock Manager
 // operates on Column Families, not their ids.
-TEST_F(PointLockManagerTest, LockNonExistingColumnFamily) {
+TEST_P(SpotLockManagerTest, LockNonExistingColumnFamily) {
   MockColumnFamilyHandle cf(1024);
   locker_->RemoveColumnFamily(&cf);
   auto txn = NewTxn();
@@ -19,7 +57,7 @@ TEST_F(PointLockManagerTest, LockNonExistingColumnFamily) {
   delete txn;
 }
 
-TEST_F(PointLockManagerTest, LockStatus) {
+TEST_P(SpotLockManagerTest, LockStatus) {
   MockColumnFamilyHandle cf1(1024), cf2(2048);
   locker_->AddColumnFamily(&cf1);
   locker_->AddColumnFamily(&cf2);
@@ -61,7 +99,7 @@ TEST_F(PointLockManagerTest, LockStatus) {
   delete txn2;
 }
 
-TEST_F(PointLockManagerTest, UnlockExclusive) {
+TEST_P(SpotLockManagerTest, UnlockExclusive) {
   MockColumnFamilyHandle cf(1);
   locker_->AddColumnFamily(&cf);
 
@@ -79,7 +117,7 @@ TEST_F(PointLockManagerTest, UnlockExclusive) {
   delete txn2;
 }
 
-TEST_F(PointLockManagerTest, UnlockShared) {
+TEST_P(SpotLockManagerTest, UnlockShared) {
   MockColumnFamilyHandle cf(1);
   locker_->AddColumnFamily(&cf);
 
@@ -100,7 +138,7 @@ TEST_F(PointLockManagerTest, UnlockShared) {
 // This test doesn't work with Range Lock Manager, because Range Lock Manager
 // doesn't support deadlock_detect_depth.
 
-TEST_F(PointLockManagerTest, DeadlockDepthExceeded) {
+TEST_P(SpotLockManagerTest, DeadlockDepthExceeded) {
   // Tests that when detecting deadlock, if the detection depth is exceeded,
   // it's also viewed as deadlock.
   MockColumnFamilyHandle cf(1);
@@ -108,7 +146,7 @@ TEST_F(PointLockManagerTest, DeadlockDepthExceeded) {
   TransactionOptions txn_opt;
   txn_opt.deadlock_detect = true;
   txn_opt.deadlock_detect_depth = 1;
-  txn_opt.lock_timeout = 1000000;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
   auto txn1 = NewTxn(txn_opt);
   auto txn2 = NewTxn(txn_opt);
   auto txn3 = NewTxn(txn_opt);
@@ -124,7 +162,8 @@ TEST_F(PointLockManagerTest, DeadlockDepthExceeded) {
   // it must have another txn waiting on it, which is txn4 in this case.
   ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
 
-  port::Thread t1 = BlockUntilWaitingTxn(wait_sync_point_name_, [&]() {
+  port::Thread t1;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t1, [&]() {
     ASSERT_OK(locker_->TryLock(txn2, 1, "k2", env_, true));
     // block because txn1 is holding a lock on k1.
     ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
@@ -132,7 +171,8 @@ TEST_F(PointLockManagerTest, DeadlockDepthExceeded) {
 
   ASSERT_OK(locker_->TryLock(txn3, 1, "k3", env_, true));
 
-  port::Thread t2 = BlockUntilWaitingTxn(wait_sync_point_name_, [&]() {
+  port::Thread t2;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t2, [&]() {
     // block because txn3 is holding a lock on k1.
     ASSERT_OK(locker_->TryLock(txn4, 1, "k3", env_, true));
   });
@@ -150,15 +190,1242 @@ TEST_F(PointLockManagerTest, DeadlockDepthExceeded) {
   t1.join();
   t2.join();
 
+  locker_->UnLock(txn2, 1, "k2", env_);
+  locker_->UnLock(txn2, 1, "k1", env_);
+  locker_->UnLock(txn4, 1, "k3", env_);
+
   delete txn4;
   delete txn3;
   delete txn2;
   delete txn1;
 }
 
+TEST_P(SpotLockManagerTest, PrioritizedLockUpgradeWithExclusiveLock) {
+  // Tests that a lock upgrade request is prioritized over other lock requests.
+
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires exclusive lock on k1.
+  // txn1 acquires exclusive locks k1 successfully
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+
+  // txn2 tries to lock k1 exclusively, will be blocked.
+  port::Thread t;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t, [this, &txn2]() {
+    // block because txn1 is holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+  });
+
+  // verify lock upgrade successfully
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  // unlock txn1, so txn2 could proceed
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  // Cleanup
+  t.join();
+
+  // Cleanup
+  locker_->UnLock(txn2, 1, "k1", env_);
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest,
+       PrioritizedLockUpgradeWithExclusiveLockAndSharedLock) {
+  // Tests that lock upgrade is prioritized when mixed with shared and exclusive
+  // locks requests
+
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires shared lock on k1.
+  // txn3 acquires exclusive lock on k1.
+  // txn1 acquires exclusive locks k1 <- request granted after txn2 release the
+  // lock
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+  auto txn3 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+  ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+
+  // txn3 tries to lock k1 exclusively, will be blocked.
+  port::Thread txn3_thread;
+  BlockUntilWaitingTxn(wait_sync_point_name_, txn3_thread, [this, &txn3]() {
+    // block because txn1 and txn2 are holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, true));
+  });
+  // Verify txn3 is blocked
+  ASSERT_TRUE(txn3_thread.joinable());
+
+  // txn1 tries to lock k1 exclusively, will be blocked.
+  port::Thread txn1_thread;
+  BlockUntilWaitingTxn(wait_sync_point_name_, txn1_thread, [this, &txn1]() {
+    // block because txn1 and txn2 are holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+  });
+  // Verify txn1 is blocked
+  ASSERT_TRUE(txn1_thread.joinable());
+
+  // Unlock txn2, so txn1 could proceed
+  locker_->UnLock(txn2, 1, "k1", env_);
+  txn1_thread.join();
+
+  // Unlock txn1, so txn3 could proceed
+  locker_->UnLock(txn1, 1, "k1", env_);
+  txn3_thread.join();
+
+  // Cleanup
+  locker_->UnLock(txn3, 1, "k1", env_);
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest, Deadlock_MultipleUpgrade) {
+  // Tests that deadlock can be detected for shared locks and exclusive locks
+  // mixed Deadlock scenario:
+
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires shared lock on k1.
+  // txn1 acquires exclusive locks k1
+  // txn2 acquires exclusive locks k1 <- dead lock detected
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+  ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+
+  // txn1 tries to lock k1 exclusively, will be blocked.
+  port::Thread t;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t, [this, &txn1]() {
+    // block because txn2 is holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+  });
+
+  auto s = locker_->TryLock(txn2, 1, "k1", env_, true);
+  ASSERT_TRUE(s.IsBusy());
+  ASSERT_EQ(s.subcode(), Status::SubCode::kDeadlock);
+
+  std::vector<DeadlockPath> deadlock_paths = locker_->GetDeadlockInfoBuffer();
+  ASSERT_EQ(deadlock_paths.size(), 1u);
+  ASSERT_FALSE(deadlock_paths[0].limit_exceeded);
+
+  std::vector<DeadlockInfo> deadlocks = deadlock_paths[0].path;
+  ASSERT_EQ(deadlocks.size(), 2u);
+
+  ASSERT_EQ(deadlocks[0].m_txn_id, txn1->GetID());
+  ASSERT_EQ(deadlocks[0].m_cf_id, 1u);
+  ASSERT_TRUE(deadlocks[0].m_exclusive);
+  ASSERT_EQ(deadlocks[0].m_waiting_key, "k1");
+
+  ASSERT_EQ(deadlocks[1].m_txn_id, txn2->GetID());
+  ASSERT_EQ(deadlocks[1].m_cf_id, 1u);
+  ASSERT_TRUE(deadlocks[1].m_exclusive);
+  ASSERT_EQ(deadlocks[1].m_waiting_key, "k1");
+
+  locker_->UnLock(txn2, 1, "k1", env_);
+  t.join();
+
+  // Cleanup
+  locker_->UnLock(txn1, 1, "k1", env_);
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest, Deadlock_MultipleUpgradeInterleaveExclusive) {
+  // Tests that deadlock can be detected for shared locks and exclusive locks
+  // mixed Deadlock scenario:
+
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires shared lock on k1.
+  // txn3 acquires exclusive lock on k1.
+  // txn1 acquires exclusive locks k1 <- request granted after txn2 release the
+  // lock.
+  // txn2 acquires exclusive locks k1 <- dead lock detected
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+  auto txn3 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+  ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+
+  // txn3 tries to lock k1 exclusively, will be blocked.
+  port::Thread txn3_thread;
+  BlockUntilWaitingTxn(wait_sync_point_name_, txn3_thread, [this, &txn3]() {
+    // block because txn1 and txn2 are holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, true));
+  });
+  // Verify txn3 is blocked
+  ASSERT_TRUE(txn3_thread.joinable());
+
+  // txn1 tries to lock k1 exclusively, will be blocked.
+  port::Thread txn1_thread;
+  BlockUntilWaitingTxn(wait_sync_point_name_, txn1_thread, [this, &txn1]() {
+    // block because txn1 and txn2 are holding a shared lock on k1.
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+  });
+  // Verify txn1 is blocked
+  ASSERT_TRUE(txn1_thread.joinable());
+
+  auto s = locker_->TryLock(txn2, 1, "k1", env_, true);
+  ASSERT_TRUE(s.IsBusy());
+  ASSERT_EQ(s.subcode(), Status::SubCode::kDeadlock);
+
+  std::vector<DeadlockPath> deadlock_paths = locker_->GetDeadlockInfoBuffer();
+  ASSERT_EQ(deadlock_paths.size(), 1u);
+  ASSERT_FALSE(deadlock_paths[0].limit_exceeded);
+
+  std::vector<DeadlockInfo> deadlocks = deadlock_paths[0].path;
+  ASSERT_EQ(deadlocks.size(), 2u);
+
+  ASSERT_EQ(deadlocks[0].m_txn_id, txn1->GetID());
+  ASSERT_EQ(deadlocks[0].m_cf_id, 1u);
+  ASSERT_TRUE(deadlocks[0].m_exclusive);
+  ASSERT_EQ(deadlocks[0].m_waiting_key, "k1");
+
+  ASSERT_EQ(deadlocks[1].m_txn_id, txn2->GetID());
+  ASSERT_EQ(deadlocks[1].m_cf_id, 1u);
+  ASSERT_TRUE(deadlocks[1].m_exclusive);
+  ASSERT_EQ(deadlocks[1].m_waiting_key, "k1");
+
+  // Unlock txn2, so txn1 could proceed
+  locker_->UnLock(txn2, 1, "k1", env_);
+  txn1_thread.join();
+
+  // Unlock txn1, so txn3 could proceed
+  locker_->UnLock(txn1, 1, "k1", env_);
+  txn3_thread.join();
+
+  // Cleanup
+  locker_->UnLock(txn3, 1, "k1", env_);
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+class PerKeyPointLockManagerTest : public PointLockManagerTest {
+ public:
+  void SetUp() override {
+    init();
+    cf_ = std::make_unique<MockColumnFamilyHandle>(1);
+    txn_opt_.deadlock_detect = true;
+    // by default use long timeout and disable expiration
+    txn_opt_.lock_timeout = kLongTxnTimeoutMs;
+    txn_opt_.expiration = -1;
+
+    // CAUTION: This test creates a separate lock manager object (right, NOT
+    // the one that the TransactionDB is using!), and runs tests on it.
+    locker_.reset(new PerKeyPointLockManager(
+        static_cast<PessimisticTransactionDB*>(db_), txndb_opt_));
+    locker_->AddColumnFamily(cf_.get());
+  }
+
+  TransactionOptions txn_opt_;
+  std::unique_ptr<MockColumnFamilyHandle> cf_;
+};
+
+TEST_F(PerKeyPointLockManagerTest, LockEfficiency) {
+  // Create multiple transactions, each acquire exclusive lock on the same key
+  std::vector<PessimisticTransaction*> txns;
+  std::vector<port::Thread> blockingThreads;
+
+  // Count the total number of wait sync point calls
+  std::atomic_int wait_sync_point_times = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_,
+      [&wait_sync_point_times](void* /*arg*/) { wait_sync_point_times++; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr auto num_of_txn = 10;
+  // create 10 transactions, each of them try to acquire exclusive lock on the
+  // same key
+  for (int i = 0; i < num_of_txn; i++) {
+    auto txn = NewTxn(txn_opt_);
+    txns.push_back(txn);
+
+    if (i == 0) {
+      // txn0 acquires the lock, so the rest of the transactions could block
+      ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, true));
+    } else {
+      blockingThreads.emplace_back([this, txn]() {
+        // block because first txn is holding an exclusive lock on k1.
+        ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, true));
+      });
+    }
+
+    // wait for transaction i to be blocked
+    while (wait_sync_point_times.load() < i) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  // unlock the key, so next transaction could take the lock.
+  locker_->UnLock(txns[0], 1, "k1", env_);
+
+  auto num_of_blocking_thread = num_of_txn - 1;
+
+  for (int i = 0; i < num_of_blocking_thread; i++) {
+    // validate the thread is finished
+    blockingThreads[i].join();
+    auto num_of_threads_completed = i + 1;
+    for (int j = 0; j < num_of_blocking_thread; j++) {
+      if (j < num_of_threads_completed) {
+        // validate the thread is no longer joinable
+        ASSERT_FALSE(blockingThreads[j].joinable());
+      } else {
+        // validate the rest of the threads are still joinable
+        ASSERT_TRUE(blockingThreads[j].joinable());
+      }
+    }
+    // unlock the key, so next transaction could take the lock.
+    locker_->UnLock(txns[i + 1], 1, "k1", env_);
+  }
+
+  ASSERT_EQ(wait_sync_point_times.load(), num_of_blocking_thread);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  for (int i = 0; i < num_of_txn; i++) {
+    delete txns[num_of_txn - i - 1];
+  }
+}
+
+TEST_F(PerKeyPointLockManagerTest, LockFairness) {
+  // Create multiple transactions requesting locks on the same key, validate
+  // that they are executed in FIFO order
+
+  // txn0 acquires exclusive lock on k1.
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires shared lock on k1.
+  // txn3 acquires exclusive lock on k1.
+  // txn4 acquires shared lock on k1.
+  // txn5 acquires exclusive lock on k1.
+  // txn6 acquires exclusive lock on k1.
+  // txn7 acquires shared lock on k1.
+  // txn8 acquires shared lock on k1.
+  // txn9 acquires exclusive lock on k1.
+
+  std::vector<PessimisticTransaction*> txns;
+  std::vector<port::Thread> blockingThreads;
+
+  // Count the total number of wait sync point calls
+  std::atomic_int wait_sync_point_times = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_,
+      [&wait_sync_point_times](void* /*arg*/) { wait_sync_point_times++; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr auto num_of_txn = 10;
+  std::vector<bool> txn_lock_types = {true, false, false, true,  false,
+                                      true, true,  false, false, true};
+  // create 10 transactions, each of them try to acquire exclusive lock on the
+  // same key
+  for (int i = 0; i < num_of_txn; i++) {
+    auto txn = NewTxn(txn_opt_);
+    txns.push_back(txn);
+
+    if (i == 0) {
+      // txn0 acquires the lock, so the rest of the transactions would block
+      ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, txn_lock_types[0]));
+    } else {
+      blockingThreads.emplace_back([this, txn, type = txn_lock_types[i]]() {
+        ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, type));
+      });
+    }
+
+    // wait for transaction i to be blocked
+    while (wait_sync_point_times.load() < i) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  auto num_of_blocking_thread = num_of_txn - 1;
+
+  auto thread_idx = 0;
+  auto txn_idx = 0;
+
+  auto unlockTxn = [&]() {
+    // unlock the key in transaction.
+    locker_->UnLock(txns[txn_idx++], 1, "k1", env_);
+  };
+
+  auto validateLockTakenByNextTxn = [&]() {
+    // validate the thread is finished
+    blockingThreads[thread_idx++].join();
+  };
+
+  auto stillWaitingForLock = [&]() {
+    // validate the thread is no longer joinable
+    ASSERT_TRUE(blockingThreads[thread_idx].joinable());
+  };
+
+  // unlock the key, so next group of transactions could take the lock.
+  unlockTxn();
+
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires shared lock on k1.
+  validateLockTakenByNextTxn();
+  validateLockTakenByNextTxn();
+
+  // txn3 acquires exclusive lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // txn4 acquires shared lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // txn5 acquires exclusive lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // txn6 acquires exclusive lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // txn7 acquires shared lock on k1.
+  // txn8 acquires shared lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+  validateLockTakenByNextTxn();
+
+  // txn9 acquires exclusive lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // clean up
+  unlockTxn();
+
+  ASSERT_EQ(wait_sync_point_times.load(), num_of_blocking_thread);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  for (int i = 0; i < num_of_txn; i++) {
+    delete txns[num_of_txn - i - 1];
+  }
+}
+
+TEST_F(PerKeyPointLockManagerTest, FIFO) {
+  // validate S, X, S lock order would be executed in FIFO order
+  // txn1 acquires shared lock on k1.
+  // txn2 acquires exclusive lock on k1.
+  // txn3 acquires shared lock on k1.
+
+  std::vector<PessimisticTransaction*> txns;
+  std::vector<port::Thread> blockingThreads;
+
+  // Count the total number of wait sync point calls
+  std::atomic_int wait_sync_point_times = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_,
+      [&wait_sync_point_times](void* /*arg*/) { wait_sync_point_times++; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr auto num_of_txn = 3;
+  std::vector<bool> txn_lock_types = {false, true, false};
+  // create 3 transactions, each of them try to acquire exclusive lock on the
+  // same key
+  for (int i = 0; i < num_of_txn; i++) {
+    auto txn = NewTxn(txn_opt_);
+    txns.push_back(txn);
+
+    if (i == 0) {
+      // txn0 acquires the lock, so the rest of the transactions would block
+      ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, txn_lock_types[0]));
+    } else {
+      blockingThreads.emplace_back([this, txn, type = txn_lock_types[i]]() {
+        ASSERT_OK(locker_->TryLock(txn, 1, "k1", env_, type));
+      });
+    }
+
+    // wait for transaction i to be blocked
+    while (wait_sync_point_times.load() < i) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+  }
+
+  auto num_of_blocking_thread = num_of_txn - 1;
+
+  auto thread_idx = 0;
+  auto txn_idx = 0;
+
+  auto unlockTxn = [&]() {
+    // unlock the key in transaction.
+    locker_->UnLock(txns[txn_idx++], 1, "k1", env_);
+  };
+
+  auto validateLockTakenByNextTxn = [&]() {
+    // validate the thread is finished
+    blockingThreads[thread_idx++].join();
+  };
+
+  auto stillWaitingForLock = [&]() {
+    // validate the thread is no longer joinable
+    ASSERT_TRUE(blockingThreads[thread_idx].joinable());
+  };
+
+  // unlock the key, so next group of transactions could take the lock.
+  stillWaitingForLock();
+  unlockTxn();
+
+  // txn1 acquires exclusive lock on k1.
+  validateLockTakenByNextTxn();
+
+  // txn2 acquires shared lock on k1.
+  stillWaitingForLock();
+  unlockTxn();
+  validateLockTakenByNextTxn();
+
+  // clean up
+  unlockTxn();
+
+  ASSERT_EQ(wait_sync_point_times.load(), num_of_blocking_thread);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  for (int i = 0; i < num_of_txn; i++) {
+    delete txns[num_of_txn - i - 1];
+  }
+}
+
+TEST_P(SpotLockManagerTest, LockDownGradeWithOtherLockRequests) {
+  // Test lock down grade always succeeds, even if there are other lock requests
+  // waiting for the same lock.
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  for (bool exclusive : {true, false}) {
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+    port::Thread t;
+    BlockUntilWaitingTxn(wait_sync_point_name_, t, [this, &txn2, exclusive]() {
+      // block because txn1 is holding a exclusive lock on k1.
+      ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, exclusive));
+    });
+
+    // txn1 downgrades the lock to shared lock, so txn2 could proceed
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+
+    locker_->UnLock(txn1, 1, "k1", env_);
+    t.join();
+    locker_->UnLock(txn2, 1, "k1", env_);
+  }
+
+  // clean up
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest, LockTimeout) {
+  // Test lock timeout
+  // txn1 acquires an exclusive lock on k1 successfully.
+  // txn2 try to acquire a lock on k1, but timedout.
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kShortTxnTimeoutMs;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  for (bool exclusive : {true, false}) {
+    auto ret = locker_->TryLock(txn2, 1, "k1", env_, exclusive);
+    ASSERT_TRUE(ret.IsTimedOut());
+  }
+
+  // clean up
+  locker_->UnLock(txn1, 1, "k1", env_);
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest, ExpiredLockStolenAfterTimeout) {
+  // validate an expired lock can be stolen by another transaction that timed
+  // out on the lock.
+  // txn1 acquires an exclusive lock on k1 successfully with a short expiration
+  // time.
+  // txn2 try to acquire a shared lock on k1 with timeout that is slightly
+  // longer than the txn1 expiration.
+  // Validate txn2 will take the lock.
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.expiration = 1000;
+  txn_opt.lock_timeout = 1000 * 2;
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  port::Thread t1;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t1, [this, &txn2]() {
+    // block because txn1 is holding an exclusive lock on k1.
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+  });
+
+  t1.join();
+
+  // clean up
+  locker_->UnLock(txn2, 1, "k1", env_);
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  delete txn2;
+  delete txn1;
+}
+
+// Try to block until transaction enters waiting state.
+// However due to timing, it could fail, so return true if succeeded, false
+// otherwise.
+bool TryBlockUntilWaitingTxn(const char* sync_point_name, port::Thread& t,
+                             std::function<void()> function) {
+  std::atomic<bool> reached(false);
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      sync_point_name, [&](void* /*arg*/) { reached.store(true); });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  // As the lifetime of the complete variable could go beyond the scope of this
+  // function, so we wrap it in a shared_ptr, and copy it into the lambda
+  std::shared_ptr<std::atomic<bool>> complete =
+      std::make_shared<std::atomic<bool>>(false);
+  t = port::Thread([complete, &function]() {
+    function();
+    complete->store(true);
+  });
+
+  auto ret = false;
+
+  while (true) {
+    if (complete->load()) {
+      // function completed, before sync point was reached, return false
+      t.join();
+      ret = false;
+      break;
+    }
+    if (reached.load()) {
+      // sync point was reached before function completed, return true
+      ret = true;
+      break;
+    }
+  }
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+  return ret;
+}
+
+TEST_F(PerKeyPointLockManagerTest, LockStealAfterExpirationExclusive) {
+  // There are multiple transactions waiting for the same lock.
+  // txn1 acquires an exclusive lock on k1 successfully with a short expiration
+  // time.
+  // txn2 try to acquire an exclusive lock on k1, before expiration time,
+  // so it is blocked and waits for txn1 lock expired.
+  // txn3 try to acquire an exclusive lock on k1 after txn1 lock expires, FIFO
+  // order is respected.
+  // txn2 is woken up and takes the lock. unlock txn2, txn3 should proceed.
+
+  txn_opt_.expiration = 1000;
+  auto txn1 = NewTxn(txn_opt_);
+  txn_opt_.expiration = -1;
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+
+  port::Thread t1;
+  auto retry_times = 10;
+
+  // Use a loop to reduce test flakiness.
+  // that the test is flaky because the txn2 thread start could be delayed until
+  // txn1 lock expired. In that case, txn2 will not enter into wait state, which
+  // will defeat the test purpose. Use a loop to retry a few times, until it is
+  // able to enter into wait state.
+  while (retry_times--) {
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+    if (TryBlockUntilWaitingTxn(wait_sync_point_name_, t1, [this, &txn2]() {
+          // block because txn1 is holding a shared lock on k1.
+          ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+        })) {
+      break;
+    }
+    // failed, retry again
+    locker_->UnLock(txn1, 1, "k1", env_);
+    locker_->UnLock(txn2, 1, "k1", env_);
+  }
+  // make sure txn2 is able to reach the wait state before proceed
+  ASSERT_GT(retry_times, 0);
+
+  // txn3 try to acquire an exclusive lock on k1, FIFO order is respected.
+  port::Thread t2;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t2, [this, &txn3]() {
+    // block because txn1 is holding an exclusive lock on k1.
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, true));
+  });
+
+  // validate txn2 is woken up and takes the lock
+  t1.join();
+
+  // unlock txn2, txn3 should proceed
+  locker_->UnLock(txn2, 1, "k1", env_);
+  t2.join();
+
+  // clean up
+  locker_->UnLock(txn3, 1, "k1", env_);
+
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, LockStealAfterExpirationShared) {
+  // There are multiple transactions waiting for the same lock.
+  // txn1 acquires a shared lock on k1 successfully with a short expiration
+  // time.
+  // txn2 try to acquire an exclusive lock on k1, before expiration time,
+  // so it is blocked and waits for txn1 lock expired.
+  // txn3 try to acquire a shared lock on k1 after txn1 lock expires, FIFO
+  // order is respected.
+  // txn2 is woken up and takes the lock. unlock txn2, txn3 should proceed.
+
+  txn_opt_.expiration = 1000;
+  auto txn1 = NewTxn(txn_opt_);
+  txn_opt_.expiration = -1;
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+
+  port::Thread t1;
+  auto retry_times = 10;
+
+  // Use a loop to reduce test flakiness.
+  // that the test is flaky because the txn2 thread start could be delayed until
+  // txn1 lock expired. In that case, txn2 will not enter into wait state, which
+  // will defeat the test purpose. Use a loop to retry a few times, until it is
+  // able to enter into wait state.
+  while (retry_times--) {
+    ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+    if (TryBlockUntilWaitingTxn(wait_sync_point_name_, t1, [this, &txn2]() {
+          // block because txn1 is holding an exclusive lock on k1.
+          ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+        })) {
+      break;
+    }
+    // failed, retry again
+    locker_->UnLock(txn1, 1, "k1", env_);
+    locker_->UnLock(txn2, 1, "k1", env_);
+  }
+  // make sure txn2 is able to reach the wait state before proceed
+  ASSERT_GT(retry_times, 0);
+
+  // txn3 try to acquire an exclusive lock on k1, FIFO order is respected.
+  port::Thread t2;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t2, [this, &txn3]() {
+    // block because txn1 is holding an exclusive lock on k1.
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, false));
+  });
+
+  // validate txn2 is woken up and takes the lock
+  t1.join();
+
+  // unlock txn2, txn3 should proceed
+  locker_->UnLock(txn2, 1, "k1", env_);
+  t2.join();
+
+  // clean up
+  locker_->UnLock(txn3, 1, "k1", env_);
+
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, DeadLockOnWaiter) {
+  // Txn1 acquires exclusive lock on k1
+  // Txn3 acquires shared lock on k2
+  // Txn2 tries to acquire exclusive lock on k1, waiting in the waiter queue.
+  // Txn3 tries to acquire exclusive lock on k1, waiting in the waiter queue.
+  // Txn3 depends on both Txn1 and Txn2. Txn1 unlocks k1.
+  // Txn2 takes the lock k1, and tries to acquire lock k2.
+  // Now Txn2 depends on Txn3.
+  // Deadlock is detected, and Txn2 is aborted.
+
+  auto txn1 = NewTxn(txn_opt_);
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+  ASSERT_OK(locker_->TryLock(txn3, 1, "k2", env_, false));
+
+  port::Thread t1;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t1, [this, &txn2]() {
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+    auto s = locker_->TryLock(txn2, 1, "k2", env_, true);
+    ASSERT_TRUE(s.IsDeadlock());
+  });
+
+  port::Thread t2;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t2, [this, &txn3]() {
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, true));
+  });
+
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  t1.join();
+
+  locker_->UnLock(txn2, 1, "k1", env_);
+  t2.join();
+
+  // clean up
+  locker_->UnLock(txn3, 1, "k1", env_);
+  locker_->UnLock(txn3, 1, "k2", env_);
+
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, SharedLockRaceCondition) {
+  // Verify a shared lock race condition is handled properly.
+  // When there are waiters in the queue, and all of them are shared waiters,
+  // and no one has taken the lock and all of them just got woken up and not
+  // yet taken the lock yet. A new shared lock request should be granted
+  // directly, without wait in the queue. If it did, It would not be woken up
+  // until the last shared lock is released.
+
+  // Disable deadlock detection timeout to prevent test flakyness.
+  deadlock_timeout_us = 0;
+  auto txn1 = NewTxn(txn_opt_);
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"PerKeyPointLockManager::AcquireWithTimeout:AfterWokenUp",
+        "PerKeyPointLockManagerTest::SharedLockRaceCondition:"
+        "BeforeNewSharedLockRequest"},
+       {"PerKeyPointLockManagerTest::SharedLockRaceCondition:"
+        "AfterNewSharedLockRequest",
+        "PerKeyPointLockManager::AcquireWithTimeout:BeforeTakeLock"}});
+
+  std::atomic<bool> reached(false);
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_,
+      [&reached](void* /*arg*/) { reached.store(true); });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // txn1 acquires an exclusive lock on k1, so that the following shared lock
+  // request would be blocked
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  // txn2 try to acquire a shared lock on k1, and get blocked
+  auto t1 = port::Thread([this, &txn2]() {
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+  });
+
+  while (!reached.load()) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  }
+
+  // unlock txn1, txn2 should be woken up, but txn2 stops on the sync point
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  // Use sync point to simulate the race condition.
+  // txn3 tries to take the lock right after txn2 is woken up, but before it
+  // takes the lock
+  TEST_SYNC_POINT(
+      "PerKeyPointLockManagerTest::SharedLockRaceCondition:"
+      "BeforeNewSharedLockRequest");
+
+  // txn3 try to acquire a shared lock on k1, and get granted immediately
+  ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, false));
+
+  TEST_SYNC_POINT(
+      "PerKeyPointLockManagerTest::SharedLockRaceCondition:"
+      "AfterNewSharedLockRequest");
+
+  // validate txn2 is woken up and takes the lock
+  t1.join();
+
+  // cleanup
+  locker_->UnLock(txn2, 1, "k1", env_);
+  locker_->UnLock(txn3, 1, "k1", env_);
+
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, UpgradeLockRaceCondition) {
+  // Verify an upgrade lock race condition is handled properly.
+  // When a key is locked in exlusive mode, shared lock waiters will be enqueued
+  // as waiters.
+  // When the exclusive lock holder release the lock. The shared lock waiters
+  // are woken up to take the lock. At this point, when a new shared lock
+  // requester comes in, it will take the lock directly without waiting or
+  // queueing. This requester then immediately upgrade the lock to exclusive
+  // lock. This request will be prioritized to the head of the queue.
+  // Meantime, it should also depend on the shared lock waiters which are still
+  // in the queue that are ready to take the lock. Later, when one of the reader
+  // lock want to also upgrade its lock, it will detect a dead lock and abort.
+
+  auto txn1 = NewTxn(txn_opt_);
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"PerKeyPointLockManager::AcquireWithTimeout:AfterWokenUp",
+        "PerKeyPointLockManagerTest::UpgradeLockRaceCondition:"
+        "BeforeNewSharedLockRequest"},
+       {"PerKeyPointLockManagerTest::UpgradeLockRaceCondition:"
+        "AfterNewSharedLockRequest",
+        "PerKeyPointLockManager::AcquireWithTimeout:BeforeTakeLock"}});
+
+  std::atomic<bool> reached(false);
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_,
+      [&reached](void* /*arg*/) { reached.store(true); });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // txn1 acquires an exclusive lock on k1, so that the following shared lock
+  // request would be blocked
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  auto t1 = port::Thread([this, &txn2]() {
+    // txn2 try to acquire a shared lock on k1, and get blocked
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+  });
+
+  while (!reached.load()) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  }
+
+  // unlock txn1, txn2 should be woken up, but txn2 stops on the sync point
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  // Use sync point to simulate the race condition.
+  // txn3 tries to take the lock right after txn2 is woken up, but before it
+  // takes the lock
+  TEST_SYNC_POINT(
+      "PerKeyPointLockManagerTest::UpgradeLockRaceCondition:"
+      "BeforeNewSharedLockRequest");
+
+  // txn3 try to acquire a shared lock on k1, and get granted immediately
+  ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, false));
+
+  // txn3 try to upgrade its lock to exclusive lock and get blocked.
+  reached = false;
+  auto t2 = port::Thread([this, &txn3]() {
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, true));
+  });
+
+  while (!reached.load()) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  }
+
+  TEST_SYNC_POINT(
+      "PerKeyPointLockManagerTest::UpgradeLockRaceCondition:"
+      "AfterNewSharedLockRequest");
+
+  // validate txn2 is woken up and takes the shared lock
+  t1.join();
+
+  // validate txn2 would get deadlock when it try to upgrade its lock to
+  // exclusive
+  auto s = locker_->TryLock(txn2, 1, "k1", env_, true);
+  ASSERT_TRUE(s.IsDeadlock());
+
+  // cleanup
+  locker_->UnLock(txn2, 1, "k1", env_);
+  t2.join();
+  locker_->UnLock(txn3, 1, "k1", env_);
+
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_P(SpotLockManagerTest, Catch22) {
+  // Benchmark the overhead of one transaction depends on another in a circle
+  // repeatedly
+
+  MockColumnFamilyHandle cf(1);
+  locker_->AddColumnFamily(&cf);
+  TransactionOptions txn_opt;
+  txn_opt.deadlock_detect = true;
+  txn_opt.lock_timeout = kLongTxnTimeoutMs;
+  txn_opt.expiration = kLongTxnTimeoutMs;
+
+  auto txn1 = NewTxn(txn_opt);
+  auto txn2 = NewTxn(txn_opt);
+
+  // use a wait count to count the number of times the lock is waited inside
+  // transaction lock
+  std::atomic_int wait_count(0);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  if (GetParam().use_per_key_point_lock_manager &&
+      GetParam().deadlock_timeout_us != 0) {
+    // Use special sync point when deadlock timeout is enabled, so the test run
+    // faster
+    SyncPoint::GetInstance()->SetCallBack(
+        "PerKeyPointLockManager::AcquireWithTimeout:"
+        "WaitingTxnBeforeDeadLockDetection",
+        [&wait_count](void* /*arg*/) { wait_count++; });
+  } else {
+    // PointLockManager
+    SyncPoint::GetInstance()->SetCallBack(
+        wait_sync_point_name_, [&wait_count](void* /*arg*/) { wait_count++; });
+  }
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // txn1 X lock
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  std::mutex coordinator_mutex;
+  int iteration_count = 10000;
+
+  // txn1 try to lock X lock in a loop
+  auto t1 = port::Thread(
+      [this, &txn1, &wait_count, &coordinator_mutex, &iteration_count]() {
+        while (wait_count.load() < iteration_count) {
+          // spin wait until the other thread enters the lock waiter queue.
+          while (wait_count.load() % 2 == 0);
+          // unlock the lock, so that the other thread can acquire the lock
+          locker_->UnLock(txn1, 1, "k1", env_);
+          {
+            // Use the coordinator mutex to make sure the other thread has been
+            // waked up and acquired the lock, before this thread try to acquire
+            // the lock again.
+            std::scoped_lock<std::mutex> lock(coordinator_mutex);
+            ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+          }
+        }
+        locker_->UnLock(txn1, 1, "k1", env_);
+      });
+
+  // txn2 try to lock X lock in a loop
+  auto t2 = port::Thread(
+      [this, &txn2, &wait_count, &coordinator_mutex, &iteration_count]() {
+        while (wait_count.load() < iteration_count) {
+          {
+            // Use the coordinator mutex to make sure the other thread has been
+            // waked up and acquired the lock, before this thread try to acquire
+            // the lock again.
+            std::scoped_lock<std::mutex> lock(coordinator_mutex);
+            ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+          }
+          // spin wait until the other thread enters the lock waiter queue.
+          while (wait_count.load() % 2 == 1);
+          // unlock the lock, so that the other thread can acquire the lock
+          locker_->UnLock(txn2, 1, "k1", env_);
+        }
+      });
+
+  // clean up
+  t1.join();
+  t2.join();
+
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, LockUpgradeOrdering) {
+  // When lock is upgraded, verify that it will only upgrade its lock after all
+  // the shared lock that are before the first exclusive lock in the lock wait
+  // queue.
+
+  auto txn1 = NewTxn(txn_opt_);
+  auto txn2 = NewTxn(txn_opt_);
+  auto txn3 = NewTxn(txn_opt_);
+  auto txn4 = NewTxn(txn_opt_);
+
+  std::mutex txn4_mutex;
+  std::unique_lock<std::mutex> txn4_lock(txn4_mutex);
+  std::atomic_bool txn4_waked_up(false);
+  std::atomic_int wait_count(0);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->SetCallBack(
+      wait_sync_point_name_, [&wait_count](void* /*arg*/) { wait_count++; });
+  SyncPoint::GetInstance()->SetCallBack(
+      "PerKeyPointLockManager::AcquireWithTimeout:AfterWokenUp",
+      [&txn4, &txn4_mutex, &txn4_waked_up](void* arg) {
+        auto transaction_id = *(static_cast<TransactionID*>(arg));
+        if (transaction_id == txn4->GetID()) {
+          txn4_waked_up.store(true);
+          {
+            // wait for txn4 mutex to be released, so that this thread will be
+            // blocked.
+            std::scoped_lock<std::mutex> lock(txn4_mutex);
+          }
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Txn1 X lock
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  // Txn2,3,4 try S lock
+  port::Thread t1([this, &txn2]() {
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+  });
+  port::Thread t2([this, &txn3]() {
+    ASSERT_OK(locker_->TryLock(txn3, 1, "k1", env_, false));
+  });
+  port::Thread t3([this, &txn4]() {
+    ASSERT_OK(locker_->TryLock(txn4, 1, "k1", env_, false));
+  });
+
+  // wait for all 3 transactions to enter wait state
+  while (wait_count.load() < 3) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+  }
+
+  // Txn1 unlock
+  locker_->UnLock(txn1, 1, "k1", env_);
+
+  // Txn2,3 take S lock
+  t1.join();
+  t2.join();
+
+  // wait for txn4 to be woken up, otherwise txn2 will get deadlock
+  while (!txn4_waked_up.load()) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+  }
+
+  // Txn2 try X lock
+  std::atomic_bool txn2_exclusive_lock_acquired(false);
+  port::Thread t4([this, &txn2, &txn2_exclusive_lock_acquired]() {
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, true));
+    txn2_exclusive_lock_acquired.store(true);
+  });
+
+  // wait for txn2 to enter wait state
+  while (wait_count.load() < 4) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+  }
+
+  // Txn3 release S lock
+  locker_->UnLock(txn3, 1, "k1", env_);
+
+  // Validate Txn2 has not acquired the lock yet
+  ASSERT_FALSE(txn2_exclusive_lock_acquired.load());
+
+  // Txn4 take S lock
+  txn4_lock.unlock();
+  t3.join();
+
+  // Txn4 release S lock Txn2 upgraded to X lock Txn2
+  locker_->UnLock(txn4, 1, "k1", env_);
+  t4.join();
+  ASSERT_TRUE(txn2_exclusive_lock_acquired.load());
+
+  // release lock clean up
+  locker_->UnLock(txn2, 1, "k1", env_);
+
+  delete txn4;
+  delete txn3;
+  delete txn2;
+  delete txn1;
+}
+
+TEST_F(PerKeyPointLockManagerTest, LockDownGradeRaceCondition) {
+  // When a lock is downgraded, it should notify all the shared waiters in the
+  // queue to take the lock.
+
+  auto txn1 = NewTxn(txn_opt_);
+  auto txn2 = NewTxn(txn_opt_);
+
+  // Txn1 X lock
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, true));
+
+  // Txn2 try S lock
+  port::Thread t1;
+  BlockUntilWaitingTxn(wait_sync_point_name_, t1, [this, &txn2]() {
+    ASSERT_OK(locker_->TryLock(txn2, 1, "k1", env_, false));
+  });
+
+  // Txn1 downgrade to S lock
+  ASSERT_OK(locker_->TryLock(txn1, 1, "k1", env_, false));
+
+  // Txn2 take S lock
+  t1.join();
+
+  // clean up
+  locker_->UnLock(txn1, 1, "k1", env_);
+  locker_->UnLock(txn2, 1, "k1", env_);
+
+  delete txn2;
+  delete txn1;
+}
+
+// Run AnyLockManagerTest with PointLockManager
 INSTANTIATE_TEST_CASE_P(PointLockManager, AnyLockManagerTest,
                         ::testing::Values(nullptr));
 
+// Run AnyLockManagerTest with PerKeyPointLockManager
+template <int64_t N>
+void PerKeyPointLockManagerTestSetup(PointLockManagerTest* self) {
+  self->init();
+  self->deadlock_timeout_us = N;
+  self->UsePerKeyPointLockManager();
+}
+
+INSTANTIATE_TEST_CASE_P(
+    PerLockPointLockManager, AnyLockManagerTest,
+    ::testing::Values(PerKeyPointLockManagerTestSetup<0>,
+                      PerKeyPointLockManagerTestSetup<100>,
+                      PerKeyPointLockManagerTestSetup<1000>));
+
+// Run PointLockManagerTest with PerLockPointLockManager and PointLockManager
+INSTANTIATE_TEST_CASE_P(
+    PointLockCorrectnessCheckTestSuite, SpotLockManagerTest,
+    ::testing::ValuesIn(std::vector<SpotLockManagerTestParam>{
+        {true, 0}, {true, 100}, {true, 1000}, {false, 0}}));
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {