@nxtedition/rocksdb 7.1.20 → 7.1.21

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h

@@ -9,6 +9,7 @@
 
 #ifdef GFLAGS
 #pragma once
+
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_shared_state.h"
 
@@ -34,13 +35,10 @@ class StressTest {
   // The initialization work is split into two parts to avoid a circular
   // dependency with `SharedState`.
   virtual void FinishInitDb(SharedState*);
-
   void TrackExpectedState(SharedState* shared);
-
   void OperateDb(ThreadState* thread);
   virtual void VerifyDb(ThreadState* thread) const = 0;
   virtual void ContinuouslyVerifyDb(ThreadState* /*thread*/) const = 0;
-
   void PrintStatistics();
 
  protected:
@@ -54,6 +52,18 @@ class StressTest {
   Status SetOptions(ThreadState* thread);
 
 #ifndef ROCKSDB_LITE
+  // For transactionsDB, there can be txns prepared but not yet committeed
+  // right before previous stress run crash.
+  // They will be recovered and processed through
+  // ProcessRecoveredPreparedTxnsHelper on the start of current stress run.
+  void ProcessRecoveredPreparedTxns(SharedState* shared);
+
+  // Default implementation will first update ExpectedState to be
+  // `SharedState::UNKNOWN` for each keys in `txn` and then randomly
+  // commit or rollback `txn`.
+  virtual void ProcessRecoveredPreparedTxnsHelper(Transaction* txn,
+                                                  SharedState* shared);
+
   Status NewTxn(WriteOptions& write_opts, Transaction** txn);
 
   Status CommitTxn(Transaction* txn, ThreadState* thread = nullptr);
@@ -129,6 +139,8 @@ class StressTest {
     return column_families_[column_family_id];
   }
 
+  virtual Slice GetValueBaseSlice(Slice slice) { return slice; }
+
 #ifndef ROCKSDB_LITE
   // Generated a list of keys that close to boundaries of SST keys.
   // If there isn't any SST file in the DB, return empty list.
@@ -222,6 +234,13 @@ class StressTest {
                          int64_t key, Slice value_from_db,
                          Slice value_from_expected) const;
 
+  void VerificationAbort(SharedState* shared, int cf, int64_t key,
+                         const Slice& value, const WideColumns& columns,
+                         const WideColumns& expected_columns) const;
+
+  static std::string DebugString(const Slice& value, const WideColumns& columns,
+                                 const WideColumns& expected_columns);
+
   void PrintEnv() const;
 
   void Open(SharedState* shared);
@@ -235,7 +254,8 @@ class StressTest {
                                    TransactionDBOptions& /*txn_db_opts*/) {}
 #endif
 
-  void MaybeUseOlderTimestampForPointLookup(ThreadState* thread,
+  // Returns whether the timestamp of read_opts is updated.
+  bool MaybeUseOlderTimestampForPointLookup(ThreadState* thread,
                                             std::string& ts_str,
                                             Slice& ts_slice,
                                             ReadOptions& read_opts);

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc

@@ -286,6 +286,24 @@ int db_stress_tool(int argc, char** argv) {
     exit(1);
   }
 
+  if (FLAGS_use_txn && FLAGS_sync_fault_injection &&
+      FLAGS_txn_write_policy != 0) {
+    fprintf(stderr,
+            "For TransactionDB, correctness testing with unsync data loss is "
+            "currently compatible with only write committed policy\n");
+    exit(1);
+  }
+
+  if (FLAGS_use_put_entity_one_in > 0 &&
+      (FLAGS_ingest_external_file_one_in > 0 || FLAGS_use_merge ||
+       FLAGS_use_full_merge_v1 || FLAGS_use_txn || FLAGS_test_multi_ops_txns ||
+       FLAGS_user_timestamp_size > 0)) {
+    fprintf(stderr,
+            "PutEntity is currently incompatible with SstFileWriter, Merge,"
+            " transactions, and user-defined timestamps\n");
+    exit(1);
+  }
+
 #ifndef NDEBUG
   KillPoint* kp = KillPoint::GetInstance();
   kp->rocksdb_kill_odds = FLAGS_kill_random_test;

package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc

@@ -7,6 +7,7 @@
 
 #include "db_stress_tool/expected_state.h"
 
+#include "db/wide/wide_column_serialization.h"
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_shared_state.h"
 #include "rocksdb/trace_reader_writer.h"
@@ -343,7 +344,9 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
                                         public WriteBatch::Handler {
  public:
   ExpectedStateTraceRecordHandler(uint64_t max_write_ops, ExpectedState* state)
-      : max_write_ops_(max_write_ops), state_(state) {}
+      : max_write_ops_(max_write_ops),
+        state_(state),
+        buffered_writes_(nullptr) {}
 
   ~ExpectedStateTraceRecordHandler() { assert(IsDone()); }
 
@@ -391,12 +394,62 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
     }
     uint32_t value_id = GetValueBase(value);
 
+    bool should_buffer_write = !(buffered_writes_ == nullptr);
+    if (should_buffer_write) {
+      return WriteBatchInternal::Put(buffered_writes_.get(), column_family_id,
+                                     key, value);
+    }
+
     state_->Put(column_family_id, static_cast<int64_t>(key_id), value_id,
                 false /* pending */);
     ++num_write_ops_;
     return Status::OK();
   }
 
+  Status PutEntityCF(uint32_t column_family_id, const Slice& key_with_ts,
+                     const Slice& entity) override {
+    Slice key =
+        StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size);
+
+    uint64_t key_id = 0;
+    if (!GetIntVal(key.ToString(), &key_id)) {
+      return Status::Corruption("Unable to parse key", key.ToString());
+    }
+
+    Slice entity_copy = entity;
+    WideColumns columns;
+    if (!WideColumnSerialization::Deserialize(entity_copy, columns).ok()) {
+      return Status::Corruption("Unable to deserialize entity",
+                                entity.ToString(/* hex */ true));
+    }
+
+    if (columns.empty() || columns[0].name() != kDefaultWideColumnName) {
+      return Status::Corruption("Cannot find default column in entity",
+                                entity.ToString(/* hex */ true));
+    }
+
+    const Slice& value_of_default = columns[0].value();
+
+    const uint32_t value_base = GetValueBase(value_of_default);
+
+    if (columns != GenerateExpectedWideColumns(value_base, value_of_default)) {
+      return Status::Corruption("Wide columns in entity inconsistent",
+                                entity.ToString(/* hex */ true));
+    }
+
+    if (buffered_writes_) {
+      return WriteBatchInternal::PutEntity(buffered_writes_.get(),
+                                           column_family_id, key, columns);
+    }
+
+    state_->Put(column_family_id, static_cast<int64_t>(key_id), value_base,
+                false /* pending */);
+
+    ++num_write_ops_;
+
+    return Status::OK();
+  }
+
   Status DeleteCF(uint32_t column_family_id,
                   const Slice& key_with_ts) override {
     Slice key =
@@ -406,6 +459,12 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
       return Status::Corruption("unable to parse key", key.ToString());
     }
 
+    bool should_buffer_write = !(buffered_writes_ == nullptr);
+    if (should_buffer_write) {
+      return WriteBatchInternal::Delete(buffered_writes_.get(),
+                                        column_family_id, key);
+    }
+
     state_->Delete(column_family_id, static_cast<int64_t>(key_id),
                    false /* pending */);
     ++num_write_ops_;
@@ -414,6 +473,18 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
 
   Status SingleDeleteCF(uint32_t column_family_id,
                         const Slice& key_with_ts) override {
+    bool should_buffer_write = !(buffered_writes_ == nullptr);
+    if (should_buffer_write) {
+      Slice key =
+          StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size);
+      Slice ts =
+          ExtractTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size);
+      std::array<Slice, 2> key_with_ts_arr{{key, ts}};
+      return WriteBatchInternal::SingleDelete(
+          buffered_writes_.get(), column_family_id,
+          SliceParts(key_with_ts_arr.data(), 2));
+    }
+
     return DeleteCF(column_family_id, key_with_ts);
   }
 
@@ -433,6 +504,12 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
       return Status::Corruption("unable to parse end key", end_key.ToString());
     }
 
+    bool should_buffer_write = !(buffered_writes_ == nullptr);
+    if (should_buffer_write) {
+      return WriteBatchInternal::DeleteRange(
+          buffered_writes_.get(), column_family_id, begin_key, end_key);
+    }
+
     state_->DeleteRange(column_family_id, static_cast<int64_t>(begin_key_id),
                         static_cast<int64_t>(end_key_id), false /* pending */);
     ++num_write_ops_;
@@ -443,13 +520,64 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
                  const Slice& value) override {
     Slice key =
         StripTimestampFromUserKey(key_with_ts, FLAGS_user_timestamp_size);
+
+    bool should_buffer_write = !(buffered_writes_ == nullptr);
+    if (should_buffer_write) {
+      return WriteBatchInternal::Merge(buffered_writes_.get(), column_family_id,
+                                       key, value);
+    }
+
     return PutCF(column_family_id, key, value);
   }
 
+  Status MarkBeginPrepare(bool = false) override {
+    assert(!buffered_writes_);
+    buffered_writes_.reset(new WriteBatch());
+    return Status::OK();
+  }
+
+  Status MarkEndPrepare(const Slice& xid) override {
+    assert(buffered_writes_);
+    std::string xid_str = xid.ToString();
+    assert(xid_to_buffered_writes_.find(xid_str) ==
+           xid_to_buffered_writes_.end());
+
+    xid_to_buffered_writes_[xid_str].swap(buffered_writes_);
+
+    buffered_writes_.reset();
+
+    return Status::OK();
+  }
+
+  Status MarkCommit(const Slice& xid) override {
+    std::string xid_str = xid.ToString();
+    assert(xid_to_buffered_writes_.find(xid_str) !=
+           xid_to_buffered_writes_.end());
+    assert(xid_to_buffered_writes_.at(xid_str));
+
+    Status s = xid_to_buffered_writes_.at(xid_str)->Iterate(this);
+    xid_to_buffered_writes_.erase(xid_str);
+
+    return s;
+  }
+
+  Status MarkRollback(const Slice& xid) override {
+    std::string xid_str = xid.ToString();
+    assert(xid_to_buffered_writes_.find(xid_str) !=
+           xid_to_buffered_writes_.end());
+    assert(xid_to_buffered_writes_.at(xid_str));
+    xid_to_buffered_writes_.erase(xid_str);
+
+    return Status::OK();
+  }
+
  private:
   uint64_t num_write_ops_ = 0;
   uint64_t max_write_ops_;
   ExpectedState* state_;
+  std::unordered_map<std::string, std::unique_ptr<WriteBatch>>
+      xid_to_buffered_writes_;
+  std::unique_ptr<WriteBatch> buffered_writes_;
 };
 
 }  // anonymous namespace

package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc

@@ -329,6 +329,10 @@ void MultiOpsTxnsStressTest::FinishInitDb(SharedState* shared) {
   if (FLAGS_enable_compaction_filter) {
     // TODO (yanqin) enable compaction filter
   }
+#ifndef ROCKSDB_LITE
+  ProcessRecoveredPreparedTxns(shared);
+#endif
+
   ReopenAndPreloadDbIfNeeded(shared);
   // TODO (yanqin) parallelize if key space is large
   for (auto& key_gen : key_gen_for_a_) {
@@ -1353,6 +1357,18 @@ uint32_t MultiOpsTxnsStressTest::GenerateNextC(ThreadState* thread) {
 }
 
 #ifndef ROCKSDB_LITE
+void MultiOpsTxnsStressTest::ProcessRecoveredPreparedTxnsHelper(
+    Transaction* txn, SharedState*) {
+  thread_local Random rand(static_cast<uint32_t>(FLAGS_seed));
+  if (rand.OneIn(2)) {
+    Status s = txn->Commit();
+    assert(s.ok());
+  } else {
+    Status s = txn->Rollback();
+    assert(s.ok());
+  }
+}
+
 Status MultiOpsTxnsStressTest::WriteToCommitTimeWriteBatch(Transaction& txn) {
   WriteBatch* ctwb = txn.GetCommitTimeWriteBatch();
   assert(ctwb);
@@ -1776,6 +1792,12 @@ void CheckAndSetOptionsForMultiOpsTxnStressTest() {
       exit(1);
     }
   }
+  if (FLAGS_sync_fault_injection == 1) {
+    fprintf(stderr,
+            "Sync fault injection is currently not supported in "
+            "-test_multi_ops_txns\n");
+    exit(1);
+  }
 #else
   fprintf(stderr, "-test_multi_ops_txns not supported in ROCKSDB_LITE mode\n");
   exit(1);

package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h

@@ -345,6 +345,10 @@ class MultiOpsTxnsStressTest : public StressTest {
   uint32_t GenerateNextC(ThreadState* thread);
 
 #ifndef ROCKSDB_LITE
+  // Randomly commit or rollback `txn`
+  void ProcessRecoveredPreparedTxnsHelper(Transaction* txn,
+                                          SharedState*) override;
+
   // Some applications, e.g. MyRocks writes a KV pair to the database via
   // commit-time-write-batch (ctwb) in additional to the transaction's regular
   // write batch. The key is usually constant representing some system