@nxtedition/rocksdb 5.2.21 → 5.2.26

package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.h

@@ -15,13 +15,15 @@
 
 #include "rocksdb/env.h"
 #include "rocksdb/trace_reader_writer.h"
+#include "rocksdb/trace_record.h"
 #include "rocksdb/write_batch.h"
 #include "trace_replay/trace_replay.h"
 
 namespace ROCKSDB_NAMESPACE {
 
-class DBImpl;
-class WriteBatch;
+// Value sizes may be used as denominators. Replacing 0 value sizes with this
+// positive integer avoids division error.
+extern const size_t kShadowValueSize /* = 10*/;
 
 enum TraceOperationType : int {
   kGet = 0,
@@ -32,7 +34,8 @@ enum TraceOperationType : int {
   kMerge = 5,
   kIteratorSeek = 6,
   kIteratorSeekForPrev = 7,
-  kTaTypeNum = 8
+  kMultiGet = 8,
+  kTaTypeNum = 9
 };
 
 struct TraceUnit {
@@ -162,7 +165,8 @@ struct CfUnit {
   std::map<uint32_t, uint32_t> cf_qps;
 };
 
-class TraceAnalyzer {
+class TraceAnalyzer : private TraceRecord::Handler,
+                      private WriteBatch::Handler {
  public:
   TraceAnalyzer(std::string& trace_path, std::string& output_path,
                 AnalyzerOptions _analyzer_opts);
@@ -180,28 +184,88 @@ class TraceAnalyzer {
 
   Status WriteTraceUnit(TraceUnit& unit);
 
-  // The trace  processing functions for different type
-  Status HandleGet(uint32_t column_family_id, const std::string& key,
-                   const uint64_t& ts, const uint32_t& get_ret);
-  Status HandlePut(uint32_t column_family_id, const Slice& key,
-                   const Slice& value);
-  Status HandleDelete(uint32_t column_family_id, const Slice& key);
-  Status HandleSingleDelete(uint32_t column_family_id, const Slice& key);
-  Status HandleDeleteRange(uint32_t column_family_id, const Slice& begin_key,
-                           const Slice& end_key);
-  Status HandleMerge(uint32_t column_family_id, const Slice& key,
-                     const Slice& value);
-  Status HandleIter(uint32_t column_family_id, const std::string& key,
-                    const uint64_t& ts, TraceType& trace_type);
   std::vector<TypeUnit>& GetTaVector() { return ta_; }
 
  private:
+  using TraceRecord::Handler::Handle;
+  Status Handle(const WriteQueryTraceRecord& record,
+                std::unique_ptr<TraceRecordResult>* result) override;
+  Status Handle(const GetQueryTraceRecord& record,
+                std::unique_ptr<TraceRecordResult>* result) override;
+  Status Handle(const IteratorSeekQueryTraceRecord& record,
+                std::unique_ptr<TraceRecordResult>* result) override;
+  Status Handle(const MultiGetQueryTraceRecord& record,
+                std::unique_ptr<TraceRecordResult>* result) override;
+
+  using WriteBatch::Handler::PutCF;
+  Status PutCF(uint32_t column_family_id, const Slice& key,
+               const Slice& value) override;
+
+  using WriteBatch::Handler::DeleteCF;
+  Status DeleteCF(uint32_t column_family_id, const Slice& key) override;
+
+  using WriteBatch::Handler::SingleDeleteCF;
+  Status SingleDeleteCF(uint32_t column_family_id, const Slice& key) override;
+
+  using WriteBatch::Handler::DeleteRangeCF;
+  Status DeleteRangeCF(uint32_t column_family_id, const Slice& begin_key,
+                       const Slice& end_key) override;
+
+  using WriteBatch::Handler::MergeCF;
+  Status MergeCF(uint32_t column_family_id, const Slice& key,
+                 const Slice& value) override;
+
+  // The following hanlders are not implemented, return Status::OK() to avoid
+  // the running time assertion and other irrelevant falures.
+  using WriteBatch::Handler::PutBlobIndexCF;
+  Status PutBlobIndexCF(uint32_t /*column_family_id*/, const Slice& /*key*/,
+                        const Slice& /*value*/) override {
+    return Status::OK();
+  }
+
+  // The default implementation of LogData does nothing.
+  using WriteBatch::Handler::LogData;
+  void LogData(const Slice& /*blob*/) override {}
+
+  using WriteBatch::Handler::MarkBeginPrepare;
+  Status MarkBeginPrepare(bool = false) override { return Status::OK(); }
+
+  using WriteBatch::Handler::MarkEndPrepare;
+  Status MarkEndPrepare(const Slice& /*xid*/) override { return Status::OK(); }
+
+  using WriteBatch::Handler::MarkNoop;
+  Status MarkNoop(bool /*empty_batch*/) override { return Status::OK(); }
+
+  using WriteBatch::Handler::MarkRollback;
+  Status MarkRollback(const Slice& /*xid*/) override { return Status::OK(); }
+
+  using WriteBatch::Handler::MarkCommit;
+  Status MarkCommit(const Slice& /*xid*/) override { return Status::OK(); }
+
+  using WriteBatch::Handler::MarkCommitWithTimestamp;
+  Status MarkCommitWithTimestamp(const Slice& /*xid*/,
+                                 const Slice& /*commit_ts*/) override {
+    return Status::OK();
+  }
+
+  // Process each trace operation and output the analysis result to
+  // stdout/files.
+  Status OutputAnalysisResult(TraceOperationType op_type, uint64_t timestamp,
+                              std::vector<uint32_t> cf_ids,
+                              std::vector<Slice> keys,
+                              std::vector<size_t> value_sizes);
+
+  Status OutputAnalysisResult(TraceOperationType op_type, uint64_t timestamp,
+                              uint32_t cf_id, const Slice& key,
+                              size_t value_size);
+
   ROCKSDB_NAMESPACE::Env* env_;
   EnvOptions env_options_;
   std::unique_ptr<TraceReader> trace_reader_;
   size_t offset_;
   char buffer_[1024];
-  uint64_t c_time_;
+  // Timestamp of a WriteBatch, used in its iteration.
+  uint64_t write_batch_ts_;
   std::string trace_name_;
   std::string output_path_;
   AnalyzerOptions analyzer_opts_;
@@ -209,6 +273,9 @@ class TraceAnalyzer {
   uint64_t total_access_keys_;
   uint64_t total_gets_;
   uint64_t total_writes_;
+  uint64_t total_seeks_;
+  uint64_t total_seek_prevs_;
+  uint64_t total_multigets_;
   uint64_t trace_create_time_;
   uint64_t begin_time_;
   uint64_t end_time_;
@@ -244,45 +311,12 @@ class TraceAnalyzer {
   Status TraceUnitWriter(
       std::unique_ptr<ROCKSDB_NAMESPACE::WritableFile>& f_ptr, TraceUnit& unit);
   Status WriteTraceSequence(const uint32_t& type, const uint32_t& cf_id,
-                            const std::string& key, const size_t value_size,
+                            const Slice& key, const size_t value_size,
                             const uint64_t ts);
   Status MakeStatisticKeyStatsOrPrefix(TraceStats& stats);
   Status MakeStatisticCorrelation(TraceStats& stats, StatsUnit& unit);
   Status MakeStatisticQPS();
-};
-
-// write bach handler to be used for WriteBache iterator
-// when processing the write trace
-class TraceWriteHandler : public WriteBatch::Handler {
- public:
-  TraceWriteHandler() { ta_ptr = nullptr; }
-  explicit TraceWriteHandler(TraceAnalyzer* _ta_ptr) { ta_ptr = _ta_ptr; }
-  ~TraceWriteHandler() {}
-
-  virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-                       const Slice& value) override {
-    return ta_ptr->HandlePut(column_family_id, key, value);
-  }
-  virtual Status DeleteCF(uint32_t column_family_id,
-                          const Slice& key) override {
-    return ta_ptr->HandleDelete(column_family_id, key);
-  }
-  virtual Status SingleDeleteCF(uint32_t column_family_id,
-                                const Slice& key) override {
-    return ta_ptr->HandleSingleDelete(column_family_id, key);
-  }
-  virtual Status DeleteRangeCF(uint32_t column_family_id,
-                               const Slice& begin_key,
-                               const Slice& end_key) override {
-    return ta_ptr->HandleDeleteRange(column_family_id, begin_key, end_key);
-  }
-  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
-                         const Slice& value) override {
-    return ta_ptr->HandleMerge(column_family_id, key, value);
-  }
-
- private:
-  TraceAnalyzer* ta_ptr;
+  int db_version_;
 };
 
 int trace_analyzer_tool(int argc, char** argv);

package/deps/rocksdb/rocksdb/tools/write_stress.cc

@@ -69,6 +69,7 @@ int main() {
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/system_clock.h"
 #include "util/gflags_compat.h"
 
 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
@@ -187,8 +188,8 @@ class WriteStress {
   void IteratorHoldThread() {
     while (!stop_.load(std::memory_order_relaxed)) {
       std::unique_ptr<Iterator> iterator(db_->NewIterator(ReadOptions()));
-      Env::Default()->SleepForMicroseconds(FLAGS_iterator_hold_sec * 1000 *
-                                           1000LL);
+      SystemClock::Default()->SleepForMicroseconds(FLAGS_iterator_hold_sec *
+                                                   1000 * 1000LL);
       for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
       }
       if (!iterator->status().ok()) {
@@ -204,9 +205,8 @@ class WriteStress {
     std::uniform_real_distribution<double> dist(0, 1);
     std::uniform_int_distribution<int> char_dist('a', 'z');
     while (!stop_.load(std::memory_order_relaxed)) {
-      Env::Default()->SleepForMicroseconds(static_cast<int>(
-                                           FLAGS_prefix_mutate_period_sec *
-                                           1000 * 1000LL));
+      SystemClock::Default()->SleepForMicroseconds(
+          static_cast<int>(FLAGS_prefix_mutate_period_sec * 1000 * 1000LL));
       if (dist(rng) < FLAGS_first_char_mutate_probability) {
         key_prefix_[0].store(static_cast<char>(char_dist(rng)), std::memory_order_relaxed);
       }
@@ -227,11 +227,12 @@ class WriteStress {
     if (FLAGS_runtime_sec == -1) {
       // infinite runtime, until we get killed
       while (true) {
-        Env::Default()->SleepForMicroseconds(1000 * 1000);
+        SystemClock::Default()->SleepForMicroseconds(1000 * 1000);
       }
     }
 
-    Env::Default()->SleepForMicroseconds(FLAGS_runtime_sec * 1000 * 1000);
+    SystemClock::Default()->SleepForMicroseconds(FLAGS_runtime_sec * 1000 *
+                                                 1000);
 
     stop_.store(true, std::memory_order_relaxed);
     for (auto& t : threads_) {

package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.cc

@@ -12,6 +12,7 @@
 #include "db/db_impl/db_impl.h"
 #include "db/dbformat.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/trace_record.h"
 #include "util/coding.h"
 #include "util/hash.h"
 #include "util/string_util.h"
@@ -99,9 +100,9 @@ uint64_t BlockCacheTraceHelper::GetBlockOffsetInFile(
 }
 
 BlockCacheTraceWriter::BlockCacheTraceWriter(
-    Env* env, const TraceOptions& trace_options,
+    SystemClock* clock, const TraceOptions& trace_options,
     std::unique_ptr<TraceWriter>&& trace_writer)
-    : env_(env),
+    : clock_(clock),
       trace_options_(trace_options),
       trace_writer_(std::move(trace_writer)) {}
 
@@ -142,7 +143,7 @@ Status BlockCacheTraceWriter::WriteBlockAccess(
 
 Status BlockCacheTraceWriter::WriteHeader() {
   Trace trace;
-  trace.ts = env_->NowMicros();
+  trace.ts = clock_->NowMicros();
   trace.type = TraceType::kTraceBegin;
   PutLengthPrefixedSlice(&trace.payload, kTraceMagic);
   PutFixed32(&trace.payload, kMajorVersion);
@@ -444,7 +445,7 @@ BlockCacheTracer::BlockCacheTracer() { writer_.store(nullptr); }
 BlockCacheTracer::~BlockCacheTracer() { EndTrace(); }
 
 Status BlockCacheTracer::StartTrace(
-    Env* env, const TraceOptions& trace_options,
+    SystemClock* clock, const TraceOptions& trace_options,
     std::unique_ptr<TraceWriter>&& trace_writer) {
   InstrumentedMutexLock lock_guard(&trace_writer_mutex_);
   if (writer_.load()) {
@@ -453,7 +454,7 @@ Status BlockCacheTracer::StartTrace(
   get_id_counter_.store(1);
   trace_options_ = trace_options;
   writer_.store(
-      new BlockCacheTraceWriter(env, trace_options, std::move(trace_writer)));
+      new BlockCacheTraceWriter(clock, trace_options, std::move(trace_writer)));
   return writer_.load()->WriteHeader();
 }
 

package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h

@@ -9,13 +9,14 @@
 #include <fstream>
 
 #include "monitoring/instrumented_mutex.h"
-#include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/trace_reader_writer.h"
 #include "table/table_reader_caller.h"
 #include "trace_replay/trace_replay.h"
 
 namespace ROCKSDB_NAMESPACE {
+class Env;
+class SystemClock;
 
 extern const uint64_t kMicrosInSecond;
 extern const uint64_t kSecondInMinute;
@@ -172,7 +173,7 @@ struct BlockCacheTraceHeader {
 // payload.
 class BlockCacheTraceWriter {
  public:
-  BlockCacheTraceWriter(Env* env, const TraceOptions& trace_options,
+  BlockCacheTraceWriter(SystemClock* clock, const TraceOptions& trace_options,
                         std::unique_ptr<TraceWriter>&& trace_writer);
   ~BlockCacheTraceWriter() = default;
   // No copy and move.
@@ -191,7 +192,7 @@ class BlockCacheTraceWriter {
   Status WriteHeader();
 
  private:
-  Env* env_;
+  SystemClock* clock_;
   TraceOptions trace_options_;
   std::unique_ptr<TraceWriter> trace_writer_;
 };
@@ -266,7 +267,7 @@ class BlockCacheTracer {
   BlockCacheTracer& operator=(BlockCacheTracer&&) = delete;
 
   // Start writing block cache accesses to the trace_writer.
-  Status StartTrace(Env* env, const TraceOptions& trace_options,
+  Status StartTrace(SystemClock* clock, const TraceOptions& trace_options,
                     std::unique_ptr<TraceWriter>&& trace_writer);
 
   // Stop writing block cache accesses to the trace_writer.

package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer_test.cc

@@ -4,9 +4,12 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "trace_replay/block_cache_tracer.h"
+
+#include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/status.h"
 #include "rocksdb/trace_reader_writer.h"
+#include "rocksdb/trace_record.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 
@@ -28,6 +31,7 @@ class BlockCacheTracerTest : public testing::Test {
   BlockCacheTracerTest() {
     test_path_ = test::PerThreadDBPath("block_cache_tracer_test");
     env_ = ROCKSDB_NAMESPACE::Env::Default();
+    clock_ = env_->GetSystemClock().get();
     EXPECT_OK(env_->CreateDir(test_path_));
     trace_file_path_ = test_path_ + "/block_cache_trace";
   }
@@ -64,7 +68,7 @@ class BlockCacheTracerTest : public testing::Test {
       record.block_type = block_type;
       record.block_size = kBlockSize + key_id;
       record.block_key = (kBlockKeyPrefix + std::to_string(key_id));
-      record.access_timestamp = env_->NowMicros();
+      record.access_timestamp = clock_->NowMicros();
       record.cf_id = kCFId;
       record.cf_name = kDefaultColumnFamilyName;
       record.caller = GetCaller(key_id);
@@ -94,7 +98,7 @@ class BlockCacheTracerTest : public testing::Test {
     record.block_type = TraceType::kBlockTraceDataBlock;
     record.block_size = kBlockSize;
     record.block_key = kBlockKeyPrefix + std::to_string(key_id);
-    record.access_timestamp = env_->NowMicros();
+    record.access_timestamp = clock_->NowMicros();
     record.cf_id = kCFId;
     record.cf_name = kDefaultColumnFamilyName;
     record.caller = GetCaller(key_id);
@@ -151,6 +155,7 @@ class BlockCacheTracerTest : public testing::Test {
   }
 
   Env* env_;
+  SystemClock* clock_;
   EnvOptions env_options_;
   std::string trace_file_path_;
   std::string test_path_;
@@ -188,7 +193,7 @@ TEST_F(BlockCacheTracerTest, AtomicWrite) {
     ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
                                  &trace_writer));
     BlockCacheTracer writer;
-    ASSERT_OK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
+    ASSERT_OK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
     ASSERT_OK(writer.WriteBlockAccess(record, record.block_key, record.cf_name,
                                       record.referenced_key));
     ASSERT_OK(env_->FileExists(trace_file_path_));
@@ -214,8 +219,8 @@ TEST_F(BlockCacheTracerTest, ConsecutiveStartTrace) {
   ASSERT_OK(
       NewFileTraceWriter(env_, env_options_, trace_file_path_, &trace_writer));
   BlockCacheTracer writer;
-  ASSERT_OK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
-  ASSERT_NOK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
+  ASSERT_OK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
+  ASSERT_NOK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
   ASSERT_OK(env_->FileExists(trace_file_path_));
 }
 
@@ -227,7 +232,7 @@ TEST_F(BlockCacheTracerTest, AtomicNoWriteAfterEndTrace) {
     ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
                                  &trace_writer));
     BlockCacheTracer writer;
-    ASSERT_OK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
+    ASSERT_OK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
     ASSERT_OK(writer.WriteBlockAccess(record, record.block_key, record.cf_name,
                                       record.referenced_key));
     writer.EndTrace();
@@ -262,7 +267,7 @@ TEST_F(BlockCacheTracerTest, NextGetId) {
     // next get id should always return 0 before we call StartTrace.
     ASSERT_EQ(0, writer.NextGetId());
     ASSERT_EQ(0, writer.NextGetId());
-    ASSERT_OK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
+    ASSERT_OK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
     ASSERT_EQ(1, writer.NextGetId());
     ASSERT_EQ(2, writer.NextGetId());
     writer.EndTrace();
@@ -276,7 +281,7 @@ TEST_F(BlockCacheTracerTest, NextGetId) {
     std::unique_ptr<TraceWriter> trace_writer;
     ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
                                  &trace_writer));
-    ASSERT_OK(writer.StartTrace(env_, trace_opt, std::move(trace_writer)));
+    ASSERT_OK(writer.StartTrace(clock_, trace_opt, std::move(trace_writer)));
     ASSERT_EQ(1, writer.NextGetId());
   }
 }
@@ -288,7 +293,7 @@ TEST_F(BlockCacheTracerTest, MixedBlocks) {
     std::unique_ptr<TraceWriter> trace_writer;
     ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_,
                                  &trace_writer));
-    BlockCacheTraceWriter writer(env_, trace_opt, std::move(trace_writer));
+    BlockCacheTraceWriter writer(clock_, trace_opt, std::move(trace_writer));
     ASSERT_OK(writer.WriteHeader());
     // Write blocks of different types.
     WriteBlockAccess(&writer, 0, TraceType::kBlockTraceUncompressionDictBlock,