@nxtedition/rocksdb 5.2.21 → 5.2.26

package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache.cc

@@ -4,14 +4,16 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "rocksdb/utilities/sim_cache.h"
+
 #include <atomic>
-#include "env/composite_env_wrapper.h"
+#include <iomanip>
+
 #include "file/writable_file_writer.h"
 #include "monitoring/statistics.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
 #include "util/mutexlock.h"
-#include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -35,8 +37,7 @@ class CacheActivityLogger {
     assert(env != nullptr);
 
     Status status;
-    EnvOptions env_opts;
-    std::unique_ptr<WritableFile> log_file;
+    FileOptions file_opts;
 
     MutexLock l(&mutex_);
 
@@ -44,13 +45,11 @@ class CacheActivityLogger {
     StopLoggingInternal();
 
     // Open log file
-    status = env->NewWritableFile(activity_log_file, &log_file, env_opts);
+    status = WritableFileWriter::Create(env->GetFileSystem(), activity_log_file,
+                                        file_opts, &file_writer_, nullptr);
     if (!status.ok()) {
       return status;
     }
-    file_writer_.reset(new WritableFileWriter(
-        NewLegacyWritableFileWrapper(std::move(log_file)), activity_log_file,
-        env_opts));
 
     max_logging_size_ = max_logging_size;
     activity_logging_enabled_.store(true);
@@ -69,11 +68,12 @@ class CacheActivityLogger {
       return;
     }
 
-    std::string log_line = "LOOKUP - " + key.ToString(true) + "\n";
-
+    std::ostringstream oss;
     // line format: "LOOKUP - <KEY>"
+    oss << "LOOKUP - " << key.ToString(true) << std::endl;
+
     MutexLock l(&mutex_);
-    Status s = file_writer_->Append(log_line);
+    Status s = file_writer_->Append(oss.str());
     if (!s.ok() && bg_status_.ok()) {
       bg_status_ = s;
     }
@@ -89,15 +89,11 @@ class CacheActivityLogger {
       return;
     }
 
-    std::string log_line = "ADD - ";
-    log_line += key.ToString(true);
-    log_line += " - ";
-    AppendNumberTo(&log_line, size);
-    log_line += "\n";
-
+    std::ostringstream oss;
     // line format: "ADD - <KEY> - <KEY-SIZE>"
+    oss << "ADD - " << key.ToString(true) << " - " << size << std::endl;
     MutexLock l(&mutex_);
-    Status s = file_writer_->Append(log_line);
+    Status s = file_writer_->Append(oss.str());
     if (!s.ok() && bg_status_.ok()) {
       bg_status_ = s;
     }
@@ -168,6 +164,7 @@ class SimCacheImpl : public SimCache {
     cache_->SetStrictCapacityLimit(strict_capacity_limit);
   }
 
+  using Cache::Insert;
   Status Insert(const Slice& key, void* value, size_t charge,
                 void (*deleter)(const Slice& key, void* value), Handle** handle,
                 Priority priority) override {
@@ -194,6 +191,7 @@ class SimCacheImpl : public SimCache {
     return cache_->Insert(key, value, charge, deleter, handle, priority);
   }
 
+  using Cache::Lookup;
   Handle* Lookup(const Slice& key, Statistics* stats) override {
     Handle* h = key_only_cache_->Lookup(key);
     if (h != nullptr) {
@@ -214,8 +212,9 @@ class SimCacheImpl : public SimCache {
 
   bool Ref(Handle* handle) override { return cache_->Ref(handle); }
 
-  bool Release(Handle* handle, bool force_erase = false) override {
-    return cache_->Release(handle, force_erase);
+  using Cache::Release;
+  bool Release(Handle* handle, bool erase_if_last_ref = false) override {
+    return cache_->Release(handle, erase_if_last_ref);
   }
 
   void Erase(const Slice& key) override {
@@ -243,6 +242,10 @@ class SimCacheImpl : public SimCache {
     return cache_->GetCharge(handle);
   }
 
+  DeleterFn GetDeleter(Handle* handle) const override {
+    return cache_->GetDeleter(handle);
+  }
+
   size_t GetPinnedUsage() const override { return cache_->GetPinnedUsage(); }
 
   void DisownData() override {
@@ -256,6 +259,13 @@ class SimCacheImpl : public SimCache {
     cache_->ApplyToAllCacheEntries(callback, thread_safe);
   }
 
+  void ApplyToAllEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      const ApplyToAllEntriesOptions& opts) override {
+    cache_->ApplyToAllEntries(callback, opts);
+  }
+
   void EraseUnRefEntries() override {
     cache_->EraseUnRefEntries();
     key_only_cache_->EraseUnRefEntries();
@@ -285,25 +295,23 @@ class SimCacheImpl : public SimCache {
   }
 
   std::string ToString() const override {
-    std::string res;
-    res.append("SimCache MISSes: " + std::to_string(get_miss_counter()) + "\n");
-    res.append("SimCache HITs:    " + std::to_string(get_hit_counter()) + "\n");
-    char buff[350];
+    std::ostringstream oss;
+    oss << "SimCache MISSes:  " << get_miss_counter() << std::endl;
+    oss << "SimCache HITs:    " << get_hit_counter() << std::endl;
     auto lookups = get_miss_counter() + get_hit_counter();
-    snprintf(buff, sizeof(buff), "SimCache HITRATE: %.2f%%\n",
-             (lookups == 0 ? 0 : get_hit_counter() * 100.0f / lookups));
-    res.append(buff);
-    return res;
+    oss << "SimCache HITRATE: " << std::fixed << std::setprecision(2)
+        << (lookups == 0 ? 0 : get_hit_counter() * 100.0f / lookups)
+        << std::endl;
+    return oss.str();
   }
 
   std::string GetPrintableOptions() const override {
-    std::string ret;
-    ret.reserve(20000);
-    ret.append("    cache_options:\n");
-    ret.append(cache_->GetPrintableOptions());
-    ret.append("    sim_cache_options:\n");
-    ret.append(key_only_cache_->GetPrintableOptions());
-    return ret;
+    std::ostringstream oss;
+    oss << "    cache_options:" << std::endl;
+    oss << cache_->GetPrintableOptions();
+    oss << "    sim_cache_options:" << std::endl;
+    oss << key_only_cache_->GetPrintableOptions();
+    return oss.str();
   }
 
   Status StartActivityLogging(const std::string& activity_log_file, Env* env,

package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc

@@ -4,7 +4,9 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include "rocksdb/utilities/sim_cache.h"
+
 #include <cstdlib>
+
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
 
@@ -21,7 +23,7 @@ class SimCacheTest : public DBTestBase {
   const size_t kNumBlocks = 5;
   const size_t kValueSize = 1000;
 
-  SimCacheTest() : DBTestBase("/sim_cache_test", /*env_do_fsync=*/true) {}
+  SimCacheTest() : DBTestBase("sim_cache_test", /*env_do_fsync=*/true) {}
 
   BlockBasedTableOptions GetTableOptions() {
     BlockBasedTableOptions table_options;
@@ -87,6 +89,8 @@ TEST_F(SimCacheTest, SimCache) {
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
   Reopen(options);
   RecordCacheCounters(options);
+  // due to cache entry stats collector
+  uint64_t base_misses = simCache->get_miss_counter();
 
   std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks);
   Iterator* iter = nullptr;
@@ -99,8 +103,8 @@ TEST_F(SimCacheTest, SimCache) {
     CheckCacheCounters(options, 1, 0, 1, 0);
     iterators[i].reset(iter);
   }
-  ASSERT_EQ(kNumBlocks,
-            simCache->get_hit_counter() + simCache->get_miss_counter());
+  ASSERT_EQ(kNumBlocks, simCache->get_hit_counter() +
+                            simCache->get_miss_counter() - base_misses);
   ASSERT_EQ(0, simCache->get_hit_counter());
   size_t usage = simCache->GetUsage();
   ASSERT_LT(0, usage);
@@ -137,8 +141,8 @@ TEST_F(SimCacheTest, SimCache) {
     CheckCacheCounters(options, 1, 0, 1, 0);
   }
   ASSERT_EQ(0, simCache->GetPinnedUsage());
-  ASSERT_EQ(3 * kNumBlocks + 1,
-            simCache->get_hit_counter() + simCache->get_miss_counter());
+  ASSERT_EQ(3 * kNumBlocks + 1, simCache->get_hit_counter() +
+                                    simCache->get_miss_counter() - base_misses);
   ASSERT_EQ(6, simCache->get_hit_counter());
 }
 
@@ -173,23 +177,21 @@ TEST_F(SimCacheTest, SimCacheLogging) {
 
   std::string file_contents = "";
   ASSERT_OK(ReadFileToString(env_, log_file, &file_contents));
+  std::istringstream contents(file_contents);
 
   int lookup_num = 0;
   int add_num = 0;
-  std::string::size_type pos;
-
-  // count number of lookups
-  pos = 0;
-  while ((pos = file_contents.find("LOOKUP -", pos)) != std::string::npos) {
-    ++lookup_num;
-    pos += 1;
-  }
 
-  // count number of additions
-  pos = 0;
-  while ((pos = file_contents.find("ADD -", pos)) != std::string::npos) {
-    ++add_num;
-    pos += 1;
+  std::string line;
+  // count number of lookups and additions
+  while (std::getline(contents, line)) {
+    // check if the line starts with LOOKUP or ADD
+    if (line.rfind("LOOKUP -", 0) == 0) {
+      ++lookup_num;
+    }
+    if (line.rfind("ADD -", 0) == 0) {
+      ++add_num;
+    }
   }
 
   // We asked for every block twice

package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc

@@ -3,14 +3,19 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-#ifndef ROCKSDB_LITE
 #include "utilities/table_properties_collectors/compact_on_deletion_collector.h"
 
 #include <memory>
 #include <sstream>
+
+#include "rocksdb/utilities/customizable_util.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
 #include "rocksdb/utilities/table_properties_collectors.h"
+#include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
+#ifndef ROCKSDB_LITE
 
 CompactOnDeletionCollector::CompactOnDeletionCollector(
     size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
@@ -93,6 +98,74 @@ Status CompactOnDeletionCollector::Finish(
   finished_ = true;
   return Status::OK();
 }
+static std::unordered_map<std::string, OptionTypeInfo>
+    on_deletion_collector_type_info = {
+#ifndef ROCKSDB_LITE
+        {"window_size",
+         {0, OptionType::kUnknown, OptionVerificationType::kNormal,
+          OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
+          [](const ConfigOptions&, const std::string&, const std::string& value,
+             void* addr) {
+            auto* factory =
+                static_cast<CompactOnDeletionCollectorFactory*>(addr);
+            factory->SetWindowSize(ParseSizeT(value));
+            return Status::OK();
+          },
+          [](const ConfigOptions&, const std::string&, const void* addr,
+             std::string* value) {
+            const auto* factory =
+                static_cast<const CompactOnDeletionCollectorFactory*>(addr);
+            *value = ToString(factory->GetWindowSize());
+            return Status::OK();
+          },
+          nullptr}},
+        {"deletion_trigger",
+         {0, OptionType::kUnknown, OptionVerificationType::kNormal,
+          OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
+          [](const ConfigOptions&, const std::string&, const std::string& value,
+             void* addr) {
+            auto* factory =
+                static_cast<CompactOnDeletionCollectorFactory*>(addr);
+            factory->SetDeletionTrigger(ParseSizeT(value));
+            return Status::OK();
+          },
+          [](const ConfigOptions&, const std::string&, const void* addr,
+             std::string* value) {
+            const auto* factory =
+                static_cast<const CompactOnDeletionCollectorFactory*>(addr);
+            *value = ToString(factory->GetDeletionTrigger());
+            return Status::OK();
+          },
+          nullptr}},
+        {"deletion_ratio",
+         {0, OptionType::kUnknown, OptionVerificationType::kNormal,
+          OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
+          [](const ConfigOptions&, const std::string&, const std::string& value,
+             void* addr) {
+            auto* factory =
+                static_cast<CompactOnDeletionCollectorFactory*>(addr);
+            factory->SetDeletionRatio(ParseDouble(value));
+            return Status::OK();
+          },
+          [](const ConfigOptions&, const std::string&, const void* addr,
+             std::string* value) {
+            const auto* factory =
+                static_cast<const CompactOnDeletionCollectorFactory*>(addr);
+            *value = ToString(factory->GetDeletionRatio());
+            return Status::OK();
+          },
+          nullptr}},
+
+#endif  // ROCKSDB_LITE
+};
+
+CompactOnDeletionCollectorFactory::CompactOnDeletionCollectorFactory(
+    size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
+    : sliding_window_size_(sliding_window_size),
+      deletion_trigger_(deletion_trigger),
+      deletion_ratio_(deletion_ratio) {
+  RegisterOptions("", this, &on_deletion_collector_type_info);
+}
 
 TablePropertiesCollector*
 CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector(
@@ -118,5 +191,37 @@ NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
       new CompactOnDeletionCollectorFactory(sliding_window_size,
                                             deletion_trigger, deletion_ratio));
 }
-}  // namespace ROCKSDB_NAMESPACE
+namespace {
+static int RegisterTablePropertiesCollectorFactories(
+    ObjectLibrary& library, const std::string& /*arg*/) {
+  library.AddFactory<TablePropertiesCollectorFactory>(
+      CompactOnDeletionCollectorFactory::kClassName(),
+      [](const std::string& /*uri*/,
+         std::unique_ptr<TablePropertiesCollectorFactory>* guard,
+         std::string* /* errmsg */) {
+        // By default, create a CompactionOnDeletionCollector that is disabled.
+        // Users will need to provide configuration parameters or call the
+        // corresponding Setter to enable the factory.
+        guard->reset(new CompactOnDeletionCollectorFactory(0, 0, 0));
+        return guard->get();
+      });
+  return 1;
+}
+}  // namespace
 #endif  // !ROCKSDB_LITE
+
+Status TablePropertiesCollectorFactory::CreateFromString(
+    const ConfigOptions& options, const std::string& value,
+    std::shared_ptr<TablePropertiesCollectorFactory>* result) {
+#ifndef ROCKSDB_LITE
+  static std::once_flag once;
+  std::call_once(once, [&]() {
+    RegisterTablePropertiesCollectorFactories(*(ObjectLibrary::Default().get()),
+                                              "");
+  });
+#endif  // ROCKSDB_LITE
+  return LoadSharedObject<TablePropertiesCollectorFactory>(options, value,
+                                                           nullptr, result);
+}
+
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.cc

@@ -31,10 +31,19 @@ Status FileTraceReader::Close() {
   return Status::OK();
 }
 
+Status FileTraceReader::Reset() {
+  if (file_reader_ == nullptr) {
+    return Status::IOError("TraceReader is closed.");
+  }
+  offset_ = 0;
+  return Status::OK();
+}
+
 Status FileTraceReader::Read(std::string* data) {
   assert(file_reader_ != nullptr);
   Status s = file_reader_->Read(IOOptions(), offset_, kTraceMetadataSize,
-                                &result_, buffer_, nullptr);
+                                &result_, buffer_, nullptr,
+                                Env::IO_TOTAL /* rate_limiter_priority */);
   if (!s.ok()) {
     return s;
   }
@@ -59,7 +68,7 @@ Status FileTraceReader::Read(std::string* data) {
       bytes_to_read > kBufferSize ? kBufferSize : bytes_to_read;
   while (to_read > 0) {
     s = file_reader_->Read(IOOptions(), offset_, to_read, &result_, buffer_,
-                           nullptr);
+                           nullptr, Env::IO_TOTAL /* rate_limiter_priority */);
     if (!s.ok()) {
       return s;
     }
@@ -76,6 +85,10 @@ Status FileTraceReader::Read(std::string* data) {
   return s;
 }
 
+FileTraceWriter::FileTraceWriter(
+    std::unique_ptr<WritableFileWriter>&& file_writer)
+    : file_writer_(std::move(file_writer)) {}
+
 FileTraceWriter::~FileTraceWriter() { Close().PermitUncheckedError(); }
 
 Status FileTraceWriter::Close() {
@@ -92,15 +105,13 @@ uint64_t FileTraceWriter::GetFileSize() { return file_writer_->GetFileSize(); }
 Status NewFileTraceReader(Env* env, const EnvOptions& env_options,
                           const std::string& trace_filename,
                           std::unique_ptr<TraceReader>* trace_reader) {
-  std::unique_ptr<RandomAccessFile> trace_file;
-  Status s = env->NewRandomAccessFile(trace_filename, &trace_file, env_options);
+  std::unique_ptr<RandomAccessFileReader> file_reader;
+  Status s = RandomAccessFileReader::Create(
+      env->GetFileSystem(), trace_filename, FileOptions(env_options),
+      &file_reader, nullptr);
   if (!s.ok()) {
     return s;
   }
-
-  std::unique_ptr<RandomAccessFileReader> file_reader;
-  file_reader.reset(new RandomAccessFileReader(
-      NewLegacyRandomAccessFileWrapper(trace_file), trace_filename));
   trace_reader->reset(new FileTraceReader(std::move(file_reader)));
   return s;
 }
@@ -108,16 +119,13 @@ Status NewFileTraceReader(Env* env, const EnvOptions& env_options,
 Status NewFileTraceWriter(Env* env, const EnvOptions& env_options,
                           const std::string& trace_filename,
                           std::unique_ptr<TraceWriter>* trace_writer) {
-  std::unique_ptr<WritableFile> trace_file;
-  Status s = env->NewWritableFile(trace_filename, &trace_file, env_options);
+  std::unique_ptr<WritableFileWriter> file_writer;
+  Status s = WritableFileWriter::Create(env->GetFileSystem(), trace_filename,
+                                        FileOptions(env_options), &file_writer,
+                                        nullptr);
   if (!s.ok()) {
     return s;
   }
-
-  std::unique_ptr<WritableFileWriter> file_writer;
-  file_writer.reset(new WritableFileWriter(
-      NewLegacyWritableFileWrapper(std::move(trace_file)), trace_filename,
-      env_options));
   trace_writer->reset(new FileTraceWriter(std::move(file_writer)));
   return s;
 }

package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.h

@@ -20,6 +20,7 @@ class FileTraceReader : public TraceReader {
 
   virtual Status Read(std::string* data) override;
   virtual Status Close() override;
+  virtual Status Reset() override;
 
  private:
   std::unique_ptr<RandomAccessFileReader> file_reader_;
@@ -33,8 +34,7 @@ class FileTraceReader : public TraceReader {
 // FileTraceWriter allows writing RocksDB traces to a file.
 class FileTraceWriter : public TraceWriter {
  public:
-  explicit FileTraceWriter(std::unique_ptr<WritableFileWriter>&& file_writer)
-      : file_writer_(std::move(file_writer)) {}
+  explicit FileTraceWriter(std::unique_ptr<WritableFileWriter>&& file_writer);
   ~FileTraceWriter();
 
   virtual Status Write(const Slice& data) override;