@nxtedition/rocksdb 5.2.21 → 5.2.28

package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc

@@ -7,7 +7,9 @@
 #ifndef ROCKSDB_LITE
 
 #include "logging/auto_roll_logger.h"
+
 #include <sys/stat.h>
+
 #include <algorithm>
 #include <cmath>
 #include <fstream>
@@ -16,30 +18,19 @@
 #include <string>
 #include <thread>
 #include <vector>
+
+#include "db/db_test_util.h"
+#include "env/emulated_clock.h"
 #include "logging/logging.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/system_clock.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 
 namespace ROCKSDB_NAMESPACE {
-namespace {
-class NoSleepEnv : public EnvWrapper {
- public:
-  NoSleepEnv(Env* base) : EnvWrapper(base) {}
-  void SleepForMicroseconds(int micros) override {
-    fake_time_ += static_cast<uint64_t>(micros);
-  }
-
-  uint64_t NowNanos() override { return fake_time_ * 1000; }
-
-  uint64_t NowMicros() override { return fake_time_; }
-
- private:
-  uint64_t fake_time_ = 6666666666;
-};
-}  // namespace
 
 // In this test we only want to Log some simple log message with
 // no format. LogMessage() provides such a simple interface and
@@ -75,7 +66,9 @@ class AutoRollLoggerTest : public testing::Test {
 
   void RollLogFileBySizeTest(AutoRollLogger* logger, size_t log_max_size,
                              const std::string& log_message);
-  void RollLogFileByTimeTest(Env*, AutoRollLogger* logger, size_t time,
+  void RollLogFileByTimeTest(const std::shared_ptr<FileSystem>& fs,
+                             const std::shared_ptr<SystemClock>& sc,
+                             AutoRollLogger* logger, size_t time,
                              const std::string& log_message);
   // return list of files under kTestDir that contains "LOG"
   std::vector<std::string> GetLogFiles() {
@@ -156,21 +149,22 @@ void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger,
   ASSERT_TRUE(message_size == logger->GetLogFileSize());
 }
 
-void AutoRollLoggerTest::RollLogFileByTimeTest(Env* env, AutoRollLogger* logger,
-                                               size_t time,
-                                               const std::string& log_message) {
+void AutoRollLoggerTest::RollLogFileByTimeTest(
+    const std::shared_ptr<FileSystem>& fs,
+    const std::shared_ptr<SystemClock>& sc, AutoRollLogger* logger, size_t time,
+    const std::string& log_message) {
   uint64_t expected_ctime;
   uint64_t actual_ctime;
 
   uint64_t total_log_size;
-  EXPECT_OK(env->GetFileSize(kLogFile, &total_log_size));
+  EXPECT_OK(fs->GetFileSize(kLogFile, IOOptions(), &total_log_size, nullptr));
   expected_ctime = logger->TEST_ctime();
   logger->SetCallNowMicrosEveryNRecords(0);
 
   // -- Write to the log for several times, which is supposed
   // to be finished before time.
   for (int i = 0; i < 10; ++i) {
-    env->SleepForMicroseconds(50000);
+    sc->SleepForMicroseconds(50000);
     LogMessage(logger, log_message.c_str());
     EXPECT_OK(logger->GetStatus());
     // Make sure we always write to the same log file (by
@@ -185,7 +179,7 @@ void AutoRollLoggerTest::RollLogFileByTimeTest(Env* env, AutoRollLogger* logger,
   }
 
   // -- Make the log file expire
-  env->SleepForMicroseconds(static_cast<int>(time * 1000000));
+  sc->SleepForMicroseconds(static_cast<int>(time * 1000000));
   LogMessage(logger, log_message.c_str());
 
   // At this time, the new log file should be created.
@@ -199,15 +193,16 @@ TEST_F(AutoRollLoggerTest, RollLogFileBySize) {
     size_t log_max_size = 1024 * 5;
     size_t keep_log_file_num = 10;
 
-    AutoRollLogger logger(Env::Default(), kTestDir, "", log_max_size, 0,
-                          keep_log_file_num);
+    AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                          kTestDir, "", log_max_size, 0, keep_log_file_num);
 
     RollLogFileBySizeTest(&logger, log_max_size,
                           kSampleMessage + ":RollLogFileBySize");
 }
 
 TEST_F(AutoRollLoggerTest, RollLogFileByTime) {
-  NoSleepEnv nse(Env::Default());
+  auto nsc =
+      std::make_shared<EmulatedSystemClock>(SystemClock::Default(), true);
 
   size_t time = 2;
   size_t log_size = 1024 * 5;
@@ -216,10 +211,11 @@ TEST_F(AutoRollLoggerTest, RollLogFileByTime) {
   InitTestDb();
   // -- Test the existence of file during the server restart.
   ASSERT_EQ(Status::NotFound(), default_env->FileExists(kLogFile));
-  AutoRollLogger logger(&nse, kTestDir, "", log_size, time, keep_log_file_num);
+  AutoRollLogger logger(default_env->GetFileSystem(), nsc, kTestDir, "",
+                        log_size, time, keep_log_file_num);
   ASSERT_OK(default_env->FileExists(kLogFile));
 
-  RollLogFileByTimeTest(&nse, &logger, time,
+  RollLogFileByTimeTest(default_env->GetFileSystem(), nsc, &logger, time,
                         kSampleMessage + ":RollLogFileByTime");
 }
 
@@ -254,15 +250,17 @@ TEST_F(AutoRollLoggerTest, OpenLogFilesMultipleTimesWithOptionLog_max_size) {
   size_t log_size = 1024;
   size_t keep_log_file_num = 10;
 
-  AutoRollLogger* logger = new AutoRollLogger(Env::Default(), kTestDir, "",
-                                              log_size, 0, keep_log_file_num);
+  AutoRollLogger* logger =
+      new AutoRollLogger(FileSystem::Default(), SystemClock::Default(),
+                         kTestDir, "", log_size, 0, keep_log_file_num);
 
   LogMessage(logger, kSampleMessage.c_str());
   ASSERT_GT(logger->GetLogFileSize(), kZero);
   delete logger;
 
   // reopens the log file and an empty log file will be created.
-  logger = new AutoRollLogger(Env::Default(), kTestDir, "", log_size, 0, 10);
+  logger = new AutoRollLogger(FileSystem::Default(), SystemClock::Default(),
+                              kTestDir, "", log_size, 0, 10);
   ASSERT_EQ(logger->GetLogFileSize(), kZero);
   delete logger;
 }
@@ -273,16 +271,17 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {
 
   InitTestDb();
 
-  NoSleepEnv nse(Env::Default());
-  AutoRollLogger logger(&nse, kTestDir, "", log_max_size, time,
-                        keep_log_file_num);
+  auto nsc =
+      std::make_shared<EmulatedSystemClock>(SystemClock::Default(), true);
+  AutoRollLogger logger(FileSystem::Default(), nsc, kTestDir, "", log_max_size,
+                        time, keep_log_file_num);
 
   // Test the ability to roll by size
   RollLogFileBySizeTest(&logger, log_max_size,
                         kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
 
   // Test the ability to roll by Time
-  RollLogFileByTimeTest(&nse, &logger, time,
+  RollLogFileByTimeTest(FileSystem::Default(), nsc, &logger, time,
                         kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
 }
 
@@ -291,7 +290,10 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {
 // port
 TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
   DBOptions options;
-  NoSleepEnv nse(Env::Default());
+  auto nsc =
+      std::make_shared<EmulatedSystemClock>(SystemClock::Default(), true);
+  std::unique_ptr<Env> nse(new CompositeEnvWrapper(Env::Default(), nsc));
+
   std::shared_ptr<Logger> logger;
 
   // Normal logger
@@ -310,14 +312,15 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
       kSampleMessage + ":CreateLoggerFromOptions - size");
 
   // Only roll by Time
-  options.env = &nse;
+  options.env = nse.get();
   InitTestDb();
   options.max_log_file_size = 0;
   options.log_file_time_to_roll = 2;
   ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
   auto_roll_logger =
     dynamic_cast<AutoRollLogger*>(logger.get());
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
+  RollLogFileByTimeTest(options.env->GetFileSystem(), nsc, auto_roll_logger,
+                        options.log_file_time_to_roll,
                         kSampleMessage + ":CreateLoggerFromOptions - time");
 
   // roll by both Time and size
@@ -329,7 +332,8 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
     dynamic_cast<AutoRollLogger*>(logger.get());
   RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
                         kSampleMessage + ":CreateLoggerFromOptions - both");
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
+  RollLogFileByTimeTest(options.env->GetFileSystem(), nsc, auto_roll_logger,
+                        options.log_file_time_to_roll,
                         kSampleMessage + ":CreateLoggerFromOptions - both");
 
   // Set keep_log_file_num
@@ -402,8 +406,8 @@ TEST_F(AutoRollLoggerTest, AutoDeleting) {
     const size_t kMaxFileSize = 512;
     {
       size_t log_num = 8;
-      AutoRollLogger logger(Env::Default(), dbname, db_log_dir, kMaxFileSize, 0,
-                            log_num);
+      AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                            dbname, db_log_dir, kMaxFileSize, 0, log_num);
       RollNTimesBySize(&logger, log_num, kMaxFileSize);
 
       ASSERT_EQ(log_num, GetLogFiles().size());
@@ -411,8 +415,8 @@ TEST_F(AutoRollLoggerTest, AutoDeleting) {
     // Shrink number of files
     {
       size_t log_num = 5;
-      AutoRollLogger logger(Env::Default(), dbname, db_log_dir, kMaxFileSize, 0,
-                            log_num);
+      AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                            dbname, db_log_dir, kMaxFileSize, 0, log_num);
       ASSERT_EQ(log_num, GetLogFiles().size());
 
       RollNTimesBySize(&logger, 3, kMaxFileSize);
@@ -422,8 +426,8 @@ TEST_F(AutoRollLoggerTest, AutoDeleting) {
     // Increase number of files again.
     {
       size_t log_num = 7;
-      AutoRollLogger logger(Env::Default(), dbname, db_log_dir, kMaxFileSize, 0,
-                            log_num);
+      AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                            dbname, db_log_dir, kMaxFileSize, 0, log_num);
       ASSERT_EQ(6, GetLogFiles().size());
 
       RollNTimesBySize(&logger, 3, kMaxFileSize);
@@ -485,7 +489,8 @@ TEST_F(AutoRollLoggerTest, InfoLogLevel) {
   // an extra-scope to force the AutoRollLogger to flush the log file when it
   // becomes out of scope.
   {
-    AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, 0, 10);
+    AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                          kTestDir, "", log_size, 0, 10);
     for (int log_level = InfoLogLevel::HEADER_LEVEL;
          log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) {
       logger.SetInfoLogLevel((InfoLogLevel)log_level);
@@ -523,7 +528,8 @@ TEST_F(AutoRollLoggerTest, Close) {
 
   size_t log_size = 8192;
   size_t log_lines = 0;
-  AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, 0, 10);
+  AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(), kTestDir,
+                        "", log_size, 0, 10);
   for (int log_level = InfoLogLevel::HEADER_LEVEL;
        log_level >= InfoLogLevel::DEBUG_LEVEL; log_level--) {
     logger.SetInfoLogLevel((InfoLogLevel)log_level);
@@ -590,8 +596,9 @@ TEST_F(AutoRollLoggerTest, LogHeaderTest) {
 
     InitTestDb();
 
-    AutoRollLogger logger(Env::Default(), kTestDir, /*db_log_dir=*/"",
-                          LOG_MAX_SIZE, /*log_file_time_to_roll=*/0,
+    AutoRollLogger logger(FileSystem::Default(), SystemClock::Default(),
+                          kTestDir, /*db_log_dir=*/"", LOG_MAX_SIZE,
+                          /*log_file_time_to_roll=*/0,
                           /*keep_log_file_num=*/10);
 
     if (test_num == 0) {
@@ -665,6 +672,50 @@ TEST_F(AutoRollLoggerTest, FileCreateFailure) {
   ASSERT_NOK(CreateLoggerFromOptions("", options, &logger));
   ASSERT_TRUE(!logger);
 }
+
+TEST_F(AutoRollLoggerTest, RenameOnlyWhenExists) {
+  InitTestDb();
+  SpecialEnv env(Env::Default());
+  Options options;
+  options.env = &env;
+
+  // Originally no LOG exists. Should not see a rename.
+  {
+    std::shared_ptr<Logger> logger;
+    ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
+    ASSERT_EQ(0, env.rename_count_);
+  }
+
+  // Now a LOG exists. Create a new one should see a rename.
+  {
+    std::shared_ptr<Logger> logger;
+    ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
+    ASSERT_EQ(1, env.rename_count_);
+  }
+}
+
+TEST_F(AutoRollLoggerTest, RenameError) {
+  InitTestDb();
+  SpecialEnv env(Env::Default());
+  env.rename_error_ = true;
+  Options options;
+  options.env = &env;
+
+  // Originally no LOG exists. Should not be impacted by rename error.
+  {
+    std::shared_ptr<Logger> logger;
+    ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
+    ASSERT_TRUE(logger != nullptr);
+  }
+
+  // Now a LOG exists. Rename error should cause failure.
+  {
+    std::shared_ptr<Logger> logger;
+    ASSERT_NOK(CreateLoggerFromOptions(kTestDir, options, &logger));
+    ASSERT_TRUE(logger == nullptr);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/logging/env_logger.h

@@ -31,9 +31,10 @@ class EnvLogger : public Logger {
             const std::string& fname, const EnvOptions& options, Env* env,
             InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL)
       : Logger(log_level),
-        file_(std::move(writable_file), fname, options, env),
-        last_flush_micros_(0),
         env_(env),
+        clock_(env_->GetSystemClock().get()),
+        file_(std::move(writable_file), fname, options, clock_),
+        last_flush_micros_(0),
         flush_pending_(false) {}
 
   ~EnvLogger() {
@@ -50,7 +51,7 @@ class EnvLogger : public Logger {
       flush_pending_ = false;
       file_.Flush().PermitUncheckedError();
     }
-    last_flush_micros_ = env_->NowMicros();
+    last_flush_micros_ = clock_->NowMicros();
   }
 
   void Flush() override {
@@ -136,7 +137,7 @@ class EnvLogger : public Logger {
       // We will ignore any error returned by Append().
       file_.Append(Slice(base, p - base)).PermitUncheckedError();
       flush_pending_ = true;
-      const uint64_t now_micros = env_->NowMicros();
+      const uint64_t now_micros = clock_->NowMicros();
       if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) {
         FlushLocked();
       }
@@ -154,11 +155,12 @@ class EnvLogger : public Logger {
   }
 
  private:
+  Env* env_;
+  SystemClock* clock_;
   WritableFileWriter file_;
   mutable port::Mutex mutex_;  // Mutex to protect the shared variables below.
   const static uint64_t flush_every_seconds_ = 5;
   std::atomic_uint_fast64_t last_flush_micros_;
-  Env* env_;
   std::atomic<bool> flush_pending_;
 };
 

package/deps/rocksdb/rocksdb/logging/env_logger_test.cc

@@ -5,7 +5,6 @@
 //
 
 #include "logging/env_logger.h"
-#include "env/mock_env.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 

package/deps/rocksdb/rocksdb/logging/posix_logger.h

@@ -108,15 +108,9 @@ class PosixLogger : public Logger {
       const time_t seconds = now_tv.tv_sec;
       struct tm t;
       localtime_r(&seconds, &t);
-      p += snprintf(p, limit - p,
-                    "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ",
-                    t.tm_year + 1900,
-                    t.tm_mon + 1,
-                    t.tm_mday,
-                    t.tm_hour,
-                    t.tm_min,
-                    t.tm_sec,
-                    static_cast<int>(now_tv.tv_usec),
+      p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llu ",
+                    t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour,
+                    t.tm_min, t.tm_sec, static_cast<int>(now_tv.tv_usec),
                     static_cast<long long unsigned int>(thread_id));
 
       // Print the message

package/deps/rocksdb/rocksdb/memory/arena.cc

@@ -12,11 +12,13 @@
 #include <sys/mman.h>
 #endif
 #include <algorithm>
+
 #include "logging/logging.h"
 #include "port/malloc.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
 #include "test_util/sync_point.h"
+#include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -170,7 +172,7 @@ char* Arena::AllocateAligned(size_t bytes, size_t huge_page_size,
     if (addr == nullptr) {
       ROCKS_LOG_WARN(logger,
                      "AllocateAligned fail to allocate huge TLB pages: %s",
-                     strerror(errno));
+                     errnoStr(errno).c_str());
       // fail back to malloc
     } else {
       return addr;

package/deps/rocksdb/rocksdb/memory/arena.h

@@ -86,7 +86,7 @@ class Arena : public Allocator {
   // Number of bytes allocated in one block
   const size_t kBlockSize;
   // Array of new[] allocated memory blocks
-  typedef std::vector<char*> Blocks;
+  using Blocks = std::vector<char*>;
   Blocks blocks_;
 
   struct MmapInfo {