@nxtedition/rocksdb 5.2.21 → 5.2.28

package/deps/rocksdb/rocksdb/util/string_util.h

@@ -39,8 +39,47 @@ extern void AppendNumberTo(std::string* str, uint64_t num);
 // Escapes any non-printable characters found in "value".
 extern void AppendEscapedStringTo(std::string* str, const Slice& value);
 
-// Return a string printout of "num"
-extern std::string NumberToString(uint64_t num);
+// Put n digits from v in base kBase to (*buf)[0] to (*buf)[n-1] and
+// advance *buf to the position after what was written.
+template <size_t kBase>
+inline void PutBaseChars(char** buf, size_t n, uint64_t v, bool uppercase) {
+  const char* digitChars = uppercase ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                                     : "0123456789abcdefghijklmnopqrstuvwxyz";
+  for (size_t i = n; i > 0; --i) {
+    (*buf)[i - 1] = digitChars[static_cast<size_t>(v % kBase)];
+    v /= kBase;
+  }
+  *buf += n;
+}
+
+// Parse n digits from *buf in base kBase to *v and advance *buf to the
+// position after what was read. On success, true is returned. On failure,
+// false is returned, *buf is placed at the first bad character, and *v
+// contains the partial parsed data. Overflow is not checked but the
+// result is accurate mod 2^64. Requires the starting value of *v to be
+// zero or previously accumulated parsed digits, i.e.
+//   ParseBaseChars(&b, n, &v);
+// is equivalent to n calls to
+//   ParseBaseChars(&b, 1, &v);
+template <int kBase>
+inline bool ParseBaseChars(const char** buf, size_t n, uint64_t* v) {
+  while (n) {
+    char c = **buf;
+    *v *= static_cast<uint64_t>(kBase);
+    if (c >= '0' && (kBase >= 10 ? c <= '9' : c < '0' + kBase)) {
+      *v += static_cast<uint64_t>(c - '0');
+    } else if (kBase > 10 && c >= 'A' && c < 'A' + kBase - 10) {
+      *v += static_cast<uint64_t>(c - 'A' + 10);
+    } else if (kBase > 10 && c >= 'a' && c < 'a' + kBase - 10) {
+      *v += static_cast<uint64_t>(c - 'a' + 10);
+    } else {
+      return false;
+    }
+    --n;
+    ++*buf;
+  }
+  return true;
+}
 
 // Return a human-readable version of num.
 // for num >= 10.000, prints "xxK"
@@ -120,6 +159,8 @@ bool StartsWith(const std::string& string, const std::string& pattern);
 #ifndef ROCKSDB_LITE
 bool ParseBoolean(const std::string& type, const std::string& value);
 
+uint8_t ParseUint8(const std::string& value);
+
 uint32_t ParseUint32(const std::string& value);
 
 int32_t ParseInt32(const std::string& value);
@@ -141,4 +182,8 @@ bool SerializeIntVector(const std::vector<int>& vec, std::string* value);
 
 extern const std::string kNullptrString;
 
+// errnoStr() function returns a string that describes the error code passed in
+// the argument err
+extern std::string errnoStr(int err);
+
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/thread_guard.h

@@ -0,0 +1,41 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "port/port.h"
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Resource management object for threads that joins the thread upon
+// destruction. Has unique ownership of the thread object, so copying it is not
+// allowed, while moving it transfers ownership.
+class ThreadGuard {
+ public:
+  ThreadGuard() = default;
+
+  explicit ThreadGuard(port::Thread&& thread) : thread_(std::move(thread)) {}
+
+  ThreadGuard(const ThreadGuard&) = delete;
+  ThreadGuard& operator=(const ThreadGuard&) = delete;
+
+  ThreadGuard(ThreadGuard&&) noexcept = default;
+  ThreadGuard& operator=(ThreadGuard&&) noexcept = default;
+
+  ~ThreadGuard() {
+    if (thread_.joinable()) {
+      thread_.join();
+    }
+  }
+
+  const port::Thread& GetThread() const { return thread_; }
+  port::Thread& GetThread() { return thread_; }
+
+ private:
+  port::Thread thread_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/thread_local.h

@@ -31,7 +31,7 @@ namespace ROCKSDB_NAMESPACE {
 // is needed to avoid deadlocks. In particular, the handler shouldn't lock any
 // mutexes and shouldn't call any methods of any ThreadLocalPtr instances,
 // unless you know what you're doing.
-typedef void (*UnrefHandler)(void* ptr);
+using UnrefHandler = void (*)(void* ptr);
 
 // ThreadLocalPtr stores only values of pointer type.  Different from
 // the usual thread-local-storage, ThreadLocalPtr has the ability to
@@ -69,7 +69,7 @@ class ThreadLocalPtr {
   // data for all existing threads
   void Scrape(autovector<void*>* ptrs, void* const replacement);
 
-  typedef std::function<void(void*, void*)> FoldFunc;
+  using FoldFunc = std::function<void(void*, void*)>;
   // Update res by applying func on each thread-local value. Holds a lock that
   // prevents unref handler from running during this call, but clients must
   // still provide external synchronization since the owning thread can

package/deps/rocksdb/rocksdb/util/thread_local_test.cc

@@ -3,9 +3,11 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-#include <thread>
+#include "util/thread_local.h"
+
 #include <atomic>
 #include <string>
+#include <thread>
 
 #include "port/port.h"
 #include "rocksdb/env.h"
@@ -13,7 +15,6 @@
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 #include "util/autovector.h"
-#include "util/thread_local.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -51,10 +52,8 @@ struct Params {
 };
 
 class IDChecker : public ThreadLocalPtr {
-public:
-  static uint32_t PeekId() {
-    return TEST_PeekId();
-  }
+ public:
+  static uint32_t PeekId() { return TEST_PeekId(); }
 };
 
 }  // anonymous namespace
@@ -122,9 +121,8 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
   ASSERT_GT(IDChecker::PeekId(), base_id);
   base_id = IDChecker::PeekId();
 
-  auto func = [](void* ptr) {
-    auto& params = *static_cast<Params*>(ptr);
-
+  auto func = [](Params* ptr) {
+    Params& params = *ptr;
     ASSERT_TRUE(params.tls1.Get() == nullptr);
     params.tls1.Reset(reinterpret_cast<int*>(1));
     ASSERT_TRUE(params.tls1.Get() == reinterpret_cast<int*>(1));
@@ -146,7 +144,8 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
   for (int iter = 0; iter < 1024; ++iter) {
     ASSERT_EQ(IDChecker::PeekId(), base_id);
     // Another new thread, read/write should not see value from previous thread
-    env_->StartThread(func, static_cast<void*>(&p));
+    env_->StartThreadTyped(func, &p);
+
     mu.Lock();
     while (p.completed != iter + 1) {
       cv.Wait();
@@ -221,10 +220,10 @@ TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) {
   // Each thread local copy of the value are also different from each
   // other.
   for (int th = 0; th < p1.total; ++th) {
-    env_->StartThread(func, static_cast<void*>(&p1));
+    env_->StartThreadTyped(func, &p1);
   }
   for (int th = 0; th < p2.total; ++th) {
-    env_->StartThread(func, static_cast<void*>(&p2));
+    env_->StartThreadTyped(func, &p2);
   }
 
   mu1.Lock();
@@ -251,9 +250,8 @@ TEST_F(ThreadLocalTest, Unref) {
   };
 
   // Case 0: no unref triggered if ThreadLocalPtr is never accessed
-  auto func0 = [](void* ptr) {
-    auto& p = *static_cast<Params*>(ptr);
-
+  auto func0 = [](Params* ptr) {
+    auto& p = *ptr;
     p.mu->Lock();
     ++(p.started);
     p.cv->SignalAll();
@@ -270,15 +268,15 @@ TEST_F(ThreadLocalTest, Unref) {
     Params p(&mu, &cv, &unref_count, th, unref);
 
     for (int i = 0; i < p.total; ++i) {
-      env_->StartThread(func0, static_cast<void*>(&p));
+      env_->StartThreadTyped(func0, &p);
     }
     env_->WaitForJoin();
     ASSERT_EQ(unref_count, 0);
   }
 
   // Case 1: unref triggered by thread exit
-  auto func1 = [](void* ptr) {
-    auto& p = *static_cast<Params*>(ptr);
+  auto func1 = [](Params* ptr) {
+    auto& p = *ptr;
 
     p.mu->Lock();
     ++(p.started);
@@ -307,7 +305,7 @@ TEST_F(ThreadLocalTest, Unref) {
     p.tls2 = &tls2;
 
     for (int i = 0; i < p.total; ++i) {
-      env_->StartThread(func1, static_cast<void*>(&p));
+      env_->StartThreadTyped(func1, &p);
     }
 
     env_->WaitForJoin();
@@ -317,8 +315,8 @@ TEST_F(ThreadLocalTest, Unref) {
   }
 
   // Case 2: unref triggered by ThreadLocal instance destruction
-  auto func2 = [](void* ptr) {
-    auto& p = *static_cast<Params*>(ptr);
+  auto func2 = [](Params* ptr) {
+    auto& p = *ptr;
 
     p.mu->Lock();
     ++(p.started);
@@ -356,7 +354,7 @@ TEST_F(ThreadLocalTest, Unref) {
     p.tls2 = new ThreadLocalPtr(unref);
 
     for (int i = 0; i < p.total; ++i) {
-      env_->StartThread(func2, static_cast<void*>(&p));
+      env_->StartThreadTyped(func2, &p);
     }
 
     // Wait for all threads to finish using Params
@@ -431,7 +429,7 @@ TEST_F(ThreadLocalTest, Scrape) {
     p.tls2 = new ThreadLocalPtr(unref);
 
     for (int i = 0; i < p.total; ++i) {
-      env_->StartThread(func, static_cast<void*>(&p));
+      env_->StartThreadTyped(func, &p);
     }
 
     // Wait for all threads to finish using Params
@@ -490,7 +488,7 @@ TEST_F(ThreadLocalTest, Fold) {
   };
 
   for (int th = 0; th < params.total; ++th) {
-    env_->StartThread(func, static_cast<void*>(&params));
+    env_->StartThread(func, &params);
   }
 
   // Wait for all threads to finish using Params

package/deps/rocksdb/rocksdb/util/threadpool_imp.cc

@@ -19,6 +19,7 @@
 #endif
 
 #include <stdlib.h>
+
 #include <algorithm>
 #include <atomic>
 #include <condition_variable>
@@ -31,12 +32,13 @@
 #include "monitoring/thread_status_util.h"
 #include "port/port.h"
 #include "test_util/sync_point.h"
+#include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 void ThreadPoolImpl::PthreadCall(const char* label, int result) {
   if (result != 0) {
-    fprintf(stderr, "pthread %s: %s\n", label, strerror(result));
+    fprintf(stderr, "pthread %s: %s\n", label, errnoStr(result).c_str());
     abort();
   }
 }
@@ -198,12 +200,12 @@ void ThreadPoolImpl::Impl::BGThread(size_t thread_id) {
           queue_.empty()) {
         break;
        }
-    }
-
-    if (IsLastExcessiveThread(thread_id)) {
+    } else if (IsLastExcessiveThread(thread_id)) {
       // Current thread is the last generated one and is excessive.
       // We always terminate excessive thread in the reverse order of
-      // generation time.
+      // generation time. But not when `exit_all_threads_ == true`,
+      // otherwise `JoinThreads()` could try to `join()` a `detach()`ed
+      // thread.
       auto& terminating_thread = bgthreads_.back();
       terminating_thread.detach();
       bgthreads_.pop_back();
@@ -345,7 +347,6 @@ void ThreadPoolImpl::Impl::StartBGThreads() {
     for (char c : thread_priority) {
       thread_name_stream << static_cast<char>(tolower(c));
     }
-    thread_name_stream << bgthreads_.size();
     pthread_setname_np(th_handle, thread_name_stream.str().c_str());
 #endif
 #endif

package/deps/rocksdb/rocksdb/util/timer.h

@@ -14,7 +14,7 @@
 #include <vector>
 
 #include "monitoring/instrumented_mutex.h"
-#include "rocksdb/env.h"
+#include "rocksdb/system_clock.h"
 #include "test_util/sync_point.h"
 #include "util/mutexlock.h"
 
@@ -38,9 +38,9 @@ namespace ROCKSDB_NAMESPACE {
 // A map from a function name to the function keeps track of all the functions.
 class Timer {
  public:
-  explicit Timer(Env* env)
-      : env_(env),
-        mutex_(env),
+  explicit Timer(SystemClock* clock)
+      : clock_(clock),
+        mutex_(clock),
         cond_var_(&mutex_),
         running_(false),
         executing_task_(false) {}
@@ -48,36 +48,38 @@ class Timer {
   ~Timer() { Shutdown(); }
 
   // Add a new function to run.
-  // fn_name has to be identical, otherwise, the new one overrides the existing
-  // one, regardless if the function is pending removed (invalid) or not.
+  // fn_name has to be identical, otherwise it will fail to add and return false
   // start_after_us is the initial delay.
   // repeat_every_us is the interval between ending time of the last call and
   // starting time of the next call. For example, repeat_every_us = 2000 and
   // the function takes 1000us to run. If it starts at time [now]us, then it
   // finishes at [now]+1000us, 2nd run starting time will be at [now]+3000us.
   // repeat_every_us == 0 means do not repeat.
-  void Add(std::function<void()> fn,
-           const std::string& fn_name,
-           uint64_t start_after_us,
-           uint64_t repeat_every_us) {
-    std::unique_ptr<FunctionInfo> fn_info(
-        new FunctionInfo(std::move(fn), fn_name,
-                         env_->NowMicros() + start_after_us, repeat_every_us));
-    {
-      InstrumentedMutexLock l(&mutex_);
-      auto it = map_.find(fn_name);
-      if (it == map_.end()) {
-        heap_.push(fn_info.get());
-        map_.emplace(std::make_pair(fn_name, std::move(fn_info)));
-      } else {
-        // If it already exists, overriding it.
-        it->second->fn = std::move(fn_info->fn);
-        it->second->valid = true;
-        it->second->next_run_time_us = env_->NowMicros() + start_after_us;
-        it->second->repeat_every_us = repeat_every_us;
-      }
+  bool Add(std::function<void()> fn, const std::string& fn_name,
+           uint64_t start_after_us, uint64_t repeat_every_us) {
+    auto fn_info = std::make_unique<FunctionInfo>(std::move(fn), fn_name, 0,
+                                                  repeat_every_us);
+    InstrumentedMutexLock l(&mutex_);
+    // Assign time within mutex to make sure the next_run_time is larger than
+    // the current running one
+    fn_info->next_run_time_us = clock_->NowMicros() + start_after_us;
+    // the new task start time should never before the current task executing
+    // time, as the executing task can only be running if it's next_run_time_us
+    // is due (<= clock_->NowMicros()).
+    if (executing_task_ &&
+        fn_info->next_run_time_us < heap_.top()->next_run_time_us) {
+      return false;
+    }
+    auto it = map_.find(fn_name);
+    if (it == map_.end()) {
+      heap_.push(fn_info.get());
+      map_.try_emplace(fn_name, std::move(fn_info));
+    } else {
+      // timer doesn't support duplicated function name
+      return false;
     }
     cond_var_.SignalAll();
+    return true;
   }
 
   void Cancel(const std::string& fn_name) {
@@ -116,7 +118,7 @@ class Timer {
     }
 
     running_ = true;
-    thread_.reset(new port::Thread(&Timer::Run, this));
+    thread_ = std::make_unique<port::Thread>(&Timer::Run, this);
     return true;
   }
 
@@ -140,8 +142,8 @@ class Timer {
 
   bool HasPendingTask() const {
     InstrumentedMutexLock l(&mutex_);
-    for (auto it = map_.begin(); it != map_.end(); it++) {
-      if (it->second->IsValid()) {
+    for (const auto& fn_info : map_) {
+      if (fn_info.second->IsValid()) {
         return true;
       }
     }
@@ -151,34 +153,34 @@ class Timer {
 #ifndef NDEBUG
   // Wait until Timer starting waiting, call the optional callback, then wait
   // for Timer waiting again.
-  // Tests can provide a custom env object to mock time, and use the callback
+  // Tests can provide a custom Clock object to mock time, and use the callback
   // here to bump current time and trigger Timer. See timer_test for example.
   //
   // Note: only support one caller of this method.
-  void TEST_WaitForRun(std::function<void()> callback = nullptr) {
+  void TEST_WaitForRun(const std::function<void()>& callback = nullptr) {
     InstrumentedMutexLock l(&mutex_);
     // It act as a spin lock
     while (executing_task_ ||
            (!heap_.empty() &&
-            heap_.top()->next_run_time_us <= env_->NowMicros())) {
-      cond_var_.TimedWait(env_->NowMicros() + 1000);
+            heap_.top()->next_run_time_us <= clock_->NowMicros())) {
+      cond_var_.TimedWait(clock_->NowMicros() + 1000);
     }
     if (callback != nullptr) {
       callback();
     }
     cond_var_.SignalAll();
     do {
-      cond_var_.TimedWait(env_->NowMicros() + 1000);
-    } while (
-        executing_task_ ||
-        (!heap_.empty() && heap_.top()->next_run_time_us <= env_->NowMicros()));
+      cond_var_.TimedWait(clock_->NowMicros() + 1000);
+    } while (executing_task_ ||
+             (!heap_.empty() &&
+              heap_.top()->next_run_time_us <= clock_->NowMicros()));
   }
 
   size_t TEST_GetPendingTaskNum() const {
     InstrumentedMutexLock l(&mutex_);
     size_t ret = 0;
-    for (auto it = map_.begin(); it != map_.end(); it++) {
-      if (it->second->IsValid()) {
+    for (const auto& fn_info : map_) {
+      if (fn_info.second->IsValid()) {
         ret++;
       }
     }
@@ -208,7 +210,7 @@ class Timer {
         continue;
       }
 
-      if (current_fn->next_run_time_us <= env_->NowMicros()) {
+      if (current_fn->next_run_time_us <= clock_->NowMicros()) {
         // make a copy of the function so it won't be changed after
         // mutex_.unlock.
         std::function<void()> fn = current_fn->fn;
@@ -220,20 +222,27 @@ class Timer {
         executing_task_ = false;
         cond_var_.SignalAll();
 
-        // Remove the work from the heap once it is done executing.
+        // Remove the work from the heap once it is done executing, make sure
+        // it's the same function after executing the work while mutex is
+        // released.
         // Note that we are just removing the pointer from the heap. Its
         // memory is still managed in the map (as it holds a unique ptr).
         // So current_fn is still a valid ptr.
+        assert(heap_.top() == current_fn);
         heap_.pop();
 
         // current_fn may be cancelled already.
         if (current_fn->IsValid() && current_fn->repeat_every_us > 0) {
           assert(running_);
-          current_fn->next_run_time_us = env_->NowMicros() +
-              current_fn->repeat_every_us;
+          current_fn->next_run_time_us =
+              clock_->NowMicros() + current_fn->repeat_every_us;
 
           // Schedule new work into the heap with new time.
           heap_.push(current_fn);
+        } else {
+          // if current_fn is cancelled or no need to repeat, remove it from the
+          // map to avoid leak.
+          map_.erase(current_fn->name);
         }
       } else {
         cond_var_.TimedWait(current_fn->next_run_time_us);
@@ -280,10 +289,10 @@ class Timer {
     // calls `Cancel()`.
     bool valid;
 
-    FunctionInfo(std::function<void()>&& _fn, const std::string& _name,
+    FunctionInfo(std::function<void()>&& _fn, std::string _name,
                  const uint64_t _next_run_time_us, uint64_t _repeat_every_us)
         : fn(std::move(_fn)),
-          name(_name),
+          name(std::move(_name)),
           next_run_time_us(_next_run_time_us),
           repeat_every_us(_repeat_every_us),
           valid(true) {}
@@ -310,7 +319,7 @@ class Timer {
     }
   };
 
-  Env* const env_;
+  SystemClock* clock_;
   // This mutex controls both the heap_ and the map_. It needs to be held for
   // making any changes in them.
   mutable InstrumentedMutex mutex_;