@nxtedition/rocksdb 8.1.4 → 8.1.6

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h

@@ -11,6 +11,7 @@
 #ifndef ROCKSDB_LITE
 
 #include <cstdint>
+#include <forward_list>
 #include <functional>
 #include <map>
 #include <string>
@@ -23,6 +24,8 @@
 #include "rocksdb/status.h"
 
 namespace ROCKSDB_NAMESPACE {
+class BackupEngineReadOnlyBase;
+class BackupEngine;
 
 // The default DB file checksum function name.
 constexpr char kDbFileChecksumFuncName[] = "FileChecksumCrc32c";
@@ -270,6 +273,28 @@ inline BackupEngineOptions::ShareFilesNaming operator|(
   return static_cast<BackupEngineOptions::ShareFilesNaming>(l | r);
 }
 
+// Identifying information about a backup shared file that is (or might be)
+// excluded from a backup using exclude_files_callback.
+struct BackupExcludedFileInfo {
+  explicit BackupExcludedFileInfo(const std::string& _relative_file)
+      : relative_file(_relative_file) {}
+
+  // File name and path relative to the backup dir.
+  std::string relative_file;
+};
+
+// An auxiliary structure for exclude_files_callback
+struct MaybeExcludeBackupFile {
+  explicit MaybeExcludeBackupFile(BackupExcludedFileInfo&& _info)
+      : info(std::move(_info)) {}
+
+  // Identifying information about a backup shared file that could be excluded
+  const BackupExcludedFileInfo info;
+
+  // API user sets to true if the file should be excluded from this backup
+  bool exclude_decision = false;
+};
+
 struct CreateBackupOptions {
   // Flush will always trigger if 2PC is enabled.
   // If write-ahead logs are disabled, set flush_before_backup=true to
@@ -278,10 +303,31 @@ struct CreateBackupOptions {
 
   // Callback for reporting progress, based on callback_trigger_interval_size.
   //
-  // RocksDB callbacks are NOT exception-safe. A callback completing with an
-  // exception can lead to undefined behavior in RocksDB, including data loss,
-  // unreported corruption, deadlocks, and more.
-  std::function<void()> progress_callback = []() {};
+  // An exception thrown from the callback will result in Status::Aborted from
+  // the operation.
+  std::function<void()> progress_callback = {};
+
+  // A callback that allows the API user to select files for exclusion, such
+  // as if the files are known to exist in an alternate backup directory.
+  // Only "shared" files can be excluded from backups. This is an advanced
+  // feature because the BackupEngine user is trusted to keep track of files
+  // such that the DB can be restored.
+  //
+  // Input to the callback is a [begin,end) range of sharable files live in
+  // the DB being backed up, and the callback implementation sets
+  // exclude_decision=true for files to exclude. A callback offers maximum
+  // flexibility, e.g. if remote files are unavailable at backup time but
+  // whose existence has been recorded somewhere. In case of an empty or
+  // no-op callback, all files are included in the backup .
+  //
+  // To restore the DB, RestoreOptions::alternate_dirs must be used to provide
+  // the excluded files.
+  //
+  // An exception thrown from the callback will result in Status::Aborted from
+  // the operation.
+  std::function<void(MaybeExcludeBackupFile* files_begin,
+                     MaybeExcludeBackupFile* files_end)>
+      exclude_files_callback = {};
 
   // If false, background_thread_cpu_priority is ignored.
   // Otherwise, the cpu priority can be decreased,
@@ -300,6 +346,11 @@ struct RestoreOptions {
   // Default: false
   bool keep_log_files;
 
+  // For backups that were created using exclude_files_callback, this
+  // option enables restoring those backups by providing BackupEngines on
+  // directories known to contain the required files.
+  std::forward_list<BackupEngineReadOnlyBase*> alternate_dirs;
+
   explicit RestoreOptions(bool _keep_log_files = false)
       : keep_log_files(_keep_log_files) {}
 };
@@ -324,9 +375,15 @@ struct BackupInfo {
   // Backup API user metadata
   std::string app_metadata;
 
-  // Backup file details, if requested with include_file_details=true
+  // Backup file details, if requested with include_file_details=true.
+  // Does not include excluded_files.
   std::vector<BackupFileInfo> file_details;
 
+  // Identifying information about shared files that were excluded from the
+  // created backup. See exclude_files_callback and alternate_dirs.
+  // This information is only provided if include_file_details=true.
+  std::vector<BackupExcludedFileInfo> excluded_files;
+
   // DB "name" (a directory in the backup_env) for opening this backup as a
   // read-only DB. This should also be used as the DBOptions::wal_dir, such
   // as by default setting wal_dir="". See also env_for_open.
@@ -348,8 +405,8 @@ struct BackupInfo {
 
   BackupInfo() {}
 
-  BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size,
-             uint32_t _number_files, const std::string& _app_metadata)
+  explicit BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size,
+                      uint32_t _number_files, const std::string& _app_metadata)
       : backup_id(_backup_id),
         timestamp(_timestamp),
         size(_size),
@@ -364,8 +421,8 @@ class BackupStatistics {
     number_fail_backup = 0;
   }
 
-  BackupStatistics(uint32_t _number_success_backup,
-                   uint32_t _number_fail_backup)
+  explicit BackupStatistics(uint32_t _number_success_backup,
+                            uint32_t _number_fail_backup)
       : number_success_backup(_number_success_backup),
         number_fail_backup(_number_fail_backup) {}
 
@@ -462,6 +519,9 @@ class BackupEngineReadOnlyBase {
   // Returns Status::OK() if all checks are good
   virtual IOStatus VerifyBackup(BackupID backup_id,
                                 bool verify_with_checksum = false) const = 0;
+
+  // Internal use only
+  virtual BackupEngine* AsBackupEngine() = 0;
 };
 
 // Append-only functions of a BackupEngine. See BackupEngine comment for

package/deps/rocksdb/rocksdb/include/rocksdb/version.h

@@ -12,7 +12,7 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 7
-#define ROCKSDB_MINOR 9
+#define ROCKSDB_MINOR 10
 #define ROCKSDB_PATCH 2
 
 // Do not use these. We made the mistake of declaring macros starting with

package/deps/rocksdb/rocksdb/memory/arena.cc

@@ -8,9 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "memory/arena.h"
-#ifndef OS_WIN
-#include <sys/mman.h>
-#endif
+
 #include <algorithm>
 
 #include "logging/logging.h"
@@ -22,16 +20,7 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-// MSVC complains that it is already defined since it is static in the header.
-#ifndef _MSC_VER
-const size_t Arena::kInlineSize;
-#endif
-
-const size_t Arena::kMinBlockSize = 4096;
-const size_t Arena::kMaxBlockSize = 2u << 30;
-static const int kAlignUnit = alignof(max_align_t);
-
-size_t OptimizeBlockSize(size_t block_size) {
+size_t Arena::OptimizeBlockSize(size_t block_size) {
   // Make sure block_size is in optimal range
   block_size = std::max(Arena::kMinBlockSize, block_size);
   block_size = std::min(Arena::kMaxBlockSize, block_size);
@@ -53,14 +42,12 @@ Arena::Arena(size_t block_size, AllocTracker* tracker, size_t huge_page_size)
   blocks_memory_ += alloc_bytes_remaining_;
   aligned_alloc_ptr_ = inline_block_;
   unaligned_alloc_ptr_ = inline_block_ + alloc_bytes_remaining_;
-#ifdef MAP_HUGETLB
-  hugetlb_size_ = huge_page_size;
-  if (hugetlb_size_ && kBlockSize > hugetlb_size_) {
-    hugetlb_size_ = ((kBlockSize - 1U) / hugetlb_size_ + 1U) * hugetlb_size_;
+  if (MemMapping::kHugePageSupported) {
+    hugetlb_size_ = huge_page_size;
+    if (hugetlb_size_ && kBlockSize > hugetlb_size_) {
+      hugetlb_size_ = ((kBlockSize - 1U) / hugetlb_size_ + 1U) * hugetlb_size_;
+    }
   }
-#else
-  (void)huge_page_size;
-#endif
   if (tracker_ != nullptr) {
     tracker_->Allocate(kInlineSize);
   }
@@ -71,21 +58,6 @@ Arena::~Arena() {
     assert(tracker_->is_freed());
     tracker_->FreeMem();
   }
-  for (const auto& block : blocks_) {
-    delete[] block;
-  }
-
-#ifdef MAP_HUGETLB
-  for (const auto& mmap_info : huge_blocks_) {
-    if (mmap_info.addr_ == nullptr) {
-      continue;
-    }
-    auto ret = munmap(mmap_info.addr_, mmap_info.length_);
-    if (ret != 0) {
-      // TODO(sdong): Better handling
-    }
-  }
-#endif
 }
 
 char* Arena::AllocateFallback(size_t bytes, bool aligned) {
@@ -99,12 +71,10 @@ char* Arena::AllocateFallback(size_t bytes, bool aligned) {
   // We waste the remaining space in the current block.
   size_t size = 0;
   char* block_head = nullptr;
-#ifdef MAP_HUGETLB
-  if (hugetlb_size_) {
+  if (MemMapping::kHugePageSupported && hugetlb_size_ > 0) {
     size = hugetlb_size_;
     block_head = AllocateFromHugePage(size);
   }
-#endif
   if (!block_head) {
     size = kBlockSize;
     block_head = AllocateNewBlock(size);
@@ -123,45 +93,22 @@ char* Arena::AllocateFallback(size_t bytes, bool aligned) {
 }
 
 char* Arena::AllocateFromHugePage(size_t bytes) {
-#ifdef MAP_HUGETLB
-  if (hugetlb_size_ == 0) {
-    return nullptr;
-  }
-  // Reserve space in `huge_blocks_` before calling `mmap`.
-  // Use `emplace_back()` instead of `reserve()` to let std::vector manage its
-  // own memory and do fewer reallocations.
-  //
-  // - If `emplace_back` throws, no memory leaks because we haven't called
-  //   `mmap` yet.
-  // - If `mmap` throws, no memory leaks because the vector will be cleaned up
-  //   via RAII.
-  huge_blocks_.emplace_back(nullptr /* addr */, 0 /* length */);
-
-  void* addr = mmap(nullptr, bytes, (PROT_READ | PROT_WRITE),
-                    (MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB), -1, 0);
-
-  if (addr == MAP_FAILED) {
-    return nullptr;
-  }
-  huge_blocks_.back() = MmapInfo(addr, bytes);
-  blocks_memory_ += bytes;
-  if (tracker_ != nullptr) {
-    tracker_->Allocate(bytes);
+  MemMapping mm = MemMapping::AllocateHuge(bytes);
+  auto addr = static_cast<char*>(mm.Get());
+  if (addr) {
+    huge_blocks_.push_back(std::move(mm));
+    blocks_memory_ += bytes;
+    if (tracker_ != nullptr) {
+      tracker_->Allocate(bytes);
+    }
   }
-  return reinterpret_cast<char*>(addr);
-#else
-  (void)bytes;
-  return nullptr;
-#endif
+  return addr;
 }
 
 char* Arena::AllocateAligned(size_t bytes, size_t huge_page_size,
                              Logger* logger) {
-  assert((kAlignUnit & (kAlignUnit - 1)) ==
-         0);  // Pointer size should be a power of 2
-
-#ifdef MAP_HUGETLB
-  if (huge_page_size > 0 && bytes > 0) {
+  if (MemMapping::kHugePageSupported && hugetlb_size_ > 0 &&
+      huge_page_size > 0 && bytes > 0) {
     // Allocate from a huge page TLB table.
     size_t reserved_size =
         ((bytes - 1U) / huge_page_size + 1U) * huge_page_size;
@@ -177,10 +124,6 @@ char* Arena::AllocateAligned(size_t bytes, size_t huge_page_size,
       return addr;
     }
   }
-#else
-  (void)huge_page_size;
-  (void)logger;
-#endif
 
   size_t current_mod =
       reinterpret_cast<uintptr_t>(aligned_alloc_ptr_) & (kAlignUnit - 1);
@@ -200,17 +143,11 @@ char* Arena::AllocateAligned(size_t bytes, size_t huge_page_size,
 }
 
 char* Arena::AllocateNewBlock(size_t block_bytes) {
-  // Reserve space in `blocks_` before allocating memory via new.
-  // Use `emplace_back()` instead of `reserve()` to let std::vector manage its
-  // own memory and do fewer reallocations.
-  //
-  // - If `emplace_back` throws, no memory leaks because we haven't called `new`
-  //   yet.
-  // - If `new` throws, no memory leaks because the vector will be cleaned up
-  //   via RAII.
-  blocks_.emplace_back(nullptr);
-
+  // NOTE: std::make_unique zero-initializes the block so is not appropriate
+  // here
   char* block = new char[block_bytes];
+  blocks_.push_back(std::unique_ptr<char[]>(block));
+
   size_t allocated_size;
 #ifdef ROCKSDB_MALLOC_USABLE_SIZE
   allocated_size = malloc_usable_size(block);
@@ -227,7 +164,6 @@ char* Arena::AllocateNewBlock(size_t block_bytes) {
   if (tracker_ != nullptr) {
     tracker_->Allocate(allocated_size);
   }
-  blocks_.back() = block;
   return block;
 }
 

package/deps/rocksdb/rocksdb/memory/arena.h

@@ -12,16 +12,13 @@
 // size, it uses malloc to directly get the requested size.
 
 #pragma once
-#ifndef OS_WIN
-#include <sys/mman.h>
-#endif
-#include <assert.h>
-#include <stdint.h>
-#include <cerrno>
+
 #include <cstddef>
-#include <vector>
+#include <deque>
+
 #include "memory/allocator.h"
-#include "util/mutexlock.h"
+#include "port/mmap.h"
+#include "rocksdb/env.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -31,9 +28,13 @@ class Arena : public Allocator {
   Arena(const Arena&) = delete;
   void operator=(const Arena&) = delete;
 
-  static const size_t kInlineSize = 2048;
-  static const size_t kMinBlockSize;
-  static const size_t kMaxBlockSize;
+  static constexpr size_t kInlineSize = 2048;
+  static constexpr size_t kMinBlockSize = 4096;
+  static constexpr size_t kMaxBlockSize = 2u << 30;
+
+  static constexpr unsigned kAlignUnit = alignof(std::max_align_t);
+  static_assert((kAlignUnit & (kAlignUnit - 1)) == 0,
+                "Pointer size should be power of 2");
 
   // huge_page_size: if 0, don't use huge page TLB. If > 0 (should set to the
   // supported hugepage size of the system), block allocation will try huge
@@ -63,7 +64,7 @@ class Arena : public Allocator {
   // by the arena (exclude the space allocated but not yet used for future
   // allocations).
   size_t ApproximateMemoryUsage() const {
-    return blocks_memory_ + blocks_.capacity() * sizeof(char*) -
+    return blocks_memory_ + blocks_.size() * sizeof(char*) -
            alloc_bytes_remaining_;
   }
 
@@ -81,21 +82,19 @@ class Arena : public Allocator {
     return blocks_.empty() && huge_blocks_.empty();
   }
 
+  // check and adjust the block_size so that the return value is
+  //  1. in the range of [kMinBlockSize, kMaxBlockSize].
+  //  2. the multiple of align unit.
+  static size_t OptimizeBlockSize(size_t block_size);
+
  private:
-  char inline_block_[kInlineSize] __attribute__((__aligned__(alignof(max_align_t))));
+  alignas(std::max_align_t) char inline_block_[kInlineSize];
   // Number of bytes allocated in one block
   const size_t kBlockSize;
-  // Array of new[] allocated memory blocks
-  using Blocks = std::vector<char*>;
-  Blocks blocks_;
-
-  struct MmapInfo {
-    void* addr_;
-    size_t length_;
-
-    MmapInfo(void* addr, size_t length) : addr_(addr), length_(length) {}
-  };
-  std::vector<MmapInfo> huge_blocks_;
+  // Allocated memory blocks
+  std::deque<std::unique_ptr<char[]>> blocks_;
+  // Huge page allocations
+  std::deque<MemMapping> huge_blocks_;
   size_t irregular_block_num = 0;
 
   // Stats for current active block.
@@ -108,15 +107,15 @@ class Arena : public Allocator {
   // How many bytes left in currently active block?
   size_t alloc_bytes_remaining_ = 0;
 
-#ifdef MAP_HUGETLB
   size_t hugetlb_size_ = 0;
-#endif  // MAP_HUGETLB
+
   char* AllocateFromHugePage(size_t bytes);
   char* AllocateFallback(size_t bytes, bool aligned);
   char* AllocateNewBlock(size_t block_bytes);
 
   // Bytes of memory in blocks allocated so far
   size_t blocks_memory_ = 0;
+  // Non-owned
   AllocTracker* tracker_;
 };
 
@@ -133,9 +132,4 @@ inline char* Arena::Allocate(size_t bytes) {
   return AllocateFallback(bytes, false /* unaligned */);
 }
 
-// check and adjust the block_size so that the return value is
-//  1. in the range of [kMinBlockSize, kMaxBlockSize].
-//  2. the multiple of align unit.
-extern size_t OptimizeBlockSize(size_t block_size);
-
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/memory/arena_test.cc

@@ -8,6 +8,11 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "memory/arena.h"
+
+#ifndef OS_WIN
+#include <sys/resource.h>
+#endif
+#include "port/port.h"
 #include "test_util/testharness.h"
 #include "util/random.h"
 
@@ -196,6 +201,91 @@ TEST_F(ArenaTest, Simple) {
   SimpleTest(0);
   SimpleTest(kHugePageSize);
 }
+
+// Number of minor page faults since last call
+size_t PopMinorPageFaultCount() {
+#ifdef RUSAGE_SELF
+  static long prev = 0;
+  struct rusage usage;
+  EXPECT_EQ(getrusage(RUSAGE_SELF, &usage), 0);
+  size_t rv = usage.ru_minflt - prev;
+  prev = usage.ru_minflt;
+  return rv;
+#else
+  // Conservative
+  return SIZE_MAX;
+#endif  // RUSAGE_SELF
+}
+
+TEST(MmapTest, AllocateLazyZeroed) {
+  // Doesn't have to be page aligned
+  constexpr size_t len = 1234567;
+  MemMapping m = MemMapping::AllocateLazyZeroed(len);
+  auto arr = static_cast<char*>(m.Get());
+
+  // Should generally work
+  ASSERT_NE(arr, nullptr);
+
+  // Start counting page faults
+  PopMinorPageFaultCount();
+
+  // Access half of the allocation
+  size_t i = 0;
+  for (; i < len / 2; ++i) {
+    ASSERT_EQ(arr[i], 0);
+    arr[i] = static_cast<char>(i & 255);
+  }
+
+  // Appropriate page faults (maybe more)
+  size_t faults = PopMinorPageFaultCount();
+  ASSERT_GE(faults, len / 2 / port::kPageSize);
+
+  // Access rest of the allocation
+  for (; i < len; ++i) {
+    ASSERT_EQ(arr[i], 0);
+    arr[i] = static_cast<char>(i & 255);
+  }
+
+  // Appropriate page faults (maybe more)
+  faults = PopMinorPageFaultCount();
+  ASSERT_GE(faults, len / 2 / port::kPageSize);
+
+  // Verify data
+  for (i = 0; i < len; ++i) {
+    ASSERT_EQ(arr[i], static_cast<char>(i & 255));
+  }
+}
+
+TEST_F(ArenaTest, UnmappedAllocation) {
+  // Verify that it's possible to get unmapped pages in large allocations,
+  // for memory efficiency and to ensure we don't accidentally waste time &
+  // space initializing the memory.
+  constexpr size_t kBlockSize = 2U << 20;
+  Arena arena(kBlockSize);
+
+  // The allocator might give us back recycled memory for a while, but
+  // shouldn't last forever.
+  for (int i = 0;; ++i) {
+    char* p = arena.Allocate(kBlockSize);
+
+    // Start counting page faults
+    PopMinorPageFaultCount();
+
+    // Overwrite the whole allocation
+    for (size_t j = 0; j < kBlockSize; ++j) {
+      p[j] = static_cast<char>(j & 255);
+    }
+
+    size_t faults = PopMinorPageFaultCount();
+    if (faults >= kBlockSize * 3 / 4 / port::kPageSize) {
+      // Most of the access generated page faults => GOOD
+      break;
+    }
+    // Should have succeeded after enough tries
+    ASSERT_LT(i, 1000);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/memory/memory_allocator.h

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include <algorithm>
+
 #include "rocksdb/memory_allocator.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -35,4 +37,11 @@ inline CacheAllocationPtr AllocateBlock(size_t size,
   return CacheAllocationPtr(new char[size]);
 }
 
+inline CacheAllocationPtr AllocateAndCopyBlock(const Slice& data,
+                                               MemoryAllocator* allocator) {
+  CacheAllocationPtr cap = AllocateBlock(data.size(), allocator);
+  std::copy_n(data.data(), data.size(), cap.get());
+  return cap;
+}
+
 }  // namespace ROCKSDB_NAMESPACE