@nxtedition/rocksdb 8.2.8 → 9.0.0

package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc

@@ -81,6 +81,7 @@ class ReduceLevelTest : public testing::Test {
 
 Status ReduceLevelTest::OpenDB(bool create_if_missing, int num_levels) {
   ROCKSDB_NAMESPACE::Options opt;
+  opt.level_compaction_dynamic_level_bytes = false;
   opt.num_levels = num_levels;
   opt.create_if_missing = create_if_missing;
   ROCKSDB_NAMESPACE::Status st =

package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc

@@ -468,4 +468,3 @@ int main(int argc, char** argv) {
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();
 }
-

package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc

@@ -419,6 +419,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
       filename = std::string(dir_or_file) + "/" + filename;
     }
 
+    if (command == "verify") {
+      verify_checksum = true;
+    }
+
     ROCKSDB_NAMESPACE::SstFileDumper dumper(
         options, filename, Temperature::kUnknown, readahead_size,
         verify_checksum, output_hex, decode_blob_index);

package/deps/rocksdb/rocksdb/unreleased_history/README.txt

@@ -0,0 +1,73 @@
+Adding release notes
+--------------------
+
+When adding release notes for the next release, add a file to one of these
+directories:
+
+unreleased_history/new_features
+unreleased_history/behavior_changes
+unreleased_history/public_api_changes
+unreleased_history/bug_fixes
+
+with a unique name that makes sense for your change, preferably using the .md
+extension for syntax highlighting.
+
+There is a script to help, as in
+
+$ unreleased_history/add.sh unreleased_history/bug_fixes/crash_in_feature.md
+
+or simply
+
+$ unreleased_history/add.sh
+
+will take you through some prompts.
+
+The file should usually contain one line of markdown, and "* " is not
+required, as it will automatically be inserted later if not included at the
+start of the first line in the file. Extra newlines or missing trailing
+newlines will also be corrected.
+
+The only times release notes should be added directly to HISTORY are if
+* A release is being amended or corrected after it is already "cut" but not
+tagged, which should be rare.
+* A single commit contains a noteworthy change and a patch release version bump
+
+
+Ordering of entries
+-------------------
+
+Within each group, entries will be included using ls sort order, so important
+entries could start their file name with a small three digit number like
+100pretty_important.md.
+
+The ordering of groups such as new_features vs. public_api_changes is
+hard-coded in unreleased_history/release.sh
+
+
+Updating HISTORY.md with release notes
+--------------------------------------
+
+The script unreleased_history/release.sh does this. Run the script before
+updating version.h to the next develpment release, so that the script will pick
+up the version being released. You might want to start with
+
+$ DRY_RUN=1 unreleased_history/release.sh | less
+
+to check for problems and preview the output. Then run
+
+$ unreleased_history/release.sh
+
+which will git rm some files and modify HISTORY.md. You still need to commit the
+changes, or revert with the command reported in the output.
+
+
+Why not update HISTORY.md directly?
+-----------------------------------
+
+First, it was common to hit unnecessary merge conflicts when adding entries to
+HISTORY.md, which slowed development. Second, when a PR was opened before a
+release cut and landed after the release cut, it was easy to add the HISTORY
+entry to the wrong version's history. This new setup completely fixes both of
+those issues, with perhaps slighly more initial work to create each entry.
+There is also now an extra step in using `git blame` to map a release note
+to its source code implementation, but that is a relatively rare operation.

package/deps/rocksdb/rocksdb/unreleased_history/add.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+set -e
+set -o pipefail
+
+if [ "$1" ]; then
+  # Target file specified on command line
+  TARGET="$1"
+else
+  # Interactively choose a group and file name
+  DIRS="`find unreleased_history/ -mindepth 1 -maxdepth 1 -type d`"
+  echo "Choose a group for new release note:"
+  echo "$DIRS" | grep -nEo '[^/]+$'
+  echo -n "Enter a number: "
+  while [ ! "$DIRNUM" ]; do read -r DIRNUM; done
+  DIR="$(echo "$DIRS" | head -n "$DIRNUM" | tail -1)"
+  echo "Choose a file name for new release note (e.g. improved_whatever.md):"
+  while [ ! "$FNAME" ]; do read -re FNAME; done
+  # Replace spaces with underscores
+  TARGET="$(echo "$DIR/$FNAME" | tr ' ' '_')"
+fi
+
+# Edit/create the file
+${EDITOR:-nano} "$TARGET"
+# Add to version control (easy to forget!)
+git add "$TARGET"

package/deps/rocksdb/rocksdb/unreleased_history/release.sh

@@ -0,0 +1,104 @@
+#! /usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+set -e
+
+if [ ! -d unreleased_history ]; then
+  echo "Can't find unreleased_history/ directory"
+  exit 1
+fi
+
+GIT_PATHS="unreleased_history/ HISTORY.md"
+if [ ! "$DRY_RUN" ]; then
+  # Check for uncommitted changes
+  UNCOMMITTED="$(git diff -- $GIT_PATHS)"
+  if [ "$UNCOMMITTED" ]; then
+    echo 'Uncommitted changes to files to be modified. Please commit first to'
+    echo 'ensure a clean revert path. You can always `git commit -a --amend`'
+    echo 'to add more changes to your commit.'
+    exit 2
+  fi
+fi
+
+# Add first part of existing HISTORY file to new version
+awk '{ print } /NOTE/ { exit(0) }' < HISTORY.md > HISTORY.new
+
+# And a blank line separator
+echo >> HISTORY.new
+
+# Add new version header
+awk '/#define ROCKSDB_MAJOR/ { major = $3 }
+     /#define ROCKSDB_MINOR/ { minor = $3 }
+     /#define ROCKSDB_PATCH/ { patch = $3 }
+     END { printf "## " major "." minor "." patch }' < include/rocksdb/version.h >> HISTORY.new
+echo " (`git log -n1 --date=format:"%m/%d/%Y" --format="%ad"`)" >> HISTORY.new
+
+function process_file () {
+  # use awk to correct
+  # * extra or missing newlines
+  # * leading or trailing whitespace
+  # * missing '* ' on first line
+  awk '/./ { gsub(/^[ \t]+/, ""); gsub(/[ \t]+$/, "");
+             if (notfirstline || $1 == "*") print;
+             else print "* " $0;
+             notfirstline=1; }' < $1 >> HISTORY.new
+  echo git rm $1
+  if [ ! "$DRY_RUN" ]; then
+    git rm $1
+  fi
+}
+
+PROCESSED_DIRECTORIES=""
+
+function process_dir () {
+  PROCESSED_DIRECTORIES="$PROCESSED_DIRECTORIES $1"
+  # ls will sort the files, including the permanent header file
+  FILES="$(ls unreleased_history/$1/)"
+  if [ "$FILES" ]; then
+    echo "### $2" >> HISTORY.new
+    for FILE in $FILES; do
+      process_file "unreleased_history/$1/$FILE"
+    done
+    echo >> HISTORY.new
+    echo "Saved entries from $1"
+  else
+    echo "Nothing new in $1"
+  fi
+}
+
+# Process dirs and files
+process_dir new_features "New Features"
+process_dir public_api_changes "Public API Changes"
+process_dir behavior_changes "Behavior Changes"
+process_dir bug_fixes "Bug Fixes"
+process_dir performance_improvements "Performance Improvements"
+
+# Check for unexpected files or dirs at top level. process_dir/process_file
+# will deal with contents of these directories
+EXPECTED_REGEX="[^/]*[.]sh|README[.]txt|$(echo $PROCESSED_DIRECTORIES | tr ' ' '|')"
+platform=`uname`
+if [ $platform = 'Darwin' ]; then
+  UNEXPECTED="$(find -E unreleased_history -mindepth 1 -maxdepth 1 -not -regex "[^/]*/($EXPECTED_REGEX)")"
+else
+  UNEXPECTED="$(find unreleased_history/ -mindepth 1 -maxdepth 1 -regextype egrep -not -regex "[^/]*/($EXPECTED_REGEX)")"
+fi
+if [ "$UNEXPECTED" ]; then
+  echo "Unexpected files I don't know how to process:"
+  echo "$UNEXPECTED"
+  rm HISTORY.new
+  exit 3
+fi
+
+# Add rest of existing HISTORY file to new version (collapsing newlines)
+awk '/./ { if (note) pr=1 }
+     /NOTE/ { note=1 }
+     { if (pr) print }' < HISTORY.md >> HISTORY.new
+
+if [ "$DRY_RUN" ]; then
+  echo '==========================================='
+  diff -U3 HISTORY.md HISTORY.new || true
+  rm HISTORY.new
+else
+  mv HISTORY.new HISTORY.md
+  echo "Done. Revert command: git checkout HEAD -- $GIT_PATHS"
+fi

package/deps/rocksdb/rocksdb/util/async_file_reader.cc

@@ -26,6 +26,11 @@ bool AsyncFileReader::MultiReadAsyncImpl(ReadAwaiter* awaiter) {
           FSReadRequest* read_req = static_cast<FSReadRequest*>(cb_arg);
           read_req->status = req.status;
           read_req->result = req.result;
+          if (req.fs_scratch != nullptr) {
+            // TODO akanksha: Revisit to remove the const in the callback.
+            FSReadRequest& req_tmp = const_cast<FSReadRequest&>(req);
+            read_req->fs_scratch = std::move(req_tmp.fs_scratch);
+          }
         },
         &awaiter->read_reqs_[i], &awaiter->io_handle_[i], &awaiter->del_fn_[i],
         /*aligned_buf=*/nullptr);

package/deps/rocksdb/rocksdb/util/bloom_impl.h

@@ -199,7 +199,7 @@ class FastLocalBloomImpl {
 
   static inline void AddHash(uint32_t h1, uint32_t h2, uint32_t len_bytes,
                              int num_probes, char *data) {
-    uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6;
+    uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     AddHashPrepared(h2, num_probes, data + bytes_to_cache_line);
   }
 
@@ -216,7 +216,7 @@ class FastLocalBloomImpl {
   static inline void PrepareHash(uint32_t h1, uint32_t len_bytes,
                                  const char *data,
                                  uint32_t /*out*/ *byte_offset) {
-    uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6;
+    uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     PREFETCH(data + bytes_to_cache_line, 0 /* rw */, 1 /* locality */);
     PREFETCH(data + bytes_to_cache_line + 63, 0 /* rw */, 1 /* locality */);
     *byte_offset = bytes_to_cache_line;
@@ -224,7 +224,7 @@ class FastLocalBloomImpl {
 
   static inline bool HashMayMatch(uint32_t h1, uint32_t h2, uint32_t len_bytes,
                                   int num_probes, const char *data) {
-    uint32_t bytes_to_cache_line = FastRange32(len_bytes >> 6, h1) << 6;
+    uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     return HashMayMatchPrepared(h2, num_probes, data + bytes_to_cache_line);
   }
 

package/deps/rocksdb/rocksdb/util/cast_util.h

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include <memory>
 #include <type_traits>
 
 #include "rocksdb/rocksdb_namespace.h"
@@ -23,6 +24,19 @@ inline DestClass* static_cast_with_check(SrcClass* x) {
   return ret;
 }
 
+template <class DestClass, class SrcClass>
+inline std::shared_ptr<DestClass> static_cast_with_check(
+    std::shared_ptr<SrcClass>&& x) {
+#if defined(ROCKSDB_USE_RTTI) && !defined(NDEBUG)
+  auto orig_raw = x.get();
+#endif
+  auto ret = std::static_pointer_cast<DestClass>(std::move(x));
+#if defined(ROCKSDB_USE_RTTI) && !defined(NDEBUG)
+  assert(ret.get() == dynamic_cast<DestClass*>(orig_raw));
+#endif
+  return ret;
+}
+
 // A wrapper around static_cast for lossless conversion between integral
 // types, including enum types. For example, this can be used for converting
 // between signed/unsigned or enum type and underlying type without fear of

package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc

@@ -12,6 +12,7 @@ void CompactionJobStats::Reset() {
   elapsed_micros = 0;
   cpu_micros = 0;
 
+  has_num_input_records = true;
   num_input_records = 0;
   num_blobs_read = 0;
   num_input_files = 0;
@@ -55,6 +56,7 @@ void CompactionJobStats::Add(const CompactionJobStats& stats) {
   elapsed_micros += stats.elapsed_micros;
   cpu_micros += stats.cpu_micros;
 
+  has_num_input_records &= stats.has_num_input_records;
   num_input_records += stats.num_input_records;
   num_blobs_read += stats.num_blobs_read;
   num_input_files += stats.num_input_files;

package/deps/rocksdb/rocksdb/util/comparator.cc

@@ -250,6 +250,12 @@ class ComparatorWithU64TsImpl : public Comparator {
 
   const char* Name() const override { return kClassName(); }
 
+  // The comparator that compares the user key without timestamp part is treated
+  // as the root comparator.
+  const Comparator* GetRootComparator() const override {
+    return &cmp_without_ts_;
+  }
+
   void FindShortSuccessor(std::string*) const override {}
   void FindShortestSeparator(std::string*, const Slice&) const override {}
   int Compare(const Slice& a, const Slice& b) const override {
@@ -316,24 +322,37 @@ const Comparator* BytewiseComparatorWithU64Ts() {
   return &comp_with_u64_ts;
 }
 
+const Comparator* ReverseBytewiseComparatorWithU64Ts() {
+  STATIC_AVOID_DESTRUCTION(
+      ComparatorWithU64TsImpl<ReverseBytewiseComparatorImpl>, comp_with_u64_ts);
+  return &comp_with_u64_ts;
+}
+
 static int RegisterBuiltinComparators(ObjectLibrary& library,
                                       const std::string& /*arg*/) {
   library.AddFactory<const Comparator>(
       BytewiseComparatorImpl::kClassName(),
       [](const std::string& /*uri*/,
-         std::unique_ptr<const Comparator>* /*guard */,
-         std::string* /* errmsg */) { return BytewiseComparator(); });
+         std::unique_ptr<const Comparator>* /*guard*/,
+         std::string* /*errmsg*/) { return BytewiseComparator(); });
   library.AddFactory<const Comparator>(
       ReverseBytewiseComparatorImpl::kClassName(),
       [](const std::string& /*uri*/,
-         std::unique_ptr<const Comparator>* /*guard */,
-         std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
+         std::unique_ptr<const Comparator>* /*guard*/,
+         std::string* /*errmsg*/) { return ReverseBytewiseComparator(); });
   library.AddFactory<const Comparator>(
       ComparatorWithU64TsImpl<BytewiseComparatorImpl>::kClassName(),
       [](const std::string& /*uri*/,
-         std::unique_ptr<const Comparator>* /*guard */,
-         std::string* /* errmsg */) { return BytewiseComparatorWithU64Ts(); });
-  return 3;
+         std::unique_ptr<const Comparator>* /*guard*/,
+         std::string* /*errmsg*/) { return BytewiseComparatorWithU64Ts(); });
+  library.AddFactory<const Comparator>(
+      ComparatorWithU64TsImpl<ReverseBytewiseComparatorImpl>::kClassName(),
+      [](const std::string& /*uri*/,
+         std::unique_ptr<const Comparator>* /*guard*/,
+         std::string* /*errmsg*/) {
+        return ReverseBytewiseComparatorWithU64Ts();
+      });
+  return 4;
 }
 
 Status Comparator::CreateFromString(const ConfigOptions& config_options,
@@ -357,6 +376,9 @@ Status Comparator::CreateFromString(const ConfigOptions& config_options,
   } else if (id ==
              ComparatorWithU64TsImpl<BytewiseComparatorImpl>::kClassName()) {
     *result = BytewiseComparatorWithU64Ts();
+  } else if (id == ComparatorWithU64TsImpl<
+                       ReverseBytewiseComparatorImpl>::kClassName()) {
+    *result = ReverseBytewiseComparatorWithU64Ts();
   } else if (value.empty()) {
     // No Id and no options.  Clear the object
     *result = nullptr;

package/deps/rocksdb/rocksdb/util/compression.cc

@@ -48,7 +48,7 @@ int ZSTDStreamingCompress::Compress(const char* input, size_t input_size,
   if (input_size == 0) {
     return 0;
   }
-#ifndef ZSTD_STREAMING
+#ifndef ZSTD_ADVANCED
   (void)input;
   (void)input_size;
   (void)output;
@@ -77,7 +77,7 @@ int ZSTDStreamingCompress::Compress(const char* input, size_t input_size,
 }
 
 void ZSTDStreamingCompress::Reset() {
-#ifdef ZSTD_STREAMING
+#ifdef ZSTD_ADVANCED
   ZSTD_CCtx_reset(cctx_, ZSTD_ResetDirective::ZSTD_reset_session_only);
   input_buffer_ = {/*src=*/nullptr, /*size=*/0, /*pos=*/0};
 #endif
@@ -91,7 +91,7 @@ int ZSTDStreamingUncompress::Uncompress(const char* input, size_t input_size,
   if (input_size == 0) {
     return 0;
   }
-#ifdef ZSTD_STREAMING
+#ifdef ZSTD_ADVANCED
   if (input) {
     // New input
     input_buffer_ = {input, input_size, /*pos=*/0};
@@ -113,7 +113,7 @@ int ZSTDStreamingUncompress::Uncompress(const char* input, size_t input_size,
 }
 
 void ZSTDStreamingUncompress::Reset() {
-#ifdef ZSTD_STREAMING
+#ifdef ZSTD_ADVANCED
   ZSTD_DCtx_reset(dctx_, ZSTD_ResetDirective::ZSTD_reset_session_only);
   input_buffer_ = {/*src=*/nullptr, /*size=*/0, /*pos=*/0};
 #endif