@nxtedition/rocksdb 7.1.20 → 7.1.21

package/deps/rocksdb/rocksdb/env/env_basic_test.cc

@@ -395,6 +395,7 @@ TEST_P(EnvMoreTestWithParam, GetChildrenIgnoresDotAndDotDot) {
 
 }  // namespace ROCKSDB_NAMESPACE
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/env/fs_posix.cc

@@ -575,7 +575,7 @@ class PosixFileSystem : public FileSystem {
     }
   }
 
-  IOStatus GetChildren(const std::string& dir, const IOOptions& /*opts*/,
+  IOStatus GetChildren(const std::string& dir, const IOOptions& opts,
                        std::vector<std::string>* result,
                        IODebugContext* /*dbg*/) override {
     result->clear();
@@ -595,12 +595,20 @@ class PosixFileSystem : public FileSystem {
     // reset errno before calling readdir()
     errno = 0;
     struct dirent* entry;
+
     while ((entry = readdir(d)) != nullptr) {
       // filter out '.' and '..' directory entries
       // which appear only on some platforms
       const bool ignore =
           entry->d_type == DT_DIR &&
-          (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0);
+          (strcmp(entry->d_name, ".") == 0 ||
+           strcmp(entry->d_name, "..") == 0
+#ifndef ASSERT_STATUS_CHECKED
+           // In case of ASSERT_STATUS_CHECKED, GetChildren support older
+           // version of API for debugging purpose.
+           || opts.do_not_recurse
+#endif
+          );
       if (!ignore) {
         result->push_back(entry->d_name);
       }

package/deps/rocksdb/rocksdb/env/io_posix_test.cc

@@ -135,6 +135,7 @@ TEST_F(LogicalBlockSizeCacheTest, Ref) {
 #endif
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/env/mock_env_test.cc

@@ -78,6 +78,7 @@ TEST_F(MockEnvTest, FakeSleeping) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/file/delete_scheduler.cc

@@ -141,7 +141,11 @@ Status DeleteScheduler::CleanupDirectory(Env* env, SstFileManagerImpl* sfm,
   Status s;
   // Check if there are any files marked as trash in this path
   std::vector<std::string> files_in_path;
-  s = env->GetChildren(path, &files_in_path);
+  const auto& fs = env->GetFileSystem();
+  IOOptions io_opts;
+  io_opts.do_not_recurse = true;
+  s = fs->GetChildren(path, io_opts, &files_in_path,
+                      /*IODebugContext*=*/nullptr);
   if (!s.ok()) {
     return s;
   }

package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc

@@ -712,6 +712,7 @@ TEST_F(DeleteSchedulerTest, IsTrashCheck) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/file/prefetch_test.cc

@@ -2079,6 +2079,7 @@ namespace {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
 
   return RUN_ALL_TESTS();

package/deps/rocksdb/rocksdb/file/writable_file_writer.h

@@ -286,7 +286,7 @@ class WritableFileWriter {
 
   bool use_direct_io() { return writable_file_->use_direct_io(); }
 
-  bool TEST_BufferIsEmpty() { return buf_.CurrentSize() == 0; }
+  bool BufferIsEmpty() { return buf_.CurrentSize() == 0; }
 
   void TEST_SetFileChecksumGenerator(
       FileChecksumGenerator* checksum_generator) {

package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h

@@ -271,7 +271,10 @@ struct AdvancedColumnFamilyOptions {
   // read amplification because a get request has to check in all of these
   // files. Also, an in-memory merge may result in writing lesser
   // data to storage if there are duplicate records in each of these
-  // individual write buffers.  Default: 1
+  // individual write buffers.
+  // If atomic flush is enabled (options.atomic_flush == true), then this
+  // option will be sanitized to 1.
+  // Default: 1
   int min_write_buffer_number_to_merge = 1;
 
   // DEPRECATED
@@ -648,6 +651,15 @@ struct AdvancedColumnFamilyOptions {
   // Default: false
   bool level_compaction_dynamic_level_bytes = false;
 
+  // Allows RocksDB to generate files that are not exactly the target_file_size
+  // only for the non-bottommost files. Which can reduce the write-amplification
+  // from compaction. The file size could be from 0 to 2x target_file_size.
+  // Once enabled, non-bottommost compaction will try to cut the files align
+  // with the next level file boundaries (grandparent level).
+  //
+  // Default: true
+  bool level_compaction_dynamic_file_size = true;
+
   // Default: 10.
   //
   // Dynamically changeable through SetOptions() API
@@ -672,6 +684,17 @@ struct AdvancedColumnFamilyOptions {
   // Dynamically changeable through SetOptions() API
   uint64_t max_compaction_bytes = 0;
 
+  // When setting up compaction input files, we ignore the
+  // `max_compaction_bytes` limit when pulling in input files that are entirely
+  // within output key range.
+  //
+  // Default: true
+  //
+  // Dynamically changeable through SetOptions() API
+  // We could remove this knob and always ignore the limit once it is proven
+  // safe.
+  bool ignore_max_compaction_bytes_for_input = true;
+
   // All writes will be slowed down to at least delayed_write_rate if estimated
   // bytes needed to be compaction exceed this threshold.
   //
@@ -899,8 +922,33 @@ struct AdvancedColumnFamilyOptions {
   //  size constrained, the size amp is going to be only for non-last levels.
   //
   // Default: 0 (disable the feature)
+  //
+  // Not dynamically changeable, change it requires db restart.
   uint64_t preclude_last_level_data_seconds = 0;
 
+  // EXPERIMENTAL
+  // If this option is set, it will preserve the internal time information about
+  // the data until it's older than the specified time here.
+  // Internally the time information is a map between sequence number and time,
+  // which is the same as `preclude_last_level_data_seconds`. But it won't
+  // preclude the data from the last level and the data in the last level won't
+  // have the sequence number zeroed out.
+  // Internally, rocksdb would sample the sequence number to time pair and store
+  // that in SST property "rocksdb.seqno.time.map". The information is currently
+  // only used for tiered storage compaction (option
+  // `preclude_last_level_data_seconds`).
+  //
+  // Note: if both `preclude_last_level_data_seconds` and this option is set, it
+  //  will preserve the max time of the 2 options and compaction still preclude
+  //  the data based on `preclude_last_level_data_seconds`.
+  //  The higher the preserve_time is, the less the sampling frequency will be (
+  //  which means less accuracy of the time estimation).
+  //
+  // Default: 0 (disable the feature)
+  //
+  // Not dynamically changeable, change it requires db restart.
+  uint64_t preserve_internal_time_seconds = 0;
+
   // When set, large values (blobs) are written to separate blob files, and
   // only pointers to them are stored in SST files. This can reduce write
   // amplification for large-value use cases at the cost of introducing a level

package/deps/rocksdb/rocksdb/include/rocksdb/cache.h

@@ -569,9 +569,11 @@ class Cache {
   // over time.
 
   // Insert a mapping from key->value into the cache and assign it
-  // the specified charge against the total cache capacity.
-  // If strict_capacity_limit is true and cache reaches its full capacity,
-  // return Status::MemoryLimit.
+  // the specified charge against the total cache capacity. If
+  // strict_capacity_limit is true and cache reaches its full capacity,
+  // return Status::MemoryLimit. `value` must be non-nullptr for this
+  // Insert() because Value() == nullptr is reserved for indicating failure
+  // with secondary-cache-compatible mappings.
   //
   // The helper argument is saved by the cache and will be used when the
   // inserted object is evicted or promoted to the secondary cache. It,
@@ -619,11 +621,31 @@ class Cache {
   // saved and used later when the object is evicted. Therefore, it must
   // outlive the cache.
   //
-  // The handle returned may not be ready. The caller should call IsReady()
-  // to check if the item value is ready, and call Wait() or WaitAll() if
-  // its not ready. The caller should then call Value() to check if the
-  // item was successfully retrieved. If unsuccessful (perhaps due to an
-  // IO error), Value() will return nullptr.
+  // ======================== Async Lookup (wait=false) ======================
+  // When wait=false, the handle returned might be in any of three states:
+  // * Present - If Value() != nullptr, then the result is present and
+  // the handle can be used just as if wait=true.
+  // * Pending, not ready (IsReady() == false) - secondary cache is still
+  // working to retrieve the value. Might become ready any time.
+  // * Pending, ready (IsReady() == true) - secondary cache has the value
+  // but it has not been loaded into primary cache. Call to Wait()/WaitAll()
+  // will not block.
+  //
+  // IMPORTANT: Pending handles are not thread-safe, and only these functions
+  // are allowed on them: Value(), IsReady(), Wait(), WaitAll(). Even Release()
+  // can only come after Wait() or WaitAll() even though a reference is held.
+  //
+  // Only Wait()/WaitAll() gets a Handle out of a Pending state. (Waiting is
+  // safe and has no effect on other handle states.) After waiting on a Handle,
+  // it is in one of two states:
+  // * Present - if Value() != nullptr
+  // * Failed - if Value() == nullptr, such as if the secondary cache
+  // initially thought it had the value but actually did not.
+  //
+  // Note that given an arbitrary Handle, the only way to distinguish the
+  // Pending+ready state from the Failed state is to Wait() on it. A cache
+  // entry not compatible with secondary cache can also have Value()==nullptr
+  // like the Failed state, but this is not generally a concern.
   virtual Handle* Lookup(const Slice& key, const CacheItemHelper* /*helper_cb*/,
                          const CreateCallback& /*create_cb*/,
                          Priority /*priority*/, bool /*wait*/,
@@ -634,27 +656,31 @@ class Cache {
   // Release a mapping returned by a previous Lookup(). The "useful"
   // parameter specifies whether the data was actually used or not,
   // which may be used by the cache implementation to decide whether
-  // to consider it as a hit for retention purposes.
+  // to consider it as a hit for retention purposes. As noted elsewhere,
+  // "pending" handles require Wait()/WaitAll() before Release().
   virtual bool Release(Handle* handle, bool /*useful*/,
                        bool erase_if_last_ref) {
     return Release(handle, erase_if_last_ref);
   }
 
-  // Determines if the handle returned by Lookup() has a valid value yet. The
-  // call is not thread safe and should be called only by someone holding a
-  // reference to the handle.
+  // Determines if the handle returned by Lookup() can give a value without
+  // blocking, though Wait()/WaitAll() might be required to publish it to
+  // Value(). See secondary cache compatible Lookup() above for details.
+  // This call is not thread safe on "pending" handles.
   virtual bool IsReady(Handle* /*handle*/) { return true; }
 
-  // If the handle returned by Lookup() is not ready yet, wait till it
-  // becomes ready.
-  // Note: A ready handle doesn't necessarily mean it has a valid value. The
-  // user should call Value() and check for nullptr.
+  // Convert a "pending" handle into a full thread-shareable handle by
+  // * If necessary, wait until secondary cache finishes loading the value.
+  // * Construct the value for primary cache and set it in the handle.
+  // Even after Wait() on a pending handle, the caller must check for
+  // Value() == nullptr in case of failure. This call is not thread-safe
+  // on pending handles. This call has no effect on non-pending handles.
+  // See secondary cache compatible Lookup() above for details.
   virtual void Wait(Handle* /*handle*/) {}
 
   // Wait for a vector of handles to become ready. As with Wait(), the user
   // should check the Value() of each handle for nullptr. This call is not
-  // thread safe and should only be called by the caller holding a reference
-  // to each of the handles.
+  // thread-safe on pending handles.
   virtual void WaitAll(std::vector<Handle*>& /*handles*/) {}
 
  private:

package/deps/rocksdb/rocksdb/include/rocksdb/db.h

@@ -481,13 +481,10 @@ class DB {
   virtual Status DeleteRange(const WriteOptions& options,
                              ColumnFamilyHandle* column_family,
                              const Slice& begin_key, const Slice& end_key);
-  virtual Status DeleteRange(const WriteOptions& /*options*/,
-                             ColumnFamilyHandle* /*column_family*/,
-                             const Slice& /*begin_key*/,
-                             const Slice& /*end_key*/, const Slice& /*ts*/) {
-    return Status::NotSupported(
-        "DeleteRange does not support user-defined timestamp yet");
-  }
+  virtual Status DeleteRange(const WriteOptions& options,
+                             ColumnFamilyHandle* column_family,
+                             const Slice& begin_key, const Slice& end_key,
+                             const Slice& ts);
 
   // Merge the database entry for "key" with "value".  Returns OK on success,
   // and a non-OK status on error. The semantics of this operation is
@@ -914,6 +911,10 @@ class DB {
     //      available in the map form.
     static const std::string kBlockCacheEntryStats;
 
+    //  "rocksdb.fast-block-cache-entry-stats" - same as above, but returns
+    //      stale values more frequently to reduce overhead and latency.
+    static const std::string kFastBlockCacheEntryStats;
+
     //  "rocksdb.num-immutable-mem-table" - returns number of immutable
     //      memtables that have not yet been flushed.
     static const std::string kNumImmutableMemTable;

package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h

@@ -112,6 +112,10 @@ struct IOOptions {
   // fsync, set this to force the fsync
   bool force_dir_fsync;
 
+  // Can be used by underlying file systems to skip recursing through sub
+  // directories and list only files in GetChildren API.
+  bool do_not_recurse;
+
   IOOptions() : IOOptions(false) {}
 
   explicit IOOptions(bool force_dir_fsync_)
@@ -119,7 +123,8 @@ struct IOOptions {
         prio(IOPriority::kIOLow),
         rate_limiter_priority(Env::IO_TOTAL),
         type(IOType::kUnknown),
-        force_dir_fsync(force_dir_fsync_) {}
+        force_dir_fsync(force_dir_fsync_),
+        do_not_recurse(false) {}
 };
 
 struct DirFsyncOptions {

package/deps/rocksdb/rocksdb/include/rocksdb/listener.h

@@ -148,6 +148,9 @@ enum class CompactionReason : int {
   kChangeTemperature,
   // Compaction scheduled to force garbage collection of blob files
   kForcedBlobGC,
+  // A special TTL compaction for RoundRobin policy, which basically the same as
+  // kLevelMaxLevelSize, but the goal is to compact TTLed files.
+  kRoundRobinTtl,
   // total number of compaction reasons, new reasons must be added above this.
   kNumOfReasons,
 };

package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h

@@ -24,7 +24,7 @@ namespace ROCKSDB_NAMESPACE {
 // handle successfullly read the item.
 class SecondaryCacheResultHandle {
  public:
-  virtual ~SecondaryCacheResultHandle() {}
+  virtual ~SecondaryCacheResultHandle() = default;
 
   // Returns whether the handle is ready or not
   virtual bool IsReady() = 0;
@@ -49,7 +49,7 @@ class SecondaryCacheResultHandle {
 // including data loss, unreported corruption, deadlocks, and more.
 class SecondaryCache : public Customizable {
  public:
-  virtual ~SecondaryCache() {}
+  ~SecondaryCache() override = default;
 
   static const char* Type() { return "SecondaryCache"; }
   static Status CreateFromString(const ConfigOptions& config_options,
@@ -83,7 +83,7 @@ class SecondaryCache : public Customizable {
       bool advise_erase, bool& is_in_sec_cache) = 0;
 
   // Indicate whether a handle can be erased in this secondary cache.
-  virtual bool SupportForceErase() const = 0;
+  [[nodiscard]] virtual bool SupportForceErase() const = 0;
 
   // At the discretion of the implementation, erase the data associated
   // with key.
@@ -92,7 +92,20 @@ class SecondaryCache : public Customizable {
   // Wait for a collection of handles to become ready.
   virtual void WaitAll(std::vector<SecondaryCacheResultHandle*> handles) = 0;
 
-  virtual std::string GetPrintableOptions() const = 0;
+  // Set the maximum configured capacity of the cache.
+  // When the new capacity is less than the old capacity and the existing usage
+  // is greater than new capacity, the implementation will do its best job to
+  // purge the released entries from the cache in order to lower the usage.
+  //
+  // The derived class can make this function no-op and return NotSupported().
+  virtual Status SetCapacity(size_t /* capacity */) {
+    return Status::NotSupported();
+  }
+
+  // The derived class can make this function no-op and return NotSupported().
+  virtual Status GetCapacity(size_t& /* capacity */) {
+    return Status::NotSupported();
+  }
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h

@@ -7,6 +7,7 @@
 
 #ifndef ROCKSDB_LITE
 
+#include "db/range_tombstone_fragmenter.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/slice.h"
@@ -30,6 +31,9 @@ class SstFileReader {
   // If "snapshot" is nullptr, the iterator returns only the latest keys.
   Iterator* NewIterator(const ReadOptions& options);
 
+  FragmentedRangeTombstoneIterator* NewRangeTombstoneIterator(
+      const ReadOptions& options);
+
   std::shared_ptr<const TableProperties> GetTableProperties() const;
 
   // Verifies whether there is corruption in this table.

package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h

@@ -148,6 +148,13 @@ class SstFileWriter {
   // REQUIRES: comparator is *not* timestamp-aware.
   Status DeleteRange(const Slice& begin_key, const Slice& end_key);
 
+  // Add a range deletion tombstone to currently opened file.
+  // REQUIRES: begin_key and end_key are user keys without timestamp.
+  // REQUIRES: the timestamp's size is equal to what is expected by
+  // the comparator.
+  Status DeleteRange(const Slice& begin_key, const Slice& end_key,
+                     const Slice& timestamp);
+
   // Finalize writing to sst file and close file.
   //
   // An optional ExternalSstFileInfo pointer can be passed to the function

package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h

@@ -59,7 +59,7 @@ class CompactionOptionsUniversal {
   //    A1...An B1...Bm C1...Ct
   // where A1 is the newest and Ct is the oldest, and we are going to compact
   // B1...Bm, we calculate the total size of all the files as total_size, as
-  // well as  the total size of C1...Ct as total_C, the compaction output file
+  // well as the total size of C1...Ct as total_C, the compaction output file
   // will be compressed iff
   //   total_C / total_size < this percentage
   // Default: -1

package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include <ostream>
 #include <tuple>
 #include <utility>
 #include <vector>
@@ -70,6 +71,14 @@ inline bool operator!=(const WideColumn& lhs, const WideColumn& rhs) {
   return !(lhs == rhs);
 }
 
+inline std::ostream& operator<<(std::ostream& os, const WideColumn& column) {
+  const bool hex =
+      (os.flags() & std::ios_base::basefield) == std::ios_base::hex;
+  os << column.name().ToString(hex) << ':' << column.value().ToString(hex);
+
+  return os;
+}
+
 // A collection of wide columns.
 using WideColumns = std::vector<WideColumn>;
 

package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h

@@ -150,12 +150,9 @@ class WriteBatch : public WriteBatchBase {
   Status DeleteRange(const Slice& begin_key, const Slice& end_key) override {
     return DeleteRange(nullptr, begin_key, end_key);
   }
-  Status DeleteRange(ColumnFamilyHandle* /*column_family*/,
-                     const Slice& /*begin_key*/, const Slice& /*end_key*/,
-                     const Slice& /*ts*/) override {
-    return Status::NotSupported(
-        "DeleteRange does not support user-defined timestamp");
-  }
+  // begin_key and end_key should be user keys without timestamp.
+  Status DeleteRange(ColumnFamilyHandle* column_family, const Slice& begin_key,
+                     const Slice& end_key, const Slice& ts) override;
 
   // variant that takes SliceParts
   Status DeleteRange(ColumnFamilyHandle* column_family,

package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc

@@ -728,6 +728,7 @@ TEST_F(AutoRollLoggerTest, RenameError) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/logging/env_logger_test.cc

@@ -156,6 +156,7 @@ TEST_F(EnvLoggerTest, ConcurrentLogging) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/logging/event_logger_test.cc

@@ -38,6 +38,7 @@ TEST_F(EventLoggerTest, SimpleTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }