@nxtedition/rocksdb 8.2.7 → 9.0.0

package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h

@@ -79,7 +79,8 @@ class SecondaryCache : public Customizable {
   // and writes it to this cache tier. OK may be returned even if the insertion
   // is not made.
   virtual Status Insert(const Slice& key, Cache::ObjectPtr obj,
-                        const Cache::CacheItemHelper* helper) = 0;
+                        const Cache::CacheItemHelper* helper,
+                        bool force_insert) = 0;
 
   // Insert a value from its saved/persistable data (typically uncompressed
   // block), as if generated by SaveToCallback/SizeCallback. This can be used
@@ -134,6 +135,17 @@ class SecondaryCache : public Customizable {
   virtual Status GetCapacity(size_t& /* capacity */) {
     return Status::NotSupported();
   }
+
+  // Temporarily decrease the cache capacity in RAM by the specified amount.
+  // The caller should call Inflate() to restore the cache capacity. This is
+  // intended to be lighter weight than SetCapacity(). The latter evenly
+  // distributes the new capacity across all shards and is meant for large
+  // changes in capacity, whereas the former is meant for relatively small
+  // changes and may be uneven by lowering capacity in a single shard.
+  virtual Status Deflate(size_t /*decrease*/) { return Status::NotSupported(); }
+
+  // Restore the capacity reduced by a prior call to Deflate().
+  virtual Status Inflate(size_t /*increase*/) { return Status::NotSupported(); }
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_manager.h

@@ -93,7 +93,8 @@ class SstFileManager {
 //
 // @param env: Pointer to Env object, please see "rocksdb/env.h".
 // @param fs: Pointer to FileSystem object (rocksdb/file_system.h"
-// @param info_log: If not nullptr, info_log will be used to log errors.
+// @param info_log: If not nullptr, info_log will be used to log messages of
+// INFO, WARN or ERROR level with respect to info_log's info level.
 //
 // == Deletion rate limiting specific arguments ==
 // @param trash_dir: Deprecated, this argument have no effect

package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h

@@ -13,6 +13,7 @@
 #include "rocksdb/options.h"
 #include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
+#include "rocksdb/wide_columns.h"
 
 #if defined(__GNUC__) || defined(__clang__)
 #define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__))
@@ -127,6 +128,10 @@ class SstFileWriter {
   // REQUIRES: timestamp's size is equal to what is expected by the comparator.
   Status Put(const Slice& user_key, const Slice& timestamp, const Slice& value);
 
+  // Add a PutEntity (key with the wide-column entity defined by "columns") to
+  // the currently opened file
+  Status PutEntity(const Slice& user_key, const WideColumns& columns);
+
   // Add a Merge key with value to currently opened file
   // REQUIRES: user_key is after any previously added point (Put/Merge/Delete)
   //           key according to the comparator.
@@ -186,4 +191,3 @@ class SstFileWriter {
   std::unique_ptr<Rep> rep_;
 };
 }  // namespace ROCKSDB_NAMESPACE
-

package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h

@@ -353,14 +353,22 @@ enum Tickers : uint32_t {
   // # of files marked as trash by sst file manager and will be deleted
   // later by background thread.
   FILES_MARKED_TRASH,
-  // # of files deleted immediately by sst file manger through delete scheduler.
+  // # of trash files deleted by the background thread from the trash queue.
+  FILES_DELETED_FROM_TRASH_QUEUE,
+  // # of files deleted immediately by sst file manager through delete
+  // scheduler.
   FILES_DELETED_IMMEDIATELY,
 
   // The counters for error handler, not that, bg_io_error is the subset of
-  // bg_error and bg_retryable_io_error is the subset of bg_io_error
+  // bg_error and bg_retryable_io_error is the subset of bg_io_error.
+  // The misspelled versions are deprecated and only kept for compatibility.
+  // TODO: remove the misspelled tickers in the next major release.
   ERROR_HANDLER_BG_ERROR_COUNT,
+  ERROR_HANDLER_BG_ERROR_COUNT_MISSPELLED,
   ERROR_HANDLER_BG_IO_ERROR_COUNT,
+  ERROR_HANDLER_BG_IO_ERROR_COUNT_MISSPELLED,
   ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT,
+  ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT_MISSPELLED,
   ERROR_HANDLER_AUTORESUME_COUNT,
   ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT,
   ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT,
@@ -503,6 +511,10 @@ enum Tickers : uint32_t {
   // compressed SST blocks from storage.
   BYTES_DECOMPRESSED_TO,
 
+  // Number of times readahead is trimmed during scans when
+  // ReadOptions.auto_readahead_size is set.
+  READAHEAD_TRIMMED,
+
   TICKER_ENUM_MAX
 };
 
@@ -547,6 +559,13 @@ enum Histograms : uint32_t {
   FILE_READ_FLUSH_MICROS,
   FILE_READ_COMPACTION_MICROS,
   FILE_READ_DB_OPEN_MICROS,
+  // The following `FILE_READ_*` require stats level greater than
+  // `StatsLevel::kExceptDetailedTimers`
+  FILE_READ_GET_MICROS,
+  FILE_READ_MULTIGET_MICROS,
+  FILE_READ_DB_ITERATOR_MICROS,
+  FILE_READ_VERIFY_DB_CHECKSUM_MICROS,
+  FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS,
 
   // The number of subcompactions actually scheduled during a compaction
   NUM_SUBCOMPACTIONS_SCHEDULED,

package/deps/rocksdb/rocksdb/include/rocksdb/table.h

@@ -47,7 +47,10 @@ struct EnvOptions;
 
 // Types of checksums to use for checking integrity of logical blocks within
 // files. All checksums currently use 32 bits of checking power (1 in 4B
-// chance of failing to detect random corruption).
+// chance of failing to detect random corruption). Traditionally, the actual
+// checking power can be far from ideal if the corruption is due to misplaced
+// data (e.g. physical blocks out of order in a file, or from another file),
+// which is fixed in format_version=6 (see below).
 enum ChecksumType : char {
   kNoChecksum = 0x0,
   kCRC32c = 0x1,
@@ -512,6 +515,9 @@ struct BlockBasedTableOptions {
   // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned
   // filters use a generally faster and more accurate Bloom filter
   // implementation, with a different schema.
+  // 6 -- Modified the file footer and checksum matching so that SST data
+  // misplaced within or between files is as likely to fail checksum
+  // verification as random corruption. Also checksum-protects SST footer.
   uint32_t format_version = 5;
 
   // Store index blocks on disk in compressed format. Changing this option to

package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h

@@ -71,6 +71,7 @@ struct TablePropertiesNames {
   static const std::string kFastCompressionEstimatedDataSize;
   static const std::string kSequenceNumberTimeMapping;
   static const std::string kTailStartOffset;
+  static const std::string kUserDefinedTimestampsPersisted;
 };
 
 // `TablePropertiesCollector` provides the mechanism for users to collect
@@ -244,6 +245,11 @@ struct TableProperties {
   // "Tail" refers to all blocks after data blocks till the end of the SST file
   uint64_t tail_start_offset = 0;
 
+  // Value of the `AdvancedColumnFamilyOptions.persist_user_defined_timestamps`
+  // when the file is created. Default to be true, only when this flag is false,
+  // it's explicitly written to meta properties block.
+  uint64_t user_defined_timestamps_persisted = 1;
+
   // DB identity
   // db_id is an identifier generated the first time the DB is created
   // If DB identity is unset or unassigned, `db_id` will be an empty string.

package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h

@@ -57,6 +57,11 @@ struct ThreadStatus {
     OP_COMPACTION,
     OP_FLUSH,
     OP_DBOPEN,
+    OP_GET,
+    OP_MULTIGET,
+    OP_DBITERATOR,
+    OP_VERIFY_DB_CHECKSUM,
+    OP_VERIFY_FILE_CHECKSUMS,
     NUM_OP_TYPES
   };
 

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -43,11 +44,42 @@ enum class OccValidationPolicy {
   kValidateParallel = 1
 };
 
+class OccLockBuckets {
+ public:
+  // Most details in internal derived class.
+  // Users should not derive from this class.
+  virtual ~OccLockBuckets() {}
+
+  virtual size_t ApproximateMemoryUsage() const = 0;
+
+ private:
+  friend class OccLockBucketsImplBase;
+  OccLockBuckets() {}
+};
+
+// An object for sharing a pool of locks across DB instances.
+//
+// Making the locks cache-aligned avoids potential false sharing, at the
+// potential cost of extra memory. The implementation has historically
+// used cache_aligned = false.
+std::shared_ptr<OccLockBuckets> MakeSharedOccLockBuckets(
+    size_t bucket_count, bool cache_aligned = false);
+
 struct OptimisticTransactionDBOptions {
   OccValidationPolicy validate_policy = OccValidationPolicy::kValidateParallel;
 
-  // works only if validate_policy == OccValidationPolicy::kValidateParallel
+  // Number of striped/bucketed mutex locks for validating transactions.
+  // Used on only if validate_policy == OccValidationPolicy::kValidateParallel
+  // and shared_lock_buckets (below) is empty. Larger number potentially
+  // reduces contention but uses more memory.
   uint32_t occ_lock_buckets = (1 << 20);
+
+  // A pool of mutex locks for validating transactions. Can be shared among
+  // DBs. Ignored if validate_policy != OccValidationPolicy::kValidateParallel.
+  // If empty and validate_policy == OccValidationPolicy::kValidateParallel,
+  // an OccLockBuckets will be created using the count in occ_lock_buckets.
+  // See MakeSharedOccLockBuckets()
+  std::shared_ptr<OccLockBuckets> shared_lock_buckets;
 };
 
 // Range deletions (including those in `WriteBatch`es passed to `Write()`) are
@@ -95,4 +127,3 @@ class OptimisticTransactionDB : public StackableDB {
 };
 
 }  // namespace ROCKSDB_NAMESPACE
-

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h

@@ -178,6 +178,15 @@ class StackableDB : public DB {
                                              import_options, metadata, handle);
   }
 
+  virtual Status CreateColumnFamilyWithImport(
+      const ColumnFamilyOptions& options, const std::string& column_family_name,
+      const ImportColumnFamilyOptions& import_options,
+      const std::vector<const ExportImportFilesMetaData*>& metadatas,
+      ColumnFamilyHandle** handle) override {
+    return db_->CreateColumnFamilyWithImport(options, column_family_name,
+                                             import_options, metadatas, handle);
+  }
+
   using DB::ClipColumnFamily;
   virtual Status ClipColumnFamily(ColumnFamilyHandle* column_family,
                                   const Slice& begin_key,
@@ -347,6 +356,11 @@ class StackableDB : public DB {
     return db_->DisableManualCompaction();
   }
 
+  virtual Status WaitForCompact(
+      const WaitForCompactOptions& wait_for_compact_options) override {
+    return db_->WaitForCompact(wait_for_compact_options);
+  }
+
   using DB::NumberLevels;
   virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
     return db_->NumberLevels(column_family);

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h

@@ -227,7 +227,8 @@ class Transaction {
   // Status::Busy() may be returned if the transaction could not guarantee
   // that there are no write conflicts.  Status::TryAgain() may be returned
   // if the memtable history size is not large enough
-  //  (See max_write_buffer_size_to_maintain).
+  // (see max_write_buffer_size_to_maintain). In either case, a Rollback()
+  // or new transaction is required to expect a different result.
   //
   // If this transaction was created by a TransactionDB(), Status::Expired()
   // may be returned if this transaction has lived for longer than
@@ -259,6 +260,7 @@ class Transaction {
       std::shared_ptr<const Snapshot>* snapshot = nullptr);
 
   // Discard all batched writes in this transaction.
+  // FIXME: what happens if this isn't called before destruction?
   virtual Status Rollback() = 0;
 
   // Records the state of the transaction for future calls to
@@ -333,8 +335,22 @@ class Transaction {
                         const size_t num_keys, const Slice* keys,
                         PinnableSlice* values, Status* statuses,
                         const bool /*sorted_input*/ = false) {
+    if (options.io_activity != Env::IOActivity::kUnknown &&
+        options.io_activity != Env::IOActivity::kMultiGet) {
+      Status s = Status::InvalidArgument(
+          "Can only call MultiGet with `ReadOptions::io_activity` is "
+          "`Env::IOActivity::kUnknown` or `Env::IOActivity::kMultiGet`");
+
+      for (size_t i = 0; i < num_keys; ++i) {
+        if (statuses[i].ok()) {
+          statuses[i] = s;
+        }
+      }
+      return;
+    }
+
     for (size_t i = 0; i < num_keys; ++i) {
-      statuses[i] = Get(options, column_family, keys[i], &values[i]);
+      statuses[i] = GetImpl(options, column_family, keys[i], &values[i]);
     }
   }
 
@@ -671,6 +687,21 @@ class Transaction {
     id_ = id;
   }
 
+  virtual Status GetImpl(const ReadOptions& /* options */,
+                         ColumnFamilyHandle* /* column_family */,
+                         const Slice& /* key */, std::string* /* value */) {
+    return Status::NotSupported("Not implemented");
+  }
+
+  virtual Status GetImpl(const ReadOptions& options,
+                         ColumnFamilyHandle* column_family, const Slice& key,
+                         PinnableSlice* pinnable_val) {
+    assert(pinnable_val != nullptr);
+    auto s = GetImpl(options, column_family, key, pinnable_val->GetSelf());
+    pinnable_val->PinSelf();
+    return s;
+  }
+
   virtual uint64_t GetLastLogNumber() const { return log_number_; }
 
  private:

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h

@@ -189,9 +189,6 @@ class WriteBatchWithIndex : public WriteBatchBase {
   // Will create a new Iterator that will use WBWIIterator as a delta and
   // base_iterator as base.
   //
-  // This function is only supported if the WriteBatchWithIndex was
-  // constructed with overwrite_key=true.
-  //
   // The returned iterator should be deleted by the caller.
   // The base_iterator is now 'owned' by the returned iterator. Deleting the
   // returned iterator will also delete the base_iterator.

package/deps/rocksdb/rocksdb/include/rocksdb/version.h

@@ -12,8 +12,8 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 8
-#define ROCKSDB_MINOR 3
-#define ROCKSDB_PATCH 2
+#define ROCKSDB_MINOR 6
+#define ROCKSDB_PATCH 7
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h

@@ -356,6 +356,9 @@ class WriteBatch : public WriteBatchBase {
   // Retrieve the serialized version of this batch.
   const std::string& Data() const { return rep_; }
 
+  // Release the serialized data and clear this batch.
+  std::string Release();
+
   // Retrieve data size of the batch.
   size_t GetDataSize() const { return rep_.size(); }
 

package/deps/rocksdb/rocksdb/memory/arena_test.cc

@@ -219,21 +219,28 @@ size_t PopMinorPageFaultCount() {
 
 TEST(MmapTest, AllocateLazyZeroed) {
   // Doesn't have to be page aligned
-  constexpr size_t len = 1234567;
-  MemMapping m = MemMapping::AllocateLazyZeroed(len);
-  auto arr = static_cast<char*>(m.Get());
+  constexpr size_t len = 1234567;    // in bytes
+  constexpr size_t count = len / 8;  // in uint64_t objects
+  // Implicit conversion move
+  TypedMemMapping<uint64_t> pre_arr = MemMapping::AllocateLazyZeroed(len);
+  // Move from same type
+  TypedMemMapping<uint64_t> arr = std::move(pre_arr);
 
-  // Should generally work
-  ASSERT_NE(arr, nullptr);
+  ASSERT_NE(arr.Get(), nullptr);
+  ASSERT_EQ(arr.Get(), &arr[0]);
+  ASSERT_EQ(arr.Get(), arr.MemMapping::Get());
+
+  ASSERT_EQ(arr.Length(), len);
+  ASSERT_EQ(arr.Count(), count);
 
   // Start counting page faults
   PopMinorPageFaultCount();
 
   // Access half of the allocation
   size_t i = 0;
-  for (; i < len / 2; ++i) {
+  for (; i < count / 2; ++i) {
     ASSERT_EQ(arr[i], 0);
-    arr[i] = static_cast<char>(i & 255);
+    arr[i] = i;
   }
 
   // Appropriate page faults (maybe more)
@@ -241,9 +248,9 @@ TEST(MmapTest, AllocateLazyZeroed) {
   ASSERT_GE(faults, len / 2 / port::kPageSize);
 
   // Access rest of the allocation
-  for (; i < len; ++i) {
+  for (; i < count; ++i) {
     ASSERT_EQ(arr[i], 0);
-    arr[i] = static_cast<char>(i & 255);
+    arr[i] = i;
   }
 
   // Appropriate page faults (maybe more)
@@ -251,8 +258,8 @@ TEST(MmapTest, AllocateLazyZeroed) {
   ASSERT_GE(faults, len / 2 / port::kPageSize);
 
   // Verify data
-  for (i = 0; i < len; ++i) {
-    ASSERT_EQ(arr[i], static_cast<char>(i & 255));
+  for (i = 0; i < count; ++i) {
+    ASSERT_EQ(arr[i], i);
   }
 }
 

package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.cc

@@ -124,7 +124,8 @@ uint32_t JemallocNodumpAllocator::GetArenaIndex() const {
   // to make Random thread-safe and prevent cacheline bouncing. Whether this is
   // worthwhile is still an open question.
   thread_local Random tl_random(next_seed.fetch_add(1));
-  return arena_indexes_[FastRange32(tl_random.Next(), arena_indexes_.size())];
+  return arena_indexes_[FastRange32(
+      tl_random.Next(), static_cast<uint32_t>(arena_indexes_.size()))];
 }
 
 Status JemallocNodumpAllocator::InitializeArenas() {