@nxtedition/rocksdb 5.2.21 → 5.2.26

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h

@@ -80,6 +80,10 @@ class StackableDB : public DB {
                      const Slice& val) override {
     return db_->Put(options, column_family, key, val);
   }
+  Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family,
+             const Slice& key, const Slice& ts, const Slice& val) override {
+    return db_->Put(options, column_family, key, ts, val);
+  }
 
   using DB::Get;
   virtual Status Get(const ReadOptions& options,
@@ -166,6 +170,10 @@ class StackableDB : public DB {
                         const Slice& key) override {
     return db_->Delete(wopts, column_family, key);
   }
+  Status Delete(const WriteOptions& wopts, ColumnFamilyHandle* column_family,
+                const Slice& key, const Slice& ts) override {
+    return db_->Delete(wopts, column_family, key, ts);
+  }
 
   using DB::SingleDelete;
   virtual Status SingleDelete(const WriteOptions& wopts,
@@ -173,6 +181,18 @@ class StackableDB : public DB {
                               const Slice& key) override {
     return db_->SingleDelete(wopts, column_family, key);
   }
+  Status SingleDelete(const WriteOptions& wopts,
+                      ColumnFamilyHandle* column_family, const Slice& key,
+                      const Slice& ts) override {
+    return db_->SingleDelete(wopts, column_family, key, ts);
+  }
+
+  using DB::DeleteRange;
+  Status DeleteRange(const WriteOptions& wopts,
+                     ColumnFamilyHandle* column_family, const Slice& start_key,
+                     const Slice& end_key) override {
+    return db_->DeleteRange(wopts, column_family, start_key, end_key);
+  }
 
   using DB::Merge;
   virtual Status Merge(const WriteOptions& options,
@@ -357,6 +377,12 @@ class StackableDB : public DB {
     return db_->GetLiveFilesChecksumInfo(checksum_list);
   }
 
+  virtual Status GetLiveFilesStorageInfo(
+      const LiveFilesStorageInfoOptions& opts,
+      std::vector<LiveFileStorageInfo>* files) override {
+    return db_->GetLiveFilesStorageInfo(opts, files);
+  }
+
   virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
                                        ColumnFamilyMetaData* cf_meta) override {
     db_->GetColumnFamilyMetaData(column_family, cf_meta);
@@ -373,9 +399,9 @@ class StackableDB : public DB {
   Status EndBlockCacheTrace() override { return db_->EndBlockCacheTrace(); }
 
   using DB::StartIOTrace;
-  Status StartIOTrace(Env* env, const TraceOptions& options,
+  Status StartIOTrace(const TraceOptions& options,
                       std::unique_ptr<TraceWriter>&& trace_writer) override {
-    return db_->StartIOTrace(env, options, std::move(trace_writer));
+    return db_->StartIOTrace(options, std::move(trace_writer));
   }
 
   using DB::EndIOTrace;
@@ -390,6 +416,13 @@ class StackableDB : public DB {
   using DB::EndTrace;
   Status EndTrace() override { return db_->EndTrace(); }
 
+  using DB::NewDefaultReplayer;
+  Status NewDefaultReplayer(const std::vector<ColumnFamilyHandle*>& handles,
+                            std::unique_ptr<TraceReader>&& reader,
+                            std::unique_ptr<Replayer>* replayer) override {
+    return db_->NewDefaultReplayer(handles, std::move(reader), replayer);
+  }
+
 #endif  // ROCKSDB_LITE
 
   virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
@@ -401,9 +434,14 @@ class StackableDB : public DB {
     return db_->GetLatestSequenceNumber();
   }
 
-  virtual bool SetPreserveDeletesSequenceNumber(
-      SequenceNumber seqnum) override {
-    return db_->SetPreserveDeletesSequenceNumber(seqnum);
+  Status IncreaseFullHistoryTsLow(ColumnFamilyHandle* column_family,
+                                  std::string ts_low) override {
+    return db_->IncreaseFullHistoryTsLow(column_family, ts_low);
+  }
+
+  Status GetFullHistoryTsLow(ColumnFamilyHandle* column_family,
+                             std::string* ts_low) override {
+    return db_->GetFullHistoryTsLow(column_family, ts_low);
   }
 
   virtual Status GetSortedWalFiles(VectorLogPtr& files) override {

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h

@@ -19,6 +19,19 @@ namespace ROCKSDB_NAMESPACE {
 class CompactOnDeletionCollectorFactory
     : public TablePropertiesCollectorFactory {
  public:
+  // A factory of a table property collector that marks a SST
+  // file as need-compaction when it observe at least "D" deletion
+  // entries in any "N" consecutive entries, or the ratio of tombstone
+  // entries >= deletion_ratio.
+  //
+  // @param sliding_window_size "N"
+  // @param deletion_trigger "D"
+  // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
+  //     based on deletion ratio.
+  CompactOnDeletionCollectorFactory(size_t sliding_window_size,
+                                    size_t deletion_trigger,
+                                    double deletion_ratio);
+
   ~CompactOnDeletionCollectorFactory() {}
 
   TablePropertiesCollector* CreateTablePropertiesCollector(
@@ -29,12 +42,14 @@ class CompactOnDeletionCollectorFactory
   void SetWindowSize(size_t sliding_window_size) {
     sliding_window_size_.store(sliding_window_size);
   }
+  size_t GetWindowSize() const { return sliding_window_size_.load(); }
 
   // Change the value of deletion_trigger "D"
   void SetDeletionTrigger(size_t deletion_trigger) {
     deletion_trigger_.store(deletion_trigger);
   }
 
+  size_t GetDeletionTrigger() const { return deletion_trigger_.load(); }
   // Change deletion ratio.
   // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
   //     based on deletion ratio.
@@ -42,33 +57,13 @@ class CompactOnDeletionCollectorFactory
     deletion_ratio_.store(deletion_ratio);
   }
 
-  const char* Name() const override {
-    return "CompactOnDeletionCollector";
-  }
+  double GetDeletionRatio() const { return deletion_ratio_.load(); }
+  static const char* kClassName() { return "CompactOnDeletionCollector"; }
+  const char* Name() const override { return kClassName(); }
 
   std::string ToString() const override;
 
  private:
-  friend std::shared_ptr<CompactOnDeletionCollectorFactory>
-  NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
-                                       size_t deletion_trigger,
-                                       double deletion_ratio);
-  // A factory of a table property collector that marks a SST
-  // file as need-compaction when it observe at least "D" deletion
-  // entries in any "N" consecutive entries, or the ratio of tombstone
-  // entries >= deletion_ratio.
-  //
-  // @param sliding_window_size "N"
-  // @param deletion_trigger "D"
-  // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
-  //     based on deletion ratio.
-  CompactOnDeletionCollectorFactory(size_t sliding_window_size,
-                                    size_t deletion_trigger,
-                                    double deletion_ratio)
-      : sliding_window_size_(sliding_window_size),
-        deletion_trigger_(deletion_trigger),
-        deletion_ratio_(deletion_ratio) {}
-
   std::atomic<size_t> sliding_window_size_;
   std::atomic<size_t> deletion_trigger_;
   std::atomic<double> deletion_ratio_;

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h

@@ -7,6 +7,7 @@
 
 #ifndef ROCKSDB_LITE
 
+#include <limits>
 #include <string>
 #include <vector>
 
@@ -24,6 +25,11 @@ using TransactionName = std::string;
 
 using TransactionID = uint64_t;
 
+using TxnTimestamp = uint64_t;
+
+constexpr TxnTimestamp kMaxTxnTimestamp =
+    std::numeric_limits<TxnTimestamp>::max();
+
 /*
   class Endpoint allows to define prefix ranges.
 
@@ -551,6 +557,18 @@ class Transaction {
 
   virtual Status RebuildFromWriteBatch(WriteBatch* src_batch) = 0;
 
+  // Note: data in the commit-time-write-batch bypasses concurrency control,
+  // thus should be used with great caution.
+  // For write-prepared/write-unprepared transactions,
+  // GetCommitTimeWriteBatch() can be used only if the transaction is started
+  // with
+  // `TransactionOptions::use_only_the_last_commit_time_batch_for_recovery` set
+  // to true. Otherwise, it is possible that two uncommitted versions of the
+  // same key exist in the database due to the current implementation (see the
+  // explanation in WritePreparedTxn::CommitInternal).
+  // During bottommost compaction, RocksDB may
+  // set the sequence numbers of both to zero once becoming committed, causing
+  // output SST file to have two identical internal keys.
   virtual WriteBatch* GetCommitTimeWriteBatch() = 0;
 
   virtual void SetLogNumber(uint64_t log) { log_number_ = log; }
@@ -594,6 +612,14 @@ class Transaction {
   // to remain the same across restarts.
   uint64_t GetId() { return id_; }
 
+  virtual Status SetReadTimestampForValidation(TxnTimestamp /*ts*/) {
+    return Status::NotSupported("timestamp not supported");
+  }
+
+  virtual Status SetCommitTimestamp(TxnTimestamp /*ts*/) {
+    return Status::NotSupported("timestamp not supported");
+  }
+
  protected:
   explicit Transaction(const TransactionDB* /*db*/) {}
   Transaction() : log_number_(0), txn_state_(STARTED) {}

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h

@@ -29,7 +29,7 @@ enum TxnDBWritePolicy {
   WRITE_UNPREPARED  // write data before the prepare phase of 2pc
 };
 
-const uint32_t kInitialMaxDeadlocks = 5;
+constexpr uint32_t kInitialMaxDeadlocks = 5;
 
 class LockManager;
 struct RangeLockInfo;
@@ -97,6 +97,21 @@ class RangeLockManagerHandle : public LockManagerHandle {
   using RangeLockStatus =
       std::unordered_multimap<ColumnFamilyId, RangeLockInfo>;
 
+  // Lock Escalation barrier check function.
+  // It is called for a couple of endpoints A and B, such that A < B.
+  // If escalation_barrier_check_func(A, B)==true, then there's a lock
+  // escalation barrier between A and B, and lock escalation is not allowed
+  // to bridge the gap between A and B.
+  //
+  // The function may be called from any thread that acquires or releases
+  // locks. It should not throw exceptions. There is currently no way to return
+  // an error.
+  using EscalationBarrierFunc =
+      std::function<bool(const Endpoint& a, const Endpoint& b)>;
+
+  // Set the user-provided barrier check function
+  virtual void SetEscalationBarrierFunc(EscalationBarrierFunc func) = 0;
+
   virtual RangeLockStatus GetRangeLockStatusData() = 0;
 
   class Counters {
@@ -104,6 +119,11 @@ class RangeLockManagerHandle : public LockManagerHandle {
     // Number of times lock escalation was triggered (for all column families)
     uint64_t escalation_count;
 
+    // Number of times lock acquisition had to wait for a conflicting lock
+    // to be released. This counts both successful waits (where the desired
+    // lock was acquired) and waits that timed out or got other error.
+    uint64_t lock_wait_count;
+
     // How much memory is currently used for locks (total for all column
     // families)
     uint64_t current_lock_memory;
@@ -140,8 +160,7 @@ struct TransactionDBOptions {
 
   // Increasing this value will increase the concurrency by dividing the lock
   // table (per column family) into more sub-tables, each with their own
-  // separate
-  // mutex.
+  // separate mutex.
   size_t num_stripes = 16;
 
   // If positive, specifies the default wait timeout in milliseconds when
@@ -151,8 +170,7 @@ struct TransactionDBOptions {
   // If 0, no waiting is done if a lock cannot instantly be acquired.
   // If negative, there is no timeout.  Not using a timeout is not recommended
   // as it can lead to deadlocks.  Currently, there is no deadlock-detection to
-  // recover
-  // from a deadlock.
+  // recover from a deadlock.
   int64_t transaction_lock_timeout = 1000;  // 1 second
 
   // If positive, specifies the wait timeout in milliseconds when writing a key
@@ -194,7 +212,7 @@ struct TransactionDBOptions {
 
   // If true, the TransactionDB implementation might skip concurrency control
   // unless it is overridden by TransactionOptions or
-  // TransactionDBWriteOptimizations. This can be used in conjuction with
+  // TransactionDBWriteOptimizations. This can be used in conjunction with
   // DBOptions::unordered_write when the TransactionDB is used solely for write
   // ordering rather than concurrency control.
   bool skip_concurrency_control = false;
@@ -206,8 +224,12 @@ struct TransactionDBOptions {
 
  private:
   // 128 entries
+  // Should the default value change, please also update wp_snapshot_cache_bits
+  // in db_stress_gflags.cc
   size_t wp_snapshot_cache_bits = static_cast<size_t>(7);
   // 8m entry, 64MB size
+  // Should the default value change, please also update wp_commit_cache_bits
+  // in db_stress_gflags.cc
   size_t wp_commit_cache_bits = static_cast<size_t>(23);
 
   // For testing, whether transaction name should be auto-generated or not. This
@@ -219,6 +241,7 @@ struct TransactionDBOptions {
   friend class WritePreparedTransactionTestBase;
   friend class TransactionTestBase;
   friend class MySQLStyleTransactionTest;
+  friend class StressTest;
 };
 
 struct TransactionOptions {
@@ -236,6 +259,8 @@ struct TransactionOptions {
   // meant to be used later during recovery. It enables an optimization to
   // postpone updating the memtable with CommitTimeWriteBatch to only
   // SwitchMemtable or recovery.
+  // This option does not affect write-committed. Only
+  // write-prepared/write-unprepared transactions will be affected.
   bool use_only_the_last_commit_time_batch_for_recovery = false;
 
   // TODO(agiardullo): TransactionDB does not yet support comparators that allow
@@ -351,6 +376,7 @@ class TransactionDB : public StackableDB {
   // used and `skip_concurrency_control` must be set. When using either
   // WRITE_PREPARED or WRITE_UNPREPARED , `skip_duplicate_key_check` must
   // additionally be set.
+  using StackableDB::DeleteRange;
   virtual Status DeleteRange(const WriteOptions&, ColumnFamilyHandle*,
                              const Slice&, const Slice&) override {
     return Status::NotSupported();

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h

@@ -18,7 +18,6 @@ namespace ROCKSDB_NAMESPACE {
 //
 // To open a TransactionDB with a custom TransactionDBMutexFactory, set
 // TransactionDBOptions.custom_mutex_factory.
-
 class TransactionDBMutex {
  public:
   virtual ~TransactionDBMutex() {}
@@ -61,7 +60,7 @@ class TransactionDBCondVar {
   //
   // Returns OK if notified.
   // Returns TimedOut if timeout is reached.
-  // Returns other status if TransactionDB should otherwis stop waiting and
+  // Returns other status if TransactionDB should otherwise stop waiting and
   //  fail the operation.
   // May return OK spuriously even if not notified.
   virtual Status WaitFor(std::shared_ptr<TransactionDBMutex> mutex,

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h

@@ -110,20 +110,32 @@ class WriteBatchWithIndex : public WriteBatchBase {
 
   Status Put(const Slice& key, const Slice& value) override;
 
+  Status Put(ColumnFamilyHandle* column_family, const Slice& key,
+             const Slice& ts, const Slice& value) override;
+
   using WriteBatchBase::Merge;
   Status Merge(ColumnFamilyHandle* column_family, const Slice& key,
                const Slice& value) override;
 
   Status Merge(const Slice& key, const Slice& value) override;
+  Status Merge(ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
+               const Slice& /*ts*/, const Slice& /*value*/) override {
+    return Status::NotSupported(
+        "Merge does not support user-defined timestamp");
+  }
 
   using WriteBatchBase::Delete;
   Status Delete(ColumnFamilyHandle* column_family, const Slice& key) override;
   Status Delete(const Slice& key) override;
+  Status Delete(ColumnFamilyHandle* column_family, const Slice& key,
+                const Slice& ts) override;
 
   using WriteBatchBase::SingleDelete;
   Status SingleDelete(ColumnFamilyHandle* column_family,
                       const Slice& key) override;
   Status SingleDelete(const Slice& key) override;
+  Status SingleDelete(ColumnFamilyHandle* column_family, const Slice& key,
+                      const Slice& ts) override;
 
   using WriteBatchBase::DeleteRange;
   Status DeleteRange(ColumnFamilyHandle* /* column_family */,
@@ -137,6 +149,12 @@ class WriteBatchWithIndex : public WriteBatchBase {
     return Status::NotSupported(
         "DeleteRange unsupported in WriteBatchWithIndex");
   }
+  Status DeleteRange(ColumnFamilyHandle* /*column_family*/,
+                     const Slice& /*begin_key*/, const Slice& /*end_key*/,
+                     const Slice& /*ts*/) override {
+    return Status::NotSupported(
+        "DeleteRange unsupported in WriteBatchWithIndex");
+  }
 
   using WriteBatchBase::PutLogData;
   Status PutLogData(const Slice& blob) override;
@@ -169,7 +187,7 @@ class WriteBatchWithIndex : public WriteBatchBase {
   // returned iterator will also delete the base_iterator.
   //
   // Updating write batch with the current key of the iterator is not safe.
-  // We strongly recommand users not to do it. It will invalidate the current
+  // We strongly recommend users not to do it. It will invalidate the current
   // key() and value() of the iterator. This invalidation happens even before
   // the write batch update finishes. The state may recover after Next() is
   // called.
@@ -257,6 +275,7 @@ class WriteBatchWithIndex : public WriteBatchBase {
   friend class WritePreparedTxn;
   friend class WriteUnpreparedTxn;
   friend class WriteBatchWithIndex_SubBatchCnt_Test;
+  friend class WriteBatchWithIndexInternal;
   // Returns the number of sub-batches inside the write batch. A sub-batch
   // starts right before inserting a key that is a duplicate of a key in the
   // last sub-batch.

package/deps/rocksdb/rocksdb/include/rocksdb/version.h

@@ -4,9 +4,16 @@
 //  (found in the LICENSE.Apache file in the root directory).
 #pragma once
 
-#define ROCKSDB_MAJOR 6
-#define ROCKSDB_MINOR 17
-#define ROCKSDB_PATCH 3
+#include <string>
+#include <unordered_map>
+
+#include "rocksdb/rocksdb_namespace.h"
+
+// NOTE: in 'main' development branch, this should be the *next*
+// minor or major version number planned for release.
+#define ROCKSDB_MAJOR 7
+#define ROCKSDB_MINOR 2
+#define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these
@@ -14,3 +21,23 @@
 #define __ROCKSDB_MAJOR__ ROCKSDB_MAJOR
 #define __ROCKSDB_MINOR__ ROCKSDB_MINOR
 #define __ROCKSDB_PATCH__ ROCKSDB_PATCH
+
+namespace ROCKSDB_NAMESPACE {
+// Returns a set of properties indicating how/when/where this version of RocksDB
+// was created.
+const std::unordered_map<std::string, std::string>& GetRocksBuildProperties();
+
+// Returns the current version of RocksDB as a string (e.g. "6.16.0").
+// If with_patch is true, the patch is included (6.16.x).
+// Otherwise, only major and minor version is included (6.16)
+std::string GetRocksVersionAsString(bool with_patch = true);
+
+// Gets the set of build properties (@see GetRocksBuildProperties) into a
+// string. Properties are returned one-per-line, with the first line being:
+// "<program> from RocksDB <version>.
+// If verbose is true, the full set of properties is
+// printed. If verbose is false, only the version information (@see
+// GetRocksVersionString) is printed.
+std::string GetRocksBuildInfoAsString(const std::string& program,
+                                      bool verbose = false);
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/wal_filter.h

@@ -8,17 +8,26 @@
 #include <map>
 #include <string>
 
+#include "rocksdb/customizable.h"
 #include "rocksdb/rocksdb_namespace.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 class WriteBatch;
+struct ConfigOptions;
 
 // WALFilter allows an application to inspect write-ahead-log (WAL)
 // records or modify their processing on recovery.
 // Please see the details below.
-class WalFilter {
+//
+// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
+// because RocksDB is not exception-safe. This could cause undefined behavior
+// including data loss, unreported corruption, deadlocks, and more.
+class WalFilter : public Customizable {
  public:
+  static const char* Type() { return "WalFilter"; }
+  static Status CreateFromString(const ConfigOptions& options,
+                                 const std::string& value, WalFilter** result);
   enum class WalProcessingOption {
     // Continue processing as usual
     kContinueProcessing = 0,
@@ -96,7 +105,7 @@ class WalFilter {
 
   // Returns a name that identifies this WAL filter.
   // The name will be printed to LOG file on start up for diagnosis.
-  virtual const char* Name() const = 0;
+  virtual const char* Name() const override = 0;
 };
 
 }  // namespace ROCKSDB_NAMESPACE