@nxtedition/rocksdb 15.4.1 → 16.0.0

package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h

@@ -16,30 +16,34 @@
 #include "rocksdb/options.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
+#include "rocksdb/types.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 // Prefix for user-defined index block names
-inline const std::string kUserDefinedIndexPrefix =
+inline constexpr const char* kUserDefinedIndexPrefix =
     "rocksdb.user_defined_index.";
 
 // This is a public API for user-defined index builders.
 // It allows users to define their own index format and build custom
 // indexes during table building. Currently, only a monolithic index
 // block is supported (no partitioned index).
-//
-// This is currently supported only for a restricted set of use cases. The
-// CF must be ingest only, and only files containing Puts generated by
-// SstFileWriter are supported.
 
 // The interface for building user-defined index.
 class UserDefinedIndexBuilder {
  public:
-  // Right now, we only support Puts. In the future, we may support merges,
-  // deletions etc.
-  enum ValueType {
-    kValue,
-    kTypeMax,
+  // Indicates the type of key-value entry being added via OnKeyAdded().
+  // UDI builders that only use AddIndexEntry() (e.g., trie-based indexes)
+  // can safely ignore this.
+  enum ValueType : uint8_t {
+    kValue = 0,   // Put: the value is the full user value.
+    kDelete = 1,  // Deletion (Delete, SingleDelete, or DeleteWithTimestamp):
+                  // the value is typically empty.
+    kMerge = 2,   // Merge operand: the value is a partial update.
+    kOther = 3,   // Other types (e.g., blob reference, wide-column entity).
+                  // The value format is type-specific and may not be the
+                  // actual user data.
+    kTypeMax,     // Sentinel — must be last. Value may change across releases.
   };
 
   // File offset and size of the data block
@@ -48,41 +52,68 @@ class UserDefinedIndexBuilder {
     uint64_t size;
   };
 
+  // Optional context for AddIndexEntry providing sequence numbers at block
+  // boundaries. Passed as a struct for forward-compatible extensibility
+  // (new fields can be added without breaking existing implementations).
+  struct IndexEntryContext {
+    // Sequence number of last_key_in_current_block.
+    SequenceNumber last_key_seq = 0;
+    // Sequence number of first_key_in_next_block (valid only when
+    // first_key_in_next_block != nullptr).
+    SequenceNumber first_key_seq = 0;
+  };
+
   virtual ~UserDefinedIndexBuilder() = default;
 
-  // Add a new index entry to index block. The key for the new index entry
-  // should be >= last_key_in_current_block and < first_key_in_next_block.
-  // The previous index entry key and the new index entry key cover
-  // all the keys in the data block associated with the new index entry.
+  // Add a new index entry for a data block boundary.
+  //
+  // The keys are user keys (without the 8-byte internal key trailer).
   //
-  // The last_key_in_current_block and first_key_in_next_block will be user
-  // keys, i.e the user key string, and optionally the user timestamp if one
-  // is configured, without a sequence number suffix.
+  // The UDI is free to compute a separator between the two user keys and
+  // store it along with the block handle. The separator must satisfy:
+  //   last_key_in_current_block <= separator < first_key_in_next_block
+  // in user-key order (ignoring sequence numbers).
   //
   // Called before the OnKeyAdded() call for first_key_in_next_block.
-  // @last_key_in_current_block: The last key in the current data block
-  // @first_key_in_next_block: it will be nullptr if the entry being added is
-  //                           the last one in the table
-  // @block_handle: offset/size of the data block referenced by this index
-  //                entry. This should be stored along with the index entry
-  //                key
-  // @separator_scratch: a scratch buffer to back a computed separator between
-  //                     those, as needed. May be modified on each call.
-  // @return: the key or separator stored in the index, which could be
-  //          last_key_in_current_block or a computed separator backed by
-  //          separator_scratch.
+  // @last_key_in_current_block: The last user key in the current data block
+  // @first_key_in_next_block: First user key in the next data block, or
+  //                           nullptr if this is the last block
+  // @block_handle: offset/size of the data block
+  // @separator_scratch: scratch buffer for a computed separator
+  // @context: sequence number context for block boundaries. The sequence
+  //   numbers are needed when the same user key spans a data block boundary
+  //   (e.g., when snapshots keep multiple versions of a key). Without
+  //   sequence numbers, the UDI cannot produce a separator that distinguishes
+  //   the two blocks. This mirrors the internal index's behavior of switching
+  //   to full internal-key separators (see
+  //   ShortenedIndexBuilder::must_use_separator_with_seq_).
+  //   Implementations that don't need sequence numbers can ignore the context.
+  // @return: the separator stored in the index
   virtual Slice AddIndexEntry(const Slice& last_key_in_current_block,
                               const Slice* first_key_in_next_block,
                               const BlockHandle& block_handle,
-                              std::string* separator_scratch) = 0;
+                              std::string* separator_scratch,
+                              const IndexEntryContext& context) = 0;
 
-  // This method will be called whenever a key is added. The subclasses may
-  // override OnKeyAdded() if they need to collect additional information.
-  // The type argument indicates whether the value is a full value or partial.
-  // At the moment, only full values are supported.
+  // Called for every key-value pair added to the SST file. UDI builders may
+  // override this to collect per-key information (e.g., for secondary
+  // indexes). Builders that only use separator keys from AddIndexEntry()
+  // (e.g., trie-based indexes) can leave this as a no-op.
   //
-  // The key will be a user key. RocksDB guarantees that there will only be
-  // one entry for each key in the file/index.
+  // @key:   The user key (without sequence number or type suffix).
+  // @type:  The entry type — kValue (Put), kDelete, kMerge, or kOther.
+  //         For kDelete entries, the value may be empty. For kOther, the
+  //         value format is type-specific and may not be actual user data.
+  // @value: The associated value (may be empty for deletions).
+  //
+  // NOTE: In SST files produced by flush or compaction, there may be multiple
+  // entries for the same user key with different sequence numbers (e.g., when
+  // snapshots are active). UDI builders that use OnKeyAdded() should be
+  // prepared for this.
+  //
+  // Thread safety: For a given builder instance, OnKeyAdded() and
+  // AddIndexEntry() are always called from a single thread. Builders do
+  // not need internal synchronization.
   virtual void OnKeyAdded(const Slice& /*key*/, ValueType /*type*/,
                           const Slice& /*value*/) {}
 
@@ -104,13 +135,36 @@ class UserDefinedIndexIterator {
   // this as an opportunity to do any prefetching and buffering of results.
   virtual void Prepare(const ScanOptions scan_opts[], size_t num_opts) = 0;
 
+  // Optional context for SeekAndGetResult providing the target sequence
+  // number. Passed as a struct for forward-compatible extensibility.
+  struct SeekContext {
+    // Sequence number of the target key. Used by UDI implementations that
+    // encode sequence numbers (when the same user key spans multiple data
+    // blocks) to locate the correct block.
+    SequenceNumber target_seq = 0;
+  };
+
+  // Position the index iterator at the very first index entry. The result
+  // must be populated the same way as SeekAndGetResult.
+  //
+  // The default implementation calls SeekAndGetResult with an empty key,
+  // which works for BytewiseComparator (empty string is the smallest key).
+  // Implementations should override this if they can reach the first entry
+  // more efficiently or if they use a comparator where empty is not smallest.
+  virtual Status SeekToFirstAndGetResult(IterateResult* result) {
+    return SeekAndGetResult(Slice(), result, SeekContext{});
+  }
+
   // Given the target key, position the index iterator at the index entry
-  // with the smallest key >= target. The result must be updated with the
-  // index key, and the bound_check_result. The bound_check_result should
-  // be set to kOutOfBound if no block satisfies the target key and
-  // termination criteria, kInbound if the data block is definitely fully
-  // within bounds, or kUnknown if the data block could be partially
-  // within bounds.
+  // for the data block that may contain the target.
+  //
+  // The target is a user key.
+  //
+  // The result must be updated with the index key and bound_check_result.
+  // bound_check_result should be kOutOfBound if no block satisfies the
+  // target, kInbound if the data block is definitely within bounds, or
+  // kUnknown if partially within bounds.
+  //
   // The UDI implementation needs to be careful about returning kOutOfBound.
   // If a limit key is specified in ScanOptions, an implementation that
   // does not store the first key in the block for the corresponding index
@@ -119,8 +173,14 @@ class UserDefinedIndexIterator {
   // is out of bounds w.r.t the limit. Other termination criteria (specified
   // in property_bag) may cause the scan to terminate earlier, in which case
   // kOutOfBound can be returned earlier.
-  virtual Status SeekAndGetResult(const Slice& target,
-                                  IterateResult* result) = 0;
+  //
+  // @context: sequence number context for the seek. The sequence number is
+  //   needed when the same user key spans multiple data blocks with different
+  //   sequence numbers. Without it, the UDI cannot distinguish which block to
+  //   return for a given (user_key, seqno) target. Implementations that don't
+  //   need sequence numbers can ignore the context.
+  virtual Status SeekAndGetResult(const Slice& target, IterateResult* result,
+                                  const SeekContext& context) = 0;
 
   // Advance to the next index entry. The result must be populated similar
   // to SeekAndGetResult.
@@ -152,7 +212,7 @@ struct UserDefinedIndexOption {
 // Factory for creating user-defined index builders.
 class UserDefinedIndexFactory : public Customizable {
  public:
-  virtual ~UserDefinedIndexFactory() = default;
+  ~UserDefinedIndexFactory() override = default;
 
   static const char* Type() { return "UserDefinedIndexFactory"; }
 
@@ -174,14 +234,14 @@ class UserDefinedIndexFactory : public Customizable {
       std::unique_ptr<UserDefinedIndexBuilder>& builder) const {
     builder.reset(NewBuilder());
     return Status::OK();
-  };
+  }
 
   virtual Status NewReader(
       const UserDefinedIndexOption& /*option*/, Slice& index_block,
       std::unique_ptr<UserDefinedIndexReader>& reader) const {
     reader = NewReader(index_block);
     return Status::OK();
-  };
+  }
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h

@@ -66,7 +66,7 @@ class DBWithTTL : public StackableDB {
   virtual Status GetTtl(ColumnFamilyHandle* h, int32_t* ttl) = 0;
 
  protected:
-  explicit DBWithTTL(DB* db) : StackableDB(db) {}
+  explicit DBWithTTL(std::unique_ptr<DB>&& db) : StackableDB(std::move(db)) {}
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h

@@ -11,6 +11,7 @@
 #include <algorithm>
 #include <functional>
 #include <map>
+#include <memory>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -23,6 +24,7 @@
 #include "rocksdb/slice.h"
 #include "rocksdb/utilities/db_ttl.h"
 #include "rocksdb/utilities/ldb_cmd_execute_result.h"
+#include "rocksdb/utilities/transaction_db.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -42,6 +44,8 @@ class LDBCommand {
   static const std::string ARG_TTL;
   static const std::string ARG_TTL_START;
   static const std::string ARG_TTL_END;
+  static const std::string ARG_USE_TXN;
+  static const std::string ARG_TXN_WRITE_POLICY;
   static const std::string ARG_TIMESTAMP;
   static const std::string ARG_TRY_LOAD_OPTIONS;
   static const std::string ARG_IGNORE_UNKNOWN_OPTIONS;
@@ -53,6 +57,7 @@ class LDBCommand {
   static const std::string ARG_COMPRESSION_TYPE;
   static const std::string ARG_COMPRESSION_MAX_DICT_BYTES;
   static const std::string ARG_BLOCK_SIZE;
+  static const std::string ARG_UNIFORM_CV_THRESHOLD;
   static const std::string ARG_AUTO_COMPACTION;
   static const std::string ARG_DB_WRITE_BUFFER_SIZE;
   static const std::string ARG_WRITE_BUFFER_SIZE;
@@ -163,8 +168,9 @@ class LDBCommand {
   std::string secondary_path_;
   std::string leader_path_;
   std::string column_family_name_;
-  DB* db_;
+  std::unique_ptr<DB> db_;
   DBWithTTL* db_ttl_;
+  TransactionDB* db_txn_;
   std::map<std::string, ColumnFamilyHandle*> cf_handles_;
   std::map<uint32_t, const Comparator*> ucmps_;
 
@@ -183,6 +189,13 @@ class LDBCommand {
   /** If true, the value is treated as timestamp suffixed */
   bool is_db_ttl_;
 
+  /** If true, open the DB as TransactionDB */
+  bool is_db_txn_;
+
+  /** Transaction write policy (0=WRITE_COMMITTED, 1=WRITE_PREPARED,
+   * 2=WRITE_UNPREPARED) */
+  int txn_write_policy_;
+
   // If true, the kvs are output with their insert/modify timestamp in a ttl db
   bool timestamp_;
 

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h

@@ -6,6 +6,7 @@
 #pragma once
 
 #include <map>
+#include <memory>
 #include <string>
 #include <unordered_set>
 #include <vector>
@@ -39,8 +40,11 @@ class MemoryUtil {
   // only report the usage of the input "cache_set" without
   // including those Cache usage inside the input list "dbs"
   // of DBs.
+  //
+  // Supports vectors of DB* or unique_ptr<DB>.
+  template <typename DBPtr>
   static Status GetApproximateMemoryUsageByType(
-      const std::vector<DB*>& dbs,
+      const std::vector<DBPtr>& dbs,
       const std::unordered_set<const Cache*> cache_set,
       std::map<MemoryUtil::UsageType, uint64_t>* usage_by_type);
 };

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h

@@ -123,7 +123,8 @@ class OptimisticTransactionDB : public StackableDB {
 
  protected:
   // To Create an OptimisticTransactionDB, call Open()
-  explicit OptimisticTransactionDB(DB* db) : StackableDB(db) {}
+  explicit OptimisticTransactionDB(std::unique_ptr<DB>&& db)
+      : StackableDB(std::move(db)) {}
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h

@@ -375,6 +375,8 @@ class StackableDB : public DB {
   void DisableManualCompaction() override {
     return db_->DisableManualCompaction();
   }
+  void AbortAllCompactions() override { return db_->AbortAllCompactions(); }
+  void ResumeAllCompactions() override { return db_->ResumeAllCompactions(); }
 
   Status WaitForCompact(
       const WaitForCompactOptions& wait_for_compact_options) override {
@@ -386,11 +388,6 @@ class StackableDB : public DB {
     return db_->NumberLevels(column_family);
   }
 
-  using DB::MaxMemCompactionLevel;
-  int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) override {
-    return db_->MaxMemCompactionLevel(column_family);
-  }
-
   using DB::Level0StopWriteTrigger;
   int Level0StopWriteTrigger(ColumnFamilyHandle* column_family) override {
     return db_->Level0StopWriteTrigger(column_family);
@@ -549,10 +546,11 @@ class StackableDB : public DB {
   }
 
   using DB::SetOptions;
-  Status SetOptions(ColumnFamilyHandle* column_family_handle,
-                    const std::unordered_map<std::string, std::string>&
-                        new_options) override {
-    return db_->SetOptions(column_family_handle, new_options);
+  Status SetOptions(
+      const std::unordered_map<ColumnFamilyHandle*,
+                               std::unordered_map<std::string, std::string>>&
+          column_families_opts_map) override {
+    return db_->SetOptions(column_families_opts_map);
   }
 
   Status SetDBOptions(const std::unordered_map<std::string, std::string>&

package/deps/rocksdb/rocksdb/include/rocksdb/version.h

@@ -11,8 +11,8 @@
 
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
-#define ROCKSDB_MAJOR 10
-#define ROCKSDB_MINOR 10
+#define ROCKSDB_MAJOR 11
+#define ROCKSDB_MINOR 1
 #define ROCKSDB_PATCH 1
 
 // Make it easy to do conditional compilation based on version checks, i.e.

package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc

@@ -647,7 +647,7 @@ TEST_F(AutoRollLoggerTest, LogHeaderTest) {
 }
 
 TEST_F(AutoRollLoggerTest, LogFileExistence) {
-  ROCKSDB_NAMESPACE::DB* db;
+  std::unique_ptr<ROCKSDB_NAMESPACE::DB> db;
   ROCKSDB_NAMESPACE::Options options;
 #ifdef OS_WIN
   // Replace all slashes in the path so windows CompSpec does not
@@ -664,7 +664,6 @@ TEST_F(AutoRollLoggerTest, LogFileExistence) {
   options.create_if_missing = true;
   ASSERT_OK(ROCKSDB_NAMESPACE::DB::Open(options, kTestDir, &db));
   ASSERT_OK(default_env->FileExists(kLogFile));
-  delete db;
 }
 
 TEST_F(AutoRollLoggerTest, FileCreateFailure) {

package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc

@@ -83,7 +83,7 @@ TEST_P(MemoryAllocatorTest, DatabaseBlockCache) {
   auto cache = NewLRUCache(1024 * 1024, 6, false, 0.0, allocator_);
   table_options.block_cache = cache;
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  DB* db = nullptr;
+  std::unique_ptr<DB> db;
   Status s = DB::Open(options, dbname, &db);
   ASSERT_OK(s);
   ASSERT_NE(db, nullptr);
@@ -115,7 +115,7 @@ TEST_P(MemoryAllocatorTest, DatabaseBlockCache) {
   // Close database
   s = db->Close();
   ASSERT_OK(s);
-  delete db;
+  db.reset();
   ASSERT_OK(DestroyDB(dbname, options));
 }