@nxtedition/rocksdb 7.0.24 → 7.0.27

package/binding.cc

@@ -14,6 +14,7 @@
 #include <rocksdb/slice_transform.h>
 #include <rocksdb/table.h>
 #include <rocksdb/write_batch.h>
+#include <rocksdb/merge_operator.h>
 
 #include <array>
 #include <memory>
@@ -701,6 +702,8 @@ struct OpenWorker final : public Worker {
 
 template <typename T, typename U>
 rocksdb::Status InitOptions(napi_env env, T& columnOptions, const U& options) {
+  rocksdb::ConfigOptions configOptions;
+
   const auto memtable_memory_budget = Uint32Property(env, options, "memtableMemoryBudget").value_or(256 * 1024 * 1024);
 
   const auto compaction = StringProperty(env, options, "compaction").value_or("level");
@@ -746,18 +749,22 @@ rocksdb::Status InitOptions(napi_env env, T& columnOptions, const U& options) {
 
   const auto prefixExtractorOpt = StringProperty(env, options, "prefixExtractor");
   if (prefixExtractorOpt) {
-    rocksdb::ConfigOptions configOptions;
     ROCKS_STATUS_RETURN(
         rocksdb::SliceTransform::CreateFromString(configOptions, *prefixExtractorOpt, &columnOptions.prefix_extractor));
   }
 
   const auto comparatorOpt = StringProperty(env, options, "comparator");
   if (comparatorOpt) {
-    rocksdb::ConfigOptions configOptions;
     ROCKS_STATUS_RETURN(
         rocksdb::Comparator::CreateFromString(configOptions, *comparatorOpt, &columnOptions.comparator));
   }
 
+  const auto mergeOperatorOpt = StringProperty(env, options, "mergeOperator");
+  if (mergeOperatorOpt) {
+    ROCKS_STATUS_RETURN(
+        rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
+  }
+
   const auto cacheSize = Uint32Property(env, options, "cacheSize").value_or(8 << 20);
 
   rocksdb::BlockBasedTableOptions tableOptions;
@@ -1054,7 +1061,9 @@ struct UpdatesNextWorker final : public rocksdb::WriteBatch::Handler, public Wor
     } else {
       cache_.emplace_back(std::nullopt);
     }
-    cache_.emplace_back(GetColumnName(column_family_id));
+    if (!updates_->column_family_id_) {
+      cache_.emplace_back(GetColumnName(column_family_id));
+    }
     return rocksdb::Status::OK();
   }
 

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -627,7 +627,11 @@ set(SOURCES
         db/compaction/compaction_picker_fifo.cc
         db/compaction/compaction_picker_level.cc
         db/compaction/compaction_picker_universal.cc
+        db/compaction/compaction_service_job.cc
+        db/compaction/compaction_state.cc
+        db/compaction/compaction_outputs.cc
         db/compaction/sst_partitioner.cc
+        db/compaction/subcompaction_state.cc
         db/convenience.cc
         db/db_filesnapshot.cc
         db/db_impl/compacted_db_impl.cc
@@ -1231,6 +1235,7 @@ if(WITH_TESTS)
         db/compaction/compaction_iterator_test.cc
         db/compaction/compaction_picker_test.cc
         db/compaction/compaction_service_test.cc
+        db/compaction/tiered_compaction_test.cc
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc

package/deps/rocksdb/rocksdb/Makefile

@@ -136,6 +136,7 @@ CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
 CFLAGS +=  $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
 endif
 
+GIT_COMMAND ?= git
 ifeq ($(USE_COROUTINES), 1)
 	USE_FOLLY = 1
 	OPT += -DUSE_COROUTINES
@@ -1782,6 +1783,9 @@ write_unprepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_unpre
 timestamped_snapshot_test: $(OBJ_DIR)/utilities/transactions/timestamped_snapshot_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+tiered_compaction_test: $(OBJ_DIR)/db/compaction/tiered_compaction_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 sst_dump: $(OBJ_DIR)/tools/sst_dump.o $(TOOLS_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2356,9 +2360,9 @@ commit_prereq:
 # integration.
 checkout_folly:
 	if [ -e third-party/folly ]; then \
-		cd third-party/folly && git fetch origin; \
+		cd third-party/folly && ${GIT_COMMAND} fetch origin; \
 	else \
-		cd third-party && git clone https://github.com/facebook/folly.git; \
+		cd third-party && ${GIT_COMMAND} clone https://github.com/facebook/folly.git; \
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically

package/deps/rocksdb/rocksdb/TARGETS

@@ -38,11 +38,15 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/compaction/compaction.cc",
         "db/compaction/compaction_iterator.cc",
         "db/compaction/compaction_job.cc",
+        "db/compaction/compaction_outputs.cc",
         "db/compaction/compaction_picker.cc",
         "db/compaction/compaction_picker_fifo.cc",
         "db/compaction/compaction_picker_level.cc",
         "db/compaction/compaction_picker_universal.cc",
+        "db/compaction/compaction_service_job.cc",
+        "db/compaction/compaction_state.cc",
         "db/compaction/sst_partitioner.cc",
+        "db/compaction/subcompaction_state.cc",
         "db/convenience.cc",
         "db/db_filesnapshot.cc",
         "db/db_impl/compacted_db_impl.cc",
@@ -368,11 +372,15 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/compaction/compaction.cc",
         "db/compaction/compaction_iterator.cc",
         "db/compaction/compaction_job.cc",
+        "db/compaction/compaction_outputs.cc",
         "db/compaction/compaction_picker.cc",
         "db/compaction/compaction_picker_fifo.cc",
         "db/compaction/compaction_picker_level.cc",
         "db/compaction/compaction_picker_universal.cc",
+        "db/compaction/compaction_service_job.cc",
+        "db/compaction/compaction_state.cc",
         "db/compaction/sst_partitioner.cc",
+        "db/compaction/subcompaction_state.cc",
         "db/convenience.cc",
         "db/db_filesnapshot.cc",
         "db/db_impl/compacted_db_impl.cc",
@@ -5764,6 +5772,12 @@ cpp_unittest_wrapper(name="thread_local_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="tiered_compaction_test",
+            srcs=["db/compaction/tiered_compaction_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="timer_queue_test",
             srcs=["util/timer_queue_test.cc"],
             deps=[":rocksdb_test_lib"],

package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc

@@ -13,6 +13,7 @@
 #include <set>
 #include <sstream>
 
+#include "cache/clock_cache.h"
 #include "cache/fast_lru_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "monitoring/histogram.h"
@@ -284,7 +285,9 @@ class CacheBench {
     }
 
     if (FLAGS_cache_type == "clock_cache") {
-      cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+      cache_ = ExperimentalNewClockCache(
+          FLAGS_cache_size, FLAGS_value_bytes, FLAGS_num_shard_bits,
+          false /*strict_capacity_limit*/, kDefaultCacheMetadataChargePolicy);
       if (!cache_) {
         fprintf(stderr, "Clock cache not supported.\n");
         exit(1);

package/deps/rocksdb/rocksdb/cache/cache_helpers.h

@@ -84,6 +84,16 @@ class CacheHandleGuard {
   Cache::Handle* GetCacheHandle() const { return handle_; }
   T* GetValue() const { return value_; }
 
+  void TransferTo(Cleanable* cleanable) {
+    if (cleanable) {
+      if (handle_ != nullptr) {
+        assert(cache_);
+        cleanable->RegisterCleanup(&ReleaseCacheHandle, cache_, handle_);
+      }
+    }
+    ResetFields();
+  }
+
   void Reset() {
     ReleaseHandle();
     ResetFields();
@@ -105,6 +115,16 @@ class CacheHandleGuard {
     value_ = nullptr;
   }
 
+  static void ReleaseCacheHandle(void* arg1, void* arg2) {
+    Cache* const cache = static_cast<Cache*>(arg1);
+    assert(cache);
+
+    Cache::Handle* const cache_handle = static_cast<Cache::Handle*>(arg2);
+    assert(cache_handle);
+
+    cache->Release(cache_handle);
+  }
+
  private:
   Cache* cache_ = nullptr;
   Cache::Handle* handle_ = nullptr;

package/deps/rocksdb/rocksdb/cache/cache_reservation_manager_test.cc

@@ -147,7 +147,7 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
 
   std::size_t new_mem_used = kSmallCacheCapacity + 1;
   Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
-  EXPECT_EQ(s, Status::Incomplete())
+  EXPECT_EQ(s, Status::MemoryLimit())
       << "Failed to return status to indicate failure of dummy entry insertion "
          "during cache reservation on full cache";
   EXPECT_GE(test_cache_rev_mng->GetTotalReservedCacheSize(),
@@ -192,7 +192,7 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
   // Create cache full again for subsequent tests
   new_mem_used = kSmallCacheCapacity + 1;
   s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
-  EXPECT_EQ(s, Status::Incomplete())
+  EXPECT_EQ(s, Status::MemoryLimit())
       << "Failed to return status to indicate failure of dummy entry insertion "
          "during cache reservation on full cache";
   EXPECT_GE(test_cache_rev_mng->GetTotalReservedCacheSize(),

package/deps/rocksdb/rocksdb/cache/cache_test.cc

@@ -77,7 +77,7 @@ class CacheTest : public testing::TestWithParam<std::string> {
   static std::string type_;
 
   static void Deleter(const Slice& key, void* v) {
-    if (type_ == kFast) {
+    if (type_ == kFast || type_ == kClock) {
       current_->deleted_keys_.push_back(DecodeKey16Bytes(key));
     } else {
       current_->deleted_keys_.push_back(DecodeKey32Bits(key));
@@ -111,7 +111,9 @@ class CacheTest : public testing::TestWithParam<std::string> {
       return NewLRUCache(capacity);
     }
     if (type == kClock) {
-      return NewClockCache(capacity);
+      return ExperimentalNewClockCache(
+          capacity, 1 /*estimated_value_size*/, -1 /*num_shard_bits*/,
+          false /*strict_capacity_limit*/, kDefaultCacheMetadataChargePolicy);
     }
     if (type == kFast) {
       return NewFastLRUCache(
@@ -135,8 +137,9 @@ class CacheTest : public testing::TestWithParam<std::string> {
       return NewLRUCache(co);
     }
     if (type == kClock) {
-      return NewClockCache(capacity, num_shard_bits, strict_capacity_limit,
-                           charge_policy);
+      return ExperimentalNewClockCache(capacity, 1 /*estimated_value_size*/,
+                                       num_shard_bits, strict_capacity_limit,
+                                       charge_policy);
     }
     if (type == kFast) {
       return NewFastLRUCache(capacity, 1 /*estimated_value_size*/,
@@ -152,7 +155,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
   // LRUCache and ClockCache don't, so the encoding depends on
   // the cache type.
   std::string EncodeKey(int k) {
-    if (GetParam() == kFast) {
+    auto type = GetParam();
+    if (type == kFast || type == kClock) {
       return EncodeKey16Bytes(k);
     } else {
       return EncodeKey32Bits(k);
@@ -160,7 +164,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
   }
 
   int DecodeKey(const Slice& k) {
-    if (GetParam() == kFast) {
+    auto type = GetParam();
+    if (type == kFast || type == kClock) {
       return DecodeKey16Bytes(k);
     } else {
       return DecodeKey32Bits(k);
@@ -217,8 +222,9 @@ std::string CacheTest::type_;
 class LRUCacheTest : public CacheTest {};
 
 TEST_P(CacheTest, UsageTest) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache requires 16 byte keys.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
     return;
   }
 
@@ -266,8 +272,9 @@ TEST_P(CacheTest, UsageTest) {
 }
 
 TEST_P(CacheTest, PinnedUsageTest) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache requires 16 byte keys.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
     return;
   }
 
@@ -492,8 +499,10 @@ TEST_P(CacheTest, EvictionPolicyRef) {
   Insert(302, 103);
   Insert(303, 104);
 
-  // Insert entries much more than Cache capacity
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  // Insert entries much more than cache capacity.
+  double load_factor =
+      std::min(fast_lru_cache::kLoadFactor, clock_cache::kLoadFactor);
+  for (int i = 0; i < 2 * static_cast<int>(kCacheSize / load_factor); i++) {
     Insert(1000 + i, 2000 + i);
   }
 
@@ -523,8 +532,9 @@ TEST_P(CacheTest, EvictionPolicyRef) {
 }
 
 TEST_P(CacheTest, EvictEmptyCache) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache requires 16 byte keys.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
     return;
   }
 
@@ -534,8 +544,9 @@ TEST_P(CacheTest, EvictEmptyCache) {
 }
 
 TEST_P(CacheTest, EraseFromDeleter) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache requires 16 byte keys.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
     return;
   }
 
@@ -650,8 +661,10 @@ TEST_P(CacheTest, ReleaseWithoutErase) {
 }
 
 TEST_P(CacheTest, SetCapacity) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache doesn't support capacity adjustments.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS(
+        "FastLRUCache and ClockCache don't support capacity adjustments.");
     return;
   }
   // test1: increase capacity
@@ -702,9 +715,11 @@ TEST_P(CacheTest, SetCapacity) {
 }
 
 TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
-  if (GetParam() == kFast) {
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
     ROCKSDB_GTEST_BYPASS(
-        "FastLRUCache doesn't support an unbounded number of inserts beyond "
+        "FastLRUCache and ClockCache don't support an unbounded number of "
+        "inserts beyond "
         "capacity.");
     return;
   }
@@ -727,7 +742,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   cache->SetStrictCapacityLimit(true);
   Cache::Handle* handle;
   s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle);
-  ASSERT_TRUE(s.IsIncomplete());
+  ASSERT_TRUE(s.IsMemoryLimit());
   ASSERT_EQ(nullptr, handle);
   ASSERT_EQ(10, cache->GetUsage());
 
@@ -744,7 +759,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
     ASSERT_NE(nullptr, handles[i]);
   }
   s = cache2->Insert(extra_key, extra_value, 1, &deleter, &handle);
-  ASSERT_TRUE(s.IsIncomplete());
+  ASSERT_TRUE(s.IsMemoryLimit());
   ASSERT_EQ(nullptr, handle);
   // test insert without handle
   s = cache2->Insert(extra_key, extra_value, 1, &deleter);
@@ -759,8 +774,10 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
 }
 
 TEST_P(CacheTest, OverCapacity) {
-  if (GetParam() == kFast) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache doesn't support capacity adjustments.");
+  auto type = GetParam();
+  if (type == kFast || type == kClock) {
+    ROCKSDB_GTEST_BYPASS(
+        "FastLRUCache and ClockCache don't support capacity adjustments.");
     return;
   }
   size_t n = 10;
@@ -938,15 +955,11 @@ TEST_P(CacheTest, GetChargeAndDeleter) {
   cache_->Release(h1);
 }
 
-#ifdef SUPPORT_CLOCK_CACHE
-std::shared_ptr<Cache> (*new_clock_cache_func)(
-    size_t, int, bool, CacheMetadataChargePolicy) = NewClockCache;
+std::shared_ptr<Cache> (*new_clock_cache_func)(size_t, size_t, int, bool,
+                                               CacheMetadataChargePolicy) =
+    ExperimentalNewClockCache;
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
                         testing::Values(kLRU, kClock, kFast));
-#else
-INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
-                        testing::Values(kLRU, kFast));
-#endif  // SUPPORT_CLOCK_CACHE
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest,
                         testing::Values(kLRU, kFast));