@nxtedition/rocksdb 7.0.4 → 7.0.7

package/chained-batch.js

@@ -27,7 +27,12 @@ class ChainedBatch extends AbstractChainedBatch {
   }
 
   _write (options, callback) {
-    process.nextTick(callback, binding.batch_write(this[kDbContext], this[kBatchContext], options))
+    try {
+      binding.batch_write(this[kDbContext], this[kBatchContext], options)
+      process.nextTick(callback, null)
+    } catch (err) {
+      process.nextTick(callback, err)
+    }
   }
 
   _close (callback) {

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -587,6 +587,7 @@ include_directories(${PROJECT_SOURCE_DIR}/include)
 if(USE_FOLLY)
   include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
   add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
+  list(APPEND THIRDPARTY_LIBS glog)
 endif()
 find_package(Threads REQUIRED)
 
@@ -614,6 +615,7 @@ set(SOURCES
         db/blob/blob_log_format.cc
         db/blob/blob_log_sequential_reader.cc
         db/blob/blob_log_writer.cc
+        db/blob/blob_source.cc
         db/blob/prefetch_buffer_collection.cc
         db/builder.cc
         db/c.cc
@@ -738,7 +740,6 @@ set(SOURCES
         table/adaptive/adaptive_table_factory.cc
         table/block_based/binary_search_index_reader.cc
         table/block_based/block.cc
-        table/block_based/block_based_filter_block.cc
         table/block_based/block_based_table_builder.cc
         table/block_based/block_based_table_factory.cc
         table/block_based/block_based_table_iterator.cc
@@ -976,9 +977,13 @@ endif()
 if(USE_FOLLY)
   list(APPEND SOURCES
     third-party/folly/folly/container/detail/F14Table.cpp
+    third-party/folly/folly/detail/Futex.cpp
     third-party/folly/folly/lang/SafeAssert.cpp
     third-party/folly/folly/lang/ToAscii.cpp
-    third-party/folly/folly/ScopeGuard.cpp)
+    third-party/folly/folly/ScopeGuard.cpp
+    third-party/folly/folly/synchronization/AtomicNotification.cpp
+    third-party/folly/folly/synchronization/DistributedMutex.cpp
+    third-party/folly/folly/synchronization/ParkingLot.cpp)
 endif()
 
 set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
@@ -1213,6 +1218,7 @@ if(WITH_TESTS)
         db/blob/blob_file_garbage_test.cc
         db/blob/blob_file_reader_test.cc
         db/blob/blob_garbage_meter_test.cc
+        db/blob/blob_source_test.cc
         db/blob/db_blob_basic_test.cc
         db/blob/db_blob_compaction_test.cc
         db/blob/db_blob_corruption_test.cc
@@ -1321,7 +1327,6 @@ if(WITH_TESTS)
         options/customizable_test.cc
         options/options_settable_test.cc
         options/options_test.cc
-        table/block_based/block_based_filter_block_test.cc
         table/block_based/block_based_table_reader_test.cc
         table/block_based/block_test.cc
         table/block_based/data_block_hash_index_test.cc

package/deps/rocksdb/rocksdb/Makefile

@@ -461,6 +461,8 @@ ifeq ($(USE_FOLLY),1)
 	endif
 	PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
 	PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
+# TODO: fix linking with fbcode compiler config
+	PLATFORM_LDFLAGS += -lglog
 endif
 
 ifdef TEST_CACHE_LINE_SIZE
@@ -1594,9 +1596,6 @@ random_access_file_reader_test: $(OBJ_DIR)/file/random_access_file_reader_test.o
 file_reader_writer_test: $(OBJ_DIR)/util/file_reader_writer_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-block_based_filter_block_test: $(OBJ_DIR)/table/block_based/block_based_filter_block_test.o $(TEST_LIBRARY) $(LIBRARY)
-	$(AM_LINK)
-
 block_based_table_reader_test: table/block_based/block_based_table_reader_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -1864,6 +1863,9 @@ blob_file_garbage_test: $(OBJ_DIR)/db/blob/blob_file_garbage_test.o $(TEST_LIBRA
 blob_file_reader_test: $(OBJ_DIR)/db/blob/blob_file_reader_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+blob_source_test: $(OBJ_DIR)/db/blob/blob_source_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 blob_garbage_meter_test: $(OBJ_DIR)/db/blob/blob_garbage_meter_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2357,10 +2359,14 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 98b9b2c1124e99f50f9085ddee74ce32afffc665
+	cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba
 	@# A hack to remove boost dependency.
 	@# NOTE: this hack is not needed if using FBCODE compiler config
 	perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
+	@# NOTE: this hack is required for clang in some cases
+	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
+	@# NOTE: this hack is required for gcc in some cases
+	perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
 
 # ---------------------------------------------------------------------------
 #   Build size testing

package/deps/rocksdb/rocksdb/TARGETS

@@ -30,6 +30,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/blob/blob_log_format.cc",
         "db/blob/blob_log_sequential_reader.cc",
         "db/blob/blob_log_writer.cc",
+        "db/blob/blob_source.cc",
         "db/blob/prefetch_buffer_collection.cc",
         "db/builder.cc",
         "db/c.cc",
@@ -165,7 +166,6 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "table/adaptive/adaptive_table_factory.cc",
         "table/block_based/binary_search_index_reader.cc",
         "table/block_based/block.cc",
-        "table/block_based/block_based_filter_block.cc",
         "table/block_based/block_based_table_builder.cc",
         "table/block_based/block_based_table_factory.cc",
         "table/block_based/block_based_table_iterator.cc",
@@ -335,6 +335,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "//folly/experimental/coro:collect",
         "//folly/experimental/coro:coroutine",
         "//folly/experimental/coro:task",
+        "//folly/synchronization:distributed_mutex",
     ], headers=None, link_whole=False, extra_test_libs=False)
 
 cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
@@ -359,6 +360,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/blob/blob_log_format.cc",
         "db/blob/blob_log_sequential_reader.cc",
         "db/blob/blob_log_writer.cc",
+        "db/blob/blob_source.cc",
         "db/blob/prefetch_buffer_collection.cc",
         "db/builder.cc",
         "db/c.cc",
@@ -494,7 +496,6 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "table/adaptive/adaptive_table_factory.cc",
         "table/block_based/binary_search_index_reader.cc",
         "table/block_based/block.cc",
-        "table/block_based/block_based_filter_block.cc",
         "table/block_based/block_based_table_builder.cc",
         "table/block_based/block_based_table_factory.cc",
         "table/block_based/block_based_table_iterator.cc",
@@ -664,6 +665,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "//folly/experimental/coro:collect",
         "//folly/experimental/coro:coroutine",
         "//folly/experimental/coro:task",
+        "//folly/synchronization:distributed_mutex",
     ], headers=None, link_whole=True, extra_test_libs=False)
 
 cpp_library_wrapper(name="rocksdb_test_lib", srcs=[
@@ -4800,8 +4802,8 @@ cpp_unittest_wrapper(name="blob_garbage_meter_test",
             extra_compiler_flags=[])
 
 
-cpp_unittest_wrapper(name="block_based_filter_block_test",
-            srcs=["table/block_based/block_based_filter_block_test.cc"],
+cpp_unittest_wrapper(name="blob_source_test",
+            srcs=["db/blob/blob_source_test.cc"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 

package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc

@@ -26,6 +26,7 @@
 #include "table/block_based/block_based_table_reader.h"
 #include "table/block_based/cachable_entry.h"
 #include "util/coding.h"
+#include "util/distributed_mutex.h"
 #include "util/gflags_compat.h"
 #include "util/hash.h"
 #include "util/mutexlock.h"
@@ -587,7 +588,15 @@ class CacheBench {
   }
 
   void PrintEnv() const {
+#if defined(__GNUC__) && !defined(__OPTIMIZE__)
+    printf(
+        "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
+#endif
+#ifndef NDEBUG
+    printf("WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
+#endif
     printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
+    printf("DMutex impl name    : %s\n", DMutex::kName());
     printf("Number of threads   : %u\n", FLAGS_threads);
     printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
     printf("Cache size          : %s\n",

package/deps/rocksdb/rocksdb/cache/cache_test.cc

@@ -650,6 +650,10 @@ TEST_P(CacheTest, ReleaseWithoutErase) {
 }
 
 TEST_P(CacheTest, SetCapacity) {
+  if (GetParam() == kFast) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache doesn't support capacity adjustments.");
+    return;
+  }
   // test1: increase capacity
   // lets create a cache with capacity 5,
   // then, insert 5 elements, then increase capacity
@@ -698,6 +702,12 @@ TEST_P(CacheTest, SetCapacity) {
 }
 
 TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
+  if (GetParam() == kFast) {
+    ROCKSDB_GTEST_BYPASS(
+        "FastLRUCache doesn't support an unbounded number of inserts beyond "
+        "capacity.");
+    return;
+  }
   // test1: set the flag to false. Insert more keys than capacity. See if they
   // all go through.
   std::shared_ptr<Cache> cache = NewCache(5, 0, false);
@@ -749,6 +759,10 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
 }
 
 TEST_P(CacheTest, OverCapacity) {
+  if (GetParam() == kFast) {
+    ROCKSDB_GTEST_BYPASS("FastLRUCache doesn't support capacity adjustments.");
+    return;
+  }
   size_t n = 10;
 
   // a LRUCache with n entries and one shard only

package/deps/rocksdb/rocksdb/cache/clock_cache.cc

@@ -39,7 +39,7 @@ std::shared_ptr<Cache> NewClockCache(
 #include "port/port.h"
 #include "tbb/concurrent_hash_map.h"
 #include "util/autovector.h"
-#include "util/mutexlock.h"
+#include "util/distributed_mutex.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -368,7 +368,7 @@ class ClockCacheShard final : public CacheShard {
 
   // Guards list_, head_, and recycle_. In addition, updating table_ also has
   // to hold the mutex, to avoid the cache being in inconsistent state.
-  mutable port::Mutex mutex_;
+  mutable DMutex mutex_;
 
   // The circular list of cache handles. Initially the list is empty. Once a
   // handle is needed by insertion, and no more handles are available in
@@ -431,7 +431,7 @@ void ClockCacheShard::ApplyToSomeEntries(
                              DeleterFn deleter)>& callback,
     uint32_t average_entries_per_lock, uint32_t* state) {
   assert(average_entries_per_lock > 0);
-  MutexLock lock(&mutex_);
+  DMutexLock l(mutex_);
 
   // Figure out the range to iterate, update `state`
   size_t list_size = list_.size();
@@ -532,7 +532,7 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
     pinned_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
     // Cleanup if it is the last reference.
     if (!InCache(flags)) {
-      MutexLock l(&mutex_);
+      DMutexLock l(mutex_);
       RecycleHandle(handle, context);
     }
   }
@@ -598,7 +598,7 @@ bool ClockCacheShard::EvictFromCache(size_t charge, CleanupContext* context) {
 void ClockCacheShard::SetCapacity(size_t capacity) {
   CleanupContext context;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     capacity_.store(capacity, std::memory_order_relaxed);
     EvictFromCache(0, &context);
   }
@@ -618,7 +618,7 @@ CacheHandle* ClockCacheShard::Insert(
   uint32_t meta_charge =
       CacheHandle::CalcMetadataCharge(key, metadata_charge_policy_);
   size_t total_charge = charge + meta_charge;
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   bool success = EvictFromCache(total_charge, context);
   bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
   if (!success && (strict || !hold_reference)) {
@@ -744,7 +744,7 @@ void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
 
 bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
                                       CleanupContext* context) {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   HashTable::accessor accessor;
   bool erased = false;
   if (table_.find(accessor, ClockCacheKey(key, hash))) {
@@ -758,7 +758,7 @@ bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
 void ClockCacheShard::EraseUnRefEntries() {
   CleanupContext context;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     table_.clear();
     for (auto& handle : list_) {
       UnsetInCache(&handle, &context);