@nxtedition/rocksdb 7.1.14 → 7.1.16

package/binding.cc

@@ -958,6 +958,7 @@ NAPI_METHOD(db_get_many) {
         readOptions.snapshot = snapshot.get();
         readOptions.async_io = true;
         readOptions.ignore_range_deletions = ignoreRangeDeletions;
+        readOptions.optimize_multiget_for_io = true;
 
         const auto size = keys.size();
 

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -74,6 +74,7 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
 endif()
+option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON)
 
 if ($ENV{CIRCLECI})
   message(STATUS "Build for CircieCI env, a few tests may be disabled")
@@ -584,10 +585,61 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
+
+if(USE_COROUTINES)
+  if(USE_FOLLY OR USE_FOLLY_LITE)
+    message(FATAL_ERROR "Please specify exactly one of USE_COROUTINES,"
+    " USE_FOLLY, and USE_FOLLY_LITE")
+  endif()
+  set(CMAKE_CXX_STANDARD 20)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines -Wno-maybe-uninitialized")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-redundant-move")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-invalid-memory-model")
+  add_compile_definitions(USE_COROUTINES)
+  set(USE_FOLLY 1)
+endif()
+
 if(USE_FOLLY)
-  include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
-  add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
-  list(APPEND THIRDPARTY_LIBS glog)
+  if(USE_FOLLY_LITE)
+    message(FATAL_ERROR "Please specify one of USE_FOLLY or USE_FOLLY_LITE")
+  endif()
+  if(ROCKSDB_BUILD_SHARED)
+    message(FATAL_ERROR "Cannot build RocksDB shared library with folly")
+  endif()
+  set(ROCKSDB_BUILD_SHARED OFF)
+  set(GFLAGS_SHARED FALSE)
+  find_package(folly)
+  # If cmake could not find the folly-config.cmake file, fall back
+  # to looking in third-party/folly for folly and its dependencies
+  if(NOT FOLLY_LIBRARIES)
+    exec_program(python3 ${PROJECT_SOURCE_DIR}/third-party/folly ARGS
+    build/fbcode_builder/getdeps.py show-inst-dir OUTPUT_VARIABLE
+    FOLLY_INST_PATH)
+    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../boost* OUTPUT_VARIABLE
+    BOOST_INST_PATH)
+    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../fmt* OUTPUT_VARIABLE
+    FMT_INST_PATH)
+    exec_program(ls ARGS -d ${FOLLY_INST_PATH}/../gflags* OUTPUT_VARIABLE
+    GFLAGS_INST_PATH)
+    set(Boost_DIR ${BOOST_INST_PATH}/lib/cmake/Boost-1.78.0)
+    if(EXISTS ${FMT_INST_PATH}/lib64)
+      set(fmt_DIR ${FMT_INST_PATH}/lib64/cmake/fmt)
+    else()
+      set(fmt_DIR ${FMT_INST_PATH}/lib/cmake/fmt)
+    endif()
+    set(gflags_DIR ${GFLAGS_INST_PATH}/lib/cmake/gflags)
+
+    exec_program(sed ARGS -i 's/gflags_shared//g'
+    ${FOLLY_INST_PATH}/lib/cmake/folly/folly-targets.cmake)
+
+    include(${FOLLY_INST_PATH}/lib/cmake/folly/folly-config.cmake)
+  endif()
+
+  add_compile_definitions(USE_FOLLY FOLLY_NO_CONFIG HAVE_CXX11_ATOMIC)
+  list(APPEND THIRDPARTY_LIBS Folly::folly)
+  set(FOLLY_LIBS Folly::folly)
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--copy-dt-needed-entries")
 endif()
 find_package(Threads REQUIRED)
 
@@ -605,6 +657,7 @@ set(SOURCES
         cache/lru_cache.cc
         cache/sharded_cache.cc
         db/arena_wrapped_db_iter.cc
+        db/blob/blob_contents.cc
         db/blob/blob_fetcher.cc
         db/blob/blob_file_addition.cc
         db/blob/blob_file_builder.cc
@@ -667,7 +720,7 @@ set(SOURCES
         db/merge_helper.cc
         db/merge_operator.cc
         db/output_validator.cc
-        db/periodic_work_scheduler.cc
+        db/periodic_task_scheduler.cc
         db/range_del_aggregator.cc
         db/range_tombstone_fragmenter.cc
         db/repair.cc
@@ -827,6 +880,7 @@ set(SOURCES
         util/slice.cc
         util/file_checksum_helper.cc
         util/status.cc
+        util/stderr_logger.cc
         util/string_util.cc
         util/thread_local.cc
         util/threadpool_imp.cc
@@ -981,7 +1035,7 @@ else()
     env/io_posix.cc)
 endif()
 
-if(USE_FOLLY)
+if(USE_FOLLY_LITE)
   list(APPEND SOURCES
     third-party/folly/folly/container/detail/F14Table.cpp
     third-party/folly/folly/detail/Futex.cpp
@@ -991,13 +1045,14 @@ if(USE_FOLLY)
     third-party/folly/folly/synchronization/AtomicNotification.cpp
     third-party/folly/folly/synchronization/DistributedMutex.cpp
     third-party/folly/folly/synchronization/ParkingLot.cpp)
+  include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
+  add_definitions(-DUSE_FOLLY -DFOLLY_NO_CONFIG)
+  list(APPEND THIRDPARTY_LIBS glog)
 endif()
 
 set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
 set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
 
-option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON)
-
 
 if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
@@ -1008,7 +1063,6 @@ endif()
 set(ROCKSDB_PLUGIN_EXTERNS "")
 set(ROCKSDB_PLUGIN_BUILTINS "")
 message(STATUS "ROCKSDB PLUGINS TO BUILD ${ROCKSDB_PLUGINS}")
-list(APPEND PLUGINS ${ROCKSDB_PLUGINS})
 foreach(PLUGIN IN LISTS PLUGINS)
   set(PLUGIN_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/plugin/${PLUGIN}/")
   message("including rocksb plugin ${PLUGIN_ROOT}")
@@ -1297,7 +1351,7 @@ if(WITH_TESTS)
         db/merge_test.cc
         db/options_file_test.cc
         db/perf_context_test.cc
-        db/periodic_work_scheduler_test.cc
+        db/periodic_task_scheduler_test.cc
         db/plain_table_db_test.cc
         db/seqno_time_test.cc
         db/prefix_test.cc
@@ -1424,7 +1478,7 @@ if(WITH_TESTS)
   add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
   set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
   add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-  target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB})
+  target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB} ${FOLLY_LIBS})
   if(MSVC)
     set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
   endif()
@@ -1482,32 +1536,32 @@ if(WITH_BENCHMARK_TOOLS)
     cache/cache_bench.cc
     cache/cache_bench_tool.cc)
   target_link_libraries(cache_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(memtablerep_bench${ARTIFACT_SUFFIX}
     memtable/memtablerep_bench.cc)
   target_link_libraries(memtablerep_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(range_del_aggregator_bench${ARTIFACT_SUFFIX}
     db/range_del_aggregator_bench.cc)
   target_link_libraries(range_del_aggregator_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(table_reader_bench${ARTIFACT_SUFFIX}
     table/table_reader_bench.cc)
   target_link_libraries(table_reader_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} testharness ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} testharness ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(filter_bench${ARTIFACT_SUFFIX}
     util/filter_bench.cc)
   target_link_libraries(filter_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(hash_table_bench${ARTIFACT_SUFFIX}
     utilities/persistent_cache/hash_table_bench.cc)
   target_link_libraries(hash_table_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 endif()
 
 option(WITH_TRACE_TOOLS "build with trace tools" ON)
@@ -1515,12 +1569,12 @@ if(WITH_TRACE_TOOLS)
   add_executable(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
     tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc)
   target_link_libraries(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
   add_executable(trace_analyzer${ARTIFACT_SUFFIX}
     tools/trace_analyzer.cc)
   target_link_libraries(trace_analyzer${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+    ${ROCKSDB_LIB} ${GFLAGS_LIB} ${FOLLY_LIBS})
 
 endif()
 

package/deps/rocksdb/rocksdb/Makefile

@@ -139,9 +139,14 @@ endif
 GIT_COMMAND ?= git
 ifeq ($(USE_COROUTINES), 1)
 	USE_FOLLY = 1
-	OPT += -DUSE_COROUTINES
+	# glog/logging.h requires HAVE_CXX11_ATOMIC
+	OPT += -DUSE_COROUTINES -DHAVE_CXX11_ATOMIC
 	ROCKSDB_CXX_STANDARD = c++2a
 	USE_RTTI = 1
+ifneq ($(USE_CLANG), 1)
+	ROCKSDB_CXX_STANDARD = c++20
+	PLATFORM_CXXFLAGS += -fcoroutines
+endif
 endif
 
 # if we're compiling for release, compile without debug code (-DNDEBUG)
@@ -185,6 +190,16 @@ ifeq ($(USE_LTO), 1)
 	LDFLAGS += -flto -fuse-linker-plugin
 endif
 
+# `COERCE_CONTEXT_SWITCH=1` will inject spurious wakeup and
+# random length of sleep or context switch at critical
+# points (e.g, before acquring db mutex) in RocksDB.
+# In this way, it coerces as many excution orders as possible in the hope of
+# exposing the problematic excution order
+COERCE_CONTEXT_SWITCH ?= 0
+ifeq ($(COERCE_CONTEXT_SWITCH), 1)
+OPT += -DCOERCE_CONTEXT_SWITCH
+endif
+
 #-----------------------------------------------
 include src.mk
 
@@ -235,6 +250,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
                   export USE_CLANG="$(USE_CLANG)"; \
                   export LIB_MODE="$(LIB_MODE)"; \
 		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
+		  export USE_FOLLY="$(USE_FOLLY)"; \
                   "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
@@ -415,6 +431,10 @@ ifndef DISABLE_JEMALLOC
 			PLATFORM_CXXFLAGS += -DUSE_JEMALLOC
 			PLATFORM_CCFLAGS  += -DUSE_JEMALLOC
 		endif
+		ifeq ($(USE_FOLLY_LITE),1)
+			PLATFORM_CXXFLAGS += -DUSE_JEMALLOC
+			PLATFORM_CCFLAGS  += -DUSE_JEMALLOC
+		endif
 	endif
 	ifdef WITH_JEMALLOC_FLAG
 		PLATFORM_LDFLAGS += -ljemalloc
@@ -448,10 +468,57 @@ endif
 
 # This provides a Makefile simulation of a Meta-internal folly integration.
 # It is not validated for general use.
+#
+# USE_FOLLY links the build targets with libfolly.a. The latter could be
+# built using 'make build_folly', or built externally and specified in
+# the CXXFLAGS and EXTRA_LDFLAGS env variables. The build_detect_platform
+# script tries to detect if an external folly dependency has been specified.
+# If not, it exports FOLLY_PATH to the path of the installed Folly and
+# dependency libraries.
+#
+# USE_FOLLY_LITE cherry picks source files from Folly to include in the
+# RocksDB library. Its faster and has fewer dependencies on 3rd party
+# libraries, but with limited functionality. For example, coroutine
+# functionality is not available.
 ifeq ($(USE_FOLLY),1)
-	ifeq (,$(FOLLY_DIR))
-		FOLLY_DIR = ./third-party/folly
+ifeq ($(USE_FOLLY_LITE),1)
+$(error Please specify only one of USE_FOLLY and USE_FOLLY_LITE)
+endif
+ifneq ($(strip $(FOLLY_PATH)),)
+	BOOST_PATH = $(shell (ls -d $(FOLLY_PATH)/../boost*))
+	DBL_CONV_PATH = $(shell (ls -d $(FOLLY_PATH)/../double-conversion*))
+	GFLAGS_PATH = $(shell (ls -d $(FOLLY_PATH)/../gflags*))
+	GLOG_PATH = $(shell (ls -d $(FOLLY_PATH)/../glog*))
+	LIBEVENT_PATH = $(shell (ls -d $(FOLLY_PATH)/../libevent*))
+	XZ_PATH = $(shell (ls -d $(FOLLY_PATH)/../xz*))
+	LIBSODIUM_PATH = $(shell (ls -d $(FOLLY_PATH)/../libsodium*))
+	FMT_PATH = $(shell (ls -d $(FOLLY_PATH)/../fmt*))
+
+	# For some reason, glog and fmt libraries are under either lib or lib64
+	GLOG_LIB_PATH = $(shell (ls -d $(GLOG_PATH)/lib*))
+	FMT_LIB_PATH = $(shell (ls -d $(FMT_PATH)/lib*))
+
+	# AIX: pre-defined system headers are surrounded by an extern "C" block
+	ifeq ($(PLATFORM), OS_AIX)
+		PLATFORM_CCFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include
+		PLATFORM_CXXFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include
+	else
+		PLATFORM_CCFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include
+		PLATFORM_CXXFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include
 	endif
+
+	# Add -ldl at the end as gcc resolves a symbol in a library by searching only in libraries specified later
+	# in the command line
+	PLATFORM_LDFLAGS += $(FOLLY_PATH)/lib/libfolly.a $(BOOST_PATH)/lib/libboost_context.a $(BOOST_PATH)/lib/libboost_filesystem.a $(BOOST_PATH)/lib/libboost_atomic.a $(BOOST_PATH)/lib/libboost_program_options.a $(BOOST_PATH)/lib/libboost_regex.a $(BOOST_PATH)/lib/libboost_system.a $(BOOST_PATH)/lib/libboost_thread.a $(DBL_CONV_PATH)/lib/libdouble-conversion.a $(FMT_LIB_PATH)/libfmt.a $(GLOG_LIB_PATH)/libglog.so $(GFLAGS_PATH)/lib/libgflags.so.2.2 $(LIBEVENT_PATH)/lib/libevent-2.1.so -ldl
+	PLATFORM_LDFLAGS += -Wl,-rpath=$(GFLAGS_PATH)/lib -Wl,-rpath=$(GLOG_LIB_PATH) -Wl,-rpath=$(LIBEVENT_PATH)/lib -Wl,-rpath=$(LIBSODIUM_PATH)/lib -Wl,-rpath=$(LIBEVENT_PATH)/lib
+endif
+	PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
+	PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
+endif
+
+ifeq ($(USE_FOLLY_LITE),1)
+	# Path to the Folly source code and include files
+	FOLLY_DIR = ./third-party/folly
 	# AIX: pre-defined system headers are surrounded by an extern "C" block
 	ifeq ($(PLATFORM), OS_AIX)
 		PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
@@ -550,7 +617,7 @@ LIB_OBJECTS += $(patsubst %.c, $(OBJ_DIR)/%.o, $(LIB_SOURCES_C))
 LIB_OBJECTS += $(patsubst %.S, $(OBJ_DIR)/%.o, $(LIB_SOURCES_ASM))
 endif
 
-ifeq ($(USE_FOLLY),1)
+ifeq ($(USE_FOLLY_LITE),1)
   LIB_OBJECTS += $(patsubst %.cpp, $(OBJ_DIR)/%.o, $(FOLLY_SOURCES))
 endif
 
@@ -591,7 +658,7 @@ ifneq ($(filter check-headers, $(MAKECMDGOALS)),)
 # TODO: add/support JNI headers
 	DEV_HEADER_DIRS := $(sort include/ $(dir $(ALL_SOURCES)))
 # Some headers like in port/ are platform-specific
-	DEV_HEADERS := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | egrep -v 'port/|plugin/|lua/|range_tree/')
+	DEV_HEADERS := $(shell $(FIND) $(DEV_HEADER_DIRS) -type f -name '*.h' | grep -E -v 'port/|plugin/|lua/|range_tree/')
 else
 	DEV_HEADERS :=
 endif
@@ -728,9 +795,9 @@ TOOLS_LIBRARY=$(STATIC_TOOLS_LIBRARY)
 endif
 STRESS_LIBRARY=$(STATIC_STRESS_LIBRARY)
 
-ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
-ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
-ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_MAJOR = $(shell grep -E "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_MINOR = $(shell grep -E "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
+ROCKSDB_PATCH = $(shell grep -E "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
 
 # If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
 # the file needs to already exist or else the build will fail
@@ -1882,7 +1949,7 @@ blob_garbage_meter_test: $(OBJ_DIR)/db/blob/blob_garbage_meter_test.o $(TEST_LIB
 timer_test: $(OBJ_DIR)/util/timer_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-periodic_work_scheduler_test: $(OBJ_DIR)/db/periodic_work_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY)
+periodic_task_scheduler_test: $(OBJ_DIR)/db/periodic_task_scheduler_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
 testutil_test: $(OBJ_DIR)/test_util/testutil_test.o $(TEST_LIBRARY) $(LIBRARY)
@@ -2371,13 +2438,26 @@ checkout_folly:
 	@# need to worry about folly breaking our integration. Update periodically
 	cd third-party/folly && git reset --hard beacd86d63cd71c904632262e6c36f60874d78ba
 	@# A hack to remove boost dependency.
-	@# NOTE: this hack is not needed if using FBCODE compiler config
+	@# NOTE: this hack is only needed if building using USE_FOLLY_LITE
 	perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
 	perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
 
+build_folly:
+	FOLLY_INST_PATH=`cd third-party/folly; $(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir`; \
+	if [ "$$FOLLY_INST_PATH" ]; then \
+		rm -rf $${FOLLY_INST_PATH}/../../*; \
+	else \
+		echo "Please run checkout_folly first"; \
+		false; \
+	fi
+	# Restore the original version of Invoke.h with boost dependency
+	cd third-party/folly && ${GIT_COMMAND} checkout folly/functional/Invoke.h
+	cd third-party/folly && MAYBE_AVX2=`echo $(CXXFLAGS) | grep -o -- -DHAVE_AVX2 | sed 's/-DHAVE_AVX2/-mavx2/g' || true` && \
+		CXXFLAGS=" $$MAYBE_AVX2 -DHAVE_CXX11_ATOMIC " $(PYTHON) build/fbcode_builder/getdeps.py build --no-tests
+
 # ---------------------------------------------------------------------------
 #   Build size testing
 # ---------------------------------------------------------------------------
@@ -2462,7 +2542,7 @@ endif
 ifneq ($(SKIP_DEPENDS), 1)
 DEPFILES = $(patsubst %.cc, $(OBJ_DIR)/%.cc.d, $(ALL_SOURCES))
 DEPFILES+ = $(patsubst %.c, $(OBJ_DIR)/%.c.d, $(LIB_SOURCES_C) $(TEST_MAIN_SOURCES_C))
-ifeq ($(USE_FOLLY),1)
+ifeq ($(USE_FOLLY_LITE),1)
   DEPFILES +=$(patsubst %.cpp, $(OBJ_DIR)/%.cpp.d, $(FOLLY_SOURCES))
 endif
 endif

package/deps/rocksdb/rocksdb/TARGETS

@@ -20,6 +20,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "cache/lru_cache.cc",
         "cache/sharded_cache.cc",
         "db/arena_wrapped_db_iter.cc",
+        "db/blob/blob_contents.cc",
         "db/blob/blob_fetcher.cc",
         "db/blob/blob_file_addition.cc",
         "db/blob/blob_file_builder.cc",
@@ -82,7 +83,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/merge_helper.cc",
         "db/merge_operator.cc",
         "db/output_validator.cc",
-        "db/periodic_work_scheduler.cc",
+        "db/periodic_task_scheduler.cc",
         "db/range_del_aggregator.cc",
         "db/range_tombstone_fragmenter.cc",
         "db/repair.cc",
@@ -252,6 +253,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "util/ribbon_config.cc",
         "util/slice.cc",
         "util/status.cc",
+        "util/stderr_logger.cc",
         "util/string_util.cc",
         "util/thread_local.cc",
         "util/threadpool_imp.cc",
@@ -357,6 +359,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "cache/lru_cache.cc",
         "cache/sharded_cache.cc",
         "db/arena_wrapped_db_iter.cc",
+        "db/blob/blob_contents.cc",
         "db/blob/blob_fetcher.cc",
         "db/blob/blob_file_addition.cc",
         "db/blob/blob_file_builder.cc",
@@ -419,7 +422,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/merge_helper.cc",
         "db/merge_operator.cc",
         "db/output_validator.cc",
-        "db/periodic_work_scheduler.cc",
+        "db/periodic_task_scheduler.cc",
         "db/range_del_aggregator.cc",
         "db/range_tombstone_fragmenter.cc",
         "db/repair.cc",
@@ -589,6 +592,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "util/ribbon_config.cc",
         "util/slice.cc",
         "util/status.cc",
+        "util/stderr_logger.cc",
         "util/string_util.cc",
         "util/thread_local.cc",
         "util/threadpool_imp.cc",
@@ -5598,8 +5602,8 @@ cpp_unittest_wrapper(name="perf_context_test",
             extra_compiler_flags=[])
 
 
-cpp_unittest_wrapper(name="periodic_work_scheduler_test",
-            srcs=["db/periodic_work_scheduler_test.cc"],
+cpp_unittest_wrapper(name="periodic_task_scheduler_test",
+            srcs=["db/periodic_task_scheduler_test.cc"],
             deps=[":rocksdb_test_lib"],
             extra_compiler_flags=[])
 

package/deps/rocksdb/rocksdb/cache/cache.cc

@@ -58,6 +58,11 @@ static std::unordered_map<std::string, OptionTypeInfo>
                    compress_format_version),
           OptionType::kUInt32T, OptionVerificationType::kNormal,
           OptionTypeFlags::kMutable}},
+        {"enable_custom_split_merge",
+         {offsetof(struct CompressedSecondaryCacheOptions,
+                   enable_custom_split_merge),
+          OptionType::kBoolean, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
 };
 #endif  // ROCKSDB_LITE
 

package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc

@@ -13,7 +13,6 @@
 #include <set>
 #include <sstream>
 
-#include "cache/clock_cache.h"
 #include "cache/fast_lru_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "monitoring/histogram.h"
@@ -292,13 +291,12 @@ class CacheBench {
     }
 
     if (FLAGS_cache_type == "clock_cache") {
-      cache_ = ExperimentalNewClockCache(
-          FLAGS_cache_size, FLAGS_value_bytes, FLAGS_num_shard_bits,
-          false /*strict_capacity_limit*/, kDefaultCacheMetadataChargePolicy);
-      if (!cache_) {
-        fprintf(stderr, "Clock cache not supported.\n");
-        exit(1);
-      }
+      fprintf(stderr, "Old clock cache implementation has been removed.\n");
+      exit(1);
+    } else if (FLAGS_cache_type == "hyper_clock_cache") {
+      cache_ = HyperClockCacheOptions(FLAGS_cache_size, FLAGS_value_bytes,
+                                      FLAGS_num_shard_bits)
+                   .MakeSharedCache();
     } else if (FLAGS_cache_type == "fast_lru_cache") {
       cache_ = NewFastLRUCache(
           FLAGS_cache_size, FLAGS_value_bytes, FLAGS_num_shard_bits,
@@ -441,6 +439,8 @@ class CacheBench {
     uint64_t total_key_size = 0;
     uint64_t total_charge = 0;
     uint64_t total_entry_count = 0;
+    uint64_t table_occupancy = 0;
+    uint64_t table_size = 0;
     std::set<Cache::DeleterFn> deleters;
     StopWatchNano timer(clock);
 
@@ -456,6 +456,9 @@ class CacheBench {
             std::ostringstream ostr;
             ostr << "Most recent cache entry stats:\n"
                  << "Number of entries: " << total_entry_count << "\n"
+                 << "Table occupancy: " << table_occupancy << " / "
+                 << table_size << " = "
+                 << (100.0 * table_occupancy / table_size) << "%\n"
                  << "Total charge: " << BytesToHumanString(total_charge) << "\n"
                  << "Average key size: "
                  << (1.0 * total_key_size / total_entry_count) << "\n"
@@ -492,6 +495,8 @@ class CacheBench {
       Cache::ApplyToAllEntriesOptions opts;
       opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock;
       shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts);
+      table_occupancy = shared->GetCacheBench()->cache_->GetOccupancyCount();
+      table_size = shared->GetCacheBench()->cache_->GetTableAddressCount();
       stats_hist->Add(timer.ElapsedNanos() / 1000);
     }
   }

package/deps/rocksdb/rocksdb/cache/cache_entry_roles.cc

@@ -23,6 +23,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
     "FilterConstruction",
     "BlockBasedTableReader",
     "FileMetadata",
+    "BlobValue",
     "BlobCache",
     "Misc",
 }};
@@ -39,6 +40,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
     "filter-construction",
     "block-based-table-reader",
     "file-metadata",
+    "blob-value",
     "blob-cache",
     "misc",
 }};