@nxtedition/rocksdb 6.0.2 → 6.0.3

package/BUILDING.md

@@ -1,10 +1,18 @@
 # LINUX
 
 - Run `./configure` in `deps/liburing`.
-- Clone and build zstd with `CFLAGS="-O3 -fPIC" make -C lib libzstd.a` and copy `libzstd.a` to `/usr/lib/x86_64-linux-gnu/libzstd_pic.a`.
-- Put zstd headers at `/usr/lib/x86_64-linux-gnu/include`.
+- Update package lib, `apt-get update`.
+- Update gcc, `apt-get upgrade gcc`.
+- Clone and build zstd with `CFLAGS="-O3 -fPIC" make -C lib libzstd.a`
+  - Copy `libzstd.a` to `/usr/lib/x86_64-linux-gnu`.
+  - Copy headers to `/usr/lib/x86_64-linux-gnu/include`.
+- Clone and build folly with `python3 ./build/fbcode_builder/getdeps.py build --no-tests --extra-cmake-defines='{"CMAKE_CXX_FLAGS": "-fPIC"}'`
+  - Copy `libfolly.a` to `/usr/lib/x86_64-linux-gnu`.
+  - Copy headers to `/usr/lib/x86_64-linux-gnu/include`.
+  - Copy boost headers from folly scratchpad to `/usr/lib/x86_64-linux-gnu/include`.
+- `npx prebuildify -t -t 17.8.0 --napi --strip --arch x64`
 
 # OSX
 
-- Run `./configure` in `deps/liburing`.
-- `brew install zstd@1.5.2`
+- `brew install zstd`
+- `npx prebuildify -t -t 17.8.0 --napi --strip --arch x64`

package/binding.cc

@@ -446,6 +446,7 @@ struct BaseIterator {
     }
     options.fill_cache = fillCache_;
     options.snapshot = snapshot_.get();
+    options.async_io = true;
 
     iterator_.reset(database_->db_->NewIterator(options));
   }
@@ -790,6 +791,7 @@ struct GetManyWorker final : public Worker {
     rocksdb::ReadOptions options;
     options.fill_cache = fillCache_;
     options.snapshot = snapshot_.get();
+    options.async_io = true;
 
     const auto numKeys = keys_.size();
 

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -796,6 +796,7 @@ set(SOURCES
         trace_replay/trace_record_result.cc
         trace_replay/trace_record.cc
         trace_replay/trace_replay.cc
+        util/async_file_reader.cc
         util/cleanable.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc
@@ -1128,6 +1129,12 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
 
   install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
+  foreach (plugin ${PLUGINS})
+    foreach (header ${${plugin}_HEADERS})
+      install(FILES plugin/${plugin}/${header} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rocksdb/plugin/${plugin})
+    endforeach()
+  endforeach()
+
   install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination})
 
   install(
@@ -1216,6 +1223,7 @@ if(WITH_TESTS)
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
+        db/db_readonly_with_timestamp_test.cc
         db/db_with_timestamp_basic_test.cc
         db/db_block_cache_test.cc
         db/db_bloom_filter_test.cc
@@ -1384,6 +1392,7 @@ if(WITH_TESTS)
 
   set(TESTUTIL_SOURCE
       db/db_test_util.cc
+      db/db_with_timestamp_test_util.cc
       monitoring/thread_status_updater_debug.cc
       table/mock_table.cc
       utilities/agg_merge/test_agg_merge.cc

package/deps/rocksdb/rocksdb/Makefile

@@ -136,6 +136,13 @@ CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
 CFLAGS +=  $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
 endif
 
+ifeq ($(USE_COROUTINES), 1)
+	USE_FOLLY = 1
+	OPT += -DUSE_COROUTINES
+	ROCKSDB_CXX_STANDARD = c++2a
+	USE_RTTI = 1
+endif
+
 # if we're compiling for release, compile without debug code (-DNDEBUG)
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
@@ -226,6 +233,7 @@ dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \
                   export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \
                   export USE_CLANG="$(USE_CLANG)"; \
                   export LIB_MODE="$(LIB_MODE)"; \
+		  export ROCKSDB_CXX_STANDARD="$(ROCKSDB_CXX_STANDARD)"; \
                   "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
@@ -1370,6 +1378,9 @@ db_blob_basic_test: $(OBJ_DIR)/db/blob/db_blob_basic_test.o $(TEST_LIBRARY) $(LI
 db_blob_compaction_test: $(OBJ_DIR)/db/blob/db_blob_compaction_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+db_readonly_with_timestamp_test: $(OBJ_DIR)/db/db_readonly_with_timestamp_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 db_with_timestamp_basic_test: $(OBJ_DIR)/db/db_with_timestamp_basic_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2053,6 +2064,7 @@ ifeq ($(PLATFORM), OS_OPENBSD)
 	ROCKSDBJNILIB = librocksdbjni-openbsd$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-openbsd$(ARCH).jar
 endif
+export SHA256_CMD
 
 zlib-$(ZLIB_VER).tar.gz:
 	curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
@@ -2185,7 +2197,7 @@ JAR_CMD := jar
 endif
 endif
 rocksdbjavastatic_javalib:
-	cd java; SHA256_CMD='$(SHA256_CMD)' $(MAKE) javalib
+	cd java; $(MAKE) javalib
 	rm -f java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) \
@@ -2299,7 +2311,7 @@ rocksdbjava: $(LIB_OBJECTS)
 ifeq ($(JAVA_HOME),)
 	$(error JAVA_HOME is not set)
 endif
-	$(AM_V_GEN)cd java; SHA256_CMD='$(SHA256_CMD)' $(MAKE) javalib;
+	$(AM_V_GEN)cd java; $(MAKE) javalib;
 	$(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. -I./java/rocksjni $(JAVA_INCLUDE) $(ROCKSDB_PLUGIN_JNI_CXX_INCLUDEFLAGS) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(ALL_JNI_NATIVE_SOURCES) $(LIB_OBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
 	$(AM_V_at)cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
@@ -2311,14 +2323,13 @@ jclean:
 	cd java;$(MAKE) clean;
 
 jtest_compile: rocksdbjava
-	cd java; SHA256_CMD='$(SHA256_CMD)' $(MAKE) java_test
+	cd java;$(MAKE) java_test
 
 jtest_run:
 	cd java;$(MAKE) run_test
 
 jtest: rocksdbjava
-	cd java;$(MAKE) sample; SHA256_CMD='$(SHA256_CMD)' $(MAKE) test;
-	$(PYTHON) tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets
+	cd java;$(MAKE) sample test
 
 jdb_bench:
 	cd java;$(MAKE) db_bench;

package/deps/rocksdb/rocksdb/TARGETS

@@ -224,6 +224,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "trace_replay/trace_record_handler.cc",
         "trace_replay/trace_record_result.cc",
         "trace_replay/trace_replay.cc",
+        "util/async_file_reader.cc",
         "util/build_version.cc",
         "util/cleanable.cc",
         "util/coding.cc",
@@ -327,7 +328,13 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "utilities/wal_filter.cc",
         "utilities/write_batch_with_index/write_batch_with_index.cc",
         "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
-    ], deps=["//folly/container:f14_hash"], headers=None, link_whole=False, extra_test_libs=False)
+    ], deps=[
+        "//folly/container:f14_hash",
+        "//folly/experimental/coro:blocking_wait",
+        "//folly/experimental/coro:collect",
+        "//folly/experimental/coro:coroutine",
+        "//folly/experimental/coro:task",
+    ], headers=None, link_whole=False, extra_test_libs=False)
 
 cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "cache/cache.cc",
@@ -545,6 +552,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "trace_replay/trace_record_handler.cc",
         "trace_replay/trace_record_result.cc",
         "trace_replay/trace_replay.cc",
+        "util/async_file_reader.cc",
         "util/build_version.cc",
         "util/cleanable.cc",
         "util/coding.cc",
@@ -648,10 +656,17 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "utilities/wal_filter.cc",
         "utilities/write_batch_with_index/write_batch_with_index.cc",
         "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
-    ], deps=["//folly/container:f14_hash"], headers=None, link_whole=True, extra_test_libs=False)
+    ], deps=[
+        "//folly/container:f14_hash",
+        "//folly/experimental/coro:blocking_wait",
+        "//folly/experimental/coro:collect",
+        "//folly/experimental/coro:coroutine",
+        "//folly/experimental/coro:task",
+    ], headers=None, link_whole=True, extra_test_libs=False)
 
 cpp_library_wrapper(name="rocksdb_test_lib", srcs=[
         "db/db_test_util.cc",
+        "db/db_with_timestamp_test_util.cc",
         "table/mock_table.cc",
         "test_util/mock_time_env.cc",
         "test_util/testharness.cc",
@@ -5161,6 +5176,12 @@ cpp_unittest_wrapper(name="db_rate_limiter_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="db_readonly_with_timestamp_test",
+            srcs=["db/db_readonly_with_timestamp_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="db_secondary_test",
             srcs=["db/db_secondary_test.cc"],
             deps=[":rocksdb_test_lib"],

package/deps/rocksdb/rocksdb/cmake/modules/CxxFlags.cmake

@@ -0,0 +1,7 @@
+macro(get_cxx_std_flags FLAGS_VARIABLE)
+  if( CMAKE_CXX_STANDARD_REQUIRED )
+    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION})
+  else()
+    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION})
+  endif()
+endmacro()

package/deps/rocksdb/rocksdb/cmake/modules/FindJeMalloc.cmake

@@ -0,0 +1,29 @@
+# - Find JeMalloc library
+# Find the native JeMalloc includes and library
+#
+# JeMalloc_INCLUDE_DIRS - where to find jemalloc.h, etc.
+# JeMalloc_LIBRARIES - List of libraries when using jemalloc.
+# JeMalloc_FOUND - True if jemalloc found.
+
+find_path(JeMalloc_INCLUDE_DIRS
+  NAMES jemalloc/jemalloc.h
+  HINTS ${JEMALLOC_ROOT_DIR}/include)
+
+find_library(JeMalloc_LIBRARIES
+  NAMES jemalloc
+  HINTS ${JEMALLOC_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(JeMalloc DEFAULT_MSG JeMalloc_LIBRARIES JeMalloc_INCLUDE_DIRS)
+
+mark_as_advanced(
+  JeMalloc_LIBRARIES
+  JeMalloc_INCLUDE_DIRS)
+
+if(JeMalloc_FOUND AND NOT (TARGET JeMalloc::JeMalloc))
+  add_library (JeMalloc::JeMalloc UNKNOWN IMPORTED)
+  set_target_properties(JeMalloc::JeMalloc
+    PROPERTIES
+      IMPORTED_LOCATION ${JeMalloc_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${JeMalloc_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/FindNUMA.cmake

@@ -0,0 +1,29 @@
+# - Find NUMA
+# Find the NUMA library and includes
+#
+# NUMA_INCLUDE_DIRS - where to find numa.h, etc.
+# NUMA_LIBRARIES - List of libraries when using NUMA.
+# NUMA_FOUND - True if NUMA found.
+
+find_path(NUMA_INCLUDE_DIRS
+  NAMES numa.h numaif.h
+  HINTS ${NUMA_ROOT_DIR}/include)
+
+find_library(NUMA_LIBRARIES
+  NAMES numa
+  HINTS ${NUMA_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NUMA DEFAULT_MSG NUMA_LIBRARIES NUMA_INCLUDE_DIRS)
+
+mark_as_advanced(
+  NUMA_LIBRARIES
+  NUMA_INCLUDE_DIRS)
+
+if(NUMA_FOUND AND NOT (TARGET NUMA::NUMA))
+  add_library (NUMA::NUMA UNKNOWN IMPORTED)
+  set_target_properties(NUMA::NUMA
+    PROPERTIES
+      IMPORTED_LOCATION ${NUMA_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${NUMA_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/FindSnappy.cmake

@@ -0,0 +1,29 @@
+# - Find Snappy
+# Find the snappy compression library and includes
+#
+# Snappy_INCLUDE_DIRS - where to find snappy.h, etc.
+# Snappy_LIBRARIES - List of libraries when using snappy.
+# Snappy_FOUND - True if snappy found.
+
+find_path(Snappy_INCLUDE_DIRS
+  NAMES snappy.h
+  HINTS ${snappy_ROOT_DIR}/include)
+
+find_library(Snappy_LIBRARIES
+  NAMES snappy
+  HINTS ${snappy_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS)
+
+mark_as_advanced(
+  Snappy_LIBRARIES
+  Snappy_INCLUDE_DIRS)
+
+if(Snappy_FOUND AND NOT (TARGET Snappy::snappy))
+  add_library (Snappy::snappy UNKNOWN IMPORTED)
+  set_target_properties(Snappy::snappy
+    PROPERTIES
+      IMPORTED_LOCATION ${Snappy_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/FindTBB.cmake

@@ -0,0 +1,33 @@
+# - Find TBB
+# Find the Thread Building Blocks library and includes
+#
+# TBB_INCLUDE_DIRS - where to find tbb.h, etc.
+# TBB_LIBRARIES - List of libraries when using TBB.
+# TBB_FOUND - True if TBB found.
+
+if(NOT DEFINED TBB_ROOT_DIR)
+  set(TBB_ROOT_DIR "$ENV{TBBROOT}")
+endif()
+
+find_path(TBB_INCLUDE_DIRS
+  NAMES tbb/tbb.h
+  HINTS ${TBB_ROOT_DIR}/include)
+
+find_library(TBB_LIBRARIES
+  NAMES tbb
+  HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIRS)
+
+mark_as_advanced(
+  TBB_LIBRARIES
+  TBB_INCLUDE_DIRS)
+
+if(TBB_FOUND AND NOT (TARGET TBB::TBB))
+  add_library (TBB::TBB UNKNOWN IMPORTED)
+  set_target_properties(TBB::TBB
+    PROPERTIES
+      IMPORTED_LOCATION ${TBB_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/Findgflags.cmake

@@ -0,0 +1,29 @@
+# - Find gflags library
+# Find the gflags includes and library
+#
+# GFLAGS_INCLUDE_DIR - where to find gflags.h.
+# GFLAGS_LIBRARIES - List of libraries when using gflags.
+# gflags_FOUND - True if gflags found.
+
+find_path(GFLAGS_INCLUDE_DIR
+  NAMES gflags/gflags.h)
+
+find_library(GFLAGS_LIBRARIES
+  NAMES gflags)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(gflags
+  DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR)
+
+mark_as_advanced(
+  GFLAGS_LIBRARIES
+  GFLAGS_INCLUDE_DIR)
+
+if(gflags_FOUND AND NOT (TARGET gflags::gflags))
+  add_library(gflags::gflags UNKNOWN IMPORTED)
+  set_target_properties(gflags::gflags
+    PROPERTIES
+      IMPORTED_LOCATION ${GFLAGS_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}
+      IMPORTED_LINK_INTERFACE_LANGUAGES "CXX")
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/Findlz4.cmake

@@ -0,0 +1,29 @@
+# - Find Lz4
+# Find the lz4 compression library and includes
+#
+# lz4_INCLUDE_DIRS - where to find lz4.h, etc.
+# lz4_LIBRARIES - List of libraries when using lz4.
+# lz4_FOUND - True if lz4 found.
+
+find_path(lz4_INCLUDE_DIRS
+  NAMES lz4.h
+  HINTS ${lz4_ROOT_DIR}/include)
+
+find_library(lz4_LIBRARIES
+  NAMES lz4
+  HINTS ${lz4_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(lz4 DEFAULT_MSG lz4_LIBRARIES lz4_INCLUDE_DIRS)
+
+mark_as_advanced(
+  lz4_LIBRARIES
+  lz4_INCLUDE_DIRS)
+
+if(lz4_FOUND AND NOT (TARGET lz4::lz4))
+  add_library(lz4::lz4 UNKNOWN IMPORTED)
+  set_target_properties(lz4::lz4
+    PROPERTIES
+      IMPORTED_LOCATION ${lz4_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/Finduring.cmake

@@ -0,0 +1,26 @@
+# - Find liburing
+#
+# uring_INCLUDE_DIR - Where to find liburing.h
+# uring_LIBRARIES - List of libraries when using uring.
+# uring_FOUND - True if uring found.
+
+find_path(uring_INCLUDE_DIR
+  NAMES liburing.h)
+find_library(uring_LIBRARIES
+  NAMES liburing.a liburing)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(uring
+  DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
+
+mark_as_advanced(
+  uring_INCLUDE_DIR
+  uring_LIBRARIES)
+
+if(uring_FOUND AND NOT TARGET uring::uring)
+  add_library(uring::uring UNKNOWN IMPORTED)
+  set_target_properties(uring::uring PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${uring_LIBRARIES}")
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/Findzstd.cmake

@@ -0,0 +1,29 @@
+# - Find zstd
+# Find the zstd compression library and includes
+#
+# zstd_INCLUDE_DIRS - where to find zstd.h, etc.
+# zstd_LIBRARIES - List of libraries when using zstd.
+# zstd_FOUND - True if zstd found.
+
+find_path(zstd_INCLUDE_DIRS
+  NAMES zstd.h
+  HINTS ${zstd_ROOT_DIR}/include)
+
+find_library(zstd_LIBRARIES
+  NAMES zstd
+  HINTS ${zstd_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARIES zstd_INCLUDE_DIRS)
+
+mark_as_advanced(
+  zstd_LIBRARIES
+  zstd_INCLUDE_DIRS)
+
+if(zstd_FOUND AND NOT (TARGET zstd::zstd))
+  add_library (zstd::zstd UNKNOWN IMPORTED)
+  set_target_properties(zstd::zstd
+    PROPERTIES
+      IMPORTED_LOCATION ${zstd_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${zstd_INCLUDE_DIRS})
+endif()

package/deps/rocksdb/rocksdb/cmake/modules/ReadVersion.cmake

@@ -0,0 +1,10 @@
+# Read rocksdb version from version.h header file.
+
+function(get_rocksdb_version version_var)
+  file(READ "${CMAKE_CURRENT_SOURCE_DIR}/include/rocksdb/version.h" version_header_file)
+  foreach(component MAJOR MINOR PATCH)
+    string(REGEX MATCH "#define ROCKSDB_${component} ([0-9]+)" _ ${version_header_file})
+    set(ROCKSDB_VERSION_${component} ${CMAKE_MATCH_1})
+  endforeach()
+  set(${version_var} "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}" PARENT_SCOPE)
+endfunction()

package/deps/rocksdb/rocksdb/db/builder.cc

@@ -37,6 +37,7 @@
 #include "table/block_based/block_based_table_builder.h"
 #include "table/format.h"
 #include "table/internal_iterator.h"
+#include "table/unique_id_impl.h"
 #include "test_util/sync_point.h"
 #include "util/stop_watch.h"
 
@@ -310,6 +311,15 @@ Status BuildTable(
       meta->file_checksum_func_name = file_writer->GetFileChecksumFuncName();
       file_checksum = meta->file_checksum;
       file_checksum_func_name = meta->file_checksum_func_name;
+      // Set unique_id only if db_id and db_session_id exist
+      if (!tboptions.db_id.empty() && !tboptions.db_session_id.empty()) {
+        if (!GetSstInternalUniqueId(tboptions.db_id, tboptions.db_session_id,
+                                    meta->fd.GetNumber(), &(meta->unique_id))
+                 .ok()) {
+          // if failed to get unique id, just set it Null
+          meta->unique_id = kNullUniqueId64x2;
+        }
+      }
     }
 
     if (s.ok()) {
@@ -331,12 +341,10 @@ Status BuildTable(
     if (s.ok() && !empty) {
       // Verify that the table is usable
       // We set for_compaction to false and don't OptimizeForCompactionTableRead
-      // here because this is a special case after we finish the table building
+      // here because this is a special case after we finish the table building.
       // No matter whether use_direct_io_for_flush_and_compaction is true,
-      // we will regrad this verification as user reads since the goal is
-      // to cache it here for further user reads
+      // the goal is to cache it here for further user reads.
       ReadOptions read_options;
-      read_options.rate_limiter_priority = Env::IO_USER;
       std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
           read_options, file_options, tboptions.internal_comparator, *meta,
           nullptr /* range_del_agg */, mutable_cf_options.prefix_extractor,

package/deps/rocksdb/rocksdb/db/c.cc

@@ -2855,6 +2855,20 @@ void rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes(
   opt->rep.bottommost_compression_opts.enabled = enabled;
 }
 
+void rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt, unsigned char use_zstd_dict_trainer,
+    unsigned char enabled) {
+  opt->rep.bottommost_compression_opts.use_zstd_dict_trainer =
+      use_zstd_dict_trainer;
+  opt->rep.bottommost_compression_opts.enabled = enabled;
+}
+
+unsigned char
+rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt) {
+  return opt->rep.bottommost_compression_opts.use_zstd_dict_trainer;
+}
+
 void rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes(
     rocksdb_options_t* opt, uint64_t max_dict_buffer_bytes,
     unsigned char enabled) {
@@ -2882,6 +2896,16 @@ int rocksdb_options_get_compression_options_zstd_max_train_bytes(
   return opt->rep.compression_opts.zstd_max_train_bytes;
 }
 
+void rocksdb_options_set_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt, unsigned char use_zstd_dict_trainer) {
+  opt->rep.compression_opts.use_zstd_dict_trainer = use_zstd_dict_trainer;
+}
+
+unsigned char rocksdb_options_get_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt) {
+  return opt->rep.compression_opts.use_zstd_dict_trainer;
+}
+
 void rocksdb_options_set_compression_options_parallel_threads(
     rocksdb_options_t* opt, int value) {
   opt->rep.compression_opts.parallel_threads = value;
@@ -3673,6 +3697,8 @@ uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
       return rep->env_unlock_file_nanos;
     case rocksdb_env_new_logger_nanos:
       return rep->env_new_logger_nanos;
+    case rocksdb_number_async_seek:
+      return rep->number_async_seek;
     default:
       break;
   }

package/deps/rocksdb/rocksdb/db/c_test.c

@@ -2539,6 +2539,9 @@ int main(int argc, char** argv) {
         200 ==
         rocksdb_options_get_compression_options_max_dict_buffer_bytes(co));
 
+    rocksdb_options_set_compression_options_use_zstd_dict_trainer(co, 0);
+    CheckCondition(
+        0 == rocksdb_options_get_compression_options_use_zstd_dict_trainer(co));
     rocksdb_options_destroy(co);
   }
 

package/deps/rocksdb/rocksdb/db/column_family.cc

@@ -136,9 +136,15 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
     }
   }
   if (cf_options.compression_opts.zstd_max_train_bytes > 0) {
-    if (!ZSTD_TrainDictionarySupported()) {
+    if (cf_options.compression_opts.use_zstd_dict_trainer) {
+      if (!ZSTD_TrainDictionarySupported()) {
+        return Status::InvalidArgument(
+            "zstd dictionary trainer cannot be used because ZSTD 1.1.3+ "
+            "is not linked with the binary.");
+      }
+    } else if (!ZSTD_FinalizeDictionarySupported()) {
       return Status::InvalidArgument(
-          "zstd dictionary trainer cannot be used because ZSTD 1.1.3+ "
+          "zstd finalizeDictionary cannot be used because ZSTD 1.4.5+ "
           "is not linked with the binary.");
     }
     if (cf_options.compression_opts.max_dict_bytes == 0) {