@nxtedition/rocksdb 7.0.0-alpha.6 → 7.0.0

package/deps/rocksdb/rocksdb/options/options.cc

@@ -100,7 +100,8 @@ AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options)
           options.blob_garbage_collection_age_cutoff),
       blob_garbage_collection_force_threshold(
           options.blob_garbage_collection_force_threshold),
-      blob_compaction_readahead_size(options.blob_compaction_readahead_size) {
+      blob_compaction_readahead_size(options.blob_compaction_readahead_size),
+      blob_file_starting_level(options.blob_file_starting_level) {
   assert(memtable_factory.get() != nullptr);
   if (max_bytes_for_level_multiplier_additional.size() <
       static_cast<unsigned int>(num_levels)) {
@@ -414,6 +415,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
     ROCKS_LOG_HEADER(
         log, "         Options.blob_compaction_readahead_size: %" PRIu64,
         blob_compaction_readahead_size);
+    ROCKS_LOG_HEADER(log, "               Options.blob_file_starting_level: %d",
+                     blob_file_starting_level);
 }  // ColumnFamilyOptions::Dump
 
 void Options::Dump(Logger* log) const {

package/deps/rocksdb/rocksdb/options/options_helper.cc

@@ -254,6 +254,7 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
       moptions.blob_garbage_collection_force_threshold;
   cf_opts->blob_compaction_readahead_size =
       moptions.blob_compaction_readahead_size;
+  cf_opts->blob_file_starting_level = moptions.blob_file_starting_level;
 
   // Misc options
   cf_opts->max_sequential_skip_in_iterations =

package/deps/rocksdb/rocksdb/options/options_parser.cc

@@ -271,7 +271,7 @@ Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in,
   std::unordered_map<std::string, std::string> opt_map;
   std::string line;
   // we only support single-lined statement.
-  while (lf_reader.ReadLine(&line)) {
+  while (lf_reader.ReadLine(&line, Env::IO_TOTAL /* rate_limiter_priority */)) {
     int line_num = static_cast<int>(lf_reader.GetLineNumber());
     line = TrimAndRemoveComment(line);
     if (line.empty()) {

package/deps/rocksdb/rocksdb/options/options_settable_test.cc

@@ -520,6 +520,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       "blob_garbage_collection_age_cutoff=0.5;"
       "blob_garbage_collection_force_threshold=0.75;"
       "blob_compaction_readahead_size=262144;"
+      "blob_file_starting_level=1;"
       "bottommost_temperature=kWarm;"
       "compaction_options_fifo={max_table_files_size=3;allow_"
       "compaction=false;age_for_warm=1;};",

package/deps/rocksdb/rocksdb/options/options_test.cc

@@ -109,6 +109,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"blob_garbage_collection_age_cutoff", "0.5"},
       {"blob_garbage_collection_force_threshold", "0.75"},
       {"blob_compaction_readahead_size", "256K"},
+      {"blob_file_starting_level", "1"},
       {"bottommost_temperature", "kWarm"},
   };
 
@@ -247,6 +248,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
   ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
   ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
+  ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
 
   cf_options_map["write_buffer_size"] = "hello";
@@ -2330,6 +2332,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
       {"blob_garbage_collection_age_cutoff", "0.5"},
       {"blob_garbage_collection_force_threshold", "0.75"},
       {"blob_compaction_readahead_size", "256K"},
+      {"blob_file_starting_level", "1"},
       {"bottommost_temperature", "kWarm"},
   };
 
@@ -2462,6 +2465,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
   ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
   ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
+  ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
 
   cf_options_map["write_buffer_size"] = "hello";

package/deps/rocksdb/rocksdb/port/port_posix.h

@@ -23,8 +23,6 @@
 
 #define __declspec(S)
 
-#define ROCKSDB_NOEXCEPT noexcept
-
 #undef PLATFORM_IS_LITTLE_ENDIAN
 #if defined(OS_MACOSX)
   #include <machine/endian.h>

package/deps/rocksdb/rocksdb/port/sys_time.h

@@ -12,36 +12,52 @@
 
 #pragma once
 
-#if defined(OS_WIN) && defined(_MSC_VER)
+#include "rocksdb/rocksdb_namespace.h"
 
-#include <time.h>
+#if defined(OS_WIN) && (defined(_MSC_VER) || defined(__MINGW32__))
 
-#include "rocksdb/rocksdb_namespace.h"
+#include <time.h>
 
 namespace ROCKSDB_NAMESPACE {
 
 namespace port {
 
-// Avoid including winsock2.h for this definition
-struct timeval {
+struct TimeVal {
   long tv_sec;
   long tv_usec;
 };
 
-void gettimeofday(struct timeval* tv, struct timezone* tz);
+void GetTimeOfDay(TimeVal* tv, struct timezone* tz);
 
-inline struct tm* localtime_r(const time_t* timep, struct tm* result) {
+inline struct tm* LocalTimeR(const time_t* timep, struct tm* result) {
   errno_t ret = localtime_s(result, timep);
   return (ret == 0) ? result : NULL;
 }
-}
 
-using port::timeval;
-using port::gettimeofday;
-using port::localtime_r;
+}  // namespace port
+
 }  // namespace ROCKSDB_NAMESPACE
 
 #else
 #include <time.h>
 #include <sys/time.h>
+
+namespace ROCKSDB_NAMESPACE {
+
+namespace port {
+
+using TimeVal = struct timeval;
+
+inline void GetTimeOfDay(TimeVal* tv, struct timezone* tz) {
+  gettimeofday(tv, tz);
+}
+
+inline struct tm* LocalTimeR(const time_t* timep, struct tm* result) {
+  return localtime_r(timep, result);
+}
+
+}  // namespace port
+
+}  // namespace ROCKSDB_NAMESPACE
+
 #endif

package/deps/rocksdb/rocksdb/port/win/env_win.cc

@@ -601,7 +601,7 @@ IOStatus WinFileSystem::NewDirectory(const std::string& name,
     return s;
   }
 
-  result->reset(new WinDirectory(handle));
+  result->reset(new WinDirectory(name, handle));
 
   return s;
 }

package/deps/rocksdb/rocksdb/port/win/io_win.cc

@@ -1064,6 +1064,22 @@ IOStatus WinDirectory::Fsync(const IOOptions& /*options*/,
   return IOStatus::OK();
 }
 
+IOStatus WinDirectory::Close(const IOOptions& /*options*/,
+                             IODebugContext* /*dbg*/) {
+  IOStatus s = IOStatus::OK();
+  BOOL ret __attribute__((__unused__));
+  if (handle_ != INVALID_HANDLE_VALUE) {
+    ret = ::CloseHandle(handle_);
+    if (!ret) {
+      auto lastError = GetLastError();
+      s = IOErrorFromWindowsError("Directory closes failed for : " + GetName(),
+                                  lastError);
+    }
+    handle_ = NULL;
+  }
+  return s;
+}
+
 size_t WinDirectory::GetUniqueId(char* id, size_t max_size) const {
   return GetUniqueIdFromFile(handle_, id, max_size);
 }

package/deps/rocksdb/rocksdb/port/win/io_win.h

@@ -472,14 +472,23 @@ class WinMemoryMappedBuffer : public MemoryMappedFileBuffer {
 };
 
 class WinDirectory : public FSDirectory {
+  const std::string filename_;
   HANDLE handle_;
 
  public:
-  explicit WinDirectory(HANDLE h) noexcept : handle_(h) {
+  explicit WinDirectory(const std::string& filename, HANDLE h) noexcept
+      : filename_(filename), handle_(h) {
     assert(handle_ != INVALID_HANDLE_VALUE);
   }
-  ~WinDirectory() { ::CloseHandle(handle_); }
+  ~WinDirectory() {
+    if (handle_ != NULL) {
+      IOStatus s = WinDirectory::Close(IOOptions(), nullptr);
+      s.PermitUncheckedError();
+    }
+  }
+  const std::string& GetName() const { return filename_; }
   IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
+  IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
 
   size_t GetUniqueId(char* id, size_t max_size) const override;
 };

package/deps/rocksdb/rocksdb/port/win/port_win.cc

@@ -52,7 +52,7 @@ std::wstring utf8_to_utf16(const std::string& utf8) {
 }
 #endif
 
-void gettimeofday(struct timeval* tv, struct timezone* /* tz */) {
+void GetTimeOfDay(TimeVal* tv, struct timezone* /* tz */) {
   std::chrono::microseconds usNow(
       std::chrono::duration_cast<std::chrono::microseconds>(
           std::chrono::system_clock::now().time_since_epoch()));

package/deps/rocksdb/rocksdb/port/win/port_win.h

@@ -70,20 +70,6 @@ extern const bool kDefaultToAdaptiveMutex;
 
 namespace port {
 
-// VS < 2015
-#if defined(_MSC_VER) && (_MSC_VER < 1900)
-
-// VS 15 has snprintf
-#define snprintf _snprintf
-
-#define ROCKSDB_NOEXCEPT
-
-#else // VS >= 2015 or MinGW
-
-#define ROCKSDB_NOEXCEPT noexcept
-
-#endif //_MSC_VER
-
 // "Windows is designed to run on little-endian computer architectures."
 // https://docs.microsoft.com/en-us/windows/win32/sysinfo/registry-value-types
 constexpr bool kLittleEndian = true;
@@ -230,8 +216,8 @@ extern void InitOnce(OnceType* once, void (*initializer)());
 
 #ifdef ROCKSDB_JEMALLOC
 // Separate inlines so they can be replaced if needed
-void* jemalloc_aligned_alloc(size_t size, size_t alignment) ROCKSDB_NOEXCEPT;
-void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT;
+void* jemalloc_aligned_alloc(size_t size, size_t alignment) noexcept;
+void jemalloc_aligned_free(void* p) noexcept;
 #endif
 
 inline void *cacheline_aligned_alloc(size_t size) {

package/deps/rocksdb/rocksdb/port/win/win_jemalloc.cc

@@ -41,10 +41,10 @@ ZSTD_customMem GetJeZstdAllocationOverrides() {
 
 namespace ROCKSDB_NAMESPACE {
 namespace port {
-void* jemalloc_aligned_alloc(size_t size, size_t alignment) ROCKSDB_NOEXCEPT {
+void* jemalloc_aligned_alloc(size_t size, size_t alignment) noexcept {
   return je_aligned_alloc(alignment, size);
 }
-void jemalloc_aligned_free(void* p) ROCKSDB_NOEXCEPT { je_free(p); }
+void jemalloc_aligned_free(void* p) noexcept { je_free(p); }
 }  // namespace port
 }  // namespace ROCKSDB_NAMESPACE
 

package/deps/rocksdb/rocksdb/port/win/win_logger.cc

@@ -118,8 +118,8 @@ void WinLogger::Logv(const char* format, va_list ap) {
     char* p = base;
     char* limit = base + bufsize;
 
-    struct timeval now_tv;
-    gettimeofday(&now_tv, nullptr);
+    port::TimeVal now_tv;
+    port::GetTimeOfDay(&now_tv, nullptr);
     const time_t seconds = now_tv.tv_sec;
     struct tm t;
     localtime_s(&t, &seconds);

package/deps/rocksdb/rocksdb/rocksdb.pc.in

@@ -1,11 +1,10 @@
 prefix="@CMAKE_INSTALL_PREFIX@"
-exec_prefix="${prefix}"
-libdir="${prefix}/lib"
-includedir="${prefix}/include"
+includedir="${prefix}/@CMAKE_INSTALL_INCLUDEDIR@"
+libdir="${prefix}/@CMAKE_INSTALL_LIBDIR@"
 
 Name: @PROJECT_NAME@
-Description: @CMAKE_PROJECT_DESCRIPTION@
-URL: @CMAKE_PROJECT_HOMEPAGE_URL@
+Description: @PROJECT_DESCRIPTION@
+URL: @PROJECT_HOMEPAGE_URL@
 Version: @PROJECT_VERSION@
 Cflags: -I"${includedir}"
 Libs: -L"${libdir}" -lrocksdb

package/deps/rocksdb/rocksdb/src.mk

@@ -82,6 +82,7 @@ LIB_SOURCES =                                                   \
   db/version_set.cc                                             \
   db/wal_edit.cc                                                \
   db/wal_manager.cc                                             \
+  db/wide/wide_column_serialization.cc                          \
   db/write_batch.cc                                             \
   db/write_batch_base.cc                                        \
   db/write_controller.cc                                        \
@@ -497,6 +498,7 @@ TEST_MAIN_SOURCES =                                                     \
   db/version_edit_test.cc                                               \
   db/version_set_test.cc                                                \
   db/wal_manager_test.cc                                                \
+  db/wide/wide_column_serialization_test.cc                             \
   db/write_batch_test.cc                                                \
   db/write_callback_test.cc                                             \
   db/write_controller_test.cc                                           \
@@ -592,6 +594,7 @@ TEST_MAIN_SOURCES =                                                     \
   utilities/transactions/write_prepared_transaction_test.cc             \
   utilities/transactions/write_unprepared_transaction_test.cc           \
   utilities/transactions/write_committed_transaction_ts_test.cc         \
+  utilities/transactions/timestamped_snapshot_test.cc                   \
   utilities/ttl/ttl_test.cc                                             \
   utilities/util_merge_operators_test.cc                                \
   utilities/write_batch_with_index/write_batch_with_index_test.cc       \

package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.cc

@@ -187,9 +187,9 @@ std::unique_ptr<FilterBlockReader> BlockBasedFilterBlockReader::Create(
 
   CachableEntry<BlockContents> filter_block;
   if (prefetch || !use_cache) {
-    const Status s = ReadFilterBlock(table, prefetch_buffer, ro, use_cache,
-                                     nullptr /* get_context */, lookup_context,
-                                     &filter_block);
+    const Status s = ReadFilterBlock(
+        table, prefetch_buffer, ro, use_cache, nullptr /* get_context */,
+        lookup_context, &filter_block, BlockType::kDeprecatedFilter);
     if (!s.ok()) {
       IGNORE_STATUS_IF_ERROR(s);
       return std::unique_ptr<FilterBlockReader>();
@@ -257,7 +257,8 @@ bool BlockBasedFilterBlockReader::MayMatch(
   CachableEntry<BlockContents> filter_block;
 
   const Status s =
-      GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block);
+      GetOrReadFilterBlock(no_io, get_context, lookup_context, &filter_block,
+                           BlockType::kDeprecatedFilter);
   if (!s.ok()) {
     IGNORE_STATUS_IF_ERROR(s);
     return true;
@@ -316,7 +317,8 @@ std::string BlockBasedFilterBlockReader::ToString() const {
 
   const Status s =
       GetOrReadFilterBlock(false /* no_io */, nullptr /* get_context */,
-                           nullptr /* lookup_context */, &filter_block);
+                           nullptr /* lookup_context */, &filter_block,
+                           BlockType::kDeprecatedFilter);
   if (!s.ok()) {
     IGNORE_STATUS_IF_ERROR(s);
     return std::string("Unable to retrieve filter block");

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc

@@ -1250,8 +1250,7 @@ void BlockBasedTableBuilder::WriteRawBlock(const Slice& block_contents,
                                            CompressionType type,
                                            BlockHandle* handle,
                                            BlockType block_type,
-                                           const Slice* raw_block_contents,
-                                           bool is_top_level_filter_block) {
+                                           const Slice* raw_block_contents) {
   Rep* r = rep_;
   bool is_data_block = block_type == BlockType::kData;
   StopWatch sw(r->ioptions.clock, r->ioptions.stats, WRITE_RAW_BLOCK_MICROS);
@@ -1311,11 +1310,9 @@ void BlockBasedTableBuilder::WriteRawBlock(const Slice& block_contents,
     }
     if (warm_cache) {
       if (type == kNoCompression) {
-        s = InsertBlockInCacheHelper(block_contents, handle, block_type,
-                                     is_top_level_filter_block);
+        s = InsertBlockInCacheHelper(block_contents, handle, block_type);
       } else if (raw_block_contents != nullptr) {
-        s = InsertBlockInCacheHelper(*raw_block_contents, handle, block_type,
-                                     is_top_level_filter_block);
+        s = InsertBlockInCacheHelper(*raw_block_contents, handle, block_type);
       }
       if (!s.ok()) {
         r->SetStatus(s);
@@ -1491,25 +1488,28 @@ Status BlockBasedTableBuilder::InsertBlockInCompressedCache(
 
 Status BlockBasedTableBuilder::InsertBlockInCacheHelper(
     const Slice& block_contents, const BlockHandle* handle,
-    BlockType block_type, bool is_top_level_filter_block) {
+    BlockType block_type) {
   Status s;
-  if (block_type == BlockType::kData || block_type == BlockType::kIndex) {
-    s = InsertBlockInCache<Block>(block_contents, handle, block_type);
-  } else if (block_type == BlockType::kFilter) {
-    if (rep_->filter_builder->IsBlockBased()) {
-      // for block-based filter which is deprecated.
-      s = InsertBlockInCache<BlockContents>(block_contents, handle, block_type);
-    } else if (is_top_level_filter_block) {
-      // for top level filter block in partitioned filter.
+  switch (block_type) {
+    case BlockType::kData:
+    case BlockType::kIndex:
+    case BlockType::kFilterPartitionIndex:
       s = InsertBlockInCache<Block>(block_contents, handle, block_type);
-    } else {
-      // for second level partitioned filters and full filters.
+      break;
+    case BlockType::kDeprecatedFilter:
+      s = InsertBlockInCache<BlockContents>(block_contents, handle, block_type);
+      break;
+    case BlockType::kFilter:
       s = InsertBlockInCache<ParsedFullFilterBlock>(block_contents, handle,
                                                     block_type);
-    }
-  } else if (block_type == BlockType::kCompressionDictionary) {
-    s = InsertBlockInCache<UncompressionDict>(block_contents, handle,
-                                              block_type);
+      break;
+    case BlockType::kCompressionDictionary:
+      s = InsertBlockInCache<UncompressionDict>(block_contents, handle,
+                                                block_type);
+      break;
+    default:
+      // no-op / not cached
+      break;
   }
   return s;
 }
@@ -1563,10 +1563,14 @@ Status BlockBasedTableBuilder::InsertBlockInCache(const Slice& block_contents,
 
 void BlockBasedTableBuilder::WriteFilterBlock(
     MetaIndexBuilder* meta_index_builder) {
+  if (rep_->filter_builder == nullptr || rep_->filter_builder->IsEmpty()) {
+    // No filter block needed
+    return;
+  }
   BlockHandle filter_block_handle;
-  bool empty_filter_block =
-      (rep_->filter_builder == nullptr || rep_->filter_builder->IsEmpty());
-  if (ok() && !empty_filter_block) {
+  bool is_block_based_filter = rep_->filter_builder->IsBlockBased();
+  bool is_partitioned_filter = rep_->table_options.partition_filters;
+  if (ok()) {
     rep_->props.num_filter_entries +=
         rep_->filter_builder->EstimateEntriesAdded();
     Status s = Status::Incomplete();
@@ -1591,31 +1595,23 @@ void BlockBasedTableBuilder::WriteFilterBlock(
 
       rep_->props.filter_size += filter_content.size();
 
-      // TODO: Refactor code so that BlockType can determine both the C++ type
-      // of a block cache entry (TBlocklike) and the CacheEntryRole while
-      // inserting blocks in cache.
-      bool top_level_filter_block = false;
-      if (s.ok() && rep_->table_options.partition_filters &&
-          !rep_->filter_builder->IsBlockBased()) {
-        top_level_filter_block = true;
-      }
-      WriteRawBlock(filter_content, kNoCompression, &filter_block_handle,
-                    BlockType::kFilter, nullptr /*raw_contents*/,
-                    top_level_filter_block);
+      BlockType btype = is_block_based_filter ? BlockType::kDeprecatedFilter
+                        : is_partitioned_filter && /* last */ s.ok()
+                            ? BlockType::kFilterPartitionIndex
+                            : BlockType::kFilter;
+      WriteRawBlock(filter_content, kNoCompression, &filter_block_handle, btype,
+                    nullptr /*raw_contents*/);
     }
     rep_->filter_builder->ResetFilterBitsBuilder();
   }
-  if (ok() && !empty_filter_block) {
+  if (ok()) {
     // Add mapping from "<filter_block_prefix>.Name" to location
     // of filter data.
     std::string key;
-    if (rep_->filter_builder->IsBlockBased()) {
-      key = BlockBasedTable::kFilterBlockPrefix;
-    } else {
-      key = rep_->table_options.partition_filters
-                ? BlockBasedTable::kPartitionedFilterBlockPrefix
-                : BlockBasedTable::kFullFilterBlockPrefix;
-    }
+    key = is_block_based_filter ? BlockBasedTable::kFilterBlockPrefix
+          : is_partitioned_filter
+              ? BlockBasedTable::kPartitionedFilterBlockPrefix
+              : BlockBasedTable::kFullFilterBlockPrefix;
     key.append(rep_->table_options.filter_policy->CompatibilityName());
     meta_index_builder->Add(key, filter_block_handle);
   }

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h

@@ -119,8 +119,7 @@ class BlockBasedTableBuilder : public TableBuilder {
                   BlockType block_type);
   // Directly write data to the file.
   void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle,
-                     BlockType block_type, const Slice* raw_data = nullptr,
-                     bool is_top_level_filter_block = false);
+                     BlockType block_type, const Slice* raw_data = nullptr);
 
   void SetupCacheKeyPrefix(const TableBuilderOptions& tbo);
 
@@ -130,8 +129,7 @@ class BlockBasedTableBuilder : public TableBuilder {
 
   Status InsertBlockInCacheHelper(const Slice& block_contents,
                                   const BlockHandle* handle,
-                                  BlockType block_type,
-                                  bool is_top_level_filter_block);
+                                  BlockType block_type);
 
   Status InsertBlockInCompressedCache(const Slice& block_contents,
                                       const CompressionType type,