@nxtedition/rocksdb 7.0.0-alpha.6 → 7.0.0

package/binding.cc

@@ -526,7 +526,7 @@ struct Iterator final : public BaseIterator {
            const bool fillCache,
            const bool keyAsBuffer,
            const bool valueAsBuffer,
-           const uint32_t highWaterMarkBytes,
+           const int32_t highWaterMarkBytes,
            std::shared_ptr<const rocksdb::Snapshot> snapshot)
       : BaseIterator(database, column, reverse, lt, lte, gt, gte, limit, fillCache, snapshot),
         keys_(keys),
@@ -551,7 +551,7 @@ struct Iterator final : public BaseIterator {
   const bool values_;
   const bool keyAsBuffer_;
   const bool valueAsBuffer_;
-  const uint32_t highWaterMarkBytes_;
+  const int32_t highWaterMarkBytes_;
   bool first_ = true;
 
  private:
@@ -691,85 +691,86 @@ struct OpenWorker final : public Worker {
   std::vector<rocksdb::ColumnFamilyDescriptor> column_families_;
 };
 
-napi_status InitOptions(napi_env env, auto& options, auto options2) {
-  const auto memtable_memory_budget = Uint32Property(env, options2, "memtableMemoryBudget").value_or(256 * 1024 * 1024);
+template <typename T>
+napi_status InitOptions(napi_env env, T& columnOptions, auto options) {
+  const auto memtable_memory_budget = Uint32Property(env, options, "memtableMemoryBudget").value_or(256 * 1024 * 1024);
 
-  const auto compaction = StringProperty(env, options2, "compaction").value_or("level");
+  const auto compaction = StringProperty(env, options, "compaction").value_or("level");
 
   if (compaction == "universal") {
-    options.write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+    columnOptions.write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
     // merge two memtables when flushing to L0
-    options.min_write_buffer_number_to_merge = 2;
+    columnOptions.min_write_buffer_number_to_merge = 2;
     // this means we'll use 50% extra memory in the worst case, but will reduce
     // write stalls.
-    options.max_write_buffer_number = 6;
+    columnOptions.max_write_buffer_number = 6;
     // universal style compaction
-    options.compaction_style = rocksdb::kCompactionStyleUniversal;
-    options.compaction_options_universal.compression_size_percent = 80;
+    columnOptions.compaction_style = rocksdb::kCompactionStyleUniversal;
+    columnOptions.compaction_options_universal.compression_size_percent = 80;
   } else {
     // merge two memtables when flushing to L0
-    options.min_write_buffer_number_to_merge = 2;
+    columnOptions.min_write_buffer_number_to_merge = 2;
     // this means we'll use 50% extra memory in the worst case, but will reduce
     // write stalls.
-    options.max_write_buffer_number = 6;
+    columnOptions.max_write_buffer_number = 6;
     // start flushing L0->L1 as soon as possible. each file on level0 is
     // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
     // memtable_memory_budget.
-    options.level0_file_num_compaction_trigger = 2;
+    columnOptions.level0_file_num_compaction_trigger = 2;
     // doesn't really matter much, but we don't want to create too many files
-    options.target_file_size_base = memtable_memory_budget / 8;
+    columnOptions.target_file_size_base = memtable_memory_budget / 8;
     // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
-    options.max_bytes_for_level_base = memtable_memory_budget;
+    columnOptions.max_bytes_for_level_base = memtable_memory_budget;
 
     // level style compaction
-    options.compaction_style = rocksdb::kCompactionStyleLevel;
+    columnOptions.compaction_style = rocksdb::kCompactionStyleLevel;
 
     // TODO (perf): only compress levels >= 2
   }
 
-  options.compression =
-      BooleanProperty(env, options2, "compression").value_or((true)) ? rocksdb::kZSTD : rocksdb::kNoCompression;
-  if (options.compression == rocksdb::kZSTD) {
-    options.compression_opts.max_dict_bytes = 16 * 1024;
-    options.compression_opts.zstd_max_train_bytes = 16 * 1024 * 100;
-    // options.compression_opts.parallel_threads
+  columnOptions.compression =
+      BooleanProperty(env, options, "compression").value_or((true)) ? rocksdb::kZSTD : rocksdb::kNoCompression;
+  if (columnOptions.compression == rocksdb::kZSTD) {
+    columnOptions.compression_opts.max_dict_bytes = 16 * 1024;
+    columnOptions.compression_opts.zstd_max_train_bytes = 16 * 1024 * 100;
+    // columnOptions.compression_opts.parallel_threads
   }
 
-  const auto cacheSize = Uint32Property(env, options2, "cacheSize").value_or(8 << 20);
+  const auto cacheSize = Uint32Property(env, options, "cacheSize").value_or(8 << 20);
 
   rocksdb::BlockBasedTableOptions tableOptions;
 
   if (cacheSize) {
     tableOptions.block_cache = rocksdb::NewLRUCache(cacheSize);
     tableOptions.cache_index_and_filter_blocks =
-        BooleanProperty(env, options2, "cacheIndexAndFilterBlocks").value_or(true);
+        BooleanProperty(env, options, "cacheIndexAndFilterBlocks").value_or(true);
   } else {
     tableOptions.no_block_cache = true;
     tableOptions.cache_index_and_filter_blocks = false;
   }
 
-  const auto optimize = StringProperty(env, options2, "optimize").value_or("");
+  const auto optimize = StringProperty(env, options, "optimize").value_or("");
 
   if (optimize == "point-lookup") {
     tableOptions.data_block_index_type = rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
     tableOptions.data_block_hash_table_util_ratio = 0.75;
     tableOptions.filter_policy.reset(rocksdb::NewRibbonFilterPolicy(10, 1));
 
-    options.memtable_prefix_bloom_size_ratio = 0.02;
-    options.memtable_whole_key_filtering = true;
+    columnOptions.memtable_prefix_bloom_size_ratio = 0.02;
+    columnOptions.memtable_whole_key_filtering = true;
   } else if (optimize == "range-lookup") {
     // TODO?
   } else {
     tableOptions.filter_policy.reset(rocksdb::NewRibbonFilterPolicy(10));
   }
 
-  tableOptions.block_size = Uint32Property(env, options2, "blockSize").value_or(4096);
-  tableOptions.block_restart_interval = Uint32Property(env, options2, "blockRestartInterval").value_or(16);
+  tableOptions.block_size = Uint32Property(env, options, "blockSize").value_or(4096);
+  tableOptions.block_restart_interval = Uint32Property(env, options, "blockRestartInterval").value_or(16);
   tableOptions.format_version = 5;
   tableOptions.checksum = rocksdb::kXXH3;
-  tableOptions.optimize_filters_for_memory = BooleanProperty(env, options2, "optimizeFiltersForMemory").value_or(true);
+  tableOptions.optimize_filters_for_memory = BooleanProperty(env, options, "optimizeFiltersForMemory").value_or(true);
 
-  options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(tableOptions));
+  columnOptions.table_factory.reset(rocksdb::NewBlockBasedTableFactory(tableOptions));
 
   return napi_ok;
 }
@@ -792,13 +793,14 @@ NAPI_METHOD(db_open) {
   dbOptions.create_if_missing = BooleanProperty(env, options, "createIfMissing").value_or(true);
   dbOptions.error_if_exists = BooleanProperty(env, options, "errorIfExists").value_or(false);
   dbOptions.avoid_unnecessary_blocking_io = true;
-  dbOptions.use_adaptive_mutex = BooleanProperty(env, options, "useAdaptiveMutex").value_or(true);
-  dbOptions.enable_pipelined_write = BooleanProperty(env, options, "enablePipelinedWrite").value_or(true);
+  dbOptions.use_adaptive_mutex = true;
+  dbOptions.enable_pipelined_write = false;
   dbOptions.max_background_jobs =
       Uint32Property(env, options, "maxBackgroundJobs").value_or(std::thread::hardware_concurrency() / 4);
   dbOptions.WAL_ttl_seconds = Uint32Property(env, options, "walTTL").value_or(0) / 1e3;
   dbOptions.WAL_size_limit_MB = Uint32Property(env, options, "walSizeLimit").value_or(0) / 1e6;
   dbOptions.create_missing_column_families = true;
+  dbOptions.unordered_write = BooleanProperty(env, options, "unorderedWrite").value_or(false);
 
   const auto infoLogLevel = StringProperty(env, options, "infoLogLevel").value_or("");
   if (infoLogLevel.size() > 0) {
@@ -1374,7 +1376,7 @@ NAPI_METHOD(iterator_init) {
   const bool keyAsBuffer = EncodingIsBuffer(env, options, "keyEncoding");
   const bool valueAsBuffer = EncodingIsBuffer(env, options, "valueEncoding");
   const auto limit = Int32Property(env, options, "limit").value_or(-1);
-  const auto highWaterMarkBytes = Uint32Property(env, options, "highWaterMarkBytes").value_or(16 * 1024);
+  const auto highWaterMarkBytes = Int32Property(env, options, "highWaterMarkBytes").value_or(16 * 1024);
 
   const auto lt = StringProperty(env, options, "lt");
   const auto lte = StringProperty(env, options, "lte");
@@ -1479,7 +1481,7 @@ struct NextWorker final : public Worker {
         cache_.push_back(v.ToString());
       }
 
-      if (bytesRead > iterator_->highWaterMarkBytes_ || cache_.size() / 2 >= size_) {
+      if ((iterator_->highWaterMarkBytes_ != -1 && bytesRead > static_cast<size_t>(iterator_->highWaterMarkBytes_)) || cache_.size() / 2 >= size_) {
         finished_ = false;
         return rocksdb::Status::OK();
       }
@@ -1546,7 +1548,6 @@ NAPI_METHOD(batch_do) {
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database));
 
   const auto operations = argv[1];
-  const auto options = argv[2];
 
   rocksdb::WriteBatch batch;
 

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -89,7 +89,11 @@ include(CMakeDependentOption)
 if(MSVC)
   option(WITH_GFLAGS "build with GFlags" OFF)
   option(WITH_XPRESS "build with windows built in compression" OFF)
-  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+  option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF)
+
+  if(NOT ROCKSDB_SKIP_THIRDPARTY)
+    include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+  endif()
 else()
   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
     # FreeBSD has jemalloc as default malloc
@@ -671,6 +675,7 @@ set(SOURCES
         db/version_set.cc
         db/wal_edit.cc
         db/wal_manager.cc
+        db/wide/wide_column_serialization.cc
         db/write_batch.cc
         db/write_batch_base.cc
         db/write_controller.cc
@@ -1122,8 +1127,8 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
   )
 
   configure_file(
-    ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}.pc.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc
+    ${PROJECT_NAME}.pc.in
+    ${PROJECT_NAME}.pc
     @ONLY
   )
 
@@ -1290,6 +1295,7 @@ if(WITH_TESTS)
         db/version_set_test.cc
         db/wal_manager_test.cc
         db/wal_edit_test.cc
+        db/wide/wide_column_serialization_test.cc
         db/write_batch_test.cc
         db/write_callback_test.cc
         db/write_controller_test.cc
@@ -1384,6 +1390,7 @@ if(WITH_TESTS)
         utilities/transactions/write_prepared_transaction_test.cc
         utilities/transactions/write_unprepared_transaction_test.cc
         utilities/transactions/lock/range/range_locking_test.cc
+        utilities/transactions/timestamped_snapshot_test.cc
         utilities/ttl/ttl_test.cc
         utilities/util_merge_operators_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc

package/deps/rocksdb/rocksdb/Makefile

@@ -1777,6 +1777,9 @@ write_prepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_prepare
 write_unprepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_unprepared_transaction_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+timestamped_snapshot_test: $(OBJ_DIR)/utilities/transactions/timestamped_snapshot_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 sst_dump: $(OBJ_DIR)/tools/sst_dump.o $(TOOLS_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -1876,7 +1879,7 @@ testutil_test: $(OBJ_DIR)/test_util/testutil_test.o $(TEST_LIBRARY) $(LIBRARY)
 io_tracer_test: $(OBJ_DIR)/trace_replay/io_tracer_test.o $(OBJ_DIR)/trace_replay/io_tracer.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
-prefetch_test: $(OBJ_DIR)/file/prefetch_test.o $(TEST_LIBRARY) $(LIBRARY)
+prefetch_test: $(OBJ_DIR)/file/prefetch_test.o  $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
 io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools/io_tracer_parser_tool.o $(TEST_LIBRARY) $(LIBRARY)
@@ -1902,6 +1905,10 @@ db_basic_bench: $(OBJ_DIR)/microbench/db_basic_bench.o $(LIBRARY)
 
 cache_reservation_manager_test: $(OBJ_DIR)/cache/cache_reservation_manager_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
+
+wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local

package/deps/rocksdb/rocksdb/TARGETS

@@ -91,6 +91,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/version_set.cc",
         "db/wal_edit.cc",
         "db/wal_manager.cc",
+        "db/wide/wide_column_serialization.cc",
         "db/write_batch.cc",
         "db/write_batch_base.cc",
         "db/write_controller.cc",
@@ -419,6 +420,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/version_set.cc",
         "db/wal_edit.cc",
         "db/wal_manager.cc",
+        "db/wide/wide_column_serialization.cc",
         "db/write_batch.cc",
         "db/write_batch_base.cc",
         "db/write_controller.cc",
@@ -5766,6 +5768,12 @@ cpp_unittest_wrapper(name="timer_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="timestamped_snapshot_test",
+            srcs=["utilities/transactions/timestamped_snapshot_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="trace_analyzer_test",
             srcs=["tools/trace_analyzer_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5814,6 +5822,12 @@ cpp_unittest_wrapper(name="wal_manager_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="wide_column_serialization_test",
+            srcs=["db/wide/wide_column_serialization_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="work_queue_test",
             srcs=["util/work_queue_test.cc"],
             deps=[":rocksdb_test_lib"],

package/deps/rocksdb/rocksdb/cache/cache.cc

@@ -34,13 +34,61 @@ static std::unordered_map<std::string, OptionTypeInfo>
           OptionType::kDouble, OptionVerificationType::kNormal,
           OptionTypeFlags::kMutable}},
 };
+
+static std::unordered_map<std::string, OptionTypeInfo>
+    comp_sec_cache_options_type_info = {
+        {"capacity",
+         {offsetof(struct CompressedSecondaryCacheOptions, capacity),
+          OptionType::kSizeT, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
+        {"num_shard_bits",
+         {offsetof(struct CompressedSecondaryCacheOptions, num_shard_bits),
+          OptionType::kInt, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
+        {"compression_type",
+         {offsetof(struct CompressedSecondaryCacheOptions, compression_type),
+          OptionType::kCompressionType, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
+        {"compress_format_version",
+         {offsetof(struct CompressedSecondaryCacheOptions,
+                   compress_format_version),
+          OptionType::kUInt32T, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
+};
 #endif  // ROCKSDB_LITE
 
 Status SecondaryCache::CreateFromString(
     const ConfigOptions& config_options, const std::string& value,
     std::shared_ptr<SecondaryCache>* result) {
-  return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
-                                          result);
+  if (value.find("compressed_secondary_cache://") == 0) {
+    std::string args = value;
+    args.erase(0, std::strlen("compressed_secondary_cache://"));
+    Status status;
+    std::shared_ptr<SecondaryCache> sec_cache;
+
+#ifndef ROCKSDB_LITE
+    CompressedSecondaryCacheOptions sec_cache_opts;
+    status = OptionTypeInfo::ParseStruct(config_options, "",
+                                         &comp_sec_cache_options_type_info, "",
+                                         args, &sec_cache_opts);
+    if (status.ok()) {
+      sec_cache = NewCompressedSecondaryCache(sec_cache_opts);
+    }
+
+#else
+    (void)config_options;
+    status = Status::NotSupported(
+        "Cannot load compressed secondary cache in LITE mode ", args);
+#endif  //! ROCKSDB_LITE
+
+    if (status.ok()) {
+      result->swap(sec_cache);
+    }
+    return status;
+  } else {
+    return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
+                                            result);
+  }
 }
 
 Status Cache::CreateFromString(const ConfigOptions& config_options,

package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc

@@ -12,6 +12,7 @@
 #include <set>
 #include <sstream>
 
+#include "cache/fast_lru_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "monitoring/histogram.h"
 #include "port/port.h"
@@ -76,7 +77,7 @@ DEFINE_string(secondary_cache_uri, "",
 static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
 #endif  // ROCKSDB_LITE
 
-DEFINE_bool(use_clock_cache, false, "");
+DEFINE_string(cache_type, "lru_cache", "Type of block cache.");
 
 // ## BEGIN stress_cache_key sub-tool options ##
 // See class StressCacheKey below.
@@ -279,13 +280,15 @@ class CacheBench {
       if (max_key > (static_cast<uint64_t>(1) << max_log_)) max_log_++;
     }
 
-    if (FLAGS_use_clock_cache) {
+    if (FLAGS_cache_type == "clock_cache") {
       cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
       if (!cache_) {
         fprintf(stderr, "Clock cache not supported.\n");
         exit(1);
       }
-    } else {
+    } else if (FLAGS_cache_type == "fast_lru_cache") {
+      cache_ = NewFastLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+    } else if (FLAGS_cache_type == "lru_cache") {
       LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
 #ifndef ROCKSDB_LITE
       if (!FLAGS_secondary_cache_uri.empty()) {
@@ -303,6 +306,9 @@ class CacheBench {
 #endif  // ROCKSDB_LITE
 
       cache_ = NewLRUCache(opts);
+    } else {
+      fprintf(stderr, "Cache type not supported.");
+      exit(1);
     }
   }