@nxtedition/rocksdb 13.5.7 → 13.5.9

package/binding.cc

@@ -127,14 +127,11 @@ struct BatchIterator : public rocksdb::WriteBatch::Handler {
         keyEncoding_(keyEncoding),
         valueEncoding_(valueEncoding) {}
 
-  rocksdb::Status Read (const rocksdb::WriteBatch& batch) {
-    cache_.clear();
+  napi_status Iterate(napi_env env, const rocksdb::WriteBatch& batch, napi_value* result) {
     cache_.reserve(batch.Count());
 
-    ROCKS_STATUS_RETURN(batch.Iterate(this));
-  }
+    ROCKS_STATUS_RETURN_NAPI(batch.Iterate(this));
 
-  napi_status Write(napi_env env, napi_value* result) {
     napi_value putStr;
     NAPI_STATUS_RETURN(napi_create_string_utf8(env, "put", NAPI_AUTO_LENGTH, &putStr));
 
@@ -185,11 +182,6 @@ struct BatchIterator : public rocksdb::WriteBatch::Handler {
     return napi_ok;
   }
 
-  napi_status Iterate(napi_env env, const rocksdb::WriteBatch& batch, napi_value* result) {
-    ROCKS_STATUS_RETURN_NAPI(Read(batch));
-    return Write(env, result);
-  }
-
   rocksdb::Status PutCF(uint32_t column_family_id, const rocksdb::Slice& key, const rocksdb::Slice& value) override {
     if (column_ && column_->GetID() != column_family_id) {
       return rocksdb::Status::OK();
@@ -482,10 +474,10 @@ class Iterator final : public BaseIterator {
     rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
     NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
 
-    Encoding keyEncoding = Encoding::String;
+    Encoding keyEncoding;
     NAPI_STATUS_THROWS(GetProperty(env, options, "keyEncoding", keyEncoding));
 
-    Encoding valueEncoding = Encoding::String;
+    Encoding valueEncoding;
     NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
 
     rocksdb::ReadOptions readOptions;
@@ -577,6 +569,8 @@ class Iterator final : public BaseIterator {
               rocksdb::PinnableSlice v;
               v.PinSelf(CurrentValue());
               state.values.push_back(std::move(v));
+            } else {
+              assert(false);
             }
             state.count += 1;
 
@@ -615,8 +609,7 @@ class Iterator final : public BaseIterator {
               NAPI_STATUS_RETURN(napi_get_undefined(env, &key));
               NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val));
             } else {
-              NAPI_STATUS_RETURN(napi_get_undefined(env, &key));
-              NAPI_STATUS_RETURN(napi_get_undefined(env, &val));
+              assert(false);
             }
 
             NAPI_STATUS_RETURN(napi_set_element(env, rows, n * 2 + 0, key));
@@ -688,7 +681,7 @@ class Iterator final : public BaseIterator {
         break;
       }
 
-      if (n & 0xf == 0 && deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
+      if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
         break;
       }
     }
@@ -736,7 +729,7 @@ NAPI_METHOD(db_init) {
   napi_valuetype type;
   NAPI_STATUS_THROWS(napi_typeof(env, argv[0], &type));
 
-  napi_value result = 0;
+  napi_value result;
 
   if (type == napi_string) {
     std::string location;
@@ -1153,8 +1146,6 @@ NAPI_METHOD(db_open) {
 
     NAPI_STATUS_THROWS(GetProperty(env, options, "writeDbIdToManifest", dbOptions.write_dbid_to_manifest));
 
-    NAPI_STATUS_THROWS(GetProperty(env, options, "failIfOptionsFileError", dbOptions.fail_if_options_file_error));
-
     NAPI_STATUS_THROWS(GetProperty(env, options, "adviseRandomOnOpen", dbOptions.advise_random_on_open));
 
     NAPI_STATUS_THROWS(GetProperty(env, options, "bytesPerSync", dbOptions.bytes_per_sync));
@@ -1877,8 +1868,7 @@ NAPI_METHOD(batch_iterate) {
   return result;
 }
 
-class Updates : public BatchIterator, public Closable {
-public:
+struct Updates : public BatchIterator, public Closable {
   Updates(Database* database,
           const int64_t since,
           const bool keys,
@@ -1899,54 +1889,6 @@ public:
     }
   }
 
-  napi_value nextv(napi_env env, uint32_t count, uint32_t timeout, napi_value callback) {
-    struct State {
-      uint64_t sequence = 0;
-      bool finished = false;
-    };
-    runAsync<State>("updates.next", env, callback,
-      [=](auto& state) {
-        if (!iterator_) {
-          rocksdb::TransactionLogIterator::ReadOptions options;
-          ROCKS_STATUS_RETURN(database_->db->GetUpdatesSince(start_, &iterator_, options));
-        }
-
-        ROCKS_STATUS_RETURN(iterator_->status());
-
-        if (!iterator_->Valid()) {
-          state.finished = true;
-          return rocksdb::Status::OK();
-        }
-
-        auto batchResult = iterator_->GetBatch();
-        Read(*batchResult.writeBatchPtr);
-        state.sequence = batchResult.sequence;
-
-        iterator_->Next();
-
-        return rocksdb::Status::OK();
-      },
-      [=](auto& state, auto env, auto& argv) {
-        argv.resize(2);
-
-        if (!state.finished) {
-          napi_value rows;
-          napi_value sequence;
-
-          NAPI_STATUS_RETURN(Write(env, &rows));
-          NAPI_STATUS_RETURN(napi_create_int64(env, state.sequence, &sequence));
-
-          NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
-          NAPI_STATUS_RETURN(napi_set_named_property(env, argv[1], "rows", rows));
-          NAPI_STATUS_RETURN(napi_set_named_property(env, argv[1], "seq", sequence));
-        } else {
-          NAPI_STATUS_RETURN(napi_get_null(env, &argv[1]));
-        }
-
-        return napi_ok;
-      });
-  }
-
   rocksdb::Status Close() override {
     if (iterator_) {
       iterator_.reset();
@@ -1955,10 +1897,11 @@ public:
     return rocksdb::Status::OK();
   }
 
- private:
   Database* database_;
   int64_t start_;
   std::unique_ptr<rocksdb::TransactionLogIterator> iterator_;
+
+ private:
   napi_ref ref_ = nullptr;
 };
 
@@ -2013,7 +1956,33 @@ NAPI_METHOD(updates_next) {
     Updates* updates;
     NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&updates)));
 
-    return updates->nextv(env, -1, -1, argv[1]);
+    if (!updates->iterator_) {
+      rocksdb::TransactionLogIterator::ReadOptions options;
+      ROCKS_STATUS_THROWS_NAPI(updates->database_->db->GetUpdatesSince(updates->start_, &updates->iterator_, options));
+    }
+
+    ROCKS_STATUS_THROWS_NAPI(updates->iterator_->status());
+
+    if (!updates->iterator_->Valid()) {
+      return 0;
+    }
+
+    auto batchResult = updates->iterator_->GetBatch();
+
+    napi_value rows;
+    napi_value sequence;
+
+    NAPI_STATUS_THROWS(updates->Iterate(env, *batchResult.writeBatchPtr, &rows));
+    NAPI_STATUS_THROWS(napi_create_int64(env, batchResult.sequence, &sequence));
+
+    ROCKS_STATUS_THROWS_NAPI(updates->iterator_->status());
+    updates->iterator_->Next();
+
+    napi_value ret;
+    NAPI_STATUS_THROWS(napi_create_object(env, &ret));
+    NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "rows", rows));
+    NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "seq", sequence));
+    return ret;
   } catch (const std::exception& e) {
     napi_throw_error(env, nullptr, e.what());
     return nullptr;
@@ -2035,6 +2004,213 @@ NAPI_METHOD(updates_close) {
   }
 }
 
+NAPI_METHOD(db_open_for_read_only) {
+  NAPI_ARGV(3);
+
+  Database* database;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
+
+  if (database->db) {
+    napi_value columns;
+    NAPI_STATUS_THROWS(napi_create_object(env, &columns));
+    for (auto& [id, column] : database->columns) {
+      napi_value val;
+      NAPI_STATUS_THROWS(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+      NAPI_STATUS_THROWS(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+    }
+    return columns;
+  } else {
+    rocksdb::Options dbOptions;
+
+    const auto options = argv[1];
+
+    int parallelism = std::max<int>(1, std::thread::hardware_concurrency() / 2);
+    NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
+    dbOptions.IncreaseParallelism(parallelism);
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walDir", dbOptions.wal_dir));
+
+    uint32_t walTTL = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
+    dbOptions.WAL_ttl_seconds = walTTL / 1e3;
+
+    uint32_t walSizeLimit = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
+    dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "maxTotalWalSize", dbOptions.max_total_wal_size));
+
+    bool walCompression = true;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walCompression", walCompression));
+    dbOptions.wal_compression =
+        walCompression ? rocksdb::CompressionType::kZSTD : rocksdb::CompressionType::kNoCompression;
+
+    dbOptions.avoid_unnecessary_blocking_io = true;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "avoidUnnecessaryBlockingIO", dbOptions.avoid_unnecessary_blocking_io));
+
+    dbOptions.create_missing_column_families = true;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "createMissingColumnFamilies", dbOptions.create_missing_column_families));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "writeDbIdToManifest", dbOptions.write_dbid_to_manifest));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "adviseRandomOnOpen", dbOptions.advise_random_on_open));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "bytesPerSync", dbOptions.bytes_per_sync));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walBytesPerSync", dbOptions.wal_bytes_per_sync));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "strictBytesPerSync", dbOptions.strict_bytes_per_sync));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "delayedWriteRate", dbOptions.delayed_write_rate));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "createIfMissing", dbOptions.create_if_missing));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "errorIfExists", dbOptions.error_if_exists));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "pipelinedWrite", dbOptions.enable_pipelined_write));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "dailyOffpeakTime", dbOptions.daily_offpeak_time_utc));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "unorderedWrite", dbOptions.unordered_write));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "allowMmapReads", dbOptions.allow_mmap_reads));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "allowMmapWrites", dbOptions.allow_mmap_writes));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "memTableHugePageSize", dbOptions.memtable_huge_page_size));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "useDirectIOReads", dbOptions.use_direct_reads));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "useDirectIOForFlushAndCompaction", dbOptions.use_direct_io_for_flush_and_compaction));
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "compactionReadaheadSize", dbOptions.compaction_readahead_size));
+
+    // TODO (feat): dbOptions.listeners
+
+    std::string infoLogLevel;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "infoLogLevel", infoLogLevel));
+    if (infoLogLevel.size() > 0) {
+      rocksdb::InfoLogLevel lvl = {};
+
+      if (infoLogLevel == "debug")
+        lvl = rocksdb::InfoLogLevel::DEBUG_LEVEL;
+      else if (infoLogLevel == "info")
+        lvl = rocksdb::InfoLogLevel::INFO_LEVEL;
+      else if (infoLogLevel == "warn")
+        lvl = rocksdb::InfoLogLevel::WARN_LEVEL;
+      else if (infoLogLevel == "error")
+        lvl = rocksdb::InfoLogLevel::ERROR_LEVEL;
+      else if (infoLogLevel == "fatal")
+        lvl = rocksdb::InfoLogLevel::FATAL_LEVEL;
+      else if (infoLogLevel == "header")
+        lvl = rocksdb::InfoLogLevel::HEADER_LEVEL;
+      else
+        napi_throw_error(env, nullptr, "invalid log level");
+
+      dbOptions.info_log_level = lvl;
+    } else {
+      // In some places RocksDB checks this option to see if it should prepare
+      // debug information (ahead of logging), so set it to the highest level.
+      dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
+      dbOptions.info_log.reset(new NullLogger());
+    }
+
+    NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
+
+    std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+
+    bool hasColumns;
+    NAPI_STATUS_THROWS(napi_has_named_property(env, options, "columns", &hasColumns));
+
+    if (hasColumns) {
+      napi_value columns;
+      NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
+
+      napi_value keys;
+      NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
+
+      uint32_t len;
+      NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
+
+      descriptors.resize(len);
+      for (uint32_t n = 0; n < len; ++n) {
+        napi_value key;
+        NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
+
+        napi_value column;
+        NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
+
+        NAPI_STATUS_THROWS(InitOptions(env, descriptors[n].options, column));
+
+        NAPI_STATUS_THROWS(GetValue(env, key, descriptors[n].name));
+      }
+    }
+
+    auto callback = argv[2];
+
+    runAsync<std::vector<rocksdb::ColumnFamilyHandle*>>(
+        "leveldown.open_for_read_only", env, callback,
+        [=](auto& handles) {
+          assert(!database->db);
+
+          rocksdb::DB* db = nullptr;
+
+          const auto status = descriptors.empty()
+                                  ? rocksdb::DB::OpenForReadOnly(dbOptions, database->location, &db)
+                                  : rocksdb::DB::OpenForReadOnly(dbOptions, database->location, descriptors, &handles, &db);
+
+          database->db.reset(db);
+
+          return status;
+        },
+        [=](auto& handles, auto env, auto& argv) {
+          argv.resize(2);
+
+          NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
+
+          for (size_t n = 0; n < handles.size(); ++n) {
+            ColumnFamily column;
+            column.handle = handles[n];
+            column.descriptor = descriptors[n];
+            database->columns[column.handle->GetID()] = column;
+          }
+
+          napi_value columns = argv[1];
+          for (auto& [id, column] : database->columns) {
+            napi_value val;
+            NAPI_STATUS_RETURN(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+            NAPI_STATUS_RETURN(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+          }
+
+          return napi_ok;
+        });
+  }
+
+  return 0;
+}
+
+NAPI_METHOD(db_compact_range) {
+  NAPI_ARGV(2);
+
+  Database* database;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
+
+  std::optional<std::string> start;
+  std::optional<std::string> end;
+
+  NAPI_STATUS_THROWS(GetProperty(env, argv[1], "start", start));
+  NAPI_STATUS_THROWS(GetProperty(env, argv[1], "end", end));
+
+  rocksdb::CompactRangeOptions options;
+
+  auto begin = start ? std::make_unique<rocksdb::Slice>(*start) : nullptr;
+  auto finish = end ? std::make_unique<rocksdb::Slice>(*end) : nullptr;
+
+  ROCKS_STATUS_THROWS_NAPI(database->db->CompactRange(options, begin.get(), finish.get()));
+
+  return 0;
+}
+
 NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_init);
   NAPI_EXPORT_FUNCTION(db_open);
@@ -2068,4 +2244,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(batch_merge);
   NAPI_EXPORT_FUNCTION(batch_count);
   NAPI_EXPORT_FUNCTION(batch_iterate);
+  NAPI_EXPORT_FUNCTION(db_open_for_read_only);
+  NAPI_EXPORT_FUNCTION(db_compact_range);
 }

package/binding.gyp

@@ -16,8 +16,8 @@
                           "/usr/lib/x86_64-linux-gnu/include",
                           "/usr/lib/include",
                         ],
-                        "cflags": ["-march=znver2", '-mtune=znver3'],
-                        "cflags_cc": ["-flto", '-march=znver2', '-mtune=znver3'],
+                        "cflags": ["-march=znver1"],
+                        "ccflags": ["-flto", '-march=znver1'],
                         "cflags!": ["-fno-exceptions"],
                         "cflags_cc!": ["-fno-exceptions"],
                         "ldflags": ["-flto", "-fuse-linker-plugin"],

package/deps/rocksdb/rocksdb/BUCK

@@ -214,6 +214,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "table/cuckoo/cuckoo_table_builder.cc",
         "table/cuckoo/cuckoo_table_factory.cc",
         "table/cuckoo/cuckoo_table_reader.cc",
+        "table/external_table.cc",
         "table/format.cc",
         "table/get_context.cc",
         "table/iterator.cc",
@@ -248,6 +249,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "trace_replay/trace_record_result.cc",
         "trace_replay/trace_replay.cc",
         "util/async_file_reader.cc",
+        "util/auto_skip_compressor.cc",
         "util/build_version.cc",
         "util/cleanable.cc",
         "util/coding.cc",
@@ -266,6 +268,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "util/random.cc",
         "util/rate_limiter.cc",
         "util/ribbon_config.cc",
+        "util/simple_mixed_compressor.cc",
         "util/slice.cc",
         "util/status.cc",
         "util/stderr_logger.cc",
@@ -317,6 +320,8 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "utilities/persistent_cache/block_cache_tier_metadata.cc",
         "utilities/persistent_cache/persistent_cache_tier.cc",
         "utilities/persistent_cache/volatile_tier_impl.cc",
+        "utilities/secondary_index/secondary_index_iterator.cc",
+        "utilities/secondary_index/simple_secondary_index.cc",
         "utilities/simulator_cache/cache_simulator.cc",
         "utilities/simulator_cache/sim_cache.cc",
         "utilities/table_properties_collectors/compact_for_tiering_collector.cc",
@@ -406,6 +411,7 @@ cpp_library_wrapper(name="rocksdb_tools_lib", srcs=[
         "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
         "tools/db_bench_tool.cc",
         "tools/simulated_hybrid_file_system.cc",
+        "tools/tool_hooks.cc",
         "tools/trace_analyzer_tool.cc",
     ], deps=[":rocksdb_lib"], headers=[], link_whole=False, extra_test_libs=False)
 
@@ -4705,6 +4711,12 @@ cpp_unittest_wrapper(name="compressed_secondary_cache_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="compression_test",
+            srcs=["util/compression_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="configurable_test",
             srcs=["options/configurable_test.cc"],
             deps=[":rocksdb_test_lib"],

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -835,6 +835,7 @@ set(SOURCES
         table/cuckoo/cuckoo_table_builder.cc
         table/cuckoo/cuckoo_table_factory.cc
         table/cuckoo/cuckoo_table_reader.cc
+        table/external_table.cc
         table/format.cc
         table/get_context.cc
         table/iterator.cc
@@ -873,11 +874,13 @@ set(SOURCES
         trace_replay/trace_record.cc
         trace_replay/trace_replay.cc
         util/async_file_reader.cc
+        util/auto_skip_compressor.cc
         util/cleanable.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc
         util/comparator.cc
         util/compression.cc
+        util/simple_mixed_compressor.cc
         util/compression_context_cache.cc
         util/concurrent_task_limiter_impl.cc
         util/crc32c.cc
@@ -939,6 +942,8 @@ set(SOURCES
         utilities/persistent_cache/block_cache_tier_metadata.cc
         utilities/persistent_cache/persistent_cache_tier.cc
         utilities/persistent_cache/volatile_tier_impl.cc
+        utilities/secondary_index/secondary_index_iterator.cc
+        utilities/secondary_index/simple_secondary_index.cc
         utilities/simulator_cache/cache_simulator.cc
         utilities/simulator_cache/sim_cache.cc
         utilities/table_properties_collectors/compact_for_tiering_collector.cc
@@ -1453,6 +1458,7 @@ if(WITH_TESTS)
         util/autovector_test.cc
         util/bloom_test.cc
         util/coding_test.cc
+        util/compression_test.cc
         util/crc32c_test.cc
         util/defer_test.cc
         util/dynamic_bloom_test.cc
@@ -1571,6 +1577,7 @@ if(WITH_BENCHMARK_TOOLS)
   add_executable(db_bench${ARTIFACT_SUFFIX}
     tools/simulated_hybrid_file_system.cc
     tools/db_bench.cc
+    tools/tool_hooks.cc
     tools/db_bench_tool.cc)
   target_link_libraries(db_bench${ARTIFACT_SUFFIX}
     ${ROCKSDB_LIB} ${THIRDPARTY_LIBS})

package/deps/rocksdb/rocksdb/Makefile

@@ -1491,6 +1491,9 @@ db_test: $(OBJ_DIR)/db/db_test.o $(TEST_LIBRARY) $(LIBRARY)
 db_test2: $(OBJ_DIR)/db/db_test2.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+compression_test: $(OBJ_DIR)/util/compression_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 db_logical_block_size_cache_test: $(OBJ_DIR)/db/db_logical_block_size_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2144,14 +2147,14 @@ ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
 BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2
-SNAPPY_VER ?= 1.2.1
-SNAPPY_SHA256 ?= 736aeb64d86566d2236ddffa2865ee5d7a82d26c9016b36218fcc27ea4f09f86
+SNAPPY_VER ?= 1.2.2
+SNAPPY_SHA256 ?= 90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
-LZ4_VER ?= 1.9.4
-LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b
+LZ4_VER ?= 1.10.0
+LZ4_SHA256 ?= 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.5.5
-ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1
+ZSTD_VER ?= 1.5.7
+ZSTD_SHA256 ?= 37d7284556b20954e56e1ca85b80226768902e2edabd3b649e9e72c0c9012ee3
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 CURL_SSL_OPTS ?= --tlsv1
 
@@ -2372,27 +2375,27 @@ rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86
 
 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerx86_64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerppc64le:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerarm64v8:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockers390x:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerriscv64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_riscv64-be --platform linux/riscv64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
@@ -2400,19 +2403,19 @@ rocksdbjavastaticdockerx86musl:
 
 rocksdbjavastaticdockerx86_64musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-musl-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerppc64lemusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-musl-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerarm64v8musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-musl-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockers390xmusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-musl-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
@@ -2467,8 +2470,8 @@ jtest_run:
 jtest: rocksdbjava
 	cd java;$(MAKE) sample test
 
-jpmd: rocksdbjava rocksdbjavageneratepom
-	cd java;$(MAKE) pmd
+jpmd: rocksdbjavageneratepom
+	cd java;$(MAKE) java java_test pmd
 
 jdb_bench:
 	cd java;$(MAKE) db_bench;
@@ -2489,11 +2492,13 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 33f5b67fcaeb8705b04fd1b850873a180dc89aaa
-	@# NOTE: this hack is required for clang in some cases
-	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
-	@# NOTE: this hack is required for gcc in some cases
-	perl -pi -e 's/(__has_include.<experimental.memory_resource>.)/__cpp_rtti && $$1/' third-party/folly/folly/memory/MemoryResource.h
+	cd third-party/folly && git reset --hard d17bf897cb5bbf8f07b122a614e8cffdc38edcde
+	@# Apparently missing include
+	perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
+	@# Warning-as-error on memcpy
+	perl -pi -e 's/memcpy.&ptr/memcpy((void*)&ptr/' third-party/folly/folly/lang/Exception.cpp
+	@# const mismatch
+	perl -pi -e 's/: environ/: (const char**)(environ)/' third-party/folly/folly/Subprocess.cpp
 	@# NOTE: boost source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on boost headers
 	cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py fetch boost
 

package/deps/rocksdb/rocksdb/cache/cache.cc

@@ -118,7 +118,6 @@ Status SecondaryCache::CreateFromString(
       sec_cache = NewCompressedSecondaryCache(sec_cache_opts);
     }
 
-
     if (status.ok()) {
       result->swap(sec_cache);
     }