@nxtedition/rocksdb 15.4.1 → 16.0.0

package/binding.cc

@@ -20,6 +20,7 @@
 
 #include <re2/re2.h>
 
+#include <cmath>
 #include <iostream>
 #include <memory>
 #include <optional>
@@ -550,10 +551,10 @@ class Iterator final : public BaseIterator {
     rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
     NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
 
-    Encoding keyEncoding;
+    Encoding keyEncoding = Encoding::Buffer;
     NAPI_STATUS_THROWS(GetProperty(env, options, "keyEncoding", keyEncoding));
 
-    Encoding valueEncoding;
+    Encoding valueEncoding = Encoding::Buffer;
     NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
 
     rocksdb::ReadOptions readOptions;
@@ -600,7 +601,9 @@ class Iterator final : public BaseIterator {
       std::vector<rocksdb::PinnableSlice> keys;
       std::vector<rocksdb::PinnableSlice> values;
       size_t count = 0;
+      size_t bytes = 0;
       bool finished = false;
+      bool limited = false;
     };
 
     napi_value resourceName;
@@ -614,13 +617,16 @@ class Iterator final : public BaseIterator {
 
           const auto deadline = timeout ? database_->db->GetEnv()->NowMicros() + timeout * 1000 : 0;
 
-          size_t bytesRead = 0;
-          while (state.count < count) {
-            if (bytesRead > highWaterMarkBytes_) {
+          while (true) {
+            if (state.count >= count || state.bytes > highWaterMarkBytes_) {
+              // Batch cap (size/bytes) reached: more data may exist, so this is
+              // "limited", not "finished".
+              state.limited = true;
               break;
             }
 
             if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
+              // Timed out: neither finished nor limited; the caller may retry.
               break;
             }
 
@@ -632,12 +638,19 @@ class Iterator final : public BaseIterator {
 
             ROCKS_STATUS_RETURN(Status());
 
-            if (!Valid() || !Increment()) {
+            if (!Valid()) {
+              // Iterator naturally exhausted.
               state.finished = true;
               break;
             }
 
-            bytesRead += CurrentKey().size() + CurrentValue().size();
+            if (!Increment()) {
+              // Hit the user's `limit` option: terminal, and flag that it was a
+              // limit rather than natural exhaustion.
+              state.finished = true;
+              state.limited = true;
+              break;
+            }
 
             if (keyFilter_ && !re2::RE2::PartialMatch(CurrentKey().ToStringView(), *keyFilter_)) {
               continue;
@@ -650,18 +663,22 @@ class Iterator final : public BaseIterator {
             if (keys_ && values_) {
               rocksdb::PinnableSlice k;
               k.PinSelf(CurrentKey());
+              state.bytes += k.size();
               state.keys.push_back(std::move(k));
 
               rocksdb::PinnableSlice v;
               v.PinSelf(CurrentValue());
+              state.bytes += v.size();
               state.values.push_back(std::move(v));
             } else if (keys_) {
               rocksdb::PinnableSlice k;
               k.PinSelf(CurrentKey());
+              state.bytes += k.size();
               state.keys.push_back(std::move(k));
             } else if (values_) {
               rocksdb::PinnableSlice v;
               v.PinSelf(CurrentValue());
+              state.bytes += v.size();
               state.values.push_back(std::move(v));
             } else {
               assert(false);
@@ -675,6 +692,9 @@ class Iterator final : public BaseIterator {
           napi_value finished;
           NAPI_STATUS_RETURN(napi_get_boolean(env, state.finished, &finished));
 
+          napi_value limited;
+          NAPI_STATUS_RETURN(napi_get_boolean(env, state.limited, &limited));
+
           napi_value rows;
           NAPI_STATUS_RETURN(napi_create_array(env, &rows));
 
@@ -702,6 +722,7 @@ class Iterator final : public BaseIterator {
           NAPI_STATUS_RETURN(napi_create_object(env, result));
           NAPI_STATUS_RETURN(napi_set_named_property(env, *result, "rows", rows));
           NAPI_STATUS_RETURN(napi_set_named_property(env, *result, "finished", finished));
+          NAPI_STATUS_RETURN(napi_set_named_property(env, *result, "limited", limited));
 
           return napi_ok;
         }));
@@ -713,19 +734,27 @@ class Iterator final : public BaseIterator {
     napi_value finished;
     NAPI_STATUS_THROWS(napi_get_boolean(env, false, &finished));
 
+    napi_value limited;
+    NAPI_STATUS_THROWS(napi_get_boolean(env, false, &limited));
+
     napi_value rows;
     NAPI_STATUS_THROWS(napi_create_array(env, &rows));
 
     const auto deadline = timeout ? database_->db->GetEnv()->NowMicros() + timeout * 1000 : 0;
 
     size_t idx = 0;
-    size_t bytesRead = 0;
-    while (idx < count * 2) {
-      if (bytesRead > highWaterMarkBytes_) {
+    size_t bytes = 0;
+    while (true) {
+      if (idx >= static_cast<size_t>(count) * 2 || bytes > highWaterMarkBytes_) {
+        // Batch cap (size/bytes) reached: more data may exist, so this is
+        // "limited", not "finished". (count is uint32_t; widen before *2 so
+        // query()'s UINT32_MAX count doesn't overflow to a small cap.)
+        NAPI_STATUS_THROWS(napi_get_boolean(env, true, &limited));
         break;
       }
 
       if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
+        // Timed out: neither finished nor limited; the caller may retry.
         break;
       }
 
@@ -737,12 +766,19 @@ class Iterator final : public BaseIterator {
 
       ROCKS_STATUS_THROWS_NAPI(Status());
 
-      if (!Valid() || !Increment()) {
+      if (!Valid()) {
+        // Iterator naturally exhausted.
         NAPI_STATUS_THROWS(napi_get_boolean(env, true, &finished));
         break;
       }
 
-      bytesRead += CurrentKey().size() + CurrentValue().size();
+      if (!Increment()) {
+        // Hit the user's `limit` option: terminal, and flag that it was a limit
+        // rather than natural exhaustion.
+        NAPI_STATUS_THROWS(napi_get_boolean(env, true, &finished));
+        NAPI_STATUS_THROWS(napi_get_boolean(env, true, &limited));
+        break;
+      }
 
       if (keyFilter_ && !re2::RE2::PartialMatch(CurrentKey().ToStringView(), *keyFilter_)) {
         continue;
@@ -756,12 +792,15 @@ class Iterator final : public BaseIterator {
       napi_value val;
 
       if (keys_ && values_) {
+        bytes += CurrentKey().size() + CurrentValue().size();
         NAPI_STATUS_THROWS(Convert(env, CurrentKey(), keyEncoding_, key, unsafe_));
         NAPI_STATUS_THROWS(Convert(env, CurrentValue(), valueEncoding_, val, unsafe_));
       } else if (keys_) {
+        bytes += CurrentKey().size();
         NAPI_STATUS_THROWS(Convert(env, CurrentKey(), keyEncoding_, key, unsafe_));
         NAPI_STATUS_THROWS(napi_get_undefined(env, &val));
       } else if (values_) {
+        bytes += CurrentValue().size();
         NAPI_STATUS_THROWS(napi_get_undefined(env, &key));
         NAPI_STATUS_THROWS(Convert(env, CurrentValue(), valueEncoding_, val, unsafe_));
       } else {
@@ -776,6 +815,7 @@ class Iterator final : public BaseIterator {
     NAPI_STATUS_THROWS(napi_create_object(env, &ret));
     NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "rows", rows));
     NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "finished", finished));
+    NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "limited", limited));
     return ret;
   }
 };
@@ -1203,7 +1243,7 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
       GetProperty(env, options, "cacheIndexAndFilterBlocks", tableOptions.cache_index_and_filter_blocks));
   NAPI_STATUS_RETURN(GetProperty(env, options, "cacheIndexAndFilterBlocksWithHighPriority",
                                  tableOptions.cache_index_and_filter_blocks_with_high_priority));
-  NAPI_STATUS_RETURN(GetProperty(env, options, "decouplePartitionedFilters", tableOptions.block_restart_interval));
+  NAPI_STATUS_RETURN(GetProperty(env, options, "decouplePartitionedFilters", tableOptions.decouple_partitioned_filters));
   NAPI_STATUS_RETURN(GetProperty(env, options, "optimizeFiltersForMemory", tableOptions.optimize_filters_for_memory));
   NAPI_STATUS_RETURN(GetProperty(env, options, "maxAutoReadaheadSize", tableOptions.max_auto_readahead_size));
   NAPI_STATUS_RETURN(GetProperty(env, options, "initialAutoReadaheadSize", tableOptions.initial_auto_readahead_size));
@@ -1221,6 +1261,11 @@ NAPI_METHOD(db_get_identity) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
+  if (!database->db) {
+    napi_throw_error(env, "LEVEL_DATABASE_NOT_OPEN", "Database is not open");
+    return NULL;
+  }
+
   std::string identity;
   ROCKS_STATUS_THROWS_NAPI(database->db->GetDbIdentity(identity));
 
@@ -1258,11 +1303,11 @@ NAPI_METHOD(db_open) {
 
     uint32_t walTTL = 0;
     NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
-    dbOptions.WAL_ttl_seconds = walTTL / 1e3;
+    dbOptions.WAL_ttl_seconds = static_cast<uint32_t>(std::ceil(walTTL / 1e3));
 
     uint32_t walSizeLimit = 0;
     NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
-    dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
+    dbOptions.WAL_size_limit_MB = static_cast<uint32_t>(std::ceil(walSizeLimit / 1e6));
 
     NAPI_STATUS_THROWS(GetProperty(env, options, "maxTotalWalSize", dbOptions.max_total_wal_size));
 
@@ -1392,13 +1437,10 @@ NAPI_METHOD(db_open) {
         [=](auto& handles) {
           assert(!database->db);
 
-          rocksdb::DB* db = nullptr;
-
-          const auto status = descriptors.empty()
-                                  ? rocksdb::DB::Open(dbOptions, database->location, &db)
-                                  : rocksdb::DB::Open(dbOptions, database->location, descriptors, &handles, &db);
-
-          database->db.reset(db);
+          const auto status =
+              descriptors.empty()
+                  ? rocksdb::DB::Open(dbOptions, database->location, &database->db)
+                  : rocksdb::DB::Open(dbOptions, database->location, descriptors, &handles, &database->db);
 
           return status;
         },
@@ -1724,6 +1766,11 @@ NAPI_METHOD(db_get_latest_sequence) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
+  if (!database->db) {
+    napi_throw_error(env, "LEVEL_DATABASE_NOT_OPEN", "Database is not open");
+    return NULL;
+  }
+
   const auto seq = database->db->GetLatestSequenceNumber();
 
   napi_value result;
@@ -1907,7 +1954,7 @@ NAPI_METHOD(batch_put) {
 }
 
 NAPI_METHOD(batch_put_log_data) {
-  NAPI_ARGV(3);
+  NAPI_ARGV(2);
 
   rocksdb::WriteBatch* batch;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&batch)));

package/deps/rocksdb/rocksdb/.clang-tidy

@@ -0,0 +1,86 @@
+# When making changes, verify the output of:
+#   clang-tidy -list-checks
+---
+Checks: "-*,\
+  bugprone-argument-comment,\
+  bugprone-dangling-handle,\
+  bugprone-fold-init-type,\
+  bugprone-forward-declaration-namespace,\
+  bugprone-forwarding-reference-overload,\
+  bugprone-shadow,\
+  bugprone-sizeof-*,\
+  bugprone-string-constructor,\
+  bugprone-undefined-memory-manipulation,\
+  bugprone-unused-return-value,\
+  bugprone-use-after-move,\
+  cert-env33-c,\
+  cert-err58-cpp,\
+  cert-msc30-c,\
+  cert-msc50-cpp,\
+  clang-analyzer-*,\
+  clang-diagnostic-*,\
+  -clang-diagnostic-missing-designated-field-initializers,\
+  concurrency-mt-unsafe,\
+  cppcoreguidelines-avoid-non-const-global-variables,\
+  cppcoreguidelines-missing-std-forward,\
+  cppcoreguidelines-pro-type-member-init,\
+  cppcoreguidelines-special-member-functions,\
+  cppcoreguidelines-virtual-class-destructor,\
+  google-build-using-namespace,\
+  google-explicit-constructor,\
+  google-readability-avoid-underscore-in-googletest-name,\
+  misc-definitions-in-headers,\
+  misc-redundant-expression,\
+  modernize-make-shared,\
+  modernize-use-emplace,\
+  modernize-use-noexcept,\
+  modernize-use-override,\
+  modernize-use-using,\
+  performance-faster-string-find,\
+  performance-for-range-copy,\
+  performance-implicit-conversion-in-loop,\
+  performance-inefficient-algorithm,\
+  performance-inefficient-string-concatenation,\
+  performance-inefficient-vector-operation,\
+  performance-move-const-arg,\
+  performance-move-constructor-init,\
+  performance-no-automatic-move,\
+  performance-no-int-to-ptr,\
+  performance-noexcept-move-constructor,\
+  performance-noexcept-swap,\
+  performance-trivially-destructible,\
+  performance-type-promotion-in-math-fn,\
+  performance-unnecessary-copy-initialization,\
+  performance-unnecessary-value-param,\
+  readability-braces-around-statements,\
+  readability-duplicate-include,\
+  readability-isolate-declaration,\
+  readability-operators-representation,\
+  readability-redundant-string-init"
+
+WarningsAsErrors: "bugprone-use-after-move"
+
+CheckOptions:
+- key: bugprone-easily-swappable-parameters.MinimumLength
+  value: 4
+- key: cppcoreguidelines-avoid-non-const-global-variables.AllowThreadLocal
+  value: true
+- key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
+  value: true
+- key: cppcoreguidelines-special-member-functions.AllowImplicitlyDeletedCopyOrMove
+  value: true
+- key: modernize-use-using.IgnoreExternC
+  value: true
+- key: performance-move-const-arg.CheckTriviallyCopyableMove
+  value: false
+- key: performance-unnecessary-value-param.AllowedTypes
+  value: '[Pp]ointer$;[Pp]tr$;[Rr]ef(erence)?$'
+- key: performance-unnecessary-copy-initialization.AllowedTypes
+  value: '[Pp]ointer$;[Pp]tr$;[Rr]ef(erence)?$'
+- key: readability-operators-representation.BinaryOperators
+  value: '&&;&=;&;|;~;!;!=;||;|=;^;^='
+- key: readability-redundant-string-init.StringNames
+  value: '::std::basic_string'
+- key: readability-named-parameter.InsertPlainNamesInForwardDecls
+  value: true
+...

package/deps/rocksdb/rocksdb/BUCK

@@ -106,6 +106,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/version_edit.cc",
         "db/version_edit_handler.cc",
         "db/version_set.cc",
+        "db/version_util.cc",
         "db/wal_edit.cc",
         "db/wal_manager.cc",
         "db/wide/wide_column_serialization.cc",
@@ -207,6 +208,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "table/block_based/hash_index_reader.cc",
         "table/block_based/index_builder.cc",
         "table/block_based/index_reader_common.cc",
+        "table/block_based/multi_scan_index_iterator.cc",
         "table/block_based/parsed_full_filter_block.cc",
         "table/block_based/partitioned_filter_block.cc",
         "table/block_based/partitioned_index_iterator.cc",
@@ -268,6 +270,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "util/dynamic_bloom.cc",
         "util/file_checksum_helper.cc",
         "util/hash.cc",
+        "util/io_dispatcher_imp.cc",
         "util/murmurhash.cc",
         "util/random.cc",
         "util/rate_limiter.cc",
@@ -361,6 +364,9 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "utilities/transactions/write_prepared_txn_db.cc",
         "utilities/transactions/write_unprepared_txn.cc",
         "utilities/transactions/write_unprepared_txn_db.cc",
+        "utilities/trie_index/bitvector.cc",
+        "utilities/trie_index/louds_trie.cc",
+        "utilities/trie_index/trie_index_factory.cc",
         "utilities/ttl/db_ttl_impl.cc",
         "utilities/types_util.cc",
         "utilities/wal_filter.cc",
@@ -4822,6 +4828,12 @@ cpp_unittest_wrapper(name="db_clip_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="db_compaction_abort_test",
+            srcs=["db/db_compaction_abort_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="db_compaction_filter_test",
             srcs=["db/db_compaction_filter_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5130,6 +5142,12 @@ cpp_unittest_wrapper(name="faiss_ivf_index_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="fault_injection_fs_test",
+            srcs=["utilities/fault_injection_fs_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="fault_injection_test",
             srcs=["db/fault_injection_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5214,6 +5232,12 @@ cpp_unittest_wrapper(name="interval_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="io_dispatcher_test",
+            srcs=["util/io_dispatcher_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="io_posix_test",
             srcs=["env/io_posix_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5610,6 +5634,18 @@ cpp_unittest_wrapper(name="transaction_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="trie_index_db_test",
+            srcs=["utilities/trie_index/trie_index_db_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
+cpp_unittest_wrapper(name="trie_index_test",
+            srcs=["utilities/trie_index/trie_index_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="ttl_test",
             srcs=["utilities/ttl/ttl_test.cc"],
             deps=[":rocksdb_test_lib"],
@@ -5712,6 +5748,12 @@ cpp_unittest_wrapper(name="write_controller_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="write_prepared_transaction_seqno_test",
+            srcs=["utilities/transactions/write_prepared_transaction_seqno_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="write_prepared_transaction_test",
             srcs=["utilities/transactions/write_prepared_transaction_test.cc"],
             deps=[":rocksdb_test_lib"],

package/deps/rocksdb/rocksdb/CMakeLists.txt

@@ -766,6 +766,7 @@ set(SOURCES
         db/version_edit.cc
         db/version_edit_handler.cc
         db/version_set.cc
+        db/version_util.cc
         db/wal_edit.cc
         db/wal_manager.cc
         db/wide/wide_column_serialization.cc
@@ -842,6 +843,7 @@ set(SOURCES
         table/block_based/block_based_table_builder.cc
         table/block_based/block_based_table_factory.cc
         table/block_based/block_based_table_iterator.cc
+        table/block_based/multi_scan_index_iterator.cc
         table/block_based/block_based_table_reader.cc
         table/block_based/block_builder.cc
         table/block_based/block_cache.cc
@@ -918,6 +920,7 @@ set(SOURCES
         util/data_structure.cc
         util/dynamic_bloom.cc
         util/hash.cc
+        util/io_dispatcher_imp.cc
         util/murmurhash.cc
         util/random.cc
         util/rate_limiter.cc
@@ -998,6 +1001,9 @@ set(SOURCES
         utilities/transactions/write_prepared_txn_db.cc
         utilities/transactions/write_unprepared_txn.cc
         utilities/transactions/write_unprepared_txn_db.cc
+        utilities/trie_index/bitvector.cc
+        utilities/trie_index/louds_trie.cc
+        utilities/trie_index/trie_index_factory.cc
         utilities/types_util.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/wal_filter.cc
@@ -1381,6 +1387,7 @@ if(WITH_TESTS)
         db/db_bloom_filter_test.cc
         db/db_compaction_filter_test.cc
         db/db_compaction_test.cc
+        db/db_compaction_abort_test.cc
         db/db_clip_test.cc
         db/db_dynamic_level_test.cc
         db/db_encryption_test.cc
@@ -1529,6 +1536,7 @@ if(WITH_TESTS)
         utilities/cassandra/cassandra_serialize_test.cc
         utilities/checkpoint/checkpoint_test.cc
         utilities/env_timed_test.cc
+        utilities/fault_injection_fs_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
         utilities/object_registry_test.cc
@@ -1546,9 +1554,12 @@ if(WITH_TESTS)
         utilities/transactions/lock/point/point_lock_manager_stress_test.cc
         utilities/transactions/write_committed_transaction_ts_test.cc
         utilities/transactions/write_prepared_transaction_test.cc
+        utilities/transactions/write_prepared_transaction_seqno_test.cc
         utilities/transactions/write_unprepared_transaction_test.cc
         utilities/transactions/lock/range/range_locking_test.cc
         utilities/transactions/timestamped_snapshot_test.cc
+        utilities/trie_index/trie_index_db_test.cc
+        utilities/trie_index/trie_index_test.cc
         utilities/ttl/ttl_test.cc
         utilities/types_util_test.cc
         utilities/util_merge_operators_test.cc