@nxtedition/rocksdb 15.1.4 → 15.2.0

package/.claude/settings.local.json

@@ -9,7 +9,12 @@
       "Bash(grep:*)",
       "Bash(while read f)",
       "Bash(do basename \"$f\")",
-      "Bash(done)"
+      "Bash(done)",
+      "Bash(ls:*)",
+      "Bash(git submodule update:*)",
+      "Bash(git submodule:*)",
+      "Bash(yarn build:*)",
+      "Bash(yarn rebuild:*)"
     ]
   }
 }

package/binding.cc

@@ -18,6 +18,8 @@
 #include <rocksdb/table.h>
 #include <rocksdb/write_batch.h>
 
+#include <re2/re2.h>
+
 #include <iostream>
 #include <memory>
 #include <optional>
@@ -411,6 +413,9 @@ class Iterator final : public BaseIterator {
   bool first_ = true;
   const Encoding keyEncoding_;
   const Encoding valueEncoding_;
+  std::optional<re2::RE2> keyFilter_;
+  std::optional<re2::RE2> valueFilter_;
+  const bool unsafe_;
 
  public:
   Iterator(Database* database,
@@ -424,15 +429,32 @@ class Iterator final : public BaseIterator {
            const std::optional<std::string>& gt,
            const std::optional<std::string>& gte,
            const size_t highWaterMarkBytes,
+           std::optional<std::string> keyFilter = std::nullopt,
+           std::optional<std::string> valueFilter = std::nullopt,
            Encoding keyEncoding = Encoding::Invalid,
            Encoding valueEncoding = Encoding::Invalid,
+           const bool unsafe = false,
            rocksdb::ReadOptions readOptions = {})
       : BaseIterator(database, column, reverse, lt, lte, gt, gte, limit, readOptions),
         keys_(keys),
         values_(values),
         highWaterMarkBytes_(highWaterMarkBytes),
         keyEncoding_(keyEncoding),
-        valueEncoding_(valueEncoding) {}
+        valueEncoding_(valueEncoding),
+        unsafe_(unsafe) {
+    if (keyFilter && keyFilter->size() > 0) {
+      keyFilter_.emplace(*keyFilter);
+      if (!keyFilter_->ok()) {
+        throw std::invalid_argument("Invalid key filter regex");
+      }
+    }
+    if (valueFilter && valueFilter->size() > 0) {
+      valueFilter_.emplace(*valueFilter);
+      if (!valueFilter_->ok()) {
+        throw std::invalid_argument("Invalid value filter regex");
+      }
+    }
+  }
 
   void Seek(const rocksdb::Slice& target) override {
     first_ = true;
@@ -443,6 +465,9 @@ class Iterator final : public BaseIterator {
     Database* database;
     NAPI_STATUS_THROWS(napi_get_value_external(env, db, reinterpret_cast<void**>(&database)));
 
+    bool unsafe = false;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "unsafe", unsafe));
+
     bool reverse = false;
     NAPI_STATUS_THROWS(GetProperty(env, options, "reverse", reverse));
 
@@ -470,6 +495,12 @@ class Iterator final : public BaseIterator {
     std::optional<std::string> gte;
     NAPI_STATUS_THROWS(GetProperty(env, options, "gte", gte));
 
+    std::optional<std::string> keyFilter;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "keyFilter", keyFilter));
+
+    std::optional<std::string> valueFilter;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "valueFilter", valueFilter));
+
     rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
     NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
 
@@ -518,7 +549,8 @@ class Iterator final : public BaseIterator {
     //   : std::chrono::microseconds::zero();
 
     return std::make_unique<Iterator>(database, column, reverse, keys, values, limit, lt, lte, gt, gte,
-                                      highWaterMarkBytes, keyEncoding, valueEncoding, readOptions);
+                                      highWaterMarkBytes, keyFilter, valueFilter, keyEncoding, valueEncoding,
+                                      unsafe, readOptions);
   }
 
   napi_value nextv(napi_env env, uint32_t count, uint32_t timeout, napi_value callback) {
@@ -538,7 +570,15 @@ class Iterator final : public BaseIterator {
           const auto deadline = timeout ? database_->db->GetEnv()->NowMicros() + timeout * 1000 : 0;
 
           size_t bytesRead = 0;
-          for (int n = 0; n < count; n++) {
+          while (state.count < count) {
+            if (bytesRead > highWaterMarkBytes_) {
+              break;
+            }
+
+            if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
+              break;
+            }
+
             if (!first_) {
               Next();
             } else {
@@ -552,6 +592,16 @@ class Iterator final : public BaseIterator {
               break;
             }
 
+            bytesRead += CurrentKey().size() + CurrentValue().size();
+
+            if (keyFilter_ && !re2::RE2::PartialMatch(CurrentKey().ToStringView(), *keyFilter_)) {
+              continue;
+            }
+
+            if (valueFilter_ && !re2::RE2::PartialMatch(CurrentValue().ToStringView(), *valueFilter_)) {
+              continue;
+            }
+
             if (keys_ && values_) {
               rocksdb::PinnableSlice k;
               k.PinSelf(CurrentKey());
@@ -572,15 +622,6 @@ class Iterator final : public BaseIterator {
               assert(false);
             }
             state.count += 1;
-
-            bytesRead += CurrentKey().size() + CurrentValue().size();
-            if (bytesRead > highWaterMarkBytes_) {
-              break;
-            }
-
-            if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
-              break;
-            }
           }
 
           return rocksdb::Status::OK();
@@ -599,14 +640,14 @@ class Iterator final : public BaseIterator {
             napi_value val;
 
             if (keys_ && values_) {
-              NAPI_STATUS_RETURN(Convert(env, std::move(state.keys[n]), keyEncoding_, key));
-              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val));
+              NAPI_STATUS_RETURN(Convert(env, std::move(state.keys[n]), keyEncoding_, key, unsafe_));
+              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val, unsafe_));
             } else if (keys_) {
-              NAPI_STATUS_RETURN(Convert(env, std::move(state.keys[n]), keyEncoding_, key));
+              NAPI_STATUS_RETURN(Convert(env, std::move(state.keys[n]), keyEncoding_, key, unsafe_));
               NAPI_STATUS_RETURN(napi_get_undefined(env, &val));
             } else if (values_) {
               NAPI_STATUS_RETURN(napi_get_undefined(env, &key));
-              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val));
+              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val, unsafe_));
             } else {
               assert(false);
             }
@@ -636,7 +677,15 @@ class Iterator final : public BaseIterator {
 
     size_t idx = 0;
     size_t bytesRead = 0;
-    for (int n = 0; n < count; n++) {
+    while (idx < count * 2) {
+      if (bytesRead > highWaterMarkBytes_) {
+        break;
+      }
+
+      if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
+        break;
+      }
+
       if (!first_) {
         Next();
       } else {
@@ -650,37 +699,38 @@ class Iterator final : public BaseIterator {
         break;
       }
 
+      bytesRead += CurrentKey().size() + CurrentValue().size();
+
+      if (keyFilter_ && !re2::RE2::PartialMatch(CurrentKey().ToStringView(), *keyFilter_)) {
+        continue;
+      }
+
+      if (valueFilter_ && !re2::RE2::PartialMatch(CurrentValue().ToStringView(), *valueFilter_)) {
+        continue;
+      }
+
       napi_value key;
       napi_value val;
 
       if (keys_ && values_) {
         const auto k = CurrentKey();
         const auto v = CurrentValue();
-        NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key));
-        NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val));
+        NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key, unsafe_));
+        NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val, unsafe_));
       } else if (keys_) {
         const auto k = CurrentKey();
-        NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key));
+        NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key, unsafe_));
         NAPI_STATUS_THROWS(napi_get_undefined(env, &val));
       } else if (values_) {
         const auto v = CurrentValue();
         NAPI_STATUS_THROWS(napi_get_undefined(env, &key));
-        NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val));
+        NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val, unsafe_));
       } else {
         assert(false);
       }
 
       NAPI_STATUS_THROWS(napi_set_element(env, rows, idx++, key));
       NAPI_STATUS_THROWS(napi_set_element(env, rows, idx++, val));
-
-      bytesRead += CurrentKey().size() + CurrentValue().size();
-      if (bytesRead > highWaterMarkBytes_) {
-        break;
-      }
-
-      if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
-        break;
-      }
     }
 
     napi_value ret;
@@ -1381,11 +1431,7 @@ NAPI_METHOD(db_get_many_sync) {
       NAPI_STATUS_THROWS(napi_get_null(env, &row));
     } else {
       ROCKS_STATUS_THROWS_NAPI(statuses[n]);
-      if (unsafe) {
-        NAPI_STATUS_THROWS(ConvertUnsafe(env, std::move(values[n]), valueEncoding, row));
-      } else {
-        NAPI_STATUS_THROWS(Convert(env, std::move(values[n]), valueEncoding, row));
-      }
+      NAPI_STATUS_THROWS(Convert(env, std::move(values[n]), valueEncoding, row, unsafe));
     }
     NAPI_STATUS_THROWS(napi_set_element(env, rows, n, row));
   }
@@ -1470,11 +1516,7 @@ NAPI_METHOD(db_get_many) {
             NAPI_STATUS_RETURN(napi_get_null(env, &row));
           } else {
             ROCKS_STATUS_RETURN_NAPI(state.statuses[n]);
-            if (unsafe) {
-              NAPI_STATUS_RETURN(ConvertUnsafe(env, std::move(state.values[n]), valueEncoding, row));
-            } else {
-              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding, row));
-            }
+            NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding, row, unsafe));
           }
           NAPI_STATUS_RETURN(napi_set_element(env, argv[1], n, row));
         }
@@ -1653,14 +1695,12 @@ NAPI_METHOD(iterator_init_sync) {
 }
 
 NAPI_METHOD(iterator_seek) {
-  NAPI_ARGV(2);
+  NAPI_ARGV(3);
 
   try {
     Iterator* iterator;
     NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-    rocksdb::PinnableSlice target;
-
     struct State {
       rocksdb::PinnableSlice target;
     } state;
@@ -1674,7 +1714,7 @@ NAPI_METHOD(iterator_seek) {
         [=](auto& state) {
           iterator->Seek(state.target);
           return iterator->Status();
-         },
+        },
         [](auto& state, auto env, auto& argv) { return napi_ok; });
   } catch (const std::exception& e) {
     napi_throw_error(env, nullptr, e.what());

package/binding.gyp

@@ -55,8 +55,108 @@
                 ],
             ],
             "dependencies": ["<(module_root_dir)/deps/rocksdb/rocksdb.gyp:rocksdb"],
-            "include_dirs": ["<!(node -e \"require('napi-macros')\")"],
-            "sources": ["binding.cc"],
+            "include_dirs": [
+              "<!(node -e \"require('napi-macros')\")",
+              "deps/re2",
+              "deps/abseil-cpp"
+            ],
+            "sources": [
+              "binding.cc",
+              "deps/re2/re2/bitmap256.cc",
+              "deps/re2/re2/bitstate.cc",
+              "deps/re2/re2/compile.cc",
+              "deps/re2/re2/dfa.cc",
+              "deps/re2/re2/filtered_re2.cc",
+              "deps/re2/re2/mimics_pcre.cc",
+              "deps/re2/re2/nfa.cc",
+              "deps/re2/re2/onepass.cc",
+              "deps/re2/re2/parse.cc",
+              "deps/re2/re2/perl_groups.cc",
+              "deps/re2/re2/prefilter.cc",
+              "deps/re2/re2/prefilter_tree.cc",
+              "deps/re2/re2/prog.cc",
+              "deps/re2/re2/re2.cc",
+              "deps/re2/re2/regexp.cc",
+              "deps/re2/re2/set.cc",
+              "deps/re2/re2/simplify.cc",
+              "deps/re2/re2/tostring.cc",
+              "deps/re2/re2/unicode_casefold.cc",
+              "deps/re2/re2/unicode_groups.cc",
+              "deps/re2/util/pcre.cc",
+              "deps/re2/util/rune.cc",
+              "deps/re2/util/strutil.cc",
+              "deps/abseil-cpp/absl/base/internal/cycleclock.cc",
+              "deps/abseil-cpp/absl/base/internal/low_level_alloc.cc",
+              "deps/abseil-cpp/absl/base/internal/raw_logging.cc",
+              "deps/abseil-cpp/absl/base/internal/spinlock.cc",
+              "deps/abseil-cpp/absl/base/internal/spinlock_wait.cc",
+              "deps/abseil-cpp/absl/base/internal/strerror.cc",
+              "deps/abseil-cpp/absl/base/internal/sysinfo.cc",
+              "deps/abseil-cpp/absl/base/internal/thread_identity.cc",
+              "deps/abseil-cpp/absl/base/internal/throw_delegate.cc",
+              "deps/abseil-cpp/absl/base/internal/unscaledcycleclock.cc",
+              "deps/abseil-cpp/absl/debugging/internal/demangle.cc",
+              "deps/abseil-cpp/absl/container/internal/raw_hash_set.cc",
+              "deps/abseil-cpp/absl/debugging/internal/address_is_readable.cc",
+              "deps/abseil-cpp/absl/debugging/internal/elf_mem_image.cc",
+              "deps/abseil-cpp/absl/debugging/internal/examine_stack.cc",
+              "deps/abseil-cpp/absl/debugging/internal/vdso_support.cc",
+              "deps/abseil-cpp/absl/debugging/stacktrace.cc",
+              "deps/abseil-cpp/absl/debugging/symbolize.cc",
+              "deps/abseil-cpp/absl/flags/commandlineflag.cc",
+              "deps/abseil-cpp/absl/flags/internal/commandlineflag.cc",
+              "deps/abseil-cpp/absl/flags/internal/flag.cc",
+              "deps/abseil-cpp/absl/flags/internal/private_handle_accessor.cc",
+              "deps/abseil-cpp/absl/flags/internal/program_name.cc",
+              "deps/abseil-cpp/absl/flags/marshalling.cc",
+              "deps/abseil-cpp/absl/flags/reflection.cc",
+              "deps/abseil-cpp/absl/flags/usage_config.cc",
+              "deps/abseil-cpp/absl/hash/internal/city.cc",
+              "deps/abseil-cpp/absl/hash/internal/hash.cc",
+              "deps/abseil-cpp/absl/hash/internal/print_hash_of.cc",
+              "deps/abseil-cpp/absl/log/internal/globals.cc",
+              "deps/abseil-cpp/absl/log/internal/log_format.cc",
+              "deps/abseil-cpp/absl/log/internal/log_message.cc",
+              "deps/abseil-cpp/absl/log/internal/log_sink_set.cc",
+              "deps/abseil-cpp/absl/log/internal/nullguard.cc",
+              "deps/abseil-cpp/absl/log/internal/proto.cc",
+              "deps/abseil-cpp/absl/log/globals.cc",
+              "deps/abseil-cpp/absl/log/log_sink.cc",
+              "deps/abseil-cpp/absl/numeric/int128.cc",
+              "deps/abseil-cpp/absl/strings/ascii.cc",
+              "deps/abseil-cpp/absl/strings/charconv.cc",
+              "deps/abseil-cpp/absl/strings/internal/charconv_bigint.cc",
+              "deps/abseil-cpp/absl/strings/internal/charconv_parse.cc",
+              "deps/abseil-cpp/absl/strings/internal/memutil.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/arg.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/bind.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/extension.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/output.cc",
+              "deps/abseil-cpp/absl/strings/internal/str_format/parser.cc",
+              "deps/abseil-cpp/absl/strings/match.cc",
+              "deps/abseil-cpp/absl/strings/numbers.cc",
+              "deps/abseil-cpp/absl/strings/str_cat.cc",
+              "deps/abseil-cpp/absl/strings/str_split.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/graphcycles.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/futex_waiter.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/kernel_timeout.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc",
+              "deps/abseil-cpp/absl/synchronization/internal/waiter_base.cc",
+              "deps/abseil-cpp/absl/synchronization/mutex.cc",
+              "deps/abseil-cpp/absl/time/clock.cc",
+              "deps/abseil-cpp/absl/time/duration.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_fixed.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_if.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_impl.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/time_zone_posix.cc",
+              "deps/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc",
+              "deps/abseil-cpp/absl/time/time.cc",
+          ],
         }
     ],
 }

package/deps/abseil-cpp/BUILD.bazel

@@ -0,0 +1,35 @@
+# Copyright 2020 The Abseil Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+# Expose license for external usage through bazel.
+exports_files([
+    "AUTHORS",
+    "LICENSE",
+])
+
+# For building with clang-cl.
+# https://bazel.build/configure/windows#clang
+platform(
+    name = "x64_windows-clang-cl",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+)