@nxtedition/rocksdb 13.0.3 → 13.1.1

package/binding.cc

@@ -431,7 +431,8 @@ class Iterator final : public BaseIterator {
         values_(values),
         highWaterMarkBytes_(highWaterMarkBytes),
         keyEncoding_(keyEncoding),
-        valueEncoding_(valueEncoding) {}
+        valueEncoding_(valueEncoding) {
+  }
 
   void Seek(const rocksdb::Slice& target) override {
     first_ = true;
@@ -504,18 +505,18 @@ class Iterator final : public BaseIterator {
     readOptions.ignore_range_deletions = false;
     NAPI_STATUS_THROWS(GetProperty(env, options, "ignoreRangeDeletions", readOptions.ignore_range_deletions));
 
-    uint32_t timeout = 0;
-    NAPI_STATUS_THROWS(GetProperty(env, options, "timeout", timeout));
+    // uint32_t timeout = 0;
+    // NAPI_STATUS_THROWS(GetProperty(env, options, "timeout", timeout));
 
-    readOptions.deadline = timeout
-      ? std::chrono::microseconds(database->db->GetEnv()->NowMicros() + timeout * 1000)
-      : std::chrono::microseconds::zero();
+    // readOptions.deadline = timeout
+    //   ? std::chrono::microseconds(database->db->GetEnv()->NowMicros() + timeout * 1000)
+    //   : std::chrono::microseconds::zero();
 
     return std::make_unique<Iterator>(database, column, reverse, keys, values, limit, lt, lte, gt, gte,
                                       highWaterMarkBytes, keyEncoding, valueEncoding, readOptions);
   }
 
-  napi_value nextv(napi_env env, uint32_t count, napi_value callback) {
+  napi_value nextv(napi_env env, uint32_t count, uint32_t timeout, napi_value callback) {
     struct State {
       std::vector<rocksdb::PinnableSlice> keys;
       std::vector<rocksdb::PinnableSlice> values;
@@ -528,18 +529,18 @@ class Iterator final : public BaseIterator {
           state.keys.reserve(count);
           state.values.reserve(count);
 
+          const auto deadline = timeout
+            ? database_->db->GetEnv()->NowMicros() + timeout * 1000
+            : 0;
+
           size_t bytesRead = 0;
-          while (true) {
+          for (int n = 0; n < count; n++) {
             if (!first_) {
               Next();
             } else {
               first_ = false;
             }
 
-            if (Status().IsTimedOut()) {
-              break;
-            }
-
             ROCKS_STATUS_RETURN(Status());
 
             if (!Valid() || !Increment()) {
@@ -550,30 +551,30 @@ class Iterator final : public BaseIterator {
             if (keys_ && values_) {
               rocksdb::PinnableSlice k;
               k.PinSelf(CurrentKey());
-              bytesRead += k.size();
               state.keys.push_back(std::move(k));
 
               rocksdb::PinnableSlice v;
               v.PinSelf(CurrentValue());
-              bytesRead += v.size();
               state.values.push_back(std::move(v));
             } else if (keys_) {
               rocksdb::PinnableSlice k;
               k.PinSelf(CurrentKey());
-              bytesRead += k.size();
               state.keys.push_back(std::move(k));
             } else if (values_) {
               rocksdb::PinnableSlice v;
               v.PinSelf(CurrentValue());
-              bytesRead += v.size();
               state.values.push_back(std::move(v));
             } else {
               assert(false);
             }
-
             state.count += 1;
 
-            if (bytesRead > highWaterMarkBytes_ || state.count >= count) {
+            bytesRead += CurrentKey().size() + CurrentValue().size();
+            if (bytesRead > highWaterMarkBytes_) {
+              break;
+            }
+
+            if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
               break;
             }
           }
@@ -620,26 +621,26 @@ class Iterator final : public BaseIterator {
     return 0;
   }
 
-  napi_value nextv(napi_env env, uint32_t count) {
+  napi_value nextv(napi_env env, uint32_t count, uint32_t timeout = 0) {
     napi_value finished;
     NAPI_STATUS_THROWS(napi_get_boolean(env, false, &finished));
 
     napi_value rows;
     NAPI_STATUS_THROWS(napi_create_array(env, &rows));
 
+    const auto deadline = timeout
+      ? database_->db->GetEnv()->NowMicros() + timeout * 1000
+      : 0;
+
     size_t idx = 0;
     size_t bytesRead = 0;
-    while (true) {
+    for (int n = 0; n < count; n++) {
       if (!first_) {
         Next();
       } else {
         first_ = false;
       }
 
-      if (Status().IsTimedOut()) {
-        break;
-      }
-
       ROCKS_STATUS_THROWS_NAPI(Status());
 
       if (!Valid() || !Increment()) {
@@ -655,17 +656,14 @@ class Iterator final : public BaseIterator {
         const auto v = CurrentValue();
         NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key));
         NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val));
-        bytesRead += k.size() + v.size();
       } else if (keys_) {
         const auto k = CurrentKey();
         NAPI_STATUS_THROWS(Convert(env, &k, keyEncoding_, key));
         NAPI_STATUS_THROWS(napi_get_undefined(env, &val));
-        bytesRead += k.size();
       } else if (values_) {
         const auto v = CurrentValue();
         NAPI_STATUS_THROWS(napi_get_undefined(env, &key));
         NAPI_STATUS_THROWS(Convert(env, &v, valueEncoding_, val));
-        bytesRead += v.size();
       } else {
         assert(false);
       }
@@ -673,7 +671,12 @@ class Iterator final : public BaseIterator {
       NAPI_STATUS_THROWS(napi_set_element(env, rows, idx++, key));
       NAPI_STATUS_THROWS(napi_set_element(env, rows, idx++, val));
 
-      if (bytesRead > highWaterMarkBytes_ || idx / 2 >= count) {
+      bytesRead += CurrentKey().size() + CurrentValue().size();
+      if (bytesRead > highWaterMarkBytes_) {
+        break;
+      }
+
+      if (deadline > 0 && database_->db->GetEnv()->NowMicros() > deadline) {
         break;
       }
     }
@@ -776,7 +779,12 @@ NAPI_METHOD(db_get_location) {
 NAPI_METHOD(db_query) {
   NAPI_ARGV(2);
 
-  return Iterator::create(env, argv[0], argv[1])->nextv(env, std::numeric_limits<uint32_t>::max());
+  try {
+    return Iterator::create(env, argv[0], argv[1])->nextv(env, std::numeric_limits<uint32_t>::max());
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 }
 
 template <typename T, typename U>
@@ -786,50 +794,50 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   uint64_t memtable_memory_budget = 256 * 1024 * 1024;
   NAPI_STATUS_RETURN(GetProperty(env, options, "memtableMemoryBudget", memtable_memory_budget));
 
-  std::optional<std::string> compactionOpt;
-  NAPI_STATUS_RETURN(GetProperty(env, options, "compaction", compactionOpt));
-  if (compactionOpt) {
-    if (*compactionOpt == "universal") {
-      columnOptions.write_buffer_size = memtable_memory_budget / 4;
-      // merge two memtables when flushing to L0
-      columnOptions.min_write_buffer_number_to_merge = 2;
-      // this means we'll use 50% extra memory in the worst case, but will reduce
-      // write stalls.
-      columnOptions.max_write_buffer_number = 6;
-      // universal style compaction
-      columnOptions.compaction_style = rocksdb::kCompactionStyleUniversal;
-      columnOptions.compaction_options_universal.compression_size_percent = 80;
-    } else if (*compactionOpt == "level") {
-      columnOptions.write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
-      // merge two memtables when flushing to L0
-      columnOptions.min_write_buffer_number_to_merge = 2;
-      // this means we'll use 50% extra memory in the worst case, but will reduce
-      // write stalls.
-      columnOptions.max_write_buffer_number = 6;
-      // start flushing L0->L1 as soon as possible. each file on level0 is
-      // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
-      // memtable_memory_budget.
-      columnOptions.level0_file_num_compaction_trigger = 2;
-      // doesn't really matter much, but we don't want to create too many files
-      columnOptions.target_file_size_base = memtable_memory_budget / 8;
-      // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
-      columnOptions.max_bytes_for_level_base = memtable_memory_budget;
-
-      // level style compaction
-      columnOptions.compaction_style = rocksdb::kCompactionStyleLevel;
-
-      // only compress levels >= 2
-      columnOptions.compression_per_level.resize(columnOptions.num_levels);
-      for (int i = 0; i < columnOptions.num_levels; ++i) {
-        if (i < 2) {
-          columnOptions.compression_per_level[i] = rocksdb::kNoCompression;
-        } else {
-          columnOptions.compression_per_level[i] = rocksdb::kZSTD;
-        }
+  std::string compaction;
+  NAPI_STATUS_RETURN(GetProperty(env, options, "compaction", compaction));
+  if (compaction == "") {
+    // Do nothing...
+  } else if (compaction == "universal") {
+    columnOptions.write_buffer_size = memtable_memory_budget / 4;
+    // merge two memtables when flushing to L0
+    columnOptions.min_write_buffer_number_to_merge = 2;
+    // this means we'll use 50% extra memory in the worst case, but will reduce
+    // write stalls.
+    columnOptions.max_write_buffer_number = 6;
+    // universal style compaction
+    columnOptions.compaction_style = rocksdb::kCompactionStyleUniversal;
+    columnOptions.compaction_options_universal.compression_size_percent = 80;
+  } else if (compaction == "level") {
+    columnOptions.write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+    // merge two memtables when flushing to L0
+    columnOptions.min_write_buffer_number_to_merge = 2;
+    // this means we'll use 50% extra memory in the worst case, but will reduce
+    // write stalls.
+    columnOptions.max_write_buffer_number = 6;
+    // start flushing L0->L1 as soon as possible. each file on level0 is
+    // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
+    // memtable_memory_budget.
+    columnOptions.level0_file_num_compaction_trigger = 2;
+    // doesn't really matter much, but we don't want to create too many files
+    columnOptions.target_file_size_base = memtable_memory_budget / 8;
+    // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
+    columnOptions.max_bytes_for_level_base = memtable_memory_budget;
+
+    // level style compaction
+    columnOptions.compaction_style = rocksdb::kCompactionStyleLevel;
+
+    // only compress levels >= 2
+    columnOptions.compression_per_level.resize(columnOptions.num_levels);
+    for (int i = 0; i < columnOptions.num_levels; ++i) {
+      if (i < 2) {
+        columnOptions.compression_per_level[i] = rocksdb::kNoCompression;
+      } else {
+        columnOptions.compression_per_level[i] = rocksdb::kZSTD;
       }
-    } else {
-      return napi_invalid_arg;
     }
+  } else {
+    return napi_invalid_arg;
   }
 
   bool compression = true;
@@ -888,15 +896,17 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   columnOptions.optimize_filters_for_hits = false;
   NAPI_STATUS_RETURN(GetProperty(env, options, "optimizeFiltersForHits", columnOptions.optimize_filters_for_hits));
 
-  uint32_t cacheSize = 8 << 20;
-  NAPI_STATUS_RETURN(GetProperty(env, options, "cacheSize", cacheSize));
-
   rocksdb::BlockBasedTableOptions tableOptions;
 
-  if (cacheSize) {
-    tableOptions.block_cache = rocksdb::HyperClockCacheOptions(cacheSize, 0).MakeSharedCache();
-  } else {
-    tableOptions.no_block_cache = true;
+  {
+    uint32_t cacheSize = 8 << 20;
+    NAPI_STATUS_RETURN(GetProperty(env, options, "cacheSize", cacheSize));
+
+    if (cacheSize) {
+      tableOptions.block_cache = rocksdb::HyperClockCacheOptions(cacheSize, 0).MakeSharedCache();
+    } else {
+      tableOptions.no_block_cache = true;
+    }
   }
 
   std::string optimize = "";
@@ -915,11 +925,11 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
     tableOptions.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
   }
 
-  std::optional<std::string> filterPolicyOpt;
-  NAPI_STATUS_RETURN(GetProperty(env, options, "filterPolicy", filterPolicyOpt));
-  if (filterPolicyOpt) {
+  std::string filterPolicy;
+  NAPI_STATUS_RETURN(GetProperty(env, options, "filterPolicy", filterPolicy));
+  if (filterPolicy != "") {
     ROCKS_STATUS_RETURN_NAPI(
-        rocksdb::FilterPolicy::CreateFromString(configOptions, *filterPolicyOpt, &tableOptions.filter_policy));
+        rocksdb::FilterPolicy::CreateFromString(configOptions, filterPolicy, &tableOptions.filter_policy));
   }
 
   tableOptions.block_size = 4 * 1024;
@@ -1469,11 +1479,16 @@ NAPI_METHOD(db_get_latest_sequence) {
 NAPI_METHOD(iterator_init) {
   NAPI_ARGV(2);
 
-  auto iterator = Iterator::create(env, argv[0], argv[1]);
-
   napi_value result;
-  NAPI_STATUS_THROWS(napi_create_external(env, iterator.get(), Finalize<Iterator>, iterator.get(), &result));
-  iterator.release();
+  try {
+    auto iterator = Iterator::create(env, argv[0], argv[1]);
+
+    NAPI_STATUS_THROWS(napi_create_external(env, iterator.get(), Finalize<Iterator>, iterator.get(), &result));
+    iterator.release();
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 
   return result;
 }
@@ -1481,13 +1496,18 @@ NAPI_METHOD(iterator_init) {
 NAPI_METHOD(iterator_seek) {
   NAPI_ARGV(2);
 
-  Iterator* iterator;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
+  try {
+    Iterator* iterator;
+    NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  rocksdb::PinnableSlice target;
-  NAPI_STATUS_THROWS(GetValue(env, argv[1], target));
+    rocksdb::PinnableSlice target;
+    NAPI_STATUS_THROWS(GetValue(env, argv[1], target));
 
-  iterator->Seek(target);
+    iterator->Seek(target);
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 
   return 0;
 }
@@ -1495,36 +1515,57 @@ NAPI_METHOD(iterator_seek) {
 NAPI_METHOD(iterator_close) {
   NAPI_ARGV(1);
 
-  Iterator* iterator;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
+  try {
+    Iterator* iterator;
+    NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  ROCKS_STATUS_THROWS_NAPI(iterator->Close());
+    ROCKS_STATUS_THROWS_NAPI(iterator->Close());
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 
   return 0;
 }
 
 NAPI_METHOD(iterator_nextv) {
-  NAPI_ARGV(3);
+  NAPI_ARGV(4);
 
-  Iterator* iterator;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
+  try {
+    Iterator* iterator;
+    NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  uint32_t count;
-  NAPI_STATUS_THROWS(napi_get_value_uint32(env, argv[1], &count));
+    uint32_t count = 1024;
+    NAPI_STATUS_THROWS(GetValue(env, argv[1], count));
+
+    uint32_t timeout = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, argv[2], "timeout", timeout));
 
-  return iterator->nextv(env, count, argv[2]);
+    return iterator->nextv(env, count, timeout, argv[3]);
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 }
 
 NAPI_METHOD(iterator_nextv_sync) {
-  NAPI_ARGV(2);
+  NAPI_ARGV(3);
 
-  Iterator* iterator;
-  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
+  try {
+    Iterator* iterator;
+    NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  uint32_t count;
-  NAPI_STATUS_THROWS(napi_get_value_uint32(env, argv[1], &count));
+    uint32_t count = 1024;
+    NAPI_STATUS_THROWS(GetValue(env, argv[1], count));
 
-  return iterator->nextv(env, count);
+    uint32_t timeout = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, argv[2], "timeout", timeout));
+
+    return iterator->nextv(env, count, timeout);
+  } catch (const std::exception& e) {
+    napi_throw_error(env, nullptr, e.what());
+    return nullptr;
+  }
 }
 
 NAPI_METHOD(batch_init) {

package/binding.gyp

@@ -8,9 +8,13 @@
                 [
                     "OS == 'linux'",
                     {
+                        "direct_dependent_settings": {
+                          "libraries": [
+                          ],
+                        },
                         "include_dirs": [
-                            "/usr/lib/x86_64-linux-gnu/include",
-                            "/usr/lib/include",
+                          "/usr/lib/x86_64-linux-gnu/include",
+                          "/usr/lib/include",
                         ],
                         "cflags": ["-march=znver1"],
                         "ccflags": ["-flto", '-march=znver1'],
@@ -22,6 +26,12 @@
                 [
                     "OS == 'mac'",
                     {
+                        "direct_dependent_settings": {
+                          "libraries": [
+                          ],
+                        },
+                        "include_dirs": [
+                        ],
                         "xcode_settings": {
                             "WARNING_CFLAGS": [
                                 "-Wno-sign-compare",

package/iterator.js

@@ -96,7 +96,7 @@ class Iterator extends AbstractIterator {
       this[kFirst] = false
 
       try {
-        const { rows, finished } = binding.iterator_nextv_sync(this[kContext], size)
+        const { rows, finished } = binding.iterator_nextv_sync(this[kContext], size, null)
         this[kCache] = rows
         this[kFinished] = finished
         this[kPosition] = 0
@@ -162,7 +162,7 @@ class Iterator extends AbstractIterator {
     }
 
     this[kFirst] = false
-    const result = binding.iterator_nextv_sync(this[kContext], size)
+    const result = binding.iterator_nextv_sync(this[kContext], size, options)
     this[kFinished] = result.finished
 
     return result
@@ -182,7 +182,7 @@ class Iterator extends AbstractIterator {
         this[kFirst] = false
         this[kBusy] = true
         this[kDB][kRef]()
-        binding.iterator_nextv(this[kContext], size, (err, result) => {
+        binding.iterator_nextv(this[kContext], size, options, (err, result) => {
           this[kBusy] = false
           this[kDB][kUnref]()
           if (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "13.0.3",
+  "version": "13.1.1",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/util.h

@@ -260,6 +260,10 @@ static napi_status GetValue(napi_env env, napi_value value, rocksdb::ColumnFamil
   return napi_get_value_external(env, value, reinterpret_cast<void**>(&result));
 }
 
+static napi_status GetValue(napi_env env, napi_value value, std::shared_ptr<rocksdb::Cache>*& result) {
+  return napi_get_value_external(env, value, reinterpret_cast<void**>(&result));
+}
+
 static napi_status GetValue(napi_env env, napi_value value, Encoding& result) {
   size_t size;
   NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, value, nullptr, 0, &size));