@nxtedition/rocksdb 10.1.6 → 10.2.2

package/binding.cc

@@ -435,16 +435,12 @@ struct Iterator final : public BaseIterator {
            const std::optional<std::string>& gt,
            const std::optional<std::string>& gte,
            const bool fillCache,
-           const Encoding keyEncoding,
-           const Encoding valueEncoding,
            const size_t highWaterMarkBytes,
            std::shared_ptr<const rocksdb::Snapshot> snapshot,
            bool tailing = false)
       : BaseIterator(database, column, reverse, lt, lte, gt, gte, limit, fillCache, snapshot, tailing),
         keys_(keys),
         values_(values),
-        keyEncoding_(keyEncoding),
-        valueEncoding_(valueEncoding),
         highWaterMarkBytes_(highWaterMarkBytes) {}
 
   void Seek(const rocksdb::Slice& target) override {
@@ -454,8 +450,6 @@ struct Iterator final : public BaseIterator {
 
   const bool keys_;
   const bool values_;
-  const Encoding keyEncoding_;
-  const Encoding valueEncoding_;
   const size_t highWaterMarkBytes_;
   bool first_ = true;
 };
@@ -957,8 +951,8 @@ NAPI_METHOD(db_get_many) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-  uint32_t size;
-  NAPI_STATUS_THROWS(napi_get_array_length(env, argv[1], &size));
+  uint32_t count;
+  NAPI_STATUS_THROWS(napi_get_array_length(env, argv[1], &count));
 
   const auto options = argv[2];
 
@@ -968,29 +962,30 @@ NAPI_METHOD(db_get_many) {
   bool ignoreRangeDeletions = false;
   NAPI_STATUS_THROWS(GetProperty(env, options, "ignoreRangeDeletions", ignoreRangeDeletions));
 
-  Encoding valueEncoding = Encoding::String;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
-
   rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
   NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
 
+  bool takeSnapshot = true;
+  NAPI_STATUS_THROWS(GetProperty(env, options, "snapshot", takeSnapshot));
+
   auto callback = argv[3];
 
-  auto snapshot = std::shared_ptr<const rocksdb::Snapshot>(
-      database->db->GetSnapshot(), [database](auto ptr) { database->db->ReleaseSnapshot(ptr); });
+  std::shared_ptr<const rocksdb::Snapshot> snapshot;
+  if (takeSnapshot) {
+    snapshot.reset(database->db->GetSnapshot(), [=](const auto ptr) { database->db->ReleaseSnapshot(ptr); });
+  }
 
-  std::vector<rocksdb::PinnableSlice> keys{size};
+  std::vector<rocksdb::PinnableSlice> keys{count};
 
-  for (uint32_t n = 0; n < size; n++) {
+  for (uint32_t n = 0; n < count; n++) {
     napi_value element;
     NAPI_STATUS_THROWS(napi_get_element(env, argv[1], n, &element));
     NAPI_STATUS_THROWS(GetValue(env, element, keys[n]));
   }
 
   struct State {
-    std::vector<rocksdb::Status> statuses;
-    std::vector<rocksdb::PinnableSlice> values;
-    std::vector<rocksdb::Slice> keys;
+    std::vector<uint8_t> data;
+    std::vector<std::optional<size_t>> sizes;
   };
 
   runAsync<State>(
@@ -1003,41 +998,69 @@ NAPI_METHOD(db_get_many) {
         readOptions.ignore_range_deletions = ignoreRangeDeletions;
         readOptions.optimize_multiget_for_io = true;
 
-        state.statuses.resize(size);
-        state.values.resize(size);
-        state.keys.resize(size);
+        std::vector<rocksdb::Status> statuses{count};
+        std::vector<rocksdb::PinnableSlice> values{count};
+        std::vector<rocksdb::Slice> keys2{count};
+
+        for (auto n = 0; n < count; n++) {
+          keys2[n] = keys[n];
+        }
+
+        database->db->MultiGet(readOptions, column, count, keys2.data(), values.data(), statuses.data());
 
-        for (auto n = 0; n < size; n++) {
-          state.keys[n] = keys[n];
+        auto size = 0;
+        for (auto n = 0; n < count; n++) {
+          const auto valueSize = values[n].size();
+          size += valueSize & 0x7 ? (valueSize | 0x7) + 1 : valueSize;
         }
 
-        database->db->MultiGet(readOptions, column, size, state.keys.data(), state.values.data(),
-                               state.statuses.data());
+        state.data.reserve(size);
+
+        auto push = [&](rocksdb::Slice* slice){
+          if (slice) {
+            state.sizes.push_back(slice->size());
+            std::copy_n(slice->data(), slice->size(), std::back_inserter(state.data));
+
+            if (state.data.size() & 0x7) {
+              state.data.resize((state.data.size() | 0x7) + 1);
+            }
+          } else {
+            state.sizes.push_back(std::nullopt);
+          }
+        };
+
+        for (auto n = 0; n < count; n++) {
+          push(statuses[n].ok() ? &values[n] : nullptr);
+        }
 
         return rocksdb::Status::OK();
       },
       [=](auto& state, auto env, auto& argv) {
-        argv.resize(2);
+        argv.resize(3);
+
+        const auto count = state.sizes.size();
 
-        NAPI_STATUS_RETURN(napi_create_array_with_length(env, size, &argv[1]));
+        NAPI_STATUS_RETURN(napi_create_array_with_length(env, count, &argv[1]));
 
-        for (uint32_t idx = 0; idx < size; idx++) {
-          const auto& status = state.statuses[idx];
-          const auto& value = state.values[idx];
+        for (uint32_t idx = 0; idx < count; idx++) {
+          const auto& maybeSize = state.sizes[idx];
 
           napi_value element;
-          if (status.IsNotFound()) {
-            NAPI_STATUS_RETURN(napi_get_undefined(env, &element));
+          if (maybeSize) {
+            NAPI_STATUS_RETURN(napi_create_uint32(env, *maybeSize, &element));
           } else {
-            ROCKS_STATUS_RETURN_NAPI(status);
-            NAPI_STATUS_RETURN(Convert(env, &value, valueEncoding, element));
+            NAPI_STATUS_RETURN(napi_get_undefined(env, &element));
           }
           NAPI_STATUS_RETURN(napi_set_element(env, argv[1], idx, element));
         }
 
-        state.statuses.clear();
-        state.values.clear();
-        state.keys.clear();
+        if (state.data.size() > 0) {
+          auto data = std::make_unique<std::vector<uint8_t>>(std::move(state.data));
+          NAPI_STATUS_RETURN(napi_create_external_buffer(env, data->size(), data->data(), Finalize<std::vector<uint8_t>>, data.get(), &argv[2]));
+          data.release();
+        } else {
+          NAPI_STATUS_RETURN(napi_get_undefined(env, &argv[2]));
+        }
 
         return napi_ok;
       });
@@ -1203,12 +1226,6 @@ NAPI_METHOD(iterator_init) {
   bool fillCache = false;
   NAPI_STATUS_THROWS(GetProperty(env, options, "fillCache", fillCache));
 
-  Encoding keyEncoding = Encoding::String;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "keyEncoding", keyEncoding));
-
-  Encoding valueEncoding = Encoding::String;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "valueEncoding", valueEncoding));
-
   int32_t limit = -1;
   NAPI_STATUS_THROWS(GetProperty(env, options, "limit", limit));
 
@@ -1230,11 +1247,16 @@ NAPI_METHOD(iterator_init) {
   rocksdb::ColumnFamilyHandle* column = database->db->DefaultColumnFamily();
   NAPI_STATUS_THROWS(GetProperty(env, options, "column", column));
 
-  std::shared_ptr<const rocksdb::Snapshot> snapshot(database->db->GetSnapshot(),
-                                                    [=](const auto ptr) { database->db->ReleaseSnapshot(ptr); });
+  bool takeSnapshot = !tailing;
+  NAPI_STATUS_THROWS(GetProperty(env, options, "snapshot", takeSnapshot));
+
+  std::shared_ptr<const rocksdb::Snapshot> snapshot;
+  if (takeSnapshot) {
+    snapshot.reset(database->db->GetSnapshot(), [=](const auto ptr) { database->db->ReleaseSnapshot(ptr); });
+  }
 
   auto iterator = std::unique_ptr<Iterator>(new Iterator(database, column, reverse, keys, values, limit, lt, lte, gt,
-                                                         gte, fillCache, keyEncoding, valueEncoding, highWaterMarkBytes,
+                                                         gte, fillCache, highWaterMarkBytes,
                                                          snapshot, tailing));
 
   napi_value result;
@@ -1275,13 +1297,14 @@ NAPI_METHOD(iterator_nextv) {
   Iterator* iterator;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&iterator)));
 
-  uint32_t size;
-  NAPI_STATUS_THROWS(napi_get_value_uint32(env, argv[1], &size));
+  uint32_t count;
+  NAPI_STATUS_THROWS(napi_get_value_uint32(env, argv[1], &count));
 
   auto callback = argv[2];
 
   struct State {
-    std::vector<rocksdb::PinnableSlice> cache;
+    std::vector<uint8_t> data;
+    std::vector<std::optional<size_t>> sizes;
     bool finished = false;
   };
 
@@ -1292,61 +1315,88 @@ NAPI_METHOD(iterator_nextv) {
           iterator->SeekToRange();
         }
 
-        state.cache.reserve(size * 2);
-        size_t bytesRead = 0;
+        state.sizes.reserve(count * 2);
+        state.data.reserve(iterator->highWaterMarkBytes_);
+
+        auto bytesRead = 0;
+
+        auto push = [&](const std::optional<rocksdb::Slice>& slice){
+          if (slice) {
+            state.sizes.push_back(slice->size());
+            std::copy_n(slice->data(), slice->size(), std::back_inserter(state.data));
+
+            if (state.data.size() & 0x7) {
+              state.data.resize((state.data.size() | 0x7) + 1);
+            }
 
+            bytesRead += slice->size();
+          } else {
+            state.sizes.push_back(std::nullopt);
+          }
+        };
+
+        auto status = rocksdb::Status::OK();
         while (true) {
-          if (!iterator->first_)
+          if (!iterator->first_) {
             iterator->Next();
-          else
+          } else {
             iterator->first_ = false;
+          }
 
-          if (!iterator->Valid() || !iterator->Increment())
+          if (!iterator->Valid() || !iterator->Increment()) {
+            status = iterator->Status();
+            state.finished = true;
             break;
-
-          auto k = rocksdb::PinnableSlice();
-          auto v = rocksdb::PinnableSlice();
+          }
 
           if (iterator->keys_ && iterator->values_) {
-            k.PinSelf(iterator->CurrentKey());
-            v.PinSelf(iterator->CurrentValue());
+            push(iterator->CurrentKey());
+            push(iterator->CurrentValue());
           } else if (iterator->keys_) {
-            k.PinSelf(iterator->CurrentKey());
+            push(iterator->CurrentKey());
+            push(std::nullopt);
           } else if (iterator->values_) {
-            v.PinSelf(iterator->CurrentValue());
+            push(std::nullopt);
+            push(iterator->CurrentValue());
           }
 
-          bytesRead += k.size() + v.size();
-          state.cache.push_back(std::move(k));
-          state.cache.push_back(std::move(v));
-
-          if (bytesRead > iterator->highWaterMarkBytes_ || state.cache.size() / 2 >= size) {
+          if (bytesRead > iterator->highWaterMarkBytes_ || state.sizes.size() / 2 >= count) {
+            status = rocksdb::Status::OK();
             state.finished = false;
-            return rocksdb::Status::OK();
+            break;
           }
         }
 
-        state.finished = true;
-
-        return iterator->Status();
+        return status;
       },
       [=](auto& state, auto env, auto& argv) {
-        argv.resize(3);
+        argv.resize(4);
 
-        NAPI_STATUS_RETURN(napi_create_array_with_length(env, state.cache.size(), &argv[1]));
+        const auto count = state.sizes.size();
 
-        for (size_t n = 0; n < state.cache.size(); n += 2) {
-          napi_value key;
-          napi_value val;
+        NAPI_STATUS_RETURN(napi_create_array_with_length(env, count, &argv[1]));
 
-          NAPI_STATUS_RETURN(Convert(env, &state.cache[n + 0], iterator->keyEncoding_, key));
-          NAPI_STATUS_RETURN(Convert(env, &state.cache[n + 1], iterator->valueEncoding_, val));
+        for (uint32_t idx = 0; idx < count; idx++) {
+          const auto& maybeSize = state.sizes[idx];
 
-          NAPI_STATUS_RETURN(napi_set_element(env, argv[1], static_cast<int>(n + 0), key));
-          NAPI_STATUS_RETURN(napi_set_element(env, argv[1], static_cast<int>(n + 1), val));
+          napi_value element;
+          if (maybeSize) {
+            NAPI_STATUS_RETURN(napi_create_uint32(env, *maybeSize, &element));
+          } else {
+            NAPI_STATUS_RETURN(napi_get_undefined(env, &element));
+          }
+          NAPI_STATUS_RETURN(napi_set_element(env, argv[1], idx, element));
+        }
+
+        if (state.data.size() > 0) {
+          auto data = std::make_unique<std::vector<uint8_t>>(std::move(state.data));
+          NAPI_STATUS_RETURN(napi_create_external_buffer(env, data->size(), data->data(), Finalize<std::vector<uint8_t>>, data.get(), &argv[2]));
+          data.release();
+        } else {
+          NAPI_STATUS_RETURN(napi_get_undefined(env, &argv[2]));
         }
 
-        NAPI_STATUS_RETURN(napi_get_boolean(env, state.finished, &argv[2]));
+        NAPI_STATUS_RETURN(napi_get_boolean(env, state.finished, &argv[3]));
 
         return napi_ok;
       });

package/index.js

@@ -8,6 +8,7 @@ const { ChainedBatch } = require('./chained-batch')
 const { Iterator } = require('./iterator')
 const fs = require('node:fs')
 const assert = require('node:assert')
+const { handleNextv } = require('./util')
 
 const kContext = Symbol('context')
 const kColumns = Symbol('columns')
@@ -148,16 +149,35 @@ class RocksLevel extends AbstractLevel {
   _getMany (keys, options, callback) {
     callback = fromCallback(callback, kPromise)
 
-    const { keyEncoding } = options ?? EMPTY
-
-    if (keyEncoding !== 'buffer') {
-      keys = keys.map(key => typeof key === 'string' ? Buffer.from(key) : key)
-    }
-
+    const { valueEncoding } = options ?? EMPTY
     try {
       this[kRef]()
-      binding.db_get_many(this[kContext], keys, options ?? EMPTY, (err, val) => {
-        callback(err, val)
+      binding.db_get_many(this[kContext], keys, options ?? EMPTY, (err, sizes, buffer) => {
+        if (err) {
+          callback(err)
+        } else {
+          buffer ??= Buffer.alloc(0)
+          const val = []
+          let offset = 0
+          for (const size of sizes) {
+            if (size == null) {
+              val.push(undefined)
+            } else {
+              if (!valueEncoding || valueEncoding === 'buffer') {
+                val.push(buffer.subarray(offset, offset + size))
+              } else if (valueEncoding === 'slice') {
+                val.push({ buffer, byteOffset: offset, byteLength: size })
+              } else {
+                val.push(buffer.toString(valueEncoding, offset, offset + size))
+              }
+              offset += size
+              if (offset & 0x7) {
+                offset = (offset | 0x7) + 1
+              }
+            }
+          }
+          callback(null, val)
+        }
         this[kUnref]()
       })
     } catch (err) {
@@ -217,14 +237,10 @@ class RocksLevel extends AbstractLevel {
     return new ChainedBatch(this, this[kContext], (batch, context, options, callback) => {
       try {
         this[kRef]()
-        const sync = binding.batch_write(this[kContext], context, options, (err) => {
+        binding.batch_write(this[kContext], context, options, (err) => {
           this[kUnref]()
           callback(err)
         })
-        if (sync) {
-          this[kUnref]()
-          process.nextTick(callback, null)
-        }
       } catch (err) {
         process.nextTick(callback, err)
       }
@@ -244,7 +260,7 @@ class RocksLevel extends AbstractLevel {
         assert(false)
       }
     }
-    batch.write(callback)
+    batch._write(options, callback)
 
     return callback[kPromise]
   }
@@ -282,15 +298,17 @@ class RocksLevel extends AbstractLevel {
     const context = binding.iterator_init(this[kContext], options ?? {})
     try {
       this[kRef]()
-      return await new Promise((resolve, reject) => binding.iterator_nextv(context, options.limit, (err, rows, finished) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve({
-            rows,
-            finished
-          })
-        }
+      return await new Promise((resolve, reject) => binding.iterator_nextv(context, options.limit, (err, sizes, buffer, finished) => {
+        handleNextv(err, sizes, buffer, finished, options, (err, rows, finished) => {
+          if (err) {
+            reject(err)
+          } else {
+            resolve({
+              rows,
+              finished
+            })
+          }
+        })
       }))
     } finally {
       binding.iterator_close(context)

package/iterator.js

@@ -2,6 +2,7 @@
 
 const { fromCallback } = require('catering')
 const { AbstractIterator } = require('abstract-level')
+const { handleNextv } = require('./util')
 
 const binding = require('./binding')
 
@@ -14,6 +15,7 @@ const kPosition = Symbol('position')
 const kHandleNext = Symbol('handleNext')
 const kHandleNextv = Symbol('handleNextv')
 const kCallback = Symbol('callback')
+const kOptions = Symbol('options')
 const empty = []
 
 const registry = new FinalizationRegistry((context) => {
@@ -27,6 +29,7 @@ class Iterator extends AbstractIterator {
     this[kContext] = binding.iterator_init(context, options)
     registry.register(this, this[kContext], this[kContext])
 
+    this[kOptions] = { ...options }
     this[kHandleNext] = this[kHandleNext].bind(this)
     this[kHandleNextv] = this[kHandleNextv].bind(this)
     this[kCallback] = null
@@ -73,15 +76,19 @@ class Iterator extends AbstractIterator {
     return this
   }
 
-  [kHandleNext] (err, items, finished) {
-    const callback = this[kCallback]
-    if (err) return callback(err)
+  [kHandleNext] (err, sizes, buffer, finished) {
+    handleNextv(err, sizes, buffer, finished, this[kOptions], (err, items, finished) => {
+      const callback = this[kCallback]
+      if (err) {
+        return callback(err)
+      }
 
-    this[kCache] = items
-    this[kFinished] = finished
-    this[kPosition] = 0
+      this[kCache] = items
+      this[kFinished] = finished
+      this[kPosition] = 0
 
-    this._next(callback)
+      this._next(callback)
+    })
   }
 
   _nextv (size, options, callback) {
@@ -98,17 +105,22 @@ class Iterator extends AbstractIterator {
     return callback[kPromise]
   }
 
-  [kHandleNextv] (err, items, finished) {
-    const callback = this[kCallback]
-    if (err) return callback(err)
-    this[kFinished] = finished
+  [kHandleNextv] (err, sizes, buffer, finished) {
+    handleNextv(err, sizes, buffer, finished, this[kOptions], (err, items, finished) => {
+      const callback = this[kCallback]
+      if (err) {
+        return callback(err)
+      }
+
+      this[kFinished] = finished
 
-    const entries = []
-    for (let n = 0; n < items.length; n += 2) {
-      entries.push([items[n + 0], items[n + 1]])
-    }
+      const entries = []
+      for (let n = 0; n < items.length; n += 2) {
+        entries.push([items[n + 0], items[n + 1]])
+      }
 
-    callback(null, entries)
+      callback(null, entries)
+    })
   }
 
   _close (callback) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "10.1.6",
+  "version": "10.2.2",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/util.js

@@ -0,0 +1,37 @@
+'use strict'
+
+function handleNextv (err, sizes, buffer, finished, options, callback) {
+  if (err) {
+    callback(err)
+  } else {
+    buffer ??= Buffer.alloc(0)
+
+    const { keyEncoding, valueEncoding } = options ?? {}
+
+    const rows = []
+    let offset = 0
+    for (let n = 0; n < sizes.length; n++) {
+      const size = sizes[n]
+      const encoding = n & 1 ? valueEncoding : keyEncoding
+      if (size == null) {
+        rows.push(undefined)
+      } else {
+        if (!encoding || encoding === 'buffer') {
+          rows.push(buffer.subarray(offset, offset + size))
+        } else if (encoding === 'slice') {
+          rows.push({ buffer, byteOffset: offset, byteLength: size })
+        } else {
+          rows.push(buffer.toString(encoding, offset, offset + size))
+        }
+        offset += size
+        if (offset & 0x7) {
+          offset = (offset | 0x7) + 1
+        }
+      }
+    }
+
+    callback(null, rows, finished)
+  }
+}
+
+exports.handleNextv = handleNextv