@nxtedition/rocksdb 7.0.63 → 7.0.64

package/deps/rocksdb/rocksdb/db/wal_manager.cc

@@ -120,6 +120,10 @@ Status WalManager::GetUpdatesSince(
     return s;
   }
 
+  std::sort(std::begin(*wal_files), std::end(*wal_files), [](auto& lhs, auto& rhs) {
+    return lhs->StartSequence() > rhs->StartSequence();
+  });
+
   s = RetainProbableWalFiles(*wal_files, seq);
   if (!s.ok()) {
     return s;

package/index.js

@@ -17,6 +17,10 @@ const kColumns = Symbol('columns')
 const kLocation = Symbol('location')
 const kPromise = Symbol('promise')
 const kUpdates = Symbol('updates')
+const kRef = Symbol('ref')
+const kUnref = Symbol('unref')
+const kRefs = Symbol('refs')
+const kPendingClose = Symbol('pendingClose')
 
 const EMPTY = {}
 
@@ -66,6 +70,9 @@ class RocksLevel extends AbstractLevel {
     this[kContext] = binding.db_init()
     this[kColumns] = {}
 
+    this[kRefs] = 0
+    this[kPendingClose] = null
+
     // .updates(...) uses 'update' listener.
     this.setMaxListeners(100)
   }
@@ -101,8 +108,23 @@ class RocksLevel extends AbstractLevel {
     }
   }
 
+  [kRef] () {
+    this[kRefs]++
+  }
+
+  [kUnref] () {
+    this[kRefs]--
+    if (this[kRefs] === 0 && this[kPendingClose]) {
+      this[kPendingClose]()
+    }
+  }
+
   _close (callback) {
-    binding.db_close(this[kContext], callback)
+    if (this[kRefs]) {
+      this[kPendingClose] = callback
+    } else {
+      binding.db_close(this[kContext], callback)
+    }
   }
 
   _put (key, value, options, callback) {
@@ -137,9 +159,14 @@ class RocksLevel extends AbstractLevel {
     callback = fromCallback(callback, kPromise)
 
     try {
-      binding.db_get_many(this[kContext], keys, options ?? EMPTY, callback)
+      this[kRef]()
+      binding.db_get_many(this[kContext], keys, options ?? EMPTY, (err, val) => {
+        callback(err, val)
+        this[kUnref]()
+      })
     } catch (err) {
       process.nextTick(callback, err)
+      this[kUnref]()
     }
 
     return callback[kPromise]
@@ -222,36 +249,6 @@ class RocksLevel extends AbstractLevel {
     return binding.db_get_property(this[kContext], property)
   }
 
-  async getCurrentWALFile () {
-    if (this.status !== 'open') {
-      throw new ModuleError('Database is not open', {
-        code: 'LEVEL_DATABASE_NOT_OPEN'
-      })
-    }
-
-    return binding.db_get_current_wal_file(this[kContext])
-  }
-
-  async getSortedWALFiles () {
-    if (this.status !== 'open') {
-      throw new ModuleError('Database is not open', {
-        code: 'LEVEL_DATABASE_NOT_OPEN'
-      })
-    }
-
-    return binding.db_get_sorted_wal_files(this[kContext])
-  }
-
-  async flushWAL (options) {
-    if (this.status !== 'open') {
-      throw new ModuleError('Database is not open', {
-        code: 'LEVEL_DATABASE_NOT_OPEN'
-      })
-    }
-
-    binding.db_flush_wal(this[kContext], options)
-  }
-
   async query (options) {
     if (this.status !== 'open') {
       throw new ModuleError('Database is not open', {
@@ -259,17 +256,9 @@ class RocksLevel extends AbstractLevel {
       })
     }
 
-    const context = binding.iterator_init(this[kContext], options)
-    const resource = {
-      callback: null,
-      close (callback) {
-        this.callback = callback
-      }
-    }
-
+    const context = binding.iterator_init(this[kContext], options ?? {})
     try {
-      this.attachResource(resource)
-
+      this[kRef]()
       const limit = options.limit ?? 1000
       return await new Promise((resolve, reject) => binding.iterator_nextv(context, limit, (err, rows, finished) => {
         if (err) {
@@ -283,11 +272,8 @@ class RocksLevel extends AbstractLevel {
         }
       }))
     } finally {
-      this.detachResource(resource)
       binding.iterator_close(context)
-      if (resource.callback) {
-        resource.callback()
-      }
+      this[kUnref]()
     }
   }
 
@@ -347,29 +333,6 @@ class RocksLevel extends AbstractLevel {
 
     const db = this
 
-    async function * _updates (options) {
-      let first = true
-      for await (const update of db[kUpdates]({ ...options, since: Math.max(0, options.since - 1024) })) {
-        if (first) {
-          if (update.sequence > options.since) {
-            // HACK
-            db.emit('warning', `Invalid update sequence ${update.sequence} > ${options.since}. Starting from 0.`)
-            first = null
-            break
-          } else {
-            first = false
-          }
-        }
-        yield update
-      }
-
-      if (first === null) {
-        for await (const update of db[kUpdates]({ ...options, since: 0 })) {
-          yield update
-        }
-      }
-    }
-
     try {
       let since = options.since
       while (true) {
@@ -401,15 +364,12 @@ class RocksLevel extends AbstractLevel {
         try {
           if (since <= db.sequence) {
             let first = true
-            for await (const update of _updates({
+            for await (const update of db[kUpdates]({
               ...options,
+              signal: ac.signal,
               // HACK: https://github.com/facebook/rocksdb/issues/10476
-              since: Math.max(0, options.since - 2048)
+              since: Math.max(0, options.since)
             })) {
-              if (ac.signal.aborted) {
-                throw new AbortError()
-              }
-
               if (first) {
                 if (update.sequence > since) {
                   db.emit('warning', `Invalid updates sequence ${update.sequence} > ${options.since}.`)
@@ -435,10 +395,6 @@ class RocksLevel extends AbstractLevel {
 
         let first = true
         for await (const update of buffer) {
-          if (ac.signal.aborted) {
-            throw new AbortError()
-          }
-
           if (first) {
             if (update.sequence > since) {
               db.emit('warning', `Invalid batch sequence ${update.sequence} > ${options.since}.`)
@@ -457,72 +413,32 @@ class RocksLevel extends AbstractLevel {
     }
   }
 
-  async * [kUpdates] (options) {
-    class Updates {
-      constructor (db, options) {
-        this.context = binding.updates_init(db[kContext], options)
-        this.closed = false
-        this.promise = null
-        this.db = db
-        this.db.attachResource(this)
-      }
-
-      async next () {
-        if (this.closed) {
-          return {}
+  async * [kUpdates] ({ signal, ...options }) {
+    const context = binding.updates_init(this[kContext], options)
+    this[kRef]()
+    try {
+      while (true) {
+        if (signal?.aborted) {
+          throw new AbortError()
         }
 
-        this.promise = new Promise(resolve => binding.updates_next(this.context, (err, rows, sequence, count) => {
-          this.promise = null
+        const entry = await new Promise((resolve, reject) => binding.updates_next(context, (err, rows, sequence, count) => {
           if (err) {
-            resolve(Promise.reject(err))
+            reject(err)
           } else {
             resolve({ rows, sequence, count })
           }
         }))
 
-        return this.promise
-      }
-
-      async close (callback) {
-        try {
-          await this.promise
-        } catch {
-          // Do nothing...
-        }
-
-        try {
-          if (!this.closed) {
-            this.closed = true
-            binding.updates_close(this.context)
-          }
-
-          if (callback) {
-            process.nextTick(callback)
-          }
-        } catch (err) {
-          if (callback) {
-            process.nextTick(callback, err)
-          } else {
-            throw err
-          }
-        } finally {
-          this.db.detachResource(this)
-        }
-      }
-    }
-
-    const updates = new Updates(this, options)
-    try {
-      while (true) {
-        const entry = await updates.next()
         if (!entry.rows) {
           return
         }
+
         yield entry
       }
     } finally {
-      await updates.close()
+      binding.updates_close(context)
+      this[kUnref]()
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "7.0.63",
+  "version": "7.0.64",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/util.h

@@ -0,0 +1,338 @@
+#pragma once
+
+#include <assert.h>
+#include <napi-macros.h>
+#include <node_api.h>
+
+#include <rocksdb/db.h>
+#include <rocksdb/slice.h>
+#include <rocksdb/status.h>
+
+#include <array>
+#include <optional>
+#include <string>
+
+#define NAPI_STATUS_RETURN(call) \
+  {                              \
+    auto _status = (call);       \
+    if (_status != napi_ok) {    \
+      return _status;            \
+    }                            \
+  }
+
+#define ROCKS_STATUS_THROWS_NAPI(call)        \
+  {                                           \
+    auto _status = (call);                    \
+    if (!_status.ok()) {                      \
+      napi_throw(env, ToError(env, _status)); \
+      return NULL;                            \
+    }                                         \
+  }
+
+#define ROCKS_STATUS_RETURN_NAPI(call)        \
+  {                                           \
+    auto _status = (call);                    \
+    if (!_status.ok()) {                      \
+      napi_throw(env, ToError(env, _status)); \
+      return napi_pending_exception;          \
+    }                                         \
+  }
+
+template <typename T>
+static void Finalize(napi_env env, void* data, void* hint) {
+  if (hint) {
+    delete reinterpret_cast<T*>(hint);
+  }
+}
+
+static napi_value CreateError(napi_env env, const std::optional<std::string_view>& code, const std::string_view& msg) {
+  napi_value codeValue = nullptr;
+  if (code) {
+    NAPI_STATUS_THROWS(napi_create_string_utf8(env, code->data(), code->size(), &codeValue));
+  }
+  napi_value msgValue;
+  NAPI_STATUS_THROWS(napi_create_string_utf8(env, msg.data(), msg.size(), &msgValue));
+  napi_value error;
+  NAPI_STATUS_THROWS(napi_create_error(env, codeValue, msgValue, &error));
+  return error;
+}
+
+static napi_value ToError(napi_env env, const rocksdb::Status& status) {
+  if (status.ok()) {
+    return 0;
+  }
+
+  const auto msg = status.ToString();
+
+  if (status.IsNotFound()) {
+    return CreateError(env, "LEVEL_NOT_FOUND", msg);
+  } else if (status.IsCorruption()) {
+    return CreateError(env, "LEVEL_CORRUPTION", msg);
+  } else if (status.IsTryAgain()) {
+    return CreateError(env, "LEVEL_TRYAGAIN", msg);
+  } else if (status.IsIOError()) {
+    if (msg.find("IO error: lock ") != std::string::npos) {  // env_posix.cc
+      return CreateError(env, "LEVEL_LOCKED", msg);
+    } else if (msg.find("IO error: LockFile ") != std::string::npos) {  // env_win.cc
+      return CreateError(env, "LEVEL_LOCKED", msg);
+    } else if (msg.find("IO error: While lock file") != std::string::npos) {  // env_mac.cc
+      return CreateError(env, "LEVEL_LOCKED", msg);
+    } else {
+      return CreateError(env, "LEVEL_IO_ERROR", msg);
+    }
+  }
+
+  return CreateError(env, {}, msg);
+}
+
+static napi_status GetString(napi_env env, napi_value from, std::string& to) {
+  napi_valuetype type;
+  NAPI_STATUS_RETURN(napi_typeof(env, from, &type));
+
+  if (type == napi_string) {
+    size_t length = 0;
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, nullptr, 0, &length));
+    to.resize(length, '\0');
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, &to[0], length + 1, &length));
+  } else {
+    bool isBuffer;
+    NAPI_STATUS_RETURN(napi_is_buffer(env, from, &isBuffer));
+
+    if (isBuffer) {
+      char* buf = nullptr;
+      size_t length = 0;
+      NAPI_STATUS_RETURN(napi_get_buffer_info(env, from, reinterpret_cast<void**>(&buf), &length));
+      to.assign(buf, length);
+    } else {
+      return napi_invalid_arg;
+    }
+  }
+
+  return napi_ok;
+}
+
+void DestroyReference(void* arg1, void* arg2) {
+  auto env = reinterpret_cast<napi_env>(arg1);
+  auto ref = reinterpret_cast<napi_ref>(arg2);
+  napi_delete_reference(env, ref);
+}
+
+static napi_status GetString(napi_env env, napi_value from, rocksdb::PinnableSlice& to) {
+  napi_valuetype type;
+  NAPI_STATUS_RETURN(napi_typeof(env, from, &type));
+
+  if (type == napi_string) {
+    size_t length = 0;
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, nullptr, 0, &length));
+    to.GetSelf()->resize(length, '\0');
+    NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, from, &(*to.GetSelf())[0], length + 1, &length));
+    to.PinSelf();
+  } else {
+    bool isBuffer;
+    NAPI_STATUS_RETURN(napi_is_buffer(env, from, &isBuffer));
+
+    if (isBuffer) {
+      char* buf = nullptr;
+      size_t length = 0;
+      NAPI_STATUS_RETURN(napi_get_buffer_info(env, from, reinterpret_cast<void**>(&buf), &length));
+
+      napi_ref ref;
+      NAPI_STATUS_RETURN(napi_create_reference(env, from, 1, &ref));
+      to.PinSlice(rocksdb::Slice(buf, length), DestroyReference, env, ref);
+    } else {
+      return napi_invalid_arg;
+    }
+  }
+
+  return napi_ok;
+}
+
+enum class Encoding { Invalid, Buffer, String };
+
+static napi_status GetValue(napi_env env, napi_value value, bool& result) {
+  return napi_get_value_bool(env, value, &result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, uint32_t& result) {
+  return napi_get_value_uint32(env, value, &result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, int32_t& result) {
+  return napi_get_value_int32(env, value, &result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, int64_t& result) {
+  return napi_get_value_int64(env, value, &result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, uint64_t& result) {
+  int64_t result2;
+  NAPI_STATUS_RETURN(napi_get_value_int64(env, value, &result2));
+  result = static_cast<uint64_t>(result2);
+  return napi_ok;
+}
+
+static napi_status GetValue(napi_env env, napi_value value, unsigned long& result) {
+  int64_t result2;
+  NAPI_STATUS_RETURN(napi_get_value_int64(env, value, &result2));
+  result = static_cast<unsigned long>(result2);
+  return napi_ok;
+}
+
+static napi_status GetValue(napi_env env, napi_value value, std::string& result) {
+  return GetString(env, value, result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, rocksdb::PinnableSlice& result) {
+  return GetString(env, value, result);
+}
+
+static napi_status GetValue(napi_env env, napi_value value, rocksdb::ColumnFamilyHandle*& result) {
+  return napi_get_value_external(env, value, reinterpret_cast<void**>(&result));
+}
+
+static napi_status GetValue(napi_env env, napi_value value, Encoding& result) {
+  size_t size;
+  NAPI_STATUS_RETURN(napi_get_value_string_utf8(env, value, nullptr, 0, &size));
+
+  if (size == 6) {
+    result = Encoding::Buffer;
+  } else {
+    result = Encoding::String;
+  }
+
+  return napi_ok;
+}
+
+template <typename T>
+static napi_status GetValue(napi_env env, napi_value value, std::optional<T>& result) {
+  result = T{};
+  return GetValue(env, value, *result);
+}
+
+template <typename T>
+static napi_status GetProperty(napi_env env,
+                               napi_value obj,
+                               const std::string_view& key,
+                               T& result,
+                               bool required = false) {
+  bool has = false;
+  NAPI_STATUS_RETURN(napi_has_named_property(env, obj, key.data(), &has));
+
+  if (!has) {
+    return required ? napi_invalid_arg : napi_ok;
+  }
+
+  napi_value value;
+  NAPI_STATUS_RETURN(napi_get_named_property(env, obj, key.data(), &value));
+
+  bool nully = false;
+
+  napi_value nullVal;
+  NAPI_STATUS_RETURN(napi_get_null(env, &nullVal));
+  NAPI_STATUS_RETURN(napi_strict_equals(env, nullVal, value, &nully));
+  if (nully) {
+    return required ? napi_invalid_arg : napi_ok;
+  }
+
+  napi_value undefinedVal;
+  NAPI_STATUS_RETURN(napi_get_undefined(env, &undefinedVal));
+  NAPI_STATUS_RETURN(napi_strict_equals(env, undefinedVal, value, &nully));
+  if (nully) {
+    return required ? napi_invalid_arg : napi_ok;
+  }
+
+  return GetValue(env, value, result);
+}
+
+template <typename T>
+napi_status Convert(napi_env env, T&& s, Encoding encoding, napi_value& result) {
+  if (!s) {
+    return napi_get_null(env, &result);
+  } else if (encoding == Encoding::Buffer) {
+    // napi_create_external_buffer would be nice but is unsafe since node
+    // buffers are not read-only.
+    return napi_create_buffer_copy(env, s->size(), s->data(), nullptr, &result);
+  } else if (encoding == Encoding::String) {
+    return napi_create_string_utf8(env, s->data(), s->size(), &result);
+  } else {
+    return napi_invalid_arg;
+  }
+}
+
+template <typename State, typename T1, typename T2>
+napi_status runAsync(const std::string& name, napi_env env, napi_value callback, T1&& execute, T2&& then) {
+  struct Worker final {
+    static void Execute(napi_env env, void* data) {
+      auto worker = reinterpret_cast<Worker*>(data);
+      worker->status = worker->execute(worker->state);
+    }
+
+    static void Complete(napi_env env, napi_status status, void* data) {
+      auto worker = std::unique_ptr<Worker>(reinterpret_cast<Worker*>(data));
+
+      napi_value callback;
+      NAPI_STATUS_THROWS_VOID(napi_get_reference_value(env, worker->callbackRef, &callback));
+
+      napi_value global;
+      NAPI_STATUS_THROWS_VOID(napi_get_global(env, &global));
+
+      if (worker->status.ok()) {
+        std::vector<napi_value> argv;
+
+        argv.resize(1);
+        NAPI_STATUS_THROWS_VOID(napi_get_null(env, &argv[0]));
+
+        const auto ret = worker->then(worker->state, env, argv);
+
+        if (ret == napi_ok) {
+          NAPI_STATUS_THROWS_VOID(napi_call_function(env, global, callback, argv.size(), argv.data(), nullptr));
+        } else {
+          const napi_extended_error_info* errInfo = nullptr;
+          NAPI_STATUS_THROWS_VOID(napi_get_last_error_info(env, &errInfo));
+          auto err = CreateError(env, std::nullopt,
+                                 !errInfo || !errInfo->error_message ? "empty error message" : errInfo->error_message);
+          NAPI_STATUS_THROWS_VOID(napi_call_function(env, global, callback, 1, &err, nullptr));
+        }
+      } else {
+        auto err = ToError(env, worker->status);
+        NAPI_STATUS_THROWS_VOID(napi_call_function(env, global, callback, 1, &err, nullptr));
+      }
+    }
+
+    ~Worker() {
+      if (callbackRef) {
+        napi_delete_reference(env, callbackRef);
+        callbackRef = nullptr;
+      }
+      if (asyncWork) {
+        napi_delete_async_work(env, asyncWork);
+        asyncWork = nullptr;
+      }
+    }
+
+    napi_env env = nullptr;
+
+    typename std::decay<T1>::type execute;
+    typename std::decay<T2>::type then;
+
+    napi_ref callbackRef = nullptr;
+    napi_async_work asyncWork = nullptr;
+    rocksdb::Status status = rocksdb::Status::OK();
+    State state = State();
+  };
+
+  auto worker = std::unique_ptr<Worker>(new Worker{env, std::forward<T1>(execute), std::forward<T2>(then)});
+
+  NAPI_STATUS_RETURN(napi_create_reference(env, callback, 1, &worker->callbackRef));
+  napi_value asyncResourceName;
+  NAPI_STATUS_RETURN(napi_create_string_utf8(env, name.data(), name.size(), &asyncResourceName));
+  NAPI_STATUS_RETURN(napi_create_async_work(env, callback, asyncResourceName, Worker::Execute, Worker::Complete,
+                                            worker.get(), &worker->asyncWork));
+
+  NAPI_STATUS_RETURN(napi_queue_async_work(env, worker->asyncWork));
+
+  worker.release();
+
+  return napi_ok;
+}