@nxtedition/rocksdb 16.0.6 → 16.0.9

package/binding.cc

@@ -17,6 +17,7 @@
 #include <rocksdb/status.h>
 #include <rocksdb/table.h>
 #include <rocksdb/write_batch.h>
+#include <rocksdb/write_buffer_manager.h>
 
 #include <re2/re2.h>
 
@@ -29,6 +30,19 @@
 #include <thread>
 #include <vector>
 
+#ifdef __linux__
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include <cerrno>
+
+// Older libc headers may lack the SYS_ alias for io_uring_setup even though
+// the kernel number (__NR_) is available — keep the Linux probe a boolean.
+#if !defined(SYS_io_uring_setup) && defined(__NR_io_uring_setup)
+#define SYS_io_uring_setup __NR_io_uring_setup
+#endif
+#endif
+
 #include "max_rev_operator.h"
 #include "util.h"
 
@@ -334,6 +348,12 @@ struct BaseIterator : public Closable {
                const int limit,
                rocksdb::ReadOptions readOptions = {})
       : database_(database), column_(column), reverse_(reverse), limit_(limit) {
+    // TODO (correctness): the +'\0' byte-successor trick below converts
+    // inclusive/exclusive bounds correctly only under bytewise ordering. With a
+    // custom CF comparator (InitOptions "comparator", e.g.
+    // rocksdb.ReverseBytewiseComparator) RocksDB applies these bounds with that
+    // comparator, silently inverting the lte/gt boundary semantics. Seek()'s
+    // manual bound check below uses raw bytewise Slice::compare as well.
     if (lte) {
       upper_bound_ = rocksdb::PinnableSlice();
       *upper_bound_->GetSelf() = std::move(*lte) + '\0';
@@ -446,7 +466,17 @@ struct BaseIterator : public Closable {
     // otherwise an iterator that already yielded `limit` rows returns nothing
     // after a refresh even though every other piece of state was reset.
     count_ = 0;
-    return iterator_->Refresh();
+    ROCKS_STATUS_RETURN(iterator_->Refresh());
+    // rocksdb::Iterator::Refresh invalidates the iterator (a Seek* is required
+    // before use), so re-establish the starting position like the constructor
+    // does — otherwise the next read sees Valid()==false and reports an empty
+    // database.
+    if (reverse_) {
+      iterator_->SeekToLast();
+    } else {
+      iterator_->SeekToFirst();
+    }
+    return iterator_->status();
   }
 
   Database* database_;
@@ -701,9 +731,9 @@ class Iterator final : public BaseIterator {
               v.PinSelf(CurrentValue());
               state.bytes += v.size();
               state.values.push_back(std::move(v));
-            } else {
-              assert(false);
             }
+            // keys:false + values:false is valid per abstract-level: rows still
+            // count, each entry surfaces as [undefined, undefined].
             state.count += 1;
           }
 
@@ -733,7 +763,8 @@ class Iterator final : public BaseIterator {
               NAPI_STATUS_RETURN(napi_get_undefined(env, &key));
               NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding_, val, unsafe_));
             } else {
-              assert(false);
+              NAPI_STATUS_RETURN(napi_get_undefined(env, &key));
+              NAPI_STATUS_RETURN(napi_get_undefined(env, &val));
             }
 
             NAPI_STATUS_RETURN(napi_set_element(env, rows, n * 2 + 0, key));
@@ -827,7 +858,8 @@ class Iterator final : public BaseIterator {
         NAPI_STATUS_THROWS(napi_get_undefined(env, &key));
         NAPI_STATUS_THROWS(Convert(env, CurrentValue(), valueEncoding_, val, unsafe_));
       } else {
-        assert(false);
+        NAPI_STATUS_THROWS(napi_get_undefined(env, &key));
+        NAPI_STATUS_THROWS(napi_get_undefined(env, &val));
       }
 
       NAPI_STATUS_THROWS(napi_set_element(env, rows, idx++, key));
@@ -904,6 +936,10 @@ NAPI_METHOD(db_init) {
     int64_t value;
     bool lossless;
     NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, argv[0], &value, &lossless));
+    if (!lossless) {
+      napi_throw_error(env, nullptr, "invalid database handle");
+      return NULL;
+    }
 
     database = reinterpret_cast<Database*>(value);
     NAPI_STATUS_THROWS(napi_create_external(env, database, nullptr, nullptr, &result));
@@ -1092,6 +1128,10 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
 
   NAPI_STATUS_RETURN(GetProperty(env, options, "optimizeFiltersForHits", columnOptions.optimize_filters_for_hits));
   NAPI_STATUS_RETURN(GetProperty(env, options, "periodicCompactionSeconds", columnOptions.periodic_compaction_seconds));
+  // memtable_huge_page_size is a column-family option: when the DB is opened
+  // with explicit column descriptors the copy read into dbOptions in db_open is
+  // sliced away, so it must be settable per column to take effect at all.
+  NAPI_STATUS_RETURN(GetProperty(env, options, "memTableHugePageSize", columnOptions.memtable_huge_page_size));
 
   // Compat
   NAPI_STATUS_RETURN(GetProperty(env, options, "enableBlobFiles", columnOptions.enable_blob_files));
@@ -1131,12 +1171,18 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
       bool lossless;
       int64_t ptr;
       NAPI_STATUS_RETURN(napi_get_value_bigint_int64(env, handleValue, &ptr, &lossless));
+      if (!lossless) {
+        return napi_invalid_arg;
+      }
 
       cache = *reinterpret_cast<std::shared_ptr<rocksdb::Cache>*>(ptr);
     } else if (cacheType == napi_bigint) {
       bool lossless;
       int64_t ptr;
       NAPI_STATUS_RETURN(napi_get_value_bigint_int64(env, cacheValue, &ptr, &lossless));
+      if (!lossless) {
+        return napi_invalid_arg;
+      }
 
       cache = *reinterpret_cast<std::shared_ptr<rocksdb::Cache>*>(ptr);
     } else if (cacheType != napi_undefined && cacheType != napi_null) {
@@ -1361,6 +1407,14 @@ NAPI_METHOD(db_open) {
     NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
     dbOptions.IncreaseParallelism(parallelism);
 
+    // IncreaseParallelism sizes the (process-wide) Env LOW pool to `parallelism`
+    // but pins the HIGH pool — where every flush of every DB sharing the default
+    // Env runs — at a single thread, so flushes across DBs serialize behind one
+    // thread. Both pools are process-wide: the last opened DB's value wins.
+    int flushParallelism = std::max(1, parallelism / 4);
+    NAPI_STATUS_THROWS(GetProperty(env, options, "flushParallelism", flushParallelism));
+    dbOptions.env->SetBackgroundThreads(std::max(1, flushParallelism), rocksdb::Env::HIGH);
+
     NAPI_STATUS_THROWS(GetProperty(env, options, "walDir", dbOptions.wal_dir));
 
     // 64-bit inputs: walTTL is in ms and walSizeLimit in bytes, so a 32-bit type
@@ -1427,6 +1481,34 @@ NAPI_METHOD(db_open) {
 
     NAPI_STATUS_THROWS(GetProperty(env, options, "writeBufferSize", dbOptions.db_write_buffer_size));
 
+    {
+      napi_value wbmValue;
+      NAPI_STATUS_THROWS(napi_get_named_property(env, options, "writeBufferManager", &wbmValue));
+
+      napi_valuetype wbmType;
+      NAPI_STATUS_THROWS(napi_typeof(env, wbmValue, &wbmType));
+
+      if (wbmType == napi_object || wbmType == napi_bigint) {
+        napi_value handleValue = wbmValue;
+        if (wbmType == napi_object) {
+          NAPI_STATUS_THROWS(napi_get_named_property(env, wbmValue, "handle", &handleValue));
+        }
+
+        bool lossless;
+        int64_t ptr;
+        NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, handleValue, &ptr, &lossless));
+        if (!lossless) {
+          napi_throw_error(env, nullptr, "invalid writeBufferManager handle");
+          return NULL;
+        }
+
+        dbOptions.write_buffer_manager = *reinterpret_cast<std::shared_ptr<rocksdb::WriteBufferManager>*>(ptr);
+      } else if (wbmType != napi_undefined && wbmType != napi_null) {
+        napi_throw_error(env, nullptr, "invalid writeBufferManager");
+        return NULL;
+      }
+    }
+
     NAPI_STATUS_THROWS(GetProperty(env, options, "manualWALFlush", dbOptions.manual_wal_flush));
     NAPI_STATUS_THROWS(GetProperty(env, options, "walManualFlush", dbOptions.manual_wal_flush));
 
@@ -2444,6 +2526,128 @@ NAPI_METHOD(cache_get_handle) {
   return result;
 }
 
+NAPI_METHOD(write_buffer_manager_init) {
+  NAPI_ARGV(1);
+
+  size_t bufferSize = 256 * 1024 * 1024;  // 256 MiB
+  NAPI_STATUS_THROWS(GetProperty(env, argv[0], "bufferSize", bufferSize));
+
+  bool allowStall = false;
+  NAPI_STATUS_THROWS(GetProperty(env, argv[0], "allowStall", allowStall));
+
+  std::shared_ptr<rocksdb::Cache> cache;
+  {
+    napi_value cacheValue;
+    NAPI_STATUS_THROWS(napi_get_named_property(env, argv[0], "cache", &cacheValue));
+
+    napi_valuetype cacheType;
+    NAPI_STATUS_THROWS(napi_typeof(env, cacheValue, &cacheType));
+
+    if (cacheType == napi_object || cacheType == napi_bigint) {
+      napi_value handleValue = cacheValue;
+      if (cacheType == napi_object) {
+        NAPI_STATUS_THROWS(napi_get_named_property(env, cacheValue, "handle", &handleValue));
+      }
+
+      bool lossless;
+      int64_t ptr;
+      NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, handleValue, &ptr, &lossless));
+      if (!lossless) {
+        napi_throw_error(env, nullptr, "invalid cache handle");
+        return NULL;
+      }
+
+      cache = *reinterpret_cast<std::shared_ptr<rocksdb::Cache>*>(ptr);
+    } else if (cacheType != napi_undefined && cacheType != napi_null) {
+      napi_throw_error(env, nullptr, "invalid cache");
+      return NULL;
+    }
+  }
+
+  auto wbm = new std::shared_ptr<rocksdb::WriteBufferManager>(
+      std::make_shared<rocksdb::WriteBufferManager>(bufferSize, cache, allowStall));
+
+  napi_value result;
+  NAPI_STATUS_THROWS(
+      napi_create_external(env, wbm, Finalize<std::shared_ptr<rocksdb::WriteBufferManager>>, wbm, &result));
+
+  return result;
+}
+
+NAPI_METHOD(write_buffer_manager_get_handle) {
+  NAPI_ARGV(1);
+
+  std::shared_ptr<rocksdb::WriteBufferManager>* wbm;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&wbm)));
+
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_create_bigint_int64(env, reinterpret_cast<intptr_t>(wbm), &result));
+
+  return result;
+}
+
+NAPI_METHOD(write_buffer_manager_get_usage) {
+  NAPI_ARGV(1);
+
+  std::shared_ptr<rocksdb::WriteBufferManager>* wbm;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&wbm)));
+
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_create_object(env, &result));
+
+  napi_value memoryUsage;
+  NAPI_STATUS_THROWS(napi_create_double(env, static_cast<double>((*wbm)->memory_usage()), &memoryUsage));
+  NAPI_STATUS_THROWS(napi_set_named_property(env, result, "memoryUsage", memoryUsage));
+
+  napi_value mutableMemoryUsage;
+  NAPI_STATUS_THROWS(
+      napi_create_double(env, static_cast<double>((*wbm)->mutable_memtable_memory_usage()), &mutableMemoryUsage));
+  NAPI_STATUS_THROWS(napi_set_named_property(env, result, "mutableMemoryUsage", mutableMemoryUsage));
+
+  napi_value bufferSize;
+  NAPI_STATUS_THROWS(napi_create_double(env, static_cast<double>((*wbm)->buffer_size()), &bufferSize));
+  NAPI_STATUS_THROWS(napi_set_named_property(env, result, "bufferSize", bufferSize));
+
+  return result;
+}
+
+// Probes whether io_uring is actually usable in this process: RocksDB gates its
+// async MultiGet / prefetch I/O on io_uring_setup succeeding at runtime and
+// falls back to serial reads SILENTLY when the syscall is denied (seccomp — the
+// default Docker/containerd profiles since late 2023 — or the
+// kernel.io_uring_disabled sysctl) or missing (ENOSYS). io_uring_setup(0, NULL)
+// never succeeds; a functional kernel rejects the arguments (EINVAL/EFAULT)
+// while a blocked one fails with EPERM/EACCES/ENOSYS before looking at them.
+NAPI_METHOD(io_uring_available) {
+#if defined(__linux__) && defined(SYS_io_uring_setup)
+  errno = 0;
+  const long rc = syscall(SYS_io_uring_setup, 0, nullptr);
+  const bool available = rc >= 0 || (errno != ENOSYS && errno != EPERM && errno != EACCES);
+  if (rc >= 0) {
+    close(static_cast<int>(rc));
+  }
+
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_get_boolean(env, available, &result));
+
+  return result;
+#elif defined(__linux__)
+  // Built without any syscall number for io_uring_setup (pre-io_uring-era
+  // headers): this binary cannot use io_uring regardless of the running
+  // kernel, so report it unavailable — the Linux contract stays boolean.
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_get_boolean(env, false, &result));
+
+  return result;
+#else
+  // Not applicable on this platform.
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_get_null(env, &result));
+
+  return result;
+#endif
+}
+
 NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_init);
   NAPI_EXPORT_FUNCTION(db_open);
@@ -2486,4 +2690,10 @@ NAPI_INIT() {
 
   NAPI_EXPORT_FUNCTION(cache_init);
   NAPI_EXPORT_FUNCTION(cache_get_handle);
+
+  NAPI_EXPORT_FUNCTION(write_buffer_manager_init);
+  NAPI_EXPORT_FUNCTION(write_buffer_manager_get_handle);
+  NAPI_EXPORT_FUNCTION(write_buffer_manager_get_usage);
+
+  NAPI_EXPORT_FUNCTION(io_uring_available);
 }

package/index.js

@@ -6,6 +6,7 @@ const ModuleError = require('module-error')
 const binding = require('./binding')
 const { ChainedBatch } = require('./chained-batch')
 const { RocksCache } = require('./cache')
+const { RocksWriteBufferManager } = require('./write-buffer-manager')
 const { Iterator } = require('./iterator')
 const fs = require('node:fs')
 const assert = require('node:assert')
@@ -414,3 +415,12 @@ class RocksLevel extends AbstractLevel {
 
 exports.RocksLevel = RocksLevel
 exports.RocksCache = RocksCache
+exports.RocksWriteBufferManager = RocksWriteBufferManager
+
+// null on platforms where io_uring does not apply (non-Linux); boolean on
+// Linux, where `false` means RocksDB's async_io silently degrades to serial
+// reads (seccomp, kernel.io_uring_disabled, a kernel without io_uring, or a
+// binary built without an io_uring syscall number).
+exports.ioUringAvailable = function ioUringAvailable () {
+  return binding.io_uring_available()
+}

package/iterator.js

@@ -16,6 +16,7 @@ const kFirst = Symbol('first')
 const kPosition = Symbol('position')
 const kBusy = Symbol('busy')
 const kPendingClose = Symbol('pendingClose')
+const kHasFilter = Symbol('hasFilter')
 
 const kEmpty = Object.freeze([])
 
@@ -32,6 +33,7 @@ class Iterator extends AbstractIterator {
     this[kDB] = db
     this[kBusy] = false
     this[kPendingClose] = null
+    this[kHasFilter] = options.keyFilter != null || options.valueFilter != null
   }
 
   [Symbol.asyncDispose] () {
@@ -85,15 +87,41 @@ class Iterator extends AbstractIterator {
       const size = this[kFirst] ? 1 : 1000
       this[kFirst] = false
 
-      try {
-        const { rows, finished } = binding.iterator_nextv_sync(this[kContext], size, null)
-        this[kCache] = rows
-        this[kFinished] = finished
-        this[kPosition] = 0
-
-        setImmediate(() => this._next(callback))
-      } catch (err) {
-        process.nextTick(callback, err)
+      if (this[kHasFilter]) {
+        try {
+          this[kDB][kRef]()
+          this[kBusy] = true
+          binding.iterator_nextv(this[kContext], size, null, (err, result) => {
+            this[kBusy] = false
+            this[kDB][kUnref]()
+
+            if (err) {
+              callback(err)
+            } else {
+              this[kCache] = result.rows
+              this[kFinished] = result.finished
+              this[kPosition] = 0
+              this._next(callback)
+            }
+
+            this._flushPendingClose()
+          })
+        } catch (err) {
+          this[kBusy] = false
+          this[kDB][kUnref]()
+          process.nextTick(callback, err)
+        }
+      } else {
+        try {
+          const { rows, finished } = binding.iterator_nextv_sync(this[kContext], size, null)
+          this[kCache] = rows
+          this[kFinished] = finished
+          this[kPosition] = 0
+
+          setImmediate(() => this._next(callback))
+        } catch (err) {
+          process.nextTick(callback, err)
+        }
       }
     }
 
@@ -189,10 +217,25 @@ class Iterator extends AbstractIterator {
     return callback[kPromise]
   }
 
+  _nextvCached (size) {
+    const end = Math.min(this[kCache].length, this[kPosition] + size * 2)
+    const rows = this[kCache].slice(this[kPosition], end)
+    this[kPosition] = end
+
+    const finished = this[kFinished] && this[kPosition] >= this[kCache].length
+    const limited = !finished && rows.length >= size * 2
+
+    return { rows, finished, limited }
+  }
+
   _nextvSync (size, options) {
     assert(this[kContext])
     assert(!this[kBusy])
 
+    if (this[kPosition] < this[kCache].length) {
+      return this._nextvCached(size)
+    }
+
     if (this[kFinished]) {
       return { rows: [], finished: true }
     }
@@ -210,7 +253,9 @@ class Iterator extends AbstractIterator {
     callback = fromCallback(callback, kPromise)
 
     try {
-      if (this[kFinished]) {
+      if (this[kPosition] < this[kCache].length) {
+        process.nextTick(callback, null, this._nextvCached(size))
+      } else if (this[kFinished]) {
         process.nextTick(callback, null, { rows: [], finished: true })
       } else {
         this[kDB][kRef]()
@@ -239,6 +284,8 @@ class Iterator extends AbstractIterator {
   }
 
   _closeSync () {
+    assert(!this[kBusy])
+
     this[kCache] = kEmpty
 
     if (this[kContext]) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "16.0.6",
+  "version": "16.0.9",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/release.sh

@@ -3,12 +3,41 @@ set -e
 
 cd "$(dirname "$0")"
 
+export DOCKER_HOST="${DOCKER_HOST:-ssh://nxtop@hq-test-srv1.nxt.io}"
+
+# Fail fast: npm publish needs a valid login, so check before the slow builds.
+if ! npm whoami --registry https://registry.npmjs.org > /dev/null 2>&1; then
+  echo "Not logged in to npm, run 'npm login' first." >&2
+  exit 1
+fi
+
 # Fail fast: npm version refuses a dirty tree, so check before the slow builds.
 if [ -n "$(git status --porcelain)" ]; then
   echo "Working tree is not clean, commit or stash changes first." >&2
   exit 1
 fi
 
+# Fail fast: don't build/publish on a branch that's behind or diverged from origin.
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+echo "Fetching origin..."
+git fetch origin "$BRANCH"
+
+LOCAL=$(git rev-parse HEAD)
+REMOTE=$(git rev-parse "origin/$BRANCH")
+BASE=$(git merge-base HEAD "origin/$BRANCH")
+
+if [ "$LOCAL" = "$REMOTE" ]; then
+  : # up to date
+elif [ "$LOCAL" = "$BASE" ]; then
+  echo "Branch '$BRANCH' is behind origin, pull the latest changes first." >&2
+  exit 1
+elif [ "$REMOTE" = "$BASE" ]; then
+  : # local is ahead, fine to push
+else
+  echo "Branch '$BRANCH' has diverged from origin, reconcile before releasing." >&2
+  exit 1
+fi
+
 # Keep the local arm64 build targeting the same node version as the Docker image.
 NODE_TARGET=$(sed -n 's/^FROM node:\([0-9.]*\).*/\1/p' Dockerfile)
 if [ -z "$NODE_TARGET" ]; then

package/write-buffer-manager.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const binding = require('./binding')
+
+const kWriteBufferManagerContext = Symbol('writeBufferManagerContext')
+
+class RocksWriteBufferManager {
+  constructor (options = {}) {
+    this[kWriteBufferManagerContext] = binding.write_buffer_manager_init(options)
+  }
+
+  get handle () {
+    return binding.write_buffer_manager_get_handle(this[kWriteBufferManagerContext])
+  }
+
+  get usage () {
+    return binding.write_buffer_manager_get_usage(this[kWriteBufferManagerContext])
+  }
+}
+
+exports.RocksWriteBufferManager = RocksWriteBufferManager