@nxtedition/rocksdb 10.0.1 → 10.0.3

package/binding.cc

@@ -50,51 +50,62 @@ struct Closable {
 };
 
 struct Database final {
-	Database(std::string location) : location(std::move(location)) {}
+  Database(std::string location) : location(std::move(location)) {}
   ~Database() { assert(!db); }
 
   rocksdb::Status Close() {
+    std::lock_guard<std::mutex> lock(mutex_);
+
     if (!db) {
       return rocksdb::Status::OK();
     }
 
-    for (auto closable : closables) {
+    for (auto closable : closables_) {
       closable->Close();
     }
-    closables.clear();
+    closables_.clear();
 
     db->FlushWAL(true);
 
-		for (auto& [id, column] : columns) {
-			db->DestroyColumnFamilyHandle(column.handle);
-		}
-		columns.clear();
+    for (auto& [id, column] : columns) {
+      db->DestroyColumnFamilyHandle(column.handle);
+    }
+    columns.clear();
+
+    auto db2 = std::move(db);
+    return db2->Close();
+  }
+
+  void Ref() { refs_++; }
 
-		auto db2 = std::move(db);
-		return db2->Close();
+  void Unref() {
+    if (--refs_ == 0) {
+      Close();
+      delete this;
+    }
   }
 
-	void Ref() {
-		std::lock_guard<std::mutex> lock(mutex);
+  void Attach(Closable* closable) {
+    std::lock_guard<std::mutex> lock(mutex_);
 
-		refs++;
-	}
+    closables_.insert(closable);
+  }
+
+  void Detach(Closable* closable) {
+    std::lock_guard<std::mutex> lock(mutex_);
 
-	void Unref() {
-		std::lock_guard<std::mutex> lock(mutex);
+    closables_.erase(closable);
+  }
 
-		if (--refs == 0) {
-			Close();
-			delete this;
-		}
-	}
+  const std::string location;
 
-	std::mutex mutex;
-	int refs = 0;
-	std::string location;
   std::unique_ptr<rocksdb::DB> db;
-  std::set<Closable*> closables;
   std::map<int32_t, ColumnFamily> columns;
+
+ private:
+  mutable std::mutex mutex_;
+  std::set<Closable*> closables_;
+  std::atomic<int> refs_ = 0;
 };
 
 enum BatchOp { Empty, Put, Delete, Merge, Data };
@@ -443,12 +454,12 @@ struct Iterator final : public BaseIterator {
 
   napi_status Attach(napi_env env, napi_value context) {
     NAPI_STATUS_RETURN(napi_create_reference(env, context, 1, &ref_));
-    database_->closables.insert(this);
+    database_->Attach(this);
     return napi_ok;
   }
 
   napi_status Detach(napi_env env) {
-    database_->closables.erase(this);
+    database_->Detach(this);
     if (ref_) {
       NAPI_STATUS_RETURN(napi_delete_reference(env, ref_));
     }
@@ -466,42 +477,65 @@ struct Iterator final : public BaseIterator {
   napi_ref ref_ = nullptr;
 };
 
+/**
+ * Hook for when the environment exits. This hook will be called after
+ * already-scheduled napi_async_work items have finished, which gives us
+ * the guarantee that no db operations will be in-flight at this time.
+ */
+static void env_cleanup_hook(void* data) {
+  auto database = reinterpret_cast<Database*>(data);
+
+  // Do everything that db_close() does but synchronously. We're expecting that GC
+  // did not (yet) collect the database because that would be a user mistake (not
+  // closing their db) made during the lifetime of the environment. That's different
+  // from an environment being torn down (like the main process or a worker thread)
+  // where it's our responsibility to clean up. Note also, the following code must
+  // be a safe noop if called before db_open() or after db_close().
+  if (database) {
+    database->Unref();
+  }
+}
+
 static void FinalizeDatabase(napi_env env, void* data, void* hint) {
-  if (data) {
-    reinterpret_cast<Database*>(data)->Unref();
+  auto database = reinterpret_cast<Database*>(data);
+  if (database) {
+    napi_remove_env_cleanup_hook(env, env_cleanup_hook, database);
+    database->Unref();
   }
 }
 
 NAPI_METHOD(db_init) {
   NAPI_ARGV(1);
 
-	Database* database = nullptr;
+  Database* database = nullptr;
 
   napi_valuetype type;
   NAPI_STATUS_THROWS(napi_typeof(env, argv[0], &type));
 
   if (type == napi_string) {
-		std::string location;
+    std::string location;
     size_t length = 0;
     NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, argv[0], nullptr, 0, &length));
     location.resize(length, '\0');
     NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, argv[0], &location[0], length + 1, &length));
 
-		database = new Database(location);
+    database = new Database(location);
   } else if (type == napi_bigint) {
-		int64_t value;
-		bool lossless;
-		NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, argv[0], &value, &lossless));
+    int64_t value;
+    bool lossless;
+    NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, argv[0], &value, &lossless));
 
-		database = reinterpret_cast<Database*>(value);
+    database = reinterpret_cast<Database*>(value);
   } else {
-		NAPI_STATUS_THROWS(napi_invalid_arg);
-	}
+    NAPI_STATUS_THROWS(napi_invalid_arg);
+  }
+
+  napi_add_env_cleanup_hook(env, env_cleanup_hook, database);
 
   napi_value result;
   NAPI_STATUS_THROWS(napi_create_external(env, database, FinalizeDatabase, nullptr, &result));
 
-	database->Ref();
+  database->Ref();
 
   return result;
 }
@@ -641,7 +675,7 @@ napi_status InitOptions(napi_env env, T& columnOptions, const U& options) {
   rocksdb::BlockBasedTableOptions tableOptions;
 
   if (cacheSize) {
-    tableOptions.block_cache = rocksdb::NewLRUCache(cacheSize);
+    tableOptions.block_cache = rocksdb::HyperClockCacheOptions(cacheSize, 0).MakeSharedCache();
     tableOptions.cache_index_and_filter_blocks = true;
     NAPI_STATUS_RETURN(
         GetProperty(env, options, "cacheIndexAndFilterBlocks", tableOptions.cache_index_and_filter_blocks));
@@ -767,152 +801,151 @@ NAPI_METHOD(db_open) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-	if (database->db) {
-		napi_value columns;
-		NAPI_STATUS_THROWS(napi_create_object(env, &columns));
-		for (auto& [id, column] : database->columns) {
-			napi_value val;
-			NAPI_STATUS_THROWS(napi_create_external(env, column.handle, nullptr, nullptr, &val));
-			NAPI_STATUS_THROWS(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
-		}
-		return columns;
-	} else {
-		rocksdb::Options dbOptions;
-
-		const auto options = argv[1];
-
-		int parallelism = std::max<int>(1, std::thread::hardware_concurrency() / 2);
-		NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
-		dbOptions.IncreaseParallelism(parallelism);
-
-		uint32_t walTTL = 0;
-		NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
-		dbOptions.WAL_ttl_seconds = walTTL / 1e3;
-
-		uint32_t walSizeLimit = 0;
-		NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
-		dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
-
-		uint32_t maxTotalWalSize = 0;
-		NAPI_STATUS_THROWS(GetProperty(env, options, "walTotalSizeLimit", walSizeLimit));
-		dbOptions.max_total_wal_size = maxTotalWalSize / 1e6;
-
-		bool walCompression = false;
-		NAPI_STATUS_THROWS(GetProperty(env, options, "walCompression", walCompression));
-		dbOptions.wal_compression =
-				walCompression ? rocksdb::CompressionType::kZSTD : rocksdb::CompressionType::kNoCompression;
-
-		dbOptions.avoid_unnecessary_blocking_io = true;
-		dbOptions.write_dbid_to_manifest = true;
-		dbOptions.enable_pipelined_write = true;  // We only write in the main thread...
-		dbOptions.create_missing_column_families = true;
-		dbOptions.fail_if_options_file_error = true;
-
-		NAPI_STATUS_THROWS(GetProperty(env, options, "createIfMissing", dbOptions.create_if_missing));
-		NAPI_STATUS_THROWS(GetProperty(env, options, "errorIfExists", dbOptions.error_if_exists));
-		NAPI_STATUS_THROWS(GetProperty(env, options, "pipelinedWrite", dbOptions.enable_pipelined_write));
-
-		// TODO (feat): dbOptions.listeners
-
-		std::string infoLogLevel;
-		NAPI_STATUS_THROWS(GetProperty(env, options, "infoLogLevel", infoLogLevel));
-		if (infoLogLevel.size() > 0) {
-			rocksdb::InfoLogLevel lvl = {};
-
-			if (infoLogLevel == "debug")
-				lvl = rocksdb::InfoLogLevel::DEBUG_LEVEL;
-			else if (infoLogLevel == "info")
-				lvl = rocksdb::InfoLogLevel::INFO_LEVEL;
-			else if (infoLogLevel == "warn")
-				lvl = rocksdb::InfoLogLevel::WARN_LEVEL;
-			else if (infoLogLevel == "error")
-				lvl = rocksdb::InfoLogLevel::ERROR_LEVEL;
-			else if (infoLogLevel == "fatal")
-				lvl = rocksdb::InfoLogLevel::FATAL_LEVEL;
-			else if (infoLogLevel == "header")
-				lvl = rocksdb::InfoLogLevel::HEADER_LEVEL;
-			else
-				napi_throw_error(env, nullptr, "invalid log level");
-
-			dbOptions.info_log_level = lvl;
-		} else {
-			// In some places RocksDB checks this option to see if it should prepare
-			// debug information (ahead of logging), so set it to the highest level.
-			dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
-			dbOptions.info_log.reset(new NullLogger());
-		}
-
-		NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
-
-		std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
-
-		bool hasColumns;
-		NAPI_STATUS_THROWS(napi_has_named_property(env, options, "columns", &hasColumns));
-
-		if (hasColumns) {
-			napi_value columns;
-			NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
-
-			napi_value keys;
-			NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
-
-			uint32_t len;
-			NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
-
-			descriptors.resize(len);
-			for (uint32_t n = 0; n < len; ++n) {
-				napi_value key;
-				NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
-
-				napi_value column;
-				NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
-
-				NAPI_STATUS_THROWS(InitOptions(env, descriptors[n].options, column));
-
-				NAPI_STATUS_THROWS(GetValue(env, key, descriptors[n].name));
-			}
-		}
-
-		auto callback = argv[2];
-
-		runAsync<std::vector<rocksdb::ColumnFamilyHandle*>>(
-				"leveldown.open", env, callback,
-				[=](auto& handles) {
-					assert(!database->db);
-
-					rocksdb::DB* db = nullptr;
-
-					const auto status = descriptors.empty()
-						? rocksdb::DB::Open(dbOptions, database->location, &db)
-						: rocksdb::DB::Open(dbOptions, database->location, descriptors, &handles, &db);
-
-					database->db.reset(db);
-
-					return status;
-				},
-				[=](auto& handles, auto env, auto& argv) {
-					argv.resize(2);
-
-					NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
-
-					for (size_t n = 0; n < handles.size(); ++n) {
-						ColumnFamily column;
-						column.handle = handles[n];
-						column.descriptor = descriptors[n];
-						database->columns[column.handle->GetID()] = column;
-					}
-
-					napi_value columns;
-					NAPI_STATUS_RETURN(napi_create_object(env, &columns));
-					for (auto& [id, column] : database->columns) {
-						napi_value val;
-						NAPI_STATUS_RETURN(napi_create_external(env, column.handle, nullptr, nullptr, &val));
-						NAPI_STATUS_RETURN(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
-					}
-
-					return napi_ok;
-				});
-	}
+  if (database->db) {
+    napi_value columns;
+    NAPI_STATUS_THROWS(napi_create_object(env, &columns));
+    for (auto& [id, column] : database->columns) {
+      napi_value val;
+      NAPI_STATUS_THROWS(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+      NAPI_STATUS_THROWS(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+    }
+    return columns;
+  } else {
+    rocksdb::Options dbOptions;
+
+    const auto options = argv[1];
+
+    int parallelism = std::max<int>(1, std::thread::hardware_concurrency() / 2);
+    NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
+    dbOptions.IncreaseParallelism(parallelism);
+
+    uint32_t walTTL = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
+    dbOptions.WAL_ttl_seconds = walTTL / 1e3;
+
+    uint32_t walSizeLimit = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
+    dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
+
+    uint32_t maxTotalWalSize = 0;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walTotalSizeLimit", walSizeLimit));
+    dbOptions.max_total_wal_size = maxTotalWalSize / 1e6;
+
+    bool walCompression = false;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "walCompression", walCompression));
+    dbOptions.wal_compression =
+        walCompression ? rocksdb::CompressionType::kZSTD : rocksdb::CompressionType::kNoCompression;
+
+    dbOptions.avoid_unnecessary_blocking_io = true;
+    dbOptions.write_dbid_to_manifest = true;
+    dbOptions.enable_pipelined_write = true;  // We only write in the main thread...
+    dbOptions.create_missing_column_families = true;
+    dbOptions.fail_if_options_file_error = true;
+
+    NAPI_STATUS_THROWS(GetProperty(env, options, "createIfMissing", dbOptions.create_if_missing));
+    NAPI_STATUS_THROWS(GetProperty(env, options, "errorIfExists", dbOptions.error_if_exists));
+    NAPI_STATUS_THROWS(GetProperty(env, options, "pipelinedWrite", dbOptions.enable_pipelined_write));
+
+    // TODO (feat): dbOptions.listeners
+
+    std::string infoLogLevel;
+    NAPI_STATUS_THROWS(GetProperty(env, options, "infoLogLevel", infoLogLevel));
+    if (infoLogLevel.size() > 0) {
+      rocksdb::InfoLogLevel lvl = {};
+
+      if (infoLogLevel == "debug")
+        lvl = rocksdb::InfoLogLevel::DEBUG_LEVEL;
+      else if (infoLogLevel == "info")
+        lvl = rocksdb::InfoLogLevel::INFO_LEVEL;
+      else if (infoLogLevel == "warn")
+        lvl = rocksdb::InfoLogLevel::WARN_LEVEL;
+      else if (infoLogLevel == "error")
+        lvl = rocksdb::InfoLogLevel::ERROR_LEVEL;
+      else if (infoLogLevel == "fatal")
+        lvl = rocksdb::InfoLogLevel::FATAL_LEVEL;
+      else if (infoLogLevel == "header")
+        lvl = rocksdb::InfoLogLevel::HEADER_LEVEL;
+      else
+        napi_throw_error(env, nullptr, "invalid log level");
+
+      dbOptions.info_log_level = lvl;
+    } else {
+      // In some places RocksDB checks this option to see if it should prepare
+      // debug information (ahead of logging), so set it to the highest level.
+      dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
+      dbOptions.info_log.reset(new NullLogger());
+    }
+
+    NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
+
+    std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+
+    bool hasColumns;
+    NAPI_STATUS_THROWS(napi_has_named_property(env, options, "columns", &hasColumns));
+
+    if (hasColumns) {
+      napi_value columns;
+      NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
+
+      napi_value keys;
+      NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
+
+      uint32_t len;
+      NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
+
+      descriptors.resize(len);
+      for (uint32_t n = 0; n < len; ++n) {
+        napi_value key;
+        NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
+
+        napi_value column;
+        NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
+
+        NAPI_STATUS_THROWS(InitOptions(env, descriptors[n].options, column));
+
+        NAPI_STATUS_THROWS(GetValue(env, key, descriptors[n].name));
+      }
+    }
+
+    auto callback = argv[2];
+
+    runAsync<std::vector<rocksdb::ColumnFamilyHandle*>>(
+        "leveldown.open", env, callback,
+        [=](auto& handles) {
+          assert(!database->db);
+
+          rocksdb::DB* db = nullptr;
+
+          const auto status = descriptors.empty()
+                                  ? rocksdb::DB::Open(dbOptions, database->location, &db)
+                                  : rocksdb::DB::Open(dbOptions, database->location, descriptors, &handles, &db);
+
+          database->db.reset(db);
+
+          return status;
+        },
+        [=](auto& handles, auto env, auto& argv) {
+          argv.resize(2);
+
+          NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
+
+          for (size_t n = 0; n < handles.size(); ++n) {
+            ColumnFamily column;
+            column.handle = handles[n];
+            column.descriptor = descriptors[n];
+            database->columns[column.handle->GetID()] = column;
+          }
+
+          napi_value columns = argv[1];
+          for (auto& [id, column] : database->columns) {
+            napi_value val;
+            NAPI_STATUS_RETURN(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+            NAPI_STATUS_RETURN(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+          }
+
+          return napi_ok;
+        });
+  }
 
   return 0;
 }

package/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+export DOCKER_HOST=${DOCKER_HOST:-ssh://root@test-srv3.hq.bmux}
+
+echo "Building image..."
+docker build --iidfile prebuilds.iid .
+
+echo "Extracting prebuilds from image..."
+IMG=$(cat prebuilds.iid)
+ID=$(docker create $IMG)
+docker cp "$ID:/rocks-level/prebuilds" ./
+
+echo "Cleaning up..."
+docker rm $ID > /dev/null
+rm prebuilds.iid
+
+echo "All done!"

package/deps/rocksdb/rocksdb.gyp

@@ -87,14 +87,8 @@
               "ROCKSDB_PTHREAD_ADAPTIVE_MUTEX=1",
               "ROCKSDB_RANGESYNC_PRESENT=1",
               "ROCKSDB_SCHED_GETCPU_PRESENT=1",
-              # "ROCKSDB_IOURING_PRESENT=1",
+              "ROCKSDB_IOURING_PRESENT=1",
               "USE_FOLLY=1",
-              "FOLLY_NO_CONFIG=1"
-              "HAVE_SSE42=1",
-              "HAVE_BMI=1",
-              "HAVE_LZCNT=1",
-              "HAVE_AVX2=1",
-              "HAVE_PCLMUL=1",
               "HAVE_UINT128_EXTENSION=1",
               "HAVE_ALIGNED_NEW=1",
               # "HAVE_FULLFSYNC=1",
@@ -107,6 +101,7 @@
               "libraries": [
                 "/usr/lib/x86_64-linux-gnu/libzstd.a",
                 "/usr/lib/x86_64-linux-gnu/libfolly.a",
+                "/usr/lib/x86_64-linux-gnu/liburing.a",
                 # "/usr/lib/x86_64-linux-gnu/libjemalloc.a",
               ],
             },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "10.0.1",
+  "version": "10.0.3",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",