@nxtedition/rocksdb 9.0.1 → 10.0.0

package/binding.cc

@@ -40,8 +40,6 @@ struct Database;
 struct Iterator;
 
 struct ColumnFamily {
-  napi_ref ref;
-  napi_value val;
   rocksdb::ColumnFamilyHandle* handle;
   rocksdb::ColumnFamilyDescriptor descriptor;
 };
@@ -52,9 +50,12 @@ struct Closable {
 };
 
 struct Database final {
+	Database(std::string location) : location(std::move(location)) {}
   ~Database() { assert(!db); }
 
   rocksdb::Status Close() {
+		std::lock_guard<std::mutex> lock(mutex);
+
     if (!db) {
       return rocksdb::Status::OK();
     }
@@ -64,17 +65,35 @@ struct Database final {
     }
     closables.clear();
 
-    for (auto& [id, column] : columns) {
-      db->DestroyColumnFamilyHandle(column.handle);
-    }
-    columns.clear();
-
     db->FlushWAL(true);
 
-    auto db2 = std::move(db);
-    return db2->Close();
+		for (auto& [id, column] : columns) {
+			db->DestroyColumnFamilyHandle(column.handle);
+		}
+		columns.clear();
+
+		auto db2 = std::move(db);
+		return db2->Close();
   }
 
+	void Ref() {
+		std::lock_guard<std::mutex> lock(mutex);
+
+		refs++;
+	}
+
+	void Unref() {
+		std::lock_guard<std::mutex> lock(mutex);
+
+		if (--refs == 0) {
+			Close();
+			delete this;
+		}
+	}
+
+	int refs = 0;
+	std::mutex mutex;
+	std::string location;
   std::unique_ptr<rocksdb::DB> db;
   std::set<Closable*> closables;
   std::map<int32_t, ColumnFamily> columns;
@@ -449,32 +468,74 @@ struct Iterator final : public BaseIterator {
   napi_ref ref_ = nullptr;
 };
 
-static void env_cleanup_hook(void* arg) {
-  auto database = reinterpret_cast<Database*>(arg);
-  if (database) {
-    database->Close();
-  }
-}
-
 static void FinalizeDatabase(napi_env env, void* data, void* hint) {
   if (data) {
-    auto database = reinterpret_cast<Database*>(data);
-    database->Close();
-    napi_remove_env_cleanup_hook(env, env_cleanup_hook, database);
-    for (auto& [id, column] : database->columns) {
-      napi_delete_reference(env, column.ref);
-    }
-    delete database;
+		auto database = reinterpret_cast<Database*>(data);
+
+		std::lock_guard<std::mutex> lock(database->mutex);
+
+    database->Unref();
   }
 }
 
 NAPI_METHOD(db_init) {
-  auto database = new Database();
-  napi_add_env_cleanup_hook(env, env_cleanup_hook, database);
+  NAPI_ARGV(1);
+
+	Database* database = nullptr;
+
+  napi_valuetype type;
+  NAPI_STATUS_THROWS(napi_typeof(env, argv[0], &type));
+
+  if (type == napi_string) {
+		std::string location;
+    size_t length = 0;
+    NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, argv[0], nullptr, 0, &length));
+    location.resize(length, '\0');
+    NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, argv[0], &location[0], length + 1, &length));
+
+		database = new Database(location);
+  } else if (type == napi_bigint) {
+		int64_t value;
+		bool lossless;
+		NAPI_STATUS_THROWS(napi_get_value_bigint_int64(env, argv[0], &value, &lossless));
+
+		database = reinterpret_cast<Database*>(value);
+  } else {
+		NAPI_STATUS_THROWS(napi_invalid_arg);
+	}
 
   napi_value result;
   NAPI_STATUS_THROWS(napi_create_external(env, database, FinalizeDatabase, nullptr, &result));
 
+	{
+		std::lock_guard<std::mutex> lock(database->mutex);
+		database->Ref();
+	}
+
+  return result;
+}
+
+NAPI_METHOD(db_get_handle) {
+  NAPI_ARGV(1);
+
+  Database* database;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
+
+  napi_value result;
+  NAPI_STATUS_THROWS(napi_create_bigint_int64(env, reinterpret_cast<intptr_t>(database), &result));
+
+  return result;
+}
+
+NAPI_METHOD(db_get_location) {
+  NAPI_ARGV(1);
+
+  Database* database;
+  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
+
+  napi_value result;
+  NAPI_STATUS_THROWS(Convert(env, &database->location, Encoding::String, result));
+
   return result;
 }
 
@@ -710,141 +771,164 @@ NAPI_METHOD(db_get_identity) {
 }
 
 NAPI_METHOD(db_open) {
-  NAPI_ARGV(4);
+  NAPI_ARGV(3);
 
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
-  std::string location;
-  NAPI_STATUS_THROWS(GetValue(env, argv[1], location));
-
-  rocksdb::Options dbOptions;
-
-  const auto options = argv[2];
-
-  int parallelism = std::max<int>(1, std::thread::hardware_concurrency() / 2);
-  NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
-  dbOptions.IncreaseParallelism(parallelism);
-
-  uint32_t walTTL = 0;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
-  dbOptions.WAL_ttl_seconds = walTTL / 1e3;
-
-  uint32_t walSizeLimit = 0;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
-  dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
-
-	uint32_t maxTotalWalSize = 0;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "walTotalSizeLimit", walSizeLimit));
-  dbOptions.max_total_wal_size = maxTotalWalSize / 1e6;
-
-  bool walCompression = false;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "walCompression", walCompression));
-  dbOptions.wal_compression =
-      walCompression ? rocksdb::CompressionType::kZSTD : rocksdb::CompressionType::kNoCompression;
-
-  dbOptions.avoid_unnecessary_blocking_io = true;
-  dbOptions.write_dbid_to_manifest = true;
-  dbOptions.use_adaptive_mutex = true;      // We don't have soo many threads in the libuv thread pool...
-  dbOptions.enable_pipelined_write = true;  // We only write in the main thread...
-  dbOptions.create_missing_column_families = true;
-  dbOptions.fail_if_options_file_error = true;
-
-  NAPI_STATUS_THROWS(GetProperty(env, options, "createIfMissing", dbOptions.create_if_missing));
-  NAPI_STATUS_THROWS(GetProperty(env, options, "errorIfExists", dbOptions.error_if_exists));
-  NAPI_STATUS_THROWS(GetProperty(env, options, "unorderedWrite", dbOptions.unordered_write));
-  NAPI_STATUS_THROWS(GetProperty(env, options, "pipelinedWrite", dbOptions.enable_pipelined_write));
-
-  // TODO (feat): dbOptions.listeners
-
-  std::string infoLogLevel;
-  NAPI_STATUS_THROWS(GetProperty(env, options, "infoLogLevel", infoLogLevel));
-  if (infoLogLevel.size() > 0) {
-    rocksdb::InfoLogLevel lvl = {};
-
-    if (infoLogLevel == "debug")
-      lvl = rocksdb::InfoLogLevel::DEBUG_LEVEL;
-    else if (infoLogLevel == "info")
-      lvl = rocksdb::InfoLogLevel::INFO_LEVEL;
-    else if (infoLogLevel == "warn")
-      lvl = rocksdb::InfoLogLevel::WARN_LEVEL;
-    else if (infoLogLevel == "error")
-      lvl = rocksdb::InfoLogLevel::ERROR_LEVEL;
-    else if (infoLogLevel == "fatal")
-      lvl = rocksdb::InfoLogLevel::FATAL_LEVEL;
-    else if (infoLogLevel == "header")
-      lvl = rocksdb::InfoLogLevel::HEADER_LEVEL;
-    else
-      napi_throw_error(env, nullptr, "invalid log level");
-
-    dbOptions.info_log_level = lvl;
-  } else {
-    // In some places RocksDB checks this option to see if it should prepare
-    // debug information (ahead of logging), so set it to the highest level.
-    dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
-    dbOptions.info_log.reset(new NullLogger());
-  }
-
-  NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
-
-  std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
-
-  bool hasColumns;
-  NAPI_STATUS_THROWS(napi_has_named_property(env, options, "columns", &hasColumns));
-
-  if (hasColumns) {
-    napi_value columns;
-    NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
-
-    napi_value keys;
-    NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
-
-    uint32_t len;
-    NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
-
-    descriptors.resize(len);
-    for (uint32_t n = 0; n < len; ++n) {
-      napi_value key;
-      NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
-
-      napi_value column;
-      NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
-
-      NAPI_STATUS_THROWS(InitOptions(env, descriptors[n].options, column));
-
-      NAPI_STATUS_THROWS(GetValue(env, key, descriptors[n].name));
-    }
-  }
-
-  auto callback = argv[3];
-
-  runAsync<std::vector<rocksdb::ColumnFamilyHandle*>>(
-      "leveldown.open", env, callback,
-      [=](auto& handles) {
-        rocksdb::DB* db = nullptr;
-        const auto status = descriptors.empty() ? rocksdb::DB::Open(dbOptions, location, &db)
-                                                : rocksdb::DB::Open(dbOptions, location, descriptors, &handles, &db);
-        database->db.reset(db);
-        return status;
-      },
-      [=](auto& handles, auto env, auto& argv) {
-        argv.resize(2);
-
-        const auto size = handles.size();
-        NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
-
-        for (size_t n = 0; n < size; ++n) {
-          ColumnFamily column;
-          column.handle = handles[n];
-          column.descriptor = descriptors[n];
-          NAPI_STATUS_RETURN(napi_create_external(env, column.handle, nullptr, nullptr, &column.val));
-          NAPI_STATUS_RETURN(napi_create_reference(env, column.val, 1, &column.ref));
-          NAPI_STATUS_RETURN(napi_set_named_property(env, argv[1], descriptors[n].name.c_str(), column.val));
-          database->columns[column.handle->GetID()] = column;
-        }
-
-        return napi_ok;
-      });
+	std::lock_guard<std::mutex> lock(database->mutex);
+
+	if (database->db) {
+		napi_value columns;
+		NAPI_STATUS_THROWS(napi_create_object(env, &columns));
+		for (auto& [id, column] : database->columns) {
+			napi_value val;
+			NAPI_STATUS_THROWS(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+			NAPI_STATUS_THROWS(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+		}
+		return columns;
+	} else {
+		rocksdb::Options dbOptions;
+
+		const auto options = argv[1];
+
+		int parallelism = std::max<int>(1, std::thread::hardware_concurrency() / 2);
+		NAPI_STATUS_THROWS(GetProperty(env, options, "parallelism", parallelism));
+		dbOptions.IncreaseParallelism(parallelism);
+
+		uint32_t walTTL = 0;
+		NAPI_STATUS_THROWS(GetProperty(env, options, "walTTL", walTTL));
+		dbOptions.WAL_ttl_seconds = walTTL / 1e3;
+
+		uint32_t walSizeLimit = 0;
+		NAPI_STATUS_THROWS(GetProperty(env, options, "walSizeLimit", walSizeLimit));
+		dbOptions.WAL_size_limit_MB = walSizeLimit / 1e6;
+
+		uint32_t maxTotalWalSize = 0;
+		NAPI_STATUS_THROWS(GetProperty(env, options, "walTotalSizeLimit", walSizeLimit));
+		dbOptions.max_total_wal_size = maxTotalWalSize / 1e6;
+
+		bool walCompression = false;
+		NAPI_STATUS_THROWS(GetProperty(env, options, "walCompression", walCompression));
+		dbOptions.wal_compression =
+				walCompression ? rocksdb::CompressionType::kZSTD : rocksdb::CompressionType::kNoCompression;
+
+		dbOptions.avoid_unnecessary_blocking_io = true;
+		dbOptions.write_dbid_to_manifest = true;
+		dbOptions.enable_pipelined_write = true;  // We only write in the main thread...
+		dbOptions.create_missing_column_families = true;
+		dbOptions.fail_if_options_file_error = true;
+
+		NAPI_STATUS_THROWS(GetProperty(env, options, "createIfMissing", dbOptions.create_if_missing));
+		NAPI_STATUS_THROWS(GetProperty(env, options, "errorIfExists", dbOptions.error_if_exists));
+		NAPI_STATUS_THROWS(GetProperty(env, options, "pipelinedWrite", dbOptions.enable_pipelined_write));
+
+		// TODO (feat): dbOptions.listeners
+
+		std::string infoLogLevel;
+		NAPI_STATUS_THROWS(GetProperty(env, options, "infoLogLevel", infoLogLevel));
+		if (infoLogLevel.size() > 0) {
+			rocksdb::InfoLogLevel lvl = {};
+
+			if (infoLogLevel == "debug")
+				lvl = rocksdb::InfoLogLevel::DEBUG_LEVEL;
+			else if (infoLogLevel == "info")
+				lvl = rocksdb::InfoLogLevel::INFO_LEVEL;
+			else if (infoLogLevel == "warn")
+				lvl = rocksdb::InfoLogLevel::WARN_LEVEL;
+			else if (infoLogLevel == "error")
+				lvl = rocksdb::InfoLogLevel::ERROR_LEVEL;
+			else if (infoLogLevel == "fatal")
+				lvl = rocksdb::InfoLogLevel::FATAL_LEVEL;
+			else if (infoLogLevel == "header")
+				lvl = rocksdb::InfoLogLevel::HEADER_LEVEL;
+			else
+				napi_throw_error(env, nullptr, "invalid log level");
+
+			dbOptions.info_log_level = lvl;
+		} else {
+			// In some places RocksDB checks this option to see if it should prepare
+			// debug information (ahead of logging), so set it to the highest level.
+			dbOptions.info_log_level = rocksdb::InfoLogLevel::HEADER_LEVEL;
+			dbOptions.info_log.reset(new NullLogger());
+		}
+
+		NAPI_STATUS_THROWS(InitOptions(env, dbOptions, options));
+
+		std::vector<rocksdb::ColumnFamilyDescriptor> descriptors;
+
+		bool hasColumns;
+		NAPI_STATUS_THROWS(napi_has_named_property(env, options, "columns", &hasColumns));
+
+		if (hasColumns) {
+			napi_value columns;
+			NAPI_STATUS_THROWS(napi_get_named_property(env, options, "columns", &columns));
+
+			napi_value keys;
+			NAPI_STATUS_THROWS(napi_get_property_names(env, columns, &keys));
+
+			uint32_t len;
+			NAPI_STATUS_THROWS(napi_get_array_length(env, keys, &len));
+
+			descriptors.resize(len);
+			for (uint32_t n = 0; n < len; ++n) {
+				napi_value key;
+				NAPI_STATUS_THROWS(napi_get_element(env, keys, n, &key));
+
+				napi_value column;
+				NAPI_STATUS_THROWS(napi_get_property(env, columns, key, &column));
+
+				NAPI_STATUS_THROWS(InitOptions(env, descriptors[n].options, column));
+
+				NAPI_STATUS_THROWS(GetValue(env, key, descriptors[n].name));
+			}
+		}
+
+		auto callback = argv[2];
+
+		runAsync<std::vector<rocksdb::ColumnFamilyHandle*>>(
+				"leveldown.open", env, callback,
+				[=](auto& handles) {
+					assert(!database->db);
+
+					rocksdb::DB* db = nullptr;
+
+					// TODO (fix): There is a race condition here... Check if we are already opening the database.
+
+					const auto status = descriptors.empty()
+						? rocksdb::DB::Open(dbOptions, database->location, &db)
+						: rocksdb::DB::Open(dbOptions, database->location, descriptors, &handles, &db);
+
+					{
+						std::lock_guard<std::mutex> lock(database->mutex);
+						database->db.reset(db);
+					}
+
+					return status;
+				},
+				[=](auto& handles, auto env, auto& argv) {
+					argv.resize(2);
+
+					NAPI_STATUS_RETURN(napi_create_object(env, &argv[1]));
+
+					for (size_t n = 0; n < handles.size(); ++n) {
+						ColumnFamily column;
+						column.handle = handles[n];
+						column.descriptor = descriptors[n];
+						database->columns[column.handle->GetID()] = column;
+					}
+
+					napi_value columns;
+					NAPI_STATUS_RETURN(napi_create_object(env, &columns));
+					for (auto& [id, column] : database->columns) {
+						napi_value val;
+						NAPI_STATUS_RETURN(napi_create_external(env, column.handle, nullptr, nullptr, &val));
+						NAPI_STATUS_RETURN(napi_set_named_property(env, columns, column.descriptor.name.c_str(), val));
+					}
+
+					return napi_ok;
+				});
+	}
 
   return 0;
 }
@@ -859,6 +943,8 @@ NAPI_METHOD(db_close) {
   Database* database;
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
 
+	std::lock_guard<std::mutex> lock(database->mutex);
+
   auto callback = argv[1];
 
   struct State {};
@@ -1567,6 +1653,8 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(db_init);
   NAPI_EXPORT_FUNCTION(db_open);
   NAPI_EXPORT_FUNCTION(db_get_identity);
+  NAPI_EXPORT_FUNCTION(db_get_handle);
+  NAPI_EXPORT_FUNCTION(db_get_location);
   NAPI_EXPORT_FUNCTION(db_close);
   NAPI_EXPORT_FUNCTION(db_get_many);
   NAPI_EXPORT_FUNCTION(db_clear);

package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc

@@ -1224,13 +1224,11 @@ TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdate) {
   ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
   ASSERT_EQ(sec_capacity, 0);
 
-  ASSERT_NOK(UpdateTieredCache(tiered_cache, -1, 0.3));
-  // Only check usage for LRU cache. HCC shows a 64KB usage for some reason
-  if (std::get<0>(GetParam()) == PrimaryCacheType::kCacheTypeLRU) {
-    ASSERT_EQ(GetCache()->GetUsage(), 0);
-  }
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
   ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
-  ASSERT_EQ(sec_capacity, 0);
+  ASSERT_EQ(sec_capacity, (30 << 20));
 }
 
 TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
@@ -1316,28 +1314,50 @@ TEST_P(CompressedSecCacheTestWithTiered, DynamicUpdateWithReservation) {
   ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
   ASSERT_EQ(sec_capacity, 0);
 
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.3));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (37 << 20),
+               GetPercent(37 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (3 << 20),
+               GetPercent(3 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, 30 << 20);
+
   ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
 }
 
-TEST_P(CompressedSecCacheTestWithTiered,
-       DynamicUpdateWithReservationUnderflow) {
+TEST_P(CompressedSecCacheTestWithTiered, ReservationOverCapacity) {
+  CompressedSecondaryCache* sec_cache =
+      reinterpret_cast<CompressedSecondaryCache*>(GetSecondaryCache());
   std::shared_ptr<Cache> tiered_cache = GetTieredCache();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
-      {{"CacheWithSecondaryAdapter::Release:ChargeSecCache1",
-        "CacheWithSecondaryAdapter::UpdateCacheReservationRatio:Begin"},
-       {"CacheWithSecondaryAdapter::UpdateCacheReservationRatio:End",
-        "CacheWithSecondaryAdapter::Release:ChargeSecCache2"}});
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-
-  port::Thread reserve_release_thread([&]() {
-    EXPECT_EQ(cache_res_mgr()->UpdateCacheReservation(50), Status::OK());
-    EXPECT_EQ(cache_res_mgr()->UpdateCacheReservation(0), Status::OK());
-  });
-  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20, 0.01));
-  reserve_release_thread.join();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-
-  ASSERT_OK(UpdateTieredCache(tiered_cache, 100 << 20, 0.3));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(110 << 20));
+  // Use EXPECT_PRED3 instead of EXPECT_NEAR to void too many size_t to
+  // double explicit casts
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20),
+               GetPercent(110 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (30 << 20),
+               GetPercent(30 << 20, 1));
+  size_t sec_capacity;
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (30 << 20));
+
+  ASSERT_OK(UpdateTieredCache(tiered_cache, -1, 0.39));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (110 << 20),
+               GetPercent(110 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (39 << 20),
+               GetPercent(39 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(90 << 20));
+  EXPECT_PRED3(CacheUsageWithinBounds, GetCache()->GetUsage(), (94 << 20),
+               GetPercent(94 << 20, 1));
+  EXPECT_PRED3(CacheUsageWithinBounds, sec_cache->TEST_GetUsage(), (35 << 20),
+               GetPercent(35 << 20, 1));
+  ASSERT_OK(sec_cache->GetCapacity(sec_capacity));
+  ASSERT_EQ(sec_capacity, (39 << 20));
+
+  ASSERT_OK(cache_res_mgr()->UpdateCacheReservation(0));
 }
 
 INSTANTIATE_TEST_CASE_P(

package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc

@@ -83,7 +83,10 @@ CacheWithSecondaryAdapter::CacheWithSecondaryAdapter(
     : CacheWrapper(std::move(target)),
       secondary_cache_(std::move(secondary_cache)),
       adm_policy_(adm_policy),
-      distribute_cache_res_(distribute_cache_res) {
+      distribute_cache_res_(distribute_cache_res),
+      placeholder_usage_(0),
+      reserved_usage_(0),
+      sec_reserved_(0) {
   target_->SetEvictionCallback(
       [this](const Slice& key, Handle* handle, bool was_hit) {
         return EvictionHandler(key, handle, was_hit);
@@ -103,8 +106,7 @@ CacheWithSecondaryAdapter::CacheWithSecondaryAdapter(
     // secondary cache is freed from the reservation.
     s = pri_cache_res_->UpdateCacheReservation(sec_capacity);
     assert(s.ok());
-    sec_cache_res_ratio_.store((double)sec_capacity / target_->GetCapacity(),
-                               std::memory_order_relaxed);
+    sec_cache_res_ratio_ = (double)sec_capacity / target_->GetCapacity();
   }
 }
 
@@ -113,7 +115,7 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
   // use after free
   target_->SetEvictionCallback({});
 #ifndef NDEBUG
-  if (distribute_cache_res_ && !ratio_changed_) {
+  if (distribute_cache_res_) {
     size_t sec_capacity = 0;
     Status s = secondary_cache_->GetCapacity(sec_capacity);
     assert(s.ok());
@@ -236,13 +238,31 @@ Status CacheWithSecondaryAdapter::Insert(const Slice& key, ObjectPtr value,
                                          const Slice& compressed_value,
                                          CompressionType type) {
   Status s = target_->Insert(key, value, helper, charge, handle, priority);
-  if (s.ok() && value == nullptr && distribute_cache_res_) {
-    size_t sec_charge = static_cast<size_t>(
-        charge * (sec_cache_res_ratio_.load(std::memory_order_relaxed)));
-    s = secondary_cache_->Deflate(sec_charge);
-    assert(s.ok());
-    s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/false);
-    assert(s.ok());
+  if (s.ok() && value == nullptr && distribute_cache_res_ && handle) {
+    charge = target_->GetCharge(*handle);
+
+    MutexLock l(&cache_res_mutex_);
+    placeholder_usage_ += charge;
+    // Check if total placeholder reservation is more than the overall
+    // cache capacity. If it is, then we don't try to charge the
+    // secondary cache because we don't want to overcharge it (beyond
+    // its capacity).
+    // In order to make this a bit more lightweight, we also check if
+    // the difference between placeholder_usage_ and reserved_usage_ is
+    // atleast kReservationChunkSize and avoid any adjustments if not.
+    if ((placeholder_usage_ <= target_->GetCapacity()) &&
+        ((placeholder_usage_ - reserved_usage_) >= kReservationChunkSize)) {
+      reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1);
+      size_t new_sec_reserved =
+          static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+      size_t sec_charge = new_sec_reserved - sec_reserved_;
+      s = secondary_cache_->Deflate(sec_charge);
+      assert(s.ok());
+      s = pri_cache_res_->UpdateCacheReservation(sec_charge,
+                                                 /*increase=*/false);
+      assert(s.ok());
+      sec_reserved_ += sec_charge;
+    }
   }
   // Warm up the secondary cache with the compressed block. The secondary
   // cache may choose to ignore it based on the admission policy.
@@ -287,14 +307,27 @@ bool CacheWithSecondaryAdapter::Release(Handle* handle,
     ObjectPtr v = target_->Value(handle);
     if (v == nullptr && distribute_cache_res_) {
       size_t charge = target_->GetCharge(handle);
-      size_t sec_charge = static_cast<size_t>(
-          charge * (sec_cache_res_ratio_.load(std::memory_order_relaxed)));
-      TEST_SYNC_POINT("CacheWithSecondaryAdapter::Release:ChargeSecCache1");
-      TEST_SYNC_POINT("CacheWithSecondaryAdapter::Release:ChargeSecCache2");
-      Status s = secondary_cache_->Inflate(sec_charge);
-      assert(s.ok());
-      s = pri_cache_res_->UpdateCacheReservation(sec_charge, /*increase=*/true);
-      assert(s.ok());
+
+      MutexLock l(&cache_res_mutex_);
+      placeholder_usage_ -= charge;
+      // Check if total placeholder reservation is more than the overall
+      // cache capacity. If it is, then we do nothing as reserved_usage_ must
+      // be already maxed out
+      if ((placeholder_usage_ <= target_->GetCapacity()) &&
+          (placeholder_usage_ < reserved_usage_)) {
+        // Adjust reserved_usage_ in chunks of kReservationChunkSize, so
+        // we don't hit this slow path too often.
+        reserved_usage_ = placeholder_usage_ & ~(kReservationChunkSize - 1);
+        size_t new_sec_reserved =
+            static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
+        size_t sec_charge = sec_reserved_ - new_sec_reserved;
+        Status s = secondary_cache_->Inflate(sec_charge);
+        assert(s.ok());
+        s = pri_cache_res_->UpdateCacheReservation(sec_charge,
+                                                   /*increase=*/true);
+        assert(s.ok());
+        sec_reserved_ -= sec_charge;
+      }
     }
   }
   return target_->Release(handle, erase_if_last_ref);
@@ -441,13 +474,11 @@ const char* CacheWithSecondaryAdapter::Name() const {
 // where the new capacity < total cache reservations.
 void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
   size_t sec_capacity = static_cast<size_t>(
-      capacity * (distribute_cache_res_
-                      ? sec_cache_res_ratio_.load(std::memory_order_relaxed)
-                      : 0.0));
+      capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0));
   size_t old_sec_capacity = 0;
 
   if (distribute_cache_res_) {
-    MutexLock m(&mutex_);
+    MutexLock m(&cache_res_mutex_);
 
     Status s = secondary_cache_->GetCapacity(old_sec_capacity);
     if (!s.ok()) {
@@ -462,9 +493,17 @@ void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
       // 3. Decrease the primary cache capacity to the total budget
       s = secondary_cache_->SetCapacity(sec_capacity);
       if (s.ok()) {
+        if (placeholder_usage_ > capacity) {
+          // Adjust reserved_usage_ down
+          reserved_usage_ = capacity & ~(kReservationChunkSize - 1);
+        }
+        size_t new_sec_reserved =
+            static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
         s = pri_cache_res_->UpdateCacheReservation(
-            old_sec_capacity - sec_capacity,
+            (old_sec_capacity - sec_capacity) -
+                (sec_reserved_ - new_sec_reserved),
             /*increase=*/false);
+        sec_reserved_ = new_sec_reserved;
         assert(s.ok());
         target_->SetCapacity(capacity);
       }
@@ -498,7 +537,7 @@ Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage(
     size_t& size) const {
   Status s;
   if (distribute_cache_res_) {
-    MutexLock m(&mutex_);
+    MutexLock m(&cache_res_mutex_);
     size_t capacity = 0;
     s = secondary_cache_->GetCapacity(capacity);
     if (s.ok()) {
@@ -526,12 +565,11 @@ Status CacheWithSecondaryAdapter::GetSecondaryCachePinnedUsage(
 // in the future.
 Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
     double compressed_secondary_ratio) {
-  if (!distribute_cache_res_ ||
-      sec_cache_res_ratio_.load(std::memory_order_relaxed) == 0.0) {
+  if (!distribute_cache_res_) {
     return Status::NotSupported();
   }
 
-  MutexLock m(&mutex_);
+  MutexLock m(&cache_res_mutex_);
   size_t pri_capacity = target_->GetCapacity();
   size_t sec_capacity =
       static_cast<size_t>(pri_capacity * compressed_secondary_ratio);
@@ -541,38 +579,17 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
     return s;
   }
 
-  TEST_SYNC_POINT(
-      "CacheWithSecondaryAdapter::UpdateCacheReservationRatio:Begin");
-
-  // There's a possible race condition here. Since the read of pri_cache_res_
-  // memory used (secondary cache usage charged to primary cache), and the
-  // change to sec_cache_res_ratio_ are not guarded by a mutex, its possible
-  // that an Insert/Release in another thread might decrease/increase the
-  // pri_cache_res_ reservation by the wrong amount. This should not be a
-  // problem because updating the sec/pri ratio is a rare operation, and
-  // the worst that can happen is we may over/under charge the secondary
-  // cache usage by a little bit. But we do need to protect against
-  // underflow of old_sec_reserved.
-  // TODO: Make the accounting more accurate by tracking the total memory
-  // reservation on the primary cache. This will also allow us to remove
-  // the restriction of not being able to change the sec/pri ratio from
-  // 0.0 to higher.
-  size_t sec_charge_to_pri = pri_cache_res_->GetTotalMemoryUsed();
-  size_t old_sec_reserved = (old_sec_capacity > sec_charge_to_pri)
-                                ? (old_sec_capacity - sec_charge_to_pri)
-                                : 0;
   // Calculate the new secondary cache reservation
-  size_t sec_reserved = static_cast<size_t>(
-      old_sec_reserved *
-      (double)(compressed_secondary_ratio /
-               sec_cache_res_ratio_.load(std::memory_order_relaxed)));
-  sec_cache_res_ratio_.store(compressed_secondary_ratio,
-                             std::memory_order_relaxed);
+  // reserved_usage_ will never be > the cache capacity, so we don't
+  // have to worry about adjusting it here.
+  sec_cache_res_ratio_ = compressed_secondary_ratio;
+  size_t new_sec_reserved =
+      static_cast<size_t>(reserved_usage_ * sec_cache_res_ratio_);
   if (sec_capacity > old_sec_capacity) {
     // We're increasing the ratio, thus ending up with a larger secondary
     // cache and a smaller usable primary cache capacity. Similar to
     // SetCapacity(), we try to avoid a temporary increase in total usage
-    // beyond teh configured capacity -
+    // beyond the configured capacity -
     // 1. A higher secondary cache ratio means it gets a higher share of
     //    cache reservations. So first account for that by deflating the
     //    secondary cache
@@ -580,12 +597,13 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
     //    cache utilization (increase in capacity - increase in share of cache
     //    reservation)
     // 3. Increase secondary cache capacity
-    s = secondary_cache_->Deflate(sec_reserved - old_sec_reserved);
+    s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_);
     assert(s.ok());
     s = pri_cache_res_->UpdateCacheReservation(
-        (sec_capacity - old_sec_capacity) - (sec_reserved - old_sec_reserved),
+        (sec_capacity - old_sec_capacity) - (new_sec_reserved - sec_reserved_),
         /*increase=*/true);
     assert(s.ok());
+    sec_reserved_ = new_sec_reserved;
     s = secondary_cache_->SetCapacity(sec_capacity);
     assert(s.ok());
   } else {
@@ -599,21 +617,16 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
     s = secondary_cache_->SetCapacity(sec_capacity);
     if (s.ok()) {
       s = pri_cache_res_->UpdateCacheReservation(
-          (old_sec_capacity - sec_capacity) - (old_sec_reserved - sec_reserved),
+          (old_sec_capacity - sec_capacity) -
+              (sec_reserved_ - new_sec_reserved),
           /*increase=*/false);
       assert(s.ok());
-      s = secondary_cache_->Inflate(old_sec_reserved - sec_reserved);
+      s = secondary_cache_->Inflate(sec_reserved_ - new_sec_reserved);
       assert(s.ok());
+      sec_reserved_ = new_sec_reserved;
     }
   }
 
-  TEST_SYNC_POINT("CacheWithSecondaryAdapter::UpdateCacheReservationRatio:End");
-#ifndef NDEBUG
-  // As mentioned in the function comments, we may accumulate some erros when
-  // the ratio is changed. We set a flag here which disables some assertions
-  // in the destructor
-  ratio_changed_ = true;
-#endif
   return s;
 }
 

package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.h

@@ -60,6 +60,8 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
   SecondaryCache* TEST_GetSecondaryCache() { return secondary_cache_.get(); }
 
  private:
+  static constexpr size_t kReservationChunkSize = 1 << 20;
+
   bool EvictionHandler(const Slice& key, Handle* handle, bool was_hit);
 
   void StartAsyncLookupOnMySecondary(AsyncLookupHandle& async_handle);
@@ -84,11 +86,18 @@ class CacheWithSecondaryAdapter : public CacheWrapper {
   std::shared_ptr<ConcurrentCacheReservationManager> pri_cache_res_;
   // Fraction of a cache memory reservation to be assigned to the secondary
   // cache
-  std::atomic<double> sec_cache_res_ratio_;
-  mutable port::Mutex mutex_;
-#ifndef NDEBUG
-  bool ratio_changed_ = false;
-#endif
+  double sec_cache_res_ratio_;
+  // Mutex for use when managing cache memory reservations. Should not be used
+  // for other purposes, as it may risk causing deadlocks.
+  mutable port::Mutex cache_res_mutex_;
+  // Total memory reserved by placeholder entriesin the cache
+  size_t placeholder_usage_;
+  // Total placeholoder memory charged to both the primary and secondary
+  // caches. Will be <= placeholder_usage_.
+  size_t reserved_usage_;
+  // Amount of memory reserved in the secondary cache. This should be
+  // reserved_usage_ * sec_cache_res_ratio_ in steady state.
+  size_t sec_reserved_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc

@@ -189,7 +189,7 @@ void CompressedCacheSetCapacityThread(void* v) {
                 s.ToString().c_str());
       }
     } else if (FLAGS_compressed_secondary_cache_ratio > 0.0) {
-      if (thread->rand.OneIn(2)) {
+      if (thread->rand.OneIn(2)) {  // if (thread->rand.OneIn(2)) {
         size_t capacity = block_cache->GetCapacity();
         size_t adjustment;
         if (FLAGS_use_write_buffer_manager && FLAGS_db_write_buffer_size > 0) {

package/deps/rocksdb/rocksdb/include/rocksdb/version.h

@@ -13,7 +13,7 @@
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 8
 #define ROCKSDB_MINOR 8
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 1
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

package/index.js

@@ -3,15 +3,14 @@
 const { fromCallback } = require('catering')
 const { AbstractLevel } = require('abstract-level')
 const ModuleError = require('module-error')
-const fs = require('fs')
 const binding = require('./binding')
 const { ChainedBatch } = require('./chained-batch')
 const { Iterator } = require('./iterator')
-const os = require('os')
+const fs = require('node:fs')
+const assert = require('node:assert')
 
 const kContext = Symbol('context')
 const kColumns = Symbol('columns')
-const kLocation = Symbol('location')
 const kPromise = Symbol('promise')
 const kRef = Symbol('ref')
 const kUnref = Symbol('unref')
@@ -21,34 +20,7 @@ const kPendingClose = Symbol('pendingClose')
 const EMPTY = {}
 
 class RocksLevel extends AbstractLevel {
-  constructor (location, options, _) {
-    // To help migrating to abstract-level
-    if (typeof options === 'function' || typeof _ === 'function') {
-      throw new ModuleError('The levelup-style callback argument has been removed', {
-        code: 'LEVEL_LEGACY'
-      })
-    }
-
-    if (typeof location !== 'string' || location === '') {
-      throw new TypeError("The first argument 'location' must be a non-empty string")
-    }
-
-    options = {
-      ...options, // TODO (fix): Other defaults...
-      parallelism: options?.parallelism ?? Math.max(1, os.cpus().length / 2),
-      createIfMissing: options?.createIfMissing ?? true,
-      errorIfExists: options?.errorIfExists ?? false,
-      walTTL: options?.walTTL ?? 0,
-      walSizeLimit: options?.walSizeLimit ?? 0,
-      walCompression: options?.walCompression ?? false,
-      unorderedWrite: options?.unorderedWrite ?? false,
-      manualWalFlush: options?.manualWalFlush ?? false,
-      walTotalSizeLimit: options?.walTotalSizeLimit ?? 0,
-      infoLogLevel: options?.infoLogLevel ?? ''
-    }
-
-    // TODO (fix): Check options.
-
+  constructor (locationOrHandle, options) {
     super({
       encodings: {
         buffer: true,
@@ -61,45 +33,64 @@ class RocksLevel extends AbstractLevel {
       }
     }, options)
 
-    this[kLocation] = location
-    this[kContext] = binding.db_init()
+    this[kContext] = binding.db_init(locationOrHandle)
     this[kColumns] = {}
 
     this[kRefs] = 0
     this[kPendingClose] = null
-
-    // .updates(...) uses 'update' listener.
-    this.setMaxListeners(100)
   }
 
   get sequence () {
     return binding.db_get_latest_sequence(this[kContext])
   }
 
-  get location () {
-    return this[kLocation]
-  }
-
   get columns () {
     return this[kColumns]
   }
 
+  get handle () {
+    // TODO (fix): Support returning handle even if not open yet...
+    assert(this.status === 'open', 'Database is not open')
+
+    return binding.db_get_handle(this[kContext])
+  }
+
+  get location () {
+    return binding.db_get_location(this[kContext])
+  }
+
   _open (options, callback) {
-    const onOpen = (err, columns) => {
-      if (err) {
+    const doOpen = () => {
+      let columns
+      try {
+        columns = binding.db_open(this[kContext], options, (err, columns) => {
+          if (err) {
+            callback(err)
+          } else {
+            this[kColumns] = columns
+            callback(null)
+          }
+        })
+      } catch (err) {
         callback(err)
-      } else {
+      }
+
+      if (columns) {
         this[kColumns] = columns
         callback(null)
       }
     }
+
     if (options.createIfMissing) {
-      fs.mkdir(this[kLocation], { recursive: true }, (err) => {
-        if (err) return callback(err)
-        binding.db_open(this[kContext], this[kLocation], options, onOpen)
+      fs.mkdir(this.location, { recursive: true }, (err) => {
+        if (err) {
+          callback(err)
+        } else {
+          doOpen()
+        }
       })
     } else {
-      binding.db_open(this[kContext], this[kLocation], options, onOpen)
+      doOpen()
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "9.0.1",
+  "version": "10.0.0",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",
@@ -14,8 +14,8 @@
     "abstract-level": "^1.0.2",
     "catering": "^2.1.1",
     "module-error": "^1.0.2",
-    "napi-macros": "~2.0.0",
-    "node-gyp-build": "^4.5.0"
+    "napi-macros": "~2.2.2",
+    "node-gyp-build": "^4.8.0"
   },
   "devDependencies": {
     "@types/node": "^18.11.3",
@@ -33,7 +33,7 @@
     "readfiletree": "^1.0.0",
     "rimraf": "^3.0.0",
     "standard": "^17.0.0",
-    "tape": "^5.6.1",
+    "tape": "^5.7.5",
     "tempy": "^1.0.1"
   },
   "standard": {

package/util.h

@@ -45,6 +45,12 @@ static void Finalize(napi_env env, void* data, void* hint) {
   }
 }
 
+static void FinalizeFree(napi_env env, void* data, void* hint) {
+  if (hint) {
+    free(hint);
+  }
+}
+
 static napi_value CreateError(napi_env env, const std::optional<std::string_view>& code, const std::string_view& msg) {
   napi_value codeValue = nullptr;
   if (code) {
@@ -352,4 +358,4 @@ napi_status runAsync(const std::string& name, napi_env env, napi_value callback,
   worker.release();
 
   return napi_ok;
-}
+}