@nxtedition/rocksdb 12.0.8 → 12.0.11

package/benchmarks/get-many.mjs

@@ -0,0 +1,42 @@
+import { bench, run } from 'mitata'
+import { RocksLevel } from '../index.js'
+
+const db = new RocksLevel('./tmp', {
+  keyEncoding: 'buffer',
+  valueEncoding: 'buffer',
+  parallelism: 4,
+  pipelinedWrite: false,
+  // unorderedWrite: true,
+  columns: {
+    default: {
+      cacheSize: 128e6,
+      memtableMemoryBudget: 128e6,
+      compaction: 'level'
+      // optimize: 'point-lookup',
+    }
+  }
+})
+await db.open()
+
+
+const getOpts = {
+  keyEncoding: 'buffer',
+  valueEncoding: 'buffer',
+  fillCache: true
+}
+
+for (let size = 1024 * 8; size <= 256 * 1024; size *= 2) {
+  const keys = []
+  for (let n = 0; n < 1024; n++) {
+    const key = `${n}-${size}`
+    keys.push(Buffer.from(key))
+    await db.put(key, Buffer.allocUnsafe(size))
+  }
+
+  bench('_getManySync ' + size / 1024, async () => {
+    db._getManySync(keys, getOpts).length
+  })
+}
+
+await run()
+

package/binding.cc

@@ -1021,6 +1021,9 @@ NAPI_METHOD(db_get_many_sync) {
   int32_t highWaterMarkBytes = std::numeric_limits<int32_t>::max();
   NAPI_STATUS_THROWS(GetProperty(env, argv[2], "highWaterMarkBytes", highWaterMarkBytes));
 
+  bool unsafe = false;
+  NAPI_STATUS_THROWS(GetProperty(env, argv[2], "unsafe", unsafe));
+
   std::vector<rocksdb::Slice> keys;
   keys.resize(count);
   std::vector<rocksdb::Status> statuses;
@@ -1053,7 +1056,11 @@ NAPI_METHOD(db_get_many_sync) {
       NAPI_STATUS_THROWS(napi_get_null(env, &row));
     } else {
       ROCKS_STATUS_THROWS_NAPI(statuses[n]);
-      NAPI_STATUS_THROWS(Convert(env, &values[n], valueEncoding, row));
+      if (unsafe && values[n].size() > 32) {
+        NAPI_STATUS_THROWS(ConvertUnsafe(env, std::move(values[n]), valueEncoding, row));
+      } else {
+        NAPI_STATUS_THROWS(Convert(env, std::move(values[n]), valueEncoding, row));
+      }
     }
     NAPI_STATUS_THROWS(napi_set_element(env, rows, n, row));
   }
@@ -1082,6 +1089,9 @@ NAPI_METHOD(db_get_many) {
   int32_t highWaterMarkBytes = std::numeric_limits<int32_t>::max();
   NAPI_STATUS_THROWS(GetProperty(env, argv[2], "highWaterMarkBytes", highWaterMarkBytes));
 
+  bool unsafe = false;
+  NAPI_STATUS_THROWS(GetProperty(env, argv[2], "unsafe", unsafe));
+
   auto callback = argv[3];
 
   struct State {
@@ -1133,7 +1143,11 @@ NAPI_METHOD(db_get_many) {
             NAPI_STATUS_RETURN(napi_get_null(env, &row));
           } else {
             ROCKS_STATUS_RETURN_NAPI(state.statuses[n]);
-            NAPI_STATUS_RETURN(Convert(env, &state.values[n], valueEncoding, row));
+            if (unsafe && state.values[n].size() > 32) {
+              NAPI_STATUS_RETURN(ConvertUnsafe(env, std::move(state.values[n]), valueEncoding, row));
+            } else {
+              NAPI_STATUS_RETURN(Convert(env, std::move(state.values[n]), valueEncoding, row));
+            }
           }
           NAPI_STATUS_RETURN(napi_set_element(env, argv[1], n, row));
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "12.0.8",
+  "version": "12.0.11",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/tmp/CURRENT

@@ -1 +1 @@
-MANIFEST-000005
+MANIFEST-000546

package/tmp/{OPTIONS-000007 → OPTIONS-000455}

@@ -5,7 +5,7 @@
 #
 
 [Version]
-  rocksdb_version=9.5.0
+  rocksdb_version=9.6.1
   options_file_version=1.1
 
 [DBOptions]
@@ -26,6 +26,7 @@
   max_background_jobs=4
   delete_obsolete_files_period_micros=21600000000
   writable_file_max_buffer_size=1048576
+  wal_write_temperature=kUnknown
   follower_catchup_retry_wait_ms=100
   file_checksum_gen_factory=nullptr
   allow_data_in_errors=false
@@ -48,6 +49,7 @@
   allow_concurrent_memtable_write=true
   paranoid_checks=true
   WAL_size_limit_MB=0
+  metadata_write_temperature=kUnknown
   lowest_used_cache_tier=kNonVolatileBlockTier
   keep_log_file_num=1000
   table_cache_numshardbits=6
@@ -121,6 +123,7 @@
   inplace_update_num_locks=10000
   enable_blob_garbage_collection=false
   arena_block_size=1048576
+  paranoid_memory_checks=false
   bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}
   target_file_size_multiplier=1
   max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
@@ -135,7 +138,7 @@
   block_protection_bytes_per_key=0
   prefix_extractor=nullptr
   max_bytes_for_level_multiplier=10.000000
-  write_buffer_size=67108864
+  write_buffer_size=32000000
   uncache_aggressiveness=0
   disable_auto_compactions=false
   max_compaction_bytes=400000000
@@ -178,10 +181,11 @@
   metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
   read_amp_bytes_per_bit=0
   verify_compression=false
-  format_version=5
+  format_version=6
+  detect_filter_construct_corruption=false
   optimize_filters_for_memory=true
+  decouple_partitioned_filters=true
   partition_filters=false
-  detect_filter_construct_corruption=false
   initial_auto_readahead_size=8192
   max_auto_readahead_size=262144
   enable_index_compression=true

package/tmp/OPTIONS-000548

@@ -0,0 +1,213 @@
+# This is a RocksDB option file.
+#
+# For detailed file format spec, please refer to the example file
+# in examples/rocksdb_option_file_example.ini
+#
+
+[Version]
+  rocksdb_version=9.6.1
+  options_file_version=1.1
+
+[DBOptions]
+  max_background_flushes=-1
+  compaction_readahead_size=2097152
+  strict_bytes_per_sync=false
+  wal_bytes_per_sync=0
+  max_open_files=-1
+  stats_history_buffer_size=1048576
+  max_total_wal_size=0
+  stats_persist_period_sec=600
+  stats_dump_period_sec=600
+  avoid_flush_during_shutdown=false
+  max_subcompactions=1
+  bytes_per_sync=0
+  delayed_write_rate=16777216
+  max_background_compactions=-1
+  max_background_jobs=4
+  delete_obsolete_files_period_micros=21600000000
+  writable_file_max_buffer_size=1048576
+  wal_write_temperature=kUnknown
+  follower_catchup_retry_wait_ms=100
+  file_checksum_gen_factory=nullptr
+  allow_data_in_errors=false
+  max_bgerror_resume_count=2147483647
+  best_efforts_recovery=false
+  write_dbid_to_manifest=false
+  atomic_flush=false
+  manual_wal_flush=false
+  two_write_queues=false
+  avoid_flush_during_recovery=false
+  dump_malloc_stats=false
+  info_log_level=HEADER_LEVEL
+  write_thread_slow_yield_usec=3
+  unordered_write=false
+  allow_ingest_behind=false
+  fail_if_options_file_error=true
+  persist_stats_to_disk=false
+  WAL_ttl_seconds=0
+  bgerror_resume_retry_interval=1000000
+  allow_concurrent_memtable_write=true
+  paranoid_checks=true
+  WAL_size_limit_MB=0
+  metadata_write_temperature=kUnknown
+  lowest_used_cache_tier=kNonVolatileBlockTier
+  keep_log_file_num=1000
+  table_cache_numshardbits=6
+  max_file_opening_threads=16
+  random_access_max_buffer_size=1048576
+  follower_refresh_catchup_period_ms=10000
+  log_readahead_size=0
+  enable_pipelined_write=false
+  background_close_inactive_wals=false
+  wal_recovery_mode=kPointInTimeRecovery
+  follower_catchup_retry_count=10
+  db_write_buffer_size=0
+  allow_2pc=false
+  skip_checking_sst_file_sizes_on_db_open=false
+  skip_stats_update_on_db_open=false
+  recycle_log_file_num=0
+  db_host_id=__hostname__
+  track_and_verify_wals_in_manifest=false
+  use_fsync=false
+  wal_compression=kNoCompression
+  compaction_verify_record_count=true
+  error_if_exists=false
+  manifest_preallocation_size=4194304
+  is_fd_close_on_exec=true
+  enable_write_thread_adaptive_yield=true
+  enable_thread_tracking=false
+  avoid_unnecessary_blocking_io=true
+  allow_fallocate=true
+  max_log_file_size=0
+  advise_random_on_open=true
+  create_missing_column_families=true
+  max_write_batch_group_size_bytes=1048576
+  use_adaptive_mutex=false
+  wal_filter=nullptr
+  create_if_missing=true
+  enforce_single_del_contracts=true
+  allow_mmap_writes=false
+  verify_sst_unique_id_in_manifest=true
+  log_file_time_to_roll=0
+  use_direct_io_for_flush_and_compaction=false
+  flush_verify_memtable_count=true
+  max_manifest_file_size=1073741824
+  write_thread_max_yield_usec=100
+  use_direct_reads=false
+  allow_mmap_reads=false
+  
+
+[CFOptions "default"]
+  bottommost_file_compaction_delay=0
+  memtable_protection_bytes_per_key=0
+  compression_per_level=kNoCompression:kNoCompression:kZSTD:kZSTD:kZSTD:kZSTD:kZSTD
+  bottommost_compression=kDisableCompressionOption
+  sample_for_compression=0
+  blob_garbage_collection_age_cutoff=0.250000
+  blob_compression_type=kNoCompression
+  compression=kZSTD
+  default_write_temperature=kUnknown
+  last_level_temperature=kUnknown
+  compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;max_read_amp=-1;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;}
+  target_file_size_base=16000000
+  memtable_whole_key_filtering=false
+  blob_file_starting_level=0
+  soft_pending_compaction_bytes_limit=68719476736
+  max_write_buffer_number=6
+  ttl=2592000
+  compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;}
+  memtable_huge_page_size=0
+  max_sequential_skip_in_iterations=8
+  strict_max_successive_merges=false
+  max_successive_merges=0
+  inplace_update_num_locks=10000
+  enable_blob_garbage_collection=false
+  arena_block_size=1048576
+  paranoid_memory_checks=false
+  bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}
+  target_file_size_multiplier=1
+  max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1
+  prepopulate_blob_cache=kDisable
+  blob_compaction_readahead_size=0
+  min_blob_size=0
+  level0_stop_writes_trigger=36
+  blob_garbage_collection_force_threshold=1.000000
+  enable_blob_files=false
+  level0_slowdown_writes_trigger=20
+  level0_file_num_compaction_trigger=2
+  block_protection_bytes_per_key=0
+  prefix_extractor=nullptr
+  max_bytes_for_level_multiplier=10.000000
+  write_buffer_size=32000000
+  uncache_aggressiveness=0
+  disable_auto_compactions=false
+  max_compaction_bytes=400000000
+  memtable_max_range_deletions=0
+  compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=1638400;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=16384;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;}
+  hard_pending_compaction_bytes_limit=274877906944
+  blob_file_size=268435456
+  periodic_compaction_seconds=0
+  paranoid_file_checks=false
+  experimental_mempurge_threshold=0.000000
+  memtable_prefix_bloom_size_ratio=0.000000
+  max_bytes_for_level_base=128000000
+  report_bg_io_stats=false
+  sst_partitioner_factory=nullptr
+  compaction_pri=kMinOverlappingRatio
+  compaction_style=kCompactionStyleLevel
+  compaction_filter_factory=nullptr
+  compaction_filter=nullptr
+  memtable_factory=SkipListFactory
+  comparator=leveldb.BytewiseComparator
+  bloom_locality=0
+  min_write_buffer_number_to_merge=2
+  table_factory=BlockBasedTable
+  max_write_buffer_size_to_maintain=0
+  max_write_buffer_number_to_maintain=0
+  optimize_filters_for_hits=false
+  default_temperature=kUnknown
+  preserve_internal_time_seconds=0
+  force_consistency_checks=true
+  merge_operator=nullptr
+  num_levels=7
+  memtable_insert_with_hint_prefix_extractor=nullptr
+  level_compaction_dynamic_level_bytes=true
+  persist_user_defined_timestamps=true
+  preclude_last_level_data_seconds=0
+  inplace_update_support=false
+  
+[TableOptions/BlockBasedTable "default"]
+  num_file_reads_for_auto_readahead=2
+  metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;}
+  read_amp_bytes_per_bit=0
+  verify_compression=false
+  format_version=6
+  detect_filter_construct_corruption=false
+  optimize_filters_for_memory=true
+  decouple_partitioned_filters=true
+  partition_filters=false
+  initial_auto_readahead_size=8192
+  max_auto_readahead_size=262144
+  enable_index_compression=true
+  checksum=kXXH3
+  index_block_restart_interval=1
+  pin_top_level_index_and_filter=true
+  block_align=false
+  block_size=4096
+  index_type=kBinarySearch
+  filter_policy=bloomfilter:10:false
+  metadata_block_size=4096
+  no_block_cache=false
+  whole_key_filtering=true
+  index_shortening=kShortenSeparators
+  block_size_deviation=10
+  data_block_index_type=kDataBlockBinarySearch
+  use_delta_encoding=true
+  data_block_hash_table_util_ratio=0.750000
+  cache_index_and_filter_blocks=false
+  prepopulate_block_cache=kDisable
+  block_restart_interval=16
+  pin_l0_filter_and_index_blocks_in_cache=false
+  cache_index_and_filter_blocks_with_high_priority=true
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory
+  

package/util.h

@@ -271,6 +271,27 @@ napi_status Convert(napi_env env, T&& s, Encoding encoding, napi_value& result)
   }
 }
 
+napi_status ConvertUnsafe(napi_env env, rocksdb::PinnableSlice&& s, Encoding encoding, napi_value& result) {
+  if (encoding == Encoding::Buffer) {
+    auto s2 = new rocksdb::PinnableSlice(std::move(s));
+    return napi_create_external_buffer(env, s2->size(), const_cast<char*>(s2->data()), Finalize<rocksdb::PinnableSlice>, s2, &result);
+  } else if (encoding == Encoding::String) {
+    return napi_create_string_utf8(env, s.data(), s.size(), &result);
+  } else {
+    return napi_invalid_arg;
+  }
+}
+
+napi_status Convert(napi_env env, rocksdb::PinnableSlice&& s, Encoding encoding, napi_value& result) {
+  if (encoding == Encoding::Buffer) {
+    return napi_create_buffer_copy(env, s.size(), s.data(), nullptr, &result);
+  } else if (encoding == Encoding::String) {
+    return napi_create_string_utf8(env, s.data(), s.size(), &result);
+  } else {
+    return napi_invalid_arg;
+  }
+}
+
 template <typename State, typename T1, typename T2>
 napi_status runAsync(State&& state,
                      const std::string& name,