@nxtedition/rocksdb 7.0.27 → 7.0.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/binding.cc +170 -30
  2. package/chained-batch.js +1 -1
  3. package/deps/rocksdb/rocksdb/CMakeLists.txt +3 -0
  4. package/deps/rocksdb/rocksdb/Makefile +3 -0
  5. package/deps/rocksdb/rocksdb/TARGETS +10 -0
  6. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +17 -7
  7. package/deps/rocksdb/rocksdb/cache/cache_entry_roles.cc +2 -0
  8. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.cc +1 -0
  9. package/deps/rocksdb/rocksdb/cache/charged_cache.cc +117 -0
  10. package/deps/rocksdb/rocksdb/cache/charged_cache.h +121 -0
  11. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +270 -180
  12. package/deps/rocksdb/rocksdb/cache/clock_cache.h +412 -124
  13. package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +1 -0
  14. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +1 -1
  15. package/deps/rocksdb/rocksdb/cache/lru_cache.h +2 -2
  16. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +2 -2
  17. package/deps/rocksdb/rocksdb/cache/sharded_cache.h +1 -1
  18. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +71 -9
  19. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +11 -2
  20. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +21 -14
  21. package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +68 -7
  22. package/deps/rocksdb/rocksdb/db/blob/blob_source.h +16 -0
  23. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +519 -12
  24. package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +120 -0
  25. package/deps/rocksdb/rocksdb/db/builder.cc +15 -5
  26. package/deps/rocksdb/rocksdb/db/builder.h +3 -0
  27. package/deps/rocksdb/rocksdb/db/c.cc +18 -0
  28. package/deps/rocksdb/rocksdb/db/c_test.c +18 -0
  29. package/deps/rocksdb/rocksdb/db/column_family.h +2 -0
  30. package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +3 -2
  31. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +9 -4
  32. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +15 -10
  33. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +36 -34
  34. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +50 -13
  35. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +12 -0
  36. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +8 -1
  37. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +2 -1
  38. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +13 -17
  39. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +26 -9
  40. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +0 -11
  41. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +93 -0
  42. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +16 -1
  43. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +3 -8
  44. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +8 -1
  45. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +17 -5
  46. package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
  47. package/deps/rocksdb/rocksdb/db/db_test2.cc +39 -12
  48. package/deps/rocksdb/rocksdb/db/db_test_util.cc +9 -0
  49. package/deps/rocksdb/rocksdb/db/db_test_util.h +2 -0
  50. package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -38
  51. package/deps/rocksdb/rocksdb/db/dbformat.h +14 -13
  52. package/deps/rocksdb/rocksdb/db/dbformat_test.cc +5 -2
  53. package/deps/rocksdb/rocksdb/db/event_helpers.cc +13 -1
  54. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -10
  55. package/deps/rocksdb/rocksdb/db/flush_job.cc +19 -15
  56. package/deps/rocksdb/rocksdb/db/flush_job.h +7 -0
  57. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +21 -15
  58. package/deps/rocksdb/rocksdb/db/forward_iterator.h +4 -3
  59. package/deps/rocksdb/rocksdb/db/memtable_list.cc +9 -0
  60. package/deps/rocksdb/rocksdb/db/memtable_list.h +5 -0
  61. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.cc +53 -12
  62. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.h +14 -2
  63. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler_test.cc +10 -10
  64. package/deps/rocksdb/rocksdb/db/repair.cc +8 -6
  65. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +890 -0
  66. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +324 -0
  67. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +186 -0
  68. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +2 -0
  69. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +13 -4
  70. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +23 -2
  71. package/deps/rocksdb/rocksdb/env/env_test.cc +74 -1
  72. package/deps/rocksdb/rocksdb/env/io_posix.cc +11 -8
  73. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +28 -0
  74. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +14 -1
  75. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +4 -4
  76. package/deps/rocksdb/rocksdb/include/rocksdb/comparator.h +30 -23
  77. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -1
  78. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +3 -13
  79. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +5 -0
  80. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/debug.h +1 -2
  81. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -0
  82. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +1 -1
  83. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
  84. package/deps/rocksdb/rocksdb/options/cf_options.cc +14 -1
  85. package/deps/rocksdb/rocksdb/options/cf_options.h +5 -0
  86. package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -56
  87. package/deps/rocksdb/rocksdb/options/db_options.cc +4 -5
  88. package/deps/rocksdb/rocksdb/options/options.cc +11 -1
  89. package/deps/rocksdb/rocksdb/options/options_helper.cc +8 -0
  90. package/deps/rocksdb/rocksdb/options/options_helper.h +4 -0
  91. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +4 -0
  92. package/deps/rocksdb/rocksdb/options/options_test.cc +4 -0
  93. package/deps/rocksdb/rocksdb/src.mk +3 -0
  94. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +6 -1
  95. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +4 -0
  96. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +36 -3
  97. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +36 -1
  98. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +14 -3
  99. package/deps/rocksdb/rocksdb/table/internal_iterator.h +1 -1
  100. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +6 -0
  101. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +5 -0
  102. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.h +3 -0
  103. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +10 -7
  104. package/deps/rocksdb/rocksdb/table/table_builder.h +7 -3
  105. package/deps/rocksdb/rocksdb/table/table_properties.cc +9 -0
  106. package/deps/rocksdb/rocksdb/test_util/mock_time_env.h +3 -2
  107. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +58 -30
  108. package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +1 -0
  109. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +20 -0
  110. package/deps/rocksdb/rocksdb/util/rate_limiter.cc +29 -154
  111. package/deps/rocksdb/rocksdb/util/rate_limiter.h +16 -34
  112. package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +0 -92
  113. package/deps/rocksdb/rocksdb/util/timer.h +6 -0
  114. package/deps/rocksdb/rocksdb/util/vector_iterator.h +4 -3
  115. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -45
  116. package/deps/rocksdb/rocksdb/utilities/debug.cc +40 -0
  117. package/deps/rocksdb/rocksdb.gyp +2 -0
  118. package/index.js +19 -2
  119. package/package.json +1 -1
  120. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  121. package/prebuilds/linux-x64/node.napi.node +0 -0
package/binding.cc CHANGED
@@ -10,11 +10,11 @@
10
10
  #include <rocksdb/db.h>
11
11
  #include <rocksdb/env.h>
12
12
  #include <rocksdb/filter_policy.h>
13
+ #include <rocksdb/merge_operator.h>
13
14
  #include <rocksdb/options.h>
14
15
  #include <rocksdb/slice_transform.h>
15
16
  #include <rocksdb/table.h>
16
17
  #include <rocksdb/write_batch.h>
17
- #include <rocksdb/merge_operator.h>
18
18
 
19
19
  #include <array>
20
20
  #include <memory>
@@ -543,19 +543,8 @@ struct Updates {
543
543
  bool keys,
544
544
  bool values,
545
545
  bool data,
546
- const std::optional<std::string>& column)
547
- : database_(database), seqNumber_(seqNumber), keys_(keys), values_(values), data_(data) {
548
- if (column) {
549
- auto columns = database->columns_;
550
- auto columnIt = std::find_if(columns.begin(), columns.end(),
551
- [&](const auto& handle) { return handle->GetName() == *column; });
552
- if (columnIt != columns.end()) {
553
- column_family_id_ = (*columnIt)->GetID();
554
- } else {
555
- // TODO: Throw?
556
- }
557
- }
558
- }
546
+ const rocksdb::ColumnFamilyHandle* column)
547
+ : database_(database), seqNumber_(seqNumber), keys_(keys), values_(values), data_(data), column_(column) {}
559
548
 
560
549
  void Close() { iterator_.reset(); }
561
550
 
@@ -577,7 +566,7 @@ struct Updates {
577
566
  bool keys_;
578
567
  bool values_;
579
568
  bool data_;
580
- std::optional<uint32_t> column_family_id_;
569
+ const rocksdb::ColumnFamilyHandle* column_;
581
570
 
582
571
  private:
583
572
  napi_ref ref_ = nullptr;
@@ -586,7 +575,8 @@ struct Updates {
586
575
  static napi_status GetColumnFamily(Database* database,
587
576
  napi_env env,
588
577
  napi_value options,
589
- rocksdb::ColumnFamilyHandle** column) {
578
+ rocksdb::ColumnFamilyHandle** column,
579
+ bool fallback = true) {
590
580
  bool hasColumn = false;
591
581
  NAPI_STATUS_RETURN(napi_has_named_property(env, options, "column", &hasColumn));
592
582
 
@@ -594,8 +584,10 @@ static napi_status GetColumnFamily(Database* database,
594
584
  napi_value value = nullptr;
595
585
  NAPI_STATUS_RETURN(napi_get_named_property(env, options, "column", &value));
596
586
  NAPI_STATUS_RETURN(napi_get_value_external(env, value, reinterpret_cast<void**>(column)));
597
- } else {
587
+ } else if (fallback) {
598
588
  *column = database->db_->DefaultColumnFamily();
589
+ } else {
590
+ *column = nullptr;
599
591
  }
600
592
 
601
593
  return napi_ok;
@@ -822,8 +814,9 @@ NAPI_METHOD(db_open) {
822
814
 
823
815
  rocksdb::Options dbOptions;
824
816
 
825
- dbOptions.IncreaseParallelism(Uint32Property(env, argv[2], "parallelism")
826
- .value_or(std::max<uint32_t>(1, std::thread::hardware_concurrency() / 2)));
817
+ const auto parallelismValue = Uint32Property(env, argv[2], "parallelism")
818
+ .value_or(std::max<uint32_t>(1, std::thread::hardware_concurrency() / 2));
819
+ dbOptions.IncreaseParallelism(parallelismValue);
827
820
 
828
821
  dbOptions.create_if_missing = BooleanProperty(env, argv[2], "createIfMissing").value_or(true);
829
822
  dbOptions.error_if_exists = BooleanProperty(env, argv[2], "errorIfExists").value_or(false);
@@ -835,12 +828,53 @@ NAPI_METHOD(db_open) {
835
828
  .value_or(std::max<uint32_t>(2, std::thread::hardware_concurrency() / 8));
836
829
  dbOptions.WAL_ttl_seconds = Uint32Property(env, argv[2], "walTTL").value_or(0) / 1e3;
837
830
  dbOptions.WAL_size_limit_MB = Uint32Property(env, argv[2], "walSizeLimit").value_or(0) / 1e6;
838
- dbOptions.create_missing_column_families = true;
839
- dbOptions.unordered_write = BooleanProperty(env, argv[2], "unorderedWrite").value_or(false);
840
- dbOptions.fail_if_options_file_error = true;
841
831
  dbOptions.wal_compression = BooleanProperty(env, argv[2], "walCompression").value_or(false)
842
832
  ? rocksdb::CompressionType::kZSTD
843
833
  : rocksdb::CompressionType::kNoCompression;
834
+ dbOptions.create_missing_column_families = true;
835
+ dbOptions.unordered_write = BooleanProperty(env, argv[2], "unorderedWrite").value_or(false);
836
+ dbOptions.fail_if_options_file_error = true;
837
+ dbOptions.manual_wal_flush = BooleanProperty(env, argv[2], "manualWalFlush").value_or(false);
838
+
839
+ napi_value ret;
840
+ NAPI_STATUS_THROWS(napi_create_object(env, &ret));
841
+ {
842
+ napi_value parallelism;
843
+ NAPI_STATUS_THROWS(napi_create_int64(env, parallelismValue, &parallelism));
844
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "parallelism", parallelism));
845
+
846
+ napi_value createIfMissing;
847
+ NAPI_STATUS_THROWS(napi_get_boolean(env, dbOptions.create_if_missing, &createIfMissing));
848
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "createIfMissing", createIfMissing));
849
+
850
+ napi_value errorIfExists;
851
+ NAPI_STATUS_THROWS(napi_get_boolean(env, dbOptions.error_if_exists, &errorIfExists));
852
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "errorIfExists", errorIfExists));
853
+
854
+ napi_value maxBackgroundJobs;
855
+ NAPI_STATUS_THROWS(napi_create_int64(env, dbOptions.max_background_jobs, &maxBackgroundJobs));
856
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "maxBackgroundJobs", maxBackgroundJobs));
857
+
858
+ napi_value walTTL;
859
+ NAPI_STATUS_THROWS(napi_create_int64(env, dbOptions.WAL_ttl_seconds, &walTTL));
860
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "walTTL", walTTL));
861
+
862
+ napi_value walSizeLimit;
863
+ NAPI_STATUS_THROWS(napi_create_int64(env, dbOptions.WAL_size_limit_MB, &walSizeLimit));
864
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "walSizeLimit", walSizeLimit));
865
+
866
+ napi_value walCompression;
867
+ NAPI_STATUS_THROWS(napi_create_int64(env, dbOptions.wal_compression, &walCompression));
868
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "walCompression", walCompression));
869
+
870
+ napi_value unorderedWrite;
871
+ NAPI_STATUS_THROWS(napi_get_boolean(env, dbOptions.error_if_exists, &unorderedWrite));
872
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "unorderedWrite", unorderedWrite));
873
+
874
+ napi_value manualWalFlush;
875
+ NAPI_STATUS_THROWS(napi_create_int64(env, dbOptions.manual_wal_flush, &manualWalFlush));
876
+ NAPI_STATUS_THROWS(napi_set_named_property(env, ret, "manualWalFlush", manualWalFlush));
877
+ }
844
878
 
845
879
  // TODO (feat): dbOptions.listeners
846
880
 
@@ -905,7 +939,7 @@ NAPI_METHOD(db_open) {
905
939
  auto worker = new OpenWorker(env, database, argv[3], location, dbOptions, columnsFamilies);
906
940
  worker->Queue(env);
907
941
 
908
- return 0;
942
+ return ret;
909
943
  }
910
944
 
911
945
  struct CloseWorker final : public Worker {
@@ -1013,57 +1047,82 @@ struct UpdatesNextWorker final : public rocksdb::WriteBatch::Handler, public Wor
1013
1047
  }
1014
1048
 
1015
1049
  rocksdb::Status PutCF(uint32_t column_family_id, const rocksdb::Slice& key, const rocksdb::Slice& value) override {
1016
- if (updates_->column_family_id_ && *updates_->column_family_id_ != column_family_id) {
1050
+ if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
1017
1051
  return rocksdb::Status::OK();
1018
1052
  }
1053
+
1019
1054
  cache_.emplace_back("put");
1055
+
1020
1056
  if (updates_->keys_) {
1021
1057
  cache_.emplace_back(key.ToStringView());
1022
1058
  } else {
1023
1059
  cache_.emplace_back(std::nullopt);
1024
1060
  }
1061
+
1025
1062
  if (updates_->values_) {
1026
1063
  cache_.emplace_back(value.ToStringView());
1027
1064
  } else {
1028
1065
  cache_.emplace_back(std::nullopt);
1029
1066
  }
1030
- cache_.emplace_back(GetColumnName(column_family_id));
1067
+
1068
+ if (!updates_->column_) {
1069
+ cache_.emplace_back(GetColumnName(column_family_id));
1070
+ } else {
1071
+ cache_.emplace_back(std::nullopt);
1072
+ }
1073
+
1031
1074
  return rocksdb::Status::OK();
1032
1075
  }
1033
1076
 
1034
1077
  rocksdb::Status DeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
1035
- if (updates_->column_family_id_ && *updates_->column_family_id_ != column_family_id) {
1078
+ if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
1036
1079
  return rocksdb::Status::OK();
1037
1080
  }
1081
+
1038
1082
  cache_.emplace_back("del");
1083
+
1039
1084
  if (updates_->keys_) {
1040
1085
  cache_.emplace_back(key.ToStringView());
1041
1086
  } else {
1042
1087
  cache_.emplace_back(std::nullopt);
1043
1088
  }
1089
+
1044
1090
  cache_.emplace_back(std::nullopt);
1045
- cache_.emplace_back(GetColumnName(column_family_id));
1091
+
1092
+ if (!updates_->column_) {
1093
+ cache_.emplace_back(GetColumnName(column_family_id));
1094
+ } else {
1095
+ cache_.emplace_back(std::nullopt);
1096
+ }
1097
+
1046
1098
  return rocksdb::Status::OK();
1047
1099
  }
1048
1100
 
1049
1101
  rocksdb::Status MergeCF(uint32_t column_family_id, const rocksdb::Slice& key, const rocksdb::Slice& value) override {
1050
- if (updates_->column_family_id_ && *updates_->column_family_id_ != column_family_id) {
1102
+ if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
1051
1103
  return rocksdb::Status::OK();
1052
1104
  }
1105
+
1053
1106
  cache_.emplace_back("put");
1107
+
1054
1108
  if (updates_->keys_) {
1055
1109
  cache_.emplace_back(key.ToStringView());
1056
1110
  } else {
1057
1111
  cache_.emplace_back(std::nullopt);
1058
1112
  }
1113
+
1059
1114
  if (updates_->values_) {
1060
1115
  cache_.emplace_back(value.ToStringView());
1061
1116
  } else {
1062
1117
  cache_.emplace_back(std::nullopt);
1063
1118
  }
1064
- if (!updates_->column_family_id_) {
1119
+
1120
+ if (!updates_->column_) {
1065
1121
  cache_.emplace_back(GetColumnName(column_family_id));
1122
+ } else {
1123
+ cache_.emplace_back(std::nullopt);
1066
1124
  }
1125
+
1067
1126
  return rocksdb::Status::OK();
1068
1127
  }
1069
1128
 
@@ -1094,7 +1153,9 @@ NAPI_METHOD(updates_init) {
1094
1153
  const auto keys = BooleanProperty(env, argv[1], "keys").value_or(true);
1095
1154
  const auto values = BooleanProperty(env, argv[1], "values").value_or(true);
1096
1155
  const auto data = BooleanProperty(env, argv[1], "data").value_or(true);
1097
- const auto column = StringProperty(env, argv[1], "column");
1156
+
1157
+ rocksdb::ColumnFamilyHandle* column;
1158
+ NAPI_STATUS_THROWS(GetColumnFamily(database, env, argv[1], &column, false));
1098
1159
 
1099
1160
  auto updates = std::make_unique<Updates>(database, seqNumber, keys, values, data, column);
1100
1161
 
@@ -1858,6 +1919,81 @@ NAPI_METHOD(batch_merge) {
1858
1919
  return 0;
1859
1920
  }
1860
1921
 
1922
+ NAPI_METHOD(db_flush_wal) {
1923
+ NAPI_ARGV(2);
1924
+
1925
+ Database* database;
1926
+ NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
1927
+
1928
+ const auto flush = BooleanProperty(env, argv[1], "flush").value_or(false);
1929
+
1930
+ ROCKS_STATUS_THROWS(database->db_->FlushWAL(flush));
1931
+
1932
+ return 0;
1933
+ }
1934
+
1935
+ napi_status FromLogFile(napi_env env, const auto& file, napi_value* obj) {
1936
+ NAPI_STATUS_RETURN(napi_create_object(env, obj));
1937
+
1938
+ napi_value pathName;
1939
+ NAPI_STATUS_RETURN(napi_create_string_utf8(env, file->PathName().c_str(), NAPI_AUTO_LENGTH, &pathName));
1940
+ NAPI_STATUS_RETURN(napi_set_named_property(env, *obj, "pathName", pathName));
1941
+
1942
+ napi_value logNumber;
1943
+ NAPI_STATUS_RETURN(napi_create_int64(env, file->LogNumber(), &logNumber));
1944
+ NAPI_STATUS_RETURN(napi_set_named_property(env, *obj, "logNumber", logNumber));
1945
+
1946
+ napi_value type;
1947
+ NAPI_STATUS_RETURN(napi_create_int64(env, file->Type(), &type));
1948
+ NAPI_STATUS_RETURN(napi_set_named_property(env, *obj, "type", type));
1949
+
1950
+ napi_value startSequence;
1951
+ NAPI_STATUS_RETURN(napi_create_int64(env, file->StartSequence(), &startSequence));
1952
+ NAPI_STATUS_RETURN(napi_set_named_property(env, *obj, "startSequence", startSequence));
1953
+
1954
+ napi_value sizeFileBytes;
1955
+ NAPI_STATUS_RETURN(napi_create_int64(env, file->SizeFileBytes(), &sizeFileBytes));
1956
+ NAPI_STATUS_RETURN(napi_set_named_property(env, *obj, "sizeFileBytes", sizeFileBytes));
1957
+
1958
+ return napi_ok;
1959
+ }
1960
+
1961
+ NAPI_METHOD(db_get_sorted_wal_files) {
1962
+ NAPI_ARGV(1);
1963
+
1964
+ Database* database;
1965
+ NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
1966
+
1967
+ rocksdb::VectorLogPtr files;
1968
+ ROCKS_STATUS_THROWS(database->db_->GetSortedWalFiles(files));
1969
+
1970
+ napi_value ret;
1971
+ NAPI_STATUS_THROWS(napi_create_array_with_length(env, files.size(), &ret));
1972
+
1973
+ for (size_t n = 0; n < files.size(); ++n) {
1974
+ napi_value obj;
1975
+ NAPI_STATUS_THROWS(FromLogFile(env, files[n], &obj));
1976
+ NAPI_STATUS_THROWS(napi_set_element(env, ret, n, obj));
1977
+ }
1978
+
1979
+ return ret;
1980
+ }
1981
+
1982
+ NAPI_METHOD(db_get_current_wal_file) {
1983
+ NAPI_ARGV(1);
1984
+
1985
+ Database* database;
1986
+ NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
1987
+
1988
+ std::unique_ptr<rocksdb::LogFile> file;
1989
+ ROCKS_STATUS_THROWS(database->db_->GetCurrentWalFile(&file));
1990
+
1991
+ napi_value ret;
1992
+ NAPI_STATUS_THROWS(FromLogFile(env, file, &ret));
1993
+
1994
+ return ret;
1995
+ }
1996
+
1861
1997
  NAPI_INIT() {
1862
1998
  NAPI_EXPORT_FUNCTION(db_init);
1863
1999
  NAPI_EXPORT_FUNCTION(db_open);
@@ -1880,6 +2016,10 @@ NAPI_INIT() {
1880
2016
  NAPI_EXPORT_FUNCTION(updates_close);
1881
2017
  NAPI_EXPORT_FUNCTION(updates_next);
1882
2018
 
2019
+ NAPI_EXPORT_FUNCTION(db_flush_wal);
2020
+ NAPI_EXPORT_FUNCTION(db_get_sorted_wal_files);
2021
+ NAPI_EXPORT_FUNCTION(db_get_current_wal_file);
2022
+
1883
2023
  NAPI_EXPORT_FUNCTION(batch_do);
1884
2024
  NAPI_EXPORT_FUNCTION(batch_init);
1885
2025
  NAPI_EXPORT_FUNCTION(batch_put);
package/chained-batch.js CHANGED
@@ -44,7 +44,7 @@ class ChainedBatch extends AbstractChainedBatch {
44
44
  binding.batch_put_log_data(this[kDbContext], this[kBatchContext], data, options)
45
45
  }
46
46
 
47
- merge (key, value, options) {
47
+ merge (key, value, options = {}) {
48
48
  // TODO (fix): Check if open...
49
49
  binding.batch_merge(this[kDbContext], this[kBatchContext], key, value, options)
50
50
  }
@@ -598,6 +598,7 @@ set(SOURCES
598
598
  cache/cache_entry_roles.cc
599
599
  cache/cache_key.cc
600
600
  cache/cache_reservation_manager.cc
601
+ cache/charged_cache.cc
601
602
  cache/clock_cache.cc
602
603
  cache/compressed_secondary_cache.cc
603
604
  cache/fast_lru_cache.cc
@@ -670,6 +671,7 @@ set(SOURCES
670
671
  db/range_del_aggregator.cc
671
672
  db/range_tombstone_fragmenter.cc
672
673
  db/repair.cc
674
+ db/seqno_to_time_mapping.cc
673
675
  db/snapshot_impl.cc
674
676
  db/table_cache.cc
675
677
  db/table_properties_collector.cc
@@ -1296,6 +1298,7 @@ if(WITH_TESTS)
1296
1298
  db/perf_context_test.cc
1297
1299
  db/periodic_work_scheduler_test.cc
1298
1300
  db/plain_table_db_test.cc
1301
+ db/seqno_time_test.cc
1299
1302
  db/prefix_test.cc
1300
1303
  db/range_del_aggregator_test.cc
1301
1304
  db/range_tombstone_fragmenter_test.cc
@@ -1504,6 +1504,9 @@ db_table_properties_test: $(OBJ_DIR)/db/db_table_properties_test.o $(TEST_LIBRAR
1504
1504
  log_write_bench: $(OBJ_DIR)/util/log_write_bench.o $(TEST_LIBRARY) $(LIBRARY)
1505
1505
  $(AM_LINK) $(PROFILING_FLAGS)
1506
1506
 
1507
+ seqno_time_test: $(OBJ_DIR)/db/seqno_time_test.o $(TEST_LIBRARY) $(LIBRARY)
1508
+ $(AM_LINK)
1509
+
1507
1510
  plain_table_db_test: $(OBJ_DIR)/db/plain_table_db_test.o $(TEST_LIBRARY) $(LIBRARY)
1508
1511
  $(AM_LINK)
1509
1512
 
@@ -13,6 +13,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
13
13
  "cache/cache_entry_roles.cc",
14
14
  "cache/cache_key.cc",
15
15
  "cache/cache_reservation_manager.cc",
16
+ "cache/charged_cache.cc",
16
17
  "cache/clock_cache.cc",
17
18
  "cache/compressed_secondary_cache.cc",
18
19
  "cache/fast_lru_cache.cc",
@@ -85,6 +86,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
85
86
  "db/range_del_aggregator.cc",
86
87
  "db/range_tombstone_fragmenter.cc",
87
88
  "db/repair.cc",
89
+ "db/seqno_to_time_mapping.cc",
88
90
  "db/snapshot_impl.cc",
89
91
  "db/table_cache.cc",
90
92
  "db/table_properties_collector.cc",
@@ -347,6 +349,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
347
349
  "cache/cache_entry_roles.cc",
348
350
  "cache/cache_key.cc",
349
351
  "cache/cache_reservation_manager.cc",
352
+ "cache/charged_cache.cc",
350
353
  "cache/clock_cache.cc",
351
354
  "cache/compressed_secondary_cache.cc",
352
355
  "cache/fast_lru_cache.cc",
@@ -419,6 +422,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
419
422
  "db/range_del_aggregator.cc",
420
423
  "db/range_tombstone_fragmenter.cc",
421
424
  "db/repair.cc",
425
+ "db/seqno_to_time_mapping.cc",
422
426
  "db/snapshot_impl.cc",
423
427
  "db/table_cache.cc",
424
428
  "db/table_properties_collector.cc",
@@ -5688,6 +5692,12 @@ cpp_unittest_wrapper(name="ribbon_test",
5688
5692
  extra_compiler_flags=[])
5689
5693
 
5690
5694
 
5695
+ cpp_unittest_wrapper(name="seqno_time_test",
5696
+ srcs=["db/seqno_time_test.cc"],
5697
+ deps=[":rocksdb_test_lib"],
5698
+ extra_compiler_flags=[])
5699
+
5700
+
5691
5701
  cpp_unittest_wrapper(name="sim_cache_test",
5692
5702
  srcs=["utilities/simulator_cache/sim_cache_test.cc"],
5693
5703
  deps=[":rocksdb_test_lib"],
@@ -74,6 +74,11 @@ DEFINE_uint32(
74
74
  DEFINE_uint32(gather_stats_entries_per_lock, 256,
75
75
  "For Cache::ApplyToAllEntries");
76
76
  DEFINE_bool(skewed, false, "If true, skew the key access distribution");
77
+
78
+ DEFINE_bool(lean, false,
79
+ "If true, no additional computation is performed besides cache "
80
+ "operations.");
81
+
77
82
  #ifndef ROCKSDB_LITE
78
83
  DEFINE_string(secondary_cache_uri, "",
79
84
  "Full URI for creating a custom secondary cache object");
@@ -522,7 +527,6 @@ class CacheBench {
522
527
  StopWatchNano timer(clock);
523
528
 
524
529
  for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
525
- timer.Start();
526
530
  Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
527
531
  uint64_t random_op = thread->rnd.Next();
528
532
  Cache::CreateCallback create_cb = [](const void* buf, size_t size,
@@ -534,6 +538,8 @@ class CacheBench {
534
538
  return Status::OK();
535
539
  };
536
540
 
541
+ timer.Start();
542
+
537
543
  if (random_op < lookup_insert_threshold_) {
538
544
  if (handle) {
539
545
  cache_->Release(handle);
@@ -543,9 +549,11 @@ class CacheBench {
543
549
  handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
544
550
  true);
545
551
  if (handle) {
546
- // do something with the data
547
- result += NPHash64(static_cast<char*>(cache_->Value(handle)),
548
- FLAGS_value_bytes);
552
+ if (!FLAGS_lean) {
553
+ // do something with the data
554
+ result += NPHash64(static_cast<char*>(cache_->Value(handle)),
555
+ FLAGS_value_bytes);
556
+ }
549
557
  } else {
550
558
  // do insert
551
559
  Status s = cache_->Insert(key, createValue(thread->rnd), &helper2,
@@ -570,9 +578,11 @@ class CacheBench {
570
578
  handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
571
579
  true);
572
580
  if (handle) {
573
- // do something with the data
574
- result += NPHash64(static_cast<char*>(cache_->Value(handle)),
575
- FLAGS_value_bytes);
581
+ if (!FLAGS_lean) {
582
+ // do something with the data
583
+ result += NPHash64(static_cast<char*>(cache_->Value(handle)),
584
+ FLAGS_value_bytes);
585
+ }
576
586
  }
577
587
  } else if (random_op < erase_threshold_) {
578
588
  // do erase
@@ -23,6 +23,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
23
23
  "FilterConstruction",
24
24
  "BlockBasedTableReader",
25
25
  "FileMetadata",
26
+ "BlobCache",
26
27
  "Misc",
27
28
  }};
28
29
 
@@ -38,6 +39,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
38
39
  "filter-construction",
39
40
  "block-based-table-reader",
40
41
  "file-metadata",
42
+ "blob-cache",
41
43
  "misc",
42
44
  }};
43
45
 
@@ -181,4 +181,5 @@ template class CacheReservationManagerImpl<CacheEntryRole::kFilterConstruction>;
181
181
  template class CacheReservationManagerImpl<CacheEntryRole::kMisc>;
182
182
  template class CacheReservationManagerImpl<CacheEntryRole::kWriteBuffer>;
183
183
  template class CacheReservationManagerImpl<CacheEntryRole::kFileMetadata>;
184
+ template class CacheReservationManagerImpl<CacheEntryRole::kBlobCache>;
184
185
  } // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,117 @@
1
+ // Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ // This source code is licensed under both the GPLv2 (found in the
3
+ // COPYING file in the root directory) and Apache 2.0 License
4
+ // (found in the LICENSE.Apache file in the root directory).
5
+
6
+ #include "cache/charged_cache.h"
7
+
8
+ #include "cache/cache_reservation_manager.h"
9
+
10
+ namespace ROCKSDB_NAMESPACE {
11
+
12
+ ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,
13
+ std::shared_ptr<Cache> block_cache)
14
+ : cache_(cache),
15
+ cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
16
+ std::make_shared<
17
+ CacheReservationManagerImpl<CacheEntryRole::kBlobCache>>(
18
+ block_cache))) {}
19
+
20
+ Status ChargedCache::Insert(const Slice& key, void* value, size_t charge,
21
+ DeleterFn deleter, Handle** handle,
22
+ Priority priority) {
23
+ Status s = cache_->Insert(key, value, charge, deleter, handle, priority);
24
+ if (s.ok()) {
25
+ // Insert may cause the cache entry eviction if the cache is full. So we
26
+ // directly call the reservation manager to update the total memory used
27
+ // in the cache.
28
+ assert(cache_res_mgr_);
29
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
30
+ .PermitUncheckedError();
31
+ }
32
+ return s;
33
+ }
34
+
35
+ Status ChargedCache::Insert(const Slice& key, void* value,
36
+ const CacheItemHelper* helper, size_t charge,
37
+ Handle** handle, Priority priority) {
38
+ Status s = cache_->Insert(key, value, helper, charge, handle, priority);
39
+ if (s.ok()) {
40
+ // Insert may cause the cache entry eviction if the cache is full. So we
41
+ // directly call the reservation manager to update the total memory used
42
+ // in the cache.
43
+ assert(cache_res_mgr_);
44
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
45
+ .PermitUncheckedError();
46
+ }
47
+ return s;
48
+ }
49
+
50
+ Cache::Handle* ChargedCache::Lookup(const Slice& key, Statistics* stats) {
51
+ return cache_->Lookup(key, stats);
52
+ }
53
+
54
+ Cache::Handle* ChargedCache::Lookup(const Slice& key,
55
+ const CacheItemHelper* helper,
56
+ const CreateCallback& create_cb,
57
+ Priority priority, bool wait,
58
+ Statistics* stats) {
59
+ auto handle = cache_->Lookup(key, helper, create_cb, priority, wait, stats);
60
+ // Lookup may promote the KV pair from the secondary cache to the primary
61
+ // cache. So we directly call the reservation manager to update the total
62
+ // memory used in the cache.
63
+ assert(cache_res_mgr_);
64
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
65
+ .PermitUncheckedError();
66
+ return handle;
67
+ }
68
+
69
+ bool ChargedCache::Release(Cache::Handle* handle, bool useful,
70
+ bool erase_if_last_ref) {
71
+ size_t memory_used_delta = cache_->GetUsage(handle);
72
+ bool erased = cache_->Release(handle, useful, erase_if_last_ref);
73
+ if (erased) {
74
+ assert(cache_res_mgr_);
75
+ cache_res_mgr_
76
+ ->UpdateCacheReservation(memory_used_delta, /* increase */ false)
77
+ .PermitUncheckedError();
78
+ }
79
+ return erased;
80
+ }
81
+
82
+ bool ChargedCache::Release(Cache::Handle* handle, bool erase_if_last_ref) {
83
+ size_t memory_used_delta = cache_->GetUsage(handle);
84
+ bool erased = cache_->Release(handle, erase_if_last_ref);
85
+ if (erased) {
86
+ assert(cache_res_mgr_);
87
+ cache_res_mgr_
88
+ ->UpdateCacheReservation(memory_used_delta, /* increase */ false)
89
+ .PermitUncheckedError();
90
+ }
91
+ return erased;
92
+ }
93
+
94
+ void ChargedCache::Erase(const Slice& key) {
95
+ cache_->Erase(key);
96
+ assert(cache_res_mgr_);
97
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
98
+ .PermitUncheckedError();
99
+ }
100
+
101
+ void ChargedCache::EraseUnRefEntries() {
102
+ cache_->EraseUnRefEntries();
103
+ assert(cache_res_mgr_);
104
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
105
+ .PermitUncheckedError();
106
+ }
107
+
108
+ void ChargedCache::SetCapacity(size_t capacity) {
109
+ cache_->SetCapacity(capacity);
110
+ // SetCapacity can result in evictions when the cache capacity is decreased,
111
+ // so we would want to update the cache reservation here as well.
112
+ assert(cache_res_mgr_);
113
+ cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
114
+ .PermitUncheckedError();
115
+ }
116
+
117
+ } // namespace ROCKSDB_NAMESPACE