@nxtedition/rocksdb 7.0.26 → 7.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +67 -25
- package/chained-batch.js +1 -1
- package/deps/rocksdb/rocksdb/CMakeLists.txt +3 -0
- package/deps/rocksdb/rocksdb/Makefile +3 -0
- package/deps/rocksdb/rocksdb/TARGETS +10 -0
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +17 -7
- package/deps/rocksdb/rocksdb/cache/cache_entry_roles.cc +2 -0
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.cc +1 -0
- package/deps/rocksdb/rocksdb/cache/charged_cache.cc +117 -0
- package/deps/rocksdb/rocksdb/cache/charged_cache.h +121 -0
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +270 -180
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +412 -124
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +1 -0
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/lru_cache.h +2 -2
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/cache/sharded_cache.h +1 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +71 -9
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +11 -2
- package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +21 -14
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +68 -7
- package/deps/rocksdb/rocksdb/db/blob/blob_source.h +16 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +519 -12
- package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +120 -0
- package/deps/rocksdb/rocksdb/db/builder.cc +15 -5
- package/deps/rocksdb/rocksdb/db/builder.h +3 -0
- package/deps/rocksdb/rocksdb/db/c.cc +18 -0
- package/deps/rocksdb/rocksdb/db/c_test.c +18 -0
- package/deps/rocksdb/rocksdb/db/column_family.h +2 -0
- package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +3 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +9 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +15 -10
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +36 -34
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +50 -13
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +12 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +8 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +13 -17
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +26 -9
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +0 -11
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +93 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +16 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +3 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +8 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +17 -5
- package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_test2.cc +39 -12
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +9 -0
- package/deps/rocksdb/rocksdb/db/db_test_util.h +2 -0
- package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -38
- package/deps/rocksdb/rocksdb/db/dbformat.h +14 -13
- package/deps/rocksdb/rocksdb/db/dbformat_test.cc +5 -2
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +13 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -10
- package/deps/rocksdb/rocksdb/db/flush_job.cc +19 -15
- package/deps/rocksdb/rocksdb/db/flush_job.h +7 -0
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +21 -15
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +4 -3
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +9 -0
- package/deps/rocksdb/rocksdb/db/memtable_list.h +5 -0
- package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.cc +53 -12
- package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.h +14 -2
- package/deps/rocksdb/rocksdb/db/periodic_work_scheduler_test.cc +10 -10
- package/deps/rocksdb/rocksdb/db/repair.cc +8 -6
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +890 -0
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +324 -0
- package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +186 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +2 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +13 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +23 -2
- package/deps/rocksdb/rocksdb/env/env_test.cc +74 -1
- package/deps/rocksdb/rocksdb/env/io_posix.cc +11 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +28 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +14 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +4 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/comparator.h +30 -23
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +3 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +5 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/debug.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +1 -1
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
- package/deps/rocksdb/rocksdb/options/cf_options.cc +14 -1
- package/deps/rocksdb/rocksdb/options/cf_options.h +5 -0
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -56
- package/deps/rocksdb/rocksdb/options/db_options.cc +4 -5
- package/deps/rocksdb/rocksdb/options/options.cc +11 -1
- package/deps/rocksdb/rocksdb/options/options_helper.cc +8 -0
- package/deps/rocksdb/rocksdb/options/options_helper.h +4 -0
- package/deps/rocksdb/rocksdb/options/options_settable_test.cc +4 -0
- package/deps/rocksdb/rocksdb/options/options_test.cc +4 -0
- package/deps/rocksdb/rocksdb/src.mk +3 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +6 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +4 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +36 -3
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +36 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +14 -3
- package/deps/rocksdb/rocksdb/table/internal_iterator.h +1 -1
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +6 -0
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +5 -0
- package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.h +3 -0
- package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +10 -7
- package/deps/rocksdb/rocksdb/table/table_builder.h +7 -3
- package/deps/rocksdb/rocksdb/table/table_properties.cc +9 -0
- package/deps/rocksdb/rocksdb/test_util/mock_time_env.h +3 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +58 -30
- package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +1 -0
- package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +20 -0
- package/deps/rocksdb/rocksdb/util/rate_limiter.cc +29 -154
- package/deps/rocksdb/rocksdb/util/rate_limiter.h +16 -34
- package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +0 -92
- package/deps/rocksdb/rocksdb/util/timer.h +6 -0
- package/deps/rocksdb/rocksdb/util/vector_iterator.h +4 -3
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -45
- package/deps/rocksdb/rocksdb/utilities/debug.cc +40 -0
- package/deps/rocksdb/rocksdb.gyp +2 -0
- package/index.js +4 -0
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
package/binding.cc
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
#include <rocksdb/db.h>
|
|
11
11
|
#include <rocksdb/env.h>
|
|
12
12
|
#include <rocksdb/filter_policy.h>
|
|
13
|
+
#include <rocksdb/merge_operator.h>
|
|
13
14
|
#include <rocksdb/options.h>
|
|
14
15
|
#include <rocksdb/slice_transform.h>
|
|
15
16
|
#include <rocksdb/table.h>
|
|
@@ -542,19 +543,8 @@ struct Updates {
|
|
|
542
543
|
bool keys,
|
|
543
544
|
bool values,
|
|
544
545
|
bool data,
|
|
545
|
-
const
|
|
546
|
-
: database_(database), seqNumber_(seqNumber), keys_(keys), values_(values), data_(data) {
|
|
547
|
-
if (column) {
|
|
548
|
-
auto columns = database->columns_;
|
|
549
|
-
auto columnIt = std::find_if(columns.begin(), columns.end(),
|
|
550
|
-
[&](const auto& handle) { return handle->GetName() == *column; });
|
|
551
|
-
if (columnIt != columns.end()) {
|
|
552
|
-
column_family_id_ = (*columnIt)->GetID();
|
|
553
|
-
} else {
|
|
554
|
-
// TODO: Throw?
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
}
|
|
546
|
+
const rocksdb::ColumnFamilyHandle* column)
|
|
547
|
+
: database_(database), seqNumber_(seqNumber), keys_(keys), values_(values), data_(data), column_(column) {}
|
|
558
548
|
|
|
559
549
|
void Close() { iterator_.reset(); }
|
|
560
550
|
|
|
@@ -576,7 +566,7 @@ struct Updates {
|
|
|
576
566
|
bool keys_;
|
|
577
567
|
bool values_;
|
|
578
568
|
bool data_;
|
|
579
|
-
|
|
569
|
+
const rocksdb::ColumnFamilyHandle* column_;
|
|
580
570
|
|
|
581
571
|
private:
|
|
582
572
|
napi_ref ref_ = nullptr;
|
|
@@ -585,7 +575,8 @@ struct Updates {
|
|
|
585
575
|
static napi_status GetColumnFamily(Database* database,
|
|
586
576
|
napi_env env,
|
|
587
577
|
napi_value options,
|
|
588
|
-
rocksdb::ColumnFamilyHandle** column
|
|
578
|
+
rocksdb::ColumnFamilyHandle** column,
|
|
579
|
+
bool fallback = true) {
|
|
589
580
|
bool hasColumn = false;
|
|
590
581
|
NAPI_STATUS_RETURN(napi_has_named_property(env, options, "column", &hasColumn));
|
|
591
582
|
|
|
@@ -593,8 +584,10 @@ static napi_status GetColumnFamily(Database* database,
|
|
|
593
584
|
napi_value value = nullptr;
|
|
594
585
|
NAPI_STATUS_RETURN(napi_get_named_property(env, options, "column", &value));
|
|
595
586
|
NAPI_STATUS_RETURN(napi_get_value_external(env, value, reinterpret_cast<void**>(column)));
|
|
596
|
-
} else {
|
|
587
|
+
} else if (fallback) {
|
|
597
588
|
*column = database->db_->DefaultColumnFamily();
|
|
589
|
+
} else {
|
|
590
|
+
*column = nullptr;
|
|
598
591
|
}
|
|
599
592
|
|
|
600
593
|
return napi_ok;
|
|
@@ -701,6 +694,8 @@ struct OpenWorker final : public Worker {
|
|
|
701
694
|
|
|
702
695
|
template <typename T, typename U>
|
|
703
696
|
rocksdb::Status InitOptions(napi_env env, T& columnOptions, const U& options) {
|
|
697
|
+
rocksdb::ConfigOptions configOptions;
|
|
698
|
+
|
|
704
699
|
const auto memtable_memory_budget = Uint32Property(env, options, "memtableMemoryBudget").value_or(256 * 1024 * 1024);
|
|
705
700
|
|
|
706
701
|
const auto compaction = StringProperty(env, options, "compaction").value_or("level");
|
|
@@ -746,18 +741,22 @@ rocksdb::Status InitOptions(napi_env env, T& columnOptions, const U& options) {
|
|
|
746
741
|
|
|
747
742
|
const auto prefixExtractorOpt = StringProperty(env, options, "prefixExtractor");
|
|
748
743
|
if (prefixExtractorOpt) {
|
|
749
|
-
rocksdb::ConfigOptions configOptions;
|
|
750
744
|
ROCKS_STATUS_RETURN(
|
|
751
745
|
rocksdb::SliceTransform::CreateFromString(configOptions, *prefixExtractorOpt, &columnOptions.prefix_extractor));
|
|
752
746
|
}
|
|
753
747
|
|
|
754
748
|
const auto comparatorOpt = StringProperty(env, options, "comparator");
|
|
755
749
|
if (comparatorOpt) {
|
|
756
|
-
rocksdb::ConfigOptions configOptions;
|
|
757
750
|
ROCKS_STATUS_RETURN(
|
|
758
751
|
rocksdb::Comparator::CreateFromString(configOptions, *comparatorOpt, &columnOptions.comparator));
|
|
759
752
|
}
|
|
760
753
|
|
|
754
|
+
const auto mergeOperatorOpt = StringProperty(env, options, "mergeOperator");
|
|
755
|
+
if (mergeOperatorOpt) {
|
|
756
|
+
ROCKS_STATUS_RETURN(
|
|
757
|
+
rocksdb::MergeOperator::CreateFromString(configOptions, *mergeOperatorOpt, &columnOptions.merge_operator));
|
|
758
|
+
}
|
|
759
|
+
|
|
761
760
|
const auto cacheSize = Uint32Property(env, options, "cacheSize").value_or(8 << 20);
|
|
762
761
|
|
|
763
762
|
rocksdb::BlockBasedTableOptions tableOptions;
|
|
@@ -834,6 +833,7 @@ NAPI_METHOD(db_open) {
|
|
|
834
833
|
dbOptions.wal_compression = BooleanProperty(env, argv[2], "walCompression").value_or(false)
|
|
835
834
|
? rocksdb::CompressionType::kZSTD
|
|
836
835
|
: rocksdb::CompressionType::kNoCompression;
|
|
836
|
+
dbOptions.manual_wal_flush = BooleanProperty(env, argv[2], "manualWalFlush").value_or(false);
|
|
837
837
|
|
|
838
838
|
// TODO (feat): dbOptions.listeners
|
|
839
839
|
|
|
@@ -1006,57 +1006,82 @@ struct UpdatesNextWorker final : public rocksdb::WriteBatch::Handler, public Wor
|
|
|
1006
1006
|
}
|
|
1007
1007
|
|
|
1008
1008
|
rocksdb::Status PutCF(uint32_t column_family_id, const rocksdb::Slice& key, const rocksdb::Slice& value) override {
|
|
1009
|
-
if (updates_->
|
|
1009
|
+
if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
|
|
1010
1010
|
return rocksdb::Status::OK();
|
|
1011
1011
|
}
|
|
1012
|
+
|
|
1012
1013
|
cache_.emplace_back("put");
|
|
1014
|
+
|
|
1013
1015
|
if (updates_->keys_) {
|
|
1014
1016
|
cache_.emplace_back(key.ToStringView());
|
|
1015
1017
|
} else {
|
|
1016
1018
|
cache_.emplace_back(std::nullopt);
|
|
1017
1019
|
}
|
|
1020
|
+
|
|
1018
1021
|
if (updates_->values_) {
|
|
1019
1022
|
cache_.emplace_back(value.ToStringView());
|
|
1020
1023
|
} else {
|
|
1021
1024
|
cache_.emplace_back(std::nullopt);
|
|
1022
1025
|
}
|
|
1023
|
-
|
|
1026
|
+
|
|
1027
|
+
if (!updates_->column_) {
|
|
1028
|
+
cache_.emplace_back(GetColumnName(column_family_id));
|
|
1029
|
+
} else {
|
|
1030
|
+
cache_.emplace_back(std::nullopt);
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1024
1033
|
return rocksdb::Status::OK();
|
|
1025
1034
|
}
|
|
1026
1035
|
|
|
1027
1036
|
rocksdb::Status DeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
|
|
1028
|
-
if (updates_->
|
|
1037
|
+
if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
|
|
1029
1038
|
return rocksdb::Status::OK();
|
|
1030
1039
|
}
|
|
1040
|
+
|
|
1031
1041
|
cache_.emplace_back("del");
|
|
1042
|
+
|
|
1032
1043
|
if (updates_->keys_) {
|
|
1033
1044
|
cache_.emplace_back(key.ToStringView());
|
|
1034
1045
|
} else {
|
|
1035
1046
|
cache_.emplace_back(std::nullopt);
|
|
1036
1047
|
}
|
|
1048
|
+
|
|
1037
1049
|
cache_.emplace_back(std::nullopt);
|
|
1038
|
-
|
|
1050
|
+
|
|
1051
|
+
if (!updates_->column_) {
|
|
1052
|
+
cache_.emplace_back(GetColumnName(column_family_id));
|
|
1053
|
+
} else {
|
|
1054
|
+
cache_.emplace_back(std::nullopt);
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1039
1057
|
return rocksdb::Status::OK();
|
|
1040
1058
|
}
|
|
1041
1059
|
|
|
1042
1060
|
rocksdb::Status MergeCF(uint32_t column_family_id, const rocksdb::Slice& key, const rocksdb::Slice& value) override {
|
|
1043
|
-
if (updates_->
|
|
1061
|
+
if (updates_->column_ && updates_->column_->GetID() != column_family_id) {
|
|
1044
1062
|
return rocksdb::Status::OK();
|
|
1045
1063
|
}
|
|
1064
|
+
|
|
1046
1065
|
cache_.emplace_back("put");
|
|
1066
|
+
|
|
1047
1067
|
if (updates_->keys_) {
|
|
1048
1068
|
cache_.emplace_back(key.ToStringView());
|
|
1049
1069
|
} else {
|
|
1050
1070
|
cache_.emplace_back(std::nullopt);
|
|
1051
1071
|
}
|
|
1072
|
+
|
|
1052
1073
|
if (updates_->values_) {
|
|
1053
1074
|
cache_.emplace_back(value.ToStringView());
|
|
1054
1075
|
} else {
|
|
1055
1076
|
cache_.emplace_back(std::nullopt);
|
|
1056
1077
|
}
|
|
1057
|
-
|
|
1078
|
+
|
|
1079
|
+
if (!updates_->column_) {
|
|
1058
1080
|
cache_.emplace_back(GetColumnName(column_family_id));
|
|
1081
|
+
} else {
|
|
1082
|
+
cache_.emplace_back(std::nullopt);
|
|
1059
1083
|
}
|
|
1084
|
+
|
|
1060
1085
|
return rocksdb::Status::OK();
|
|
1061
1086
|
}
|
|
1062
1087
|
|
|
@@ -1087,7 +1112,9 @@ NAPI_METHOD(updates_init) {
|
|
|
1087
1112
|
const auto keys = BooleanProperty(env, argv[1], "keys").value_or(true);
|
|
1088
1113
|
const auto values = BooleanProperty(env, argv[1], "values").value_or(true);
|
|
1089
1114
|
const auto data = BooleanProperty(env, argv[1], "data").value_or(true);
|
|
1090
|
-
|
|
1115
|
+
|
|
1116
|
+
rocksdb::ColumnFamilyHandle* column;
|
|
1117
|
+
NAPI_STATUS_THROWS(GetColumnFamily(database, env, argv[1], &column, false));
|
|
1091
1118
|
|
|
1092
1119
|
auto updates = std::make_unique<Updates>(database, seqNumber, keys, values, data, column);
|
|
1093
1120
|
|
|
@@ -1851,6 +1878,19 @@ NAPI_METHOD(batch_merge) {
|
|
|
1851
1878
|
return 0;
|
|
1852
1879
|
}
|
|
1853
1880
|
|
|
1881
|
+
NAPI_METHOD(db_flush_wal) {
|
|
1882
|
+
NAPI_ARGV(2);
|
|
1883
|
+
|
|
1884
|
+
Database* database;
|
|
1885
|
+
NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], reinterpret_cast<void**>(&database)));
|
|
1886
|
+
|
|
1887
|
+
const auto flush = BooleanProperty(env, argv[1], "flush").value_or(false);
|
|
1888
|
+
|
|
1889
|
+
ROCKS_STATUS_THROWS(database->db_->FlushWAL(flush));
|
|
1890
|
+
|
|
1891
|
+
return 0;
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1854
1894
|
NAPI_INIT() {
|
|
1855
1895
|
NAPI_EXPORT_FUNCTION(db_init);
|
|
1856
1896
|
NAPI_EXPORT_FUNCTION(db_open);
|
|
@@ -1873,6 +1913,8 @@ NAPI_INIT() {
|
|
|
1873
1913
|
NAPI_EXPORT_FUNCTION(updates_close);
|
|
1874
1914
|
NAPI_EXPORT_FUNCTION(updates_next);
|
|
1875
1915
|
|
|
1916
|
+
NAPI_EXPORT_FUNCTION(db_flush_wal);
|
|
1917
|
+
|
|
1876
1918
|
NAPI_EXPORT_FUNCTION(batch_do);
|
|
1877
1919
|
NAPI_EXPORT_FUNCTION(batch_init);
|
|
1878
1920
|
NAPI_EXPORT_FUNCTION(batch_put);
|
package/chained-batch.js
CHANGED
|
@@ -44,7 +44,7 @@ class ChainedBatch extends AbstractChainedBatch {
|
|
|
44
44
|
binding.batch_put_log_data(this[kDbContext], this[kBatchContext], data, options)
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
merge (key, value, options) {
|
|
47
|
+
merge (key, value, options = {}) {
|
|
48
48
|
// TODO (fix): Check if open...
|
|
49
49
|
binding.batch_merge(this[kDbContext], this[kBatchContext], key, value, options)
|
|
50
50
|
}
|
|
@@ -598,6 +598,7 @@ set(SOURCES
|
|
|
598
598
|
cache/cache_entry_roles.cc
|
|
599
599
|
cache/cache_key.cc
|
|
600
600
|
cache/cache_reservation_manager.cc
|
|
601
|
+
cache/charged_cache.cc
|
|
601
602
|
cache/clock_cache.cc
|
|
602
603
|
cache/compressed_secondary_cache.cc
|
|
603
604
|
cache/fast_lru_cache.cc
|
|
@@ -670,6 +671,7 @@ set(SOURCES
|
|
|
670
671
|
db/range_del_aggregator.cc
|
|
671
672
|
db/range_tombstone_fragmenter.cc
|
|
672
673
|
db/repair.cc
|
|
674
|
+
db/seqno_to_time_mapping.cc
|
|
673
675
|
db/snapshot_impl.cc
|
|
674
676
|
db/table_cache.cc
|
|
675
677
|
db/table_properties_collector.cc
|
|
@@ -1296,6 +1298,7 @@ if(WITH_TESTS)
|
|
|
1296
1298
|
db/perf_context_test.cc
|
|
1297
1299
|
db/periodic_work_scheduler_test.cc
|
|
1298
1300
|
db/plain_table_db_test.cc
|
|
1301
|
+
db/seqno_time_test.cc
|
|
1299
1302
|
db/prefix_test.cc
|
|
1300
1303
|
db/range_del_aggregator_test.cc
|
|
1301
1304
|
db/range_tombstone_fragmenter_test.cc
|
|
@@ -1504,6 +1504,9 @@ db_table_properties_test: $(OBJ_DIR)/db/db_table_properties_test.o $(TEST_LIBRAR
|
|
|
1504
1504
|
log_write_bench: $(OBJ_DIR)/util/log_write_bench.o $(TEST_LIBRARY) $(LIBRARY)
|
|
1505
1505
|
$(AM_LINK) $(PROFILING_FLAGS)
|
|
1506
1506
|
|
|
1507
|
+
seqno_time_test: $(OBJ_DIR)/db/seqno_time_test.o $(TEST_LIBRARY) $(LIBRARY)
|
|
1508
|
+
$(AM_LINK)
|
|
1509
|
+
|
|
1507
1510
|
plain_table_db_test: $(OBJ_DIR)/db/plain_table_db_test.o $(TEST_LIBRARY) $(LIBRARY)
|
|
1508
1511
|
$(AM_LINK)
|
|
1509
1512
|
|
|
@@ -13,6 +13,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|
|
13
13
|
"cache/cache_entry_roles.cc",
|
|
14
14
|
"cache/cache_key.cc",
|
|
15
15
|
"cache/cache_reservation_manager.cc",
|
|
16
|
+
"cache/charged_cache.cc",
|
|
16
17
|
"cache/clock_cache.cc",
|
|
17
18
|
"cache/compressed_secondary_cache.cc",
|
|
18
19
|
"cache/fast_lru_cache.cc",
|
|
@@ -85,6 +86,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|
|
85
86
|
"db/range_del_aggregator.cc",
|
|
86
87
|
"db/range_tombstone_fragmenter.cc",
|
|
87
88
|
"db/repair.cc",
|
|
89
|
+
"db/seqno_to_time_mapping.cc",
|
|
88
90
|
"db/snapshot_impl.cc",
|
|
89
91
|
"db/table_cache.cc",
|
|
90
92
|
"db/table_properties_collector.cc",
|
|
@@ -347,6 +349,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
|
|
|
347
349
|
"cache/cache_entry_roles.cc",
|
|
348
350
|
"cache/cache_key.cc",
|
|
349
351
|
"cache/cache_reservation_manager.cc",
|
|
352
|
+
"cache/charged_cache.cc",
|
|
350
353
|
"cache/clock_cache.cc",
|
|
351
354
|
"cache/compressed_secondary_cache.cc",
|
|
352
355
|
"cache/fast_lru_cache.cc",
|
|
@@ -419,6 +422,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
|
|
|
419
422
|
"db/range_del_aggregator.cc",
|
|
420
423
|
"db/range_tombstone_fragmenter.cc",
|
|
421
424
|
"db/repair.cc",
|
|
425
|
+
"db/seqno_to_time_mapping.cc",
|
|
422
426
|
"db/snapshot_impl.cc",
|
|
423
427
|
"db/table_cache.cc",
|
|
424
428
|
"db/table_properties_collector.cc",
|
|
@@ -5688,6 +5692,12 @@ cpp_unittest_wrapper(name="ribbon_test",
|
|
|
5688
5692
|
extra_compiler_flags=[])
|
|
5689
5693
|
|
|
5690
5694
|
|
|
5695
|
+
cpp_unittest_wrapper(name="seqno_time_test",
|
|
5696
|
+
srcs=["db/seqno_time_test.cc"],
|
|
5697
|
+
deps=[":rocksdb_test_lib"],
|
|
5698
|
+
extra_compiler_flags=[])
|
|
5699
|
+
|
|
5700
|
+
|
|
5691
5701
|
cpp_unittest_wrapper(name="sim_cache_test",
|
|
5692
5702
|
srcs=["utilities/simulator_cache/sim_cache_test.cc"],
|
|
5693
5703
|
deps=[":rocksdb_test_lib"],
|
|
@@ -74,6 +74,11 @@ DEFINE_uint32(
|
|
|
74
74
|
DEFINE_uint32(gather_stats_entries_per_lock, 256,
|
|
75
75
|
"For Cache::ApplyToAllEntries");
|
|
76
76
|
DEFINE_bool(skewed, false, "If true, skew the key access distribution");
|
|
77
|
+
|
|
78
|
+
DEFINE_bool(lean, false,
|
|
79
|
+
"If true, no additional computation is performed besides cache "
|
|
80
|
+
"operations.");
|
|
81
|
+
|
|
77
82
|
#ifndef ROCKSDB_LITE
|
|
78
83
|
DEFINE_string(secondary_cache_uri, "",
|
|
79
84
|
"Full URI for creating a custom secondary cache object");
|
|
@@ -522,7 +527,6 @@ class CacheBench {
|
|
|
522
527
|
StopWatchNano timer(clock);
|
|
523
528
|
|
|
524
529
|
for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
|
|
525
|
-
timer.Start();
|
|
526
530
|
Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
|
|
527
531
|
uint64_t random_op = thread->rnd.Next();
|
|
528
532
|
Cache::CreateCallback create_cb = [](const void* buf, size_t size,
|
|
@@ -534,6 +538,8 @@ class CacheBench {
|
|
|
534
538
|
return Status::OK();
|
|
535
539
|
};
|
|
536
540
|
|
|
541
|
+
timer.Start();
|
|
542
|
+
|
|
537
543
|
if (random_op < lookup_insert_threshold_) {
|
|
538
544
|
if (handle) {
|
|
539
545
|
cache_->Release(handle);
|
|
@@ -543,9 +549,11 @@ class CacheBench {
|
|
|
543
549
|
handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
|
|
544
550
|
true);
|
|
545
551
|
if (handle) {
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
552
|
+
if (!FLAGS_lean) {
|
|
553
|
+
// do something with the data
|
|
554
|
+
result += NPHash64(static_cast<char*>(cache_->Value(handle)),
|
|
555
|
+
FLAGS_value_bytes);
|
|
556
|
+
}
|
|
549
557
|
} else {
|
|
550
558
|
// do insert
|
|
551
559
|
Status s = cache_->Insert(key, createValue(thread->rnd), &helper2,
|
|
@@ -570,9 +578,11 @@ class CacheBench {
|
|
|
570
578
|
handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
|
|
571
579
|
true);
|
|
572
580
|
if (handle) {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
581
|
+
if (!FLAGS_lean) {
|
|
582
|
+
// do something with the data
|
|
583
|
+
result += NPHash64(static_cast<char*>(cache_->Value(handle)),
|
|
584
|
+
FLAGS_value_bytes);
|
|
585
|
+
}
|
|
576
586
|
}
|
|
577
587
|
} else if (random_op < erase_threshold_) {
|
|
578
588
|
// do erase
|
|
@@ -23,6 +23,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
|
|
|
23
23
|
"FilterConstruction",
|
|
24
24
|
"BlockBasedTableReader",
|
|
25
25
|
"FileMetadata",
|
|
26
|
+
"BlobCache",
|
|
26
27
|
"Misc",
|
|
27
28
|
}};
|
|
28
29
|
|
|
@@ -38,6 +39,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
|
|
|
38
39
|
"filter-construction",
|
|
39
40
|
"block-based-table-reader",
|
|
40
41
|
"file-metadata",
|
|
42
|
+
"blob-cache",
|
|
41
43
|
"misc",
|
|
42
44
|
}};
|
|
43
45
|
|
|
@@ -181,4 +181,5 @@ template class CacheReservationManagerImpl<CacheEntryRole::kFilterConstruction>;
|
|
|
181
181
|
template class CacheReservationManagerImpl<CacheEntryRole::kMisc>;
|
|
182
182
|
template class CacheReservationManagerImpl<CacheEntryRole::kWriteBuffer>;
|
|
183
183
|
template class CacheReservationManagerImpl<CacheEntryRole::kFileMetadata>;
|
|
184
|
+
template class CacheReservationManagerImpl<CacheEntryRole::kBlobCache>;
|
|
184
185
|
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
2
|
+
// This source code is licensed under both the GPLv2 (found in the
|
|
3
|
+
// COPYING file in the root directory) and Apache 2.0 License
|
|
4
|
+
// (found in the LICENSE.Apache file in the root directory).
|
|
5
|
+
|
|
6
|
+
#include "cache/charged_cache.h"
|
|
7
|
+
|
|
8
|
+
#include "cache/cache_reservation_manager.h"
|
|
9
|
+
|
|
10
|
+
namespace ROCKSDB_NAMESPACE {
|
|
11
|
+
|
|
12
|
+
ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,
|
|
13
|
+
std::shared_ptr<Cache> block_cache)
|
|
14
|
+
: cache_(cache),
|
|
15
|
+
cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
|
|
16
|
+
std::make_shared<
|
|
17
|
+
CacheReservationManagerImpl<CacheEntryRole::kBlobCache>>(
|
|
18
|
+
block_cache))) {}
|
|
19
|
+
|
|
20
|
+
Status ChargedCache::Insert(const Slice& key, void* value, size_t charge,
|
|
21
|
+
DeleterFn deleter, Handle** handle,
|
|
22
|
+
Priority priority) {
|
|
23
|
+
Status s = cache_->Insert(key, value, charge, deleter, handle, priority);
|
|
24
|
+
if (s.ok()) {
|
|
25
|
+
// Insert may cause the cache entry eviction if the cache is full. So we
|
|
26
|
+
// directly call the reservation manager to update the total memory used
|
|
27
|
+
// in the cache.
|
|
28
|
+
assert(cache_res_mgr_);
|
|
29
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
30
|
+
.PermitUncheckedError();
|
|
31
|
+
}
|
|
32
|
+
return s;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
Status ChargedCache::Insert(const Slice& key, void* value,
|
|
36
|
+
const CacheItemHelper* helper, size_t charge,
|
|
37
|
+
Handle** handle, Priority priority) {
|
|
38
|
+
Status s = cache_->Insert(key, value, helper, charge, handle, priority);
|
|
39
|
+
if (s.ok()) {
|
|
40
|
+
// Insert may cause the cache entry eviction if the cache is full. So we
|
|
41
|
+
// directly call the reservation manager to update the total memory used
|
|
42
|
+
// in the cache.
|
|
43
|
+
assert(cache_res_mgr_);
|
|
44
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
45
|
+
.PermitUncheckedError();
|
|
46
|
+
}
|
|
47
|
+
return s;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
Cache::Handle* ChargedCache::Lookup(const Slice& key, Statistics* stats) {
|
|
51
|
+
return cache_->Lookup(key, stats);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
Cache::Handle* ChargedCache::Lookup(const Slice& key,
|
|
55
|
+
const CacheItemHelper* helper,
|
|
56
|
+
const CreateCallback& create_cb,
|
|
57
|
+
Priority priority, bool wait,
|
|
58
|
+
Statistics* stats) {
|
|
59
|
+
auto handle = cache_->Lookup(key, helper, create_cb, priority, wait, stats);
|
|
60
|
+
// Lookup may promote the KV pair from the secondary cache to the primary
|
|
61
|
+
// cache. So we directly call the reservation manager to update the total
|
|
62
|
+
// memory used in the cache.
|
|
63
|
+
assert(cache_res_mgr_);
|
|
64
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
65
|
+
.PermitUncheckedError();
|
|
66
|
+
return handle;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
bool ChargedCache::Release(Cache::Handle* handle, bool useful,
|
|
70
|
+
bool erase_if_last_ref) {
|
|
71
|
+
size_t memory_used_delta = cache_->GetUsage(handle);
|
|
72
|
+
bool erased = cache_->Release(handle, useful, erase_if_last_ref);
|
|
73
|
+
if (erased) {
|
|
74
|
+
assert(cache_res_mgr_);
|
|
75
|
+
cache_res_mgr_
|
|
76
|
+
->UpdateCacheReservation(memory_used_delta, /* increase */ false)
|
|
77
|
+
.PermitUncheckedError();
|
|
78
|
+
}
|
|
79
|
+
return erased;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
bool ChargedCache::Release(Cache::Handle* handle, bool erase_if_last_ref) {
|
|
83
|
+
size_t memory_used_delta = cache_->GetUsage(handle);
|
|
84
|
+
bool erased = cache_->Release(handle, erase_if_last_ref);
|
|
85
|
+
if (erased) {
|
|
86
|
+
assert(cache_res_mgr_);
|
|
87
|
+
cache_res_mgr_
|
|
88
|
+
->UpdateCacheReservation(memory_used_delta, /* increase */ false)
|
|
89
|
+
.PermitUncheckedError();
|
|
90
|
+
}
|
|
91
|
+
return erased;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
void ChargedCache::Erase(const Slice& key) {
|
|
95
|
+
cache_->Erase(key);
|
|
96
|
+
assert(cache_res_mgr_);
|
|
97
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
98
|
+
.PermitUncheckedError();
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
void ChargedCache::EraseUnRefEntries() {
|
|
102
|
+
cache_->EraseUnRefEntries();
|
|
103
|
+
assert(cache_res_mgr_);
|
|
104
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
105
|
+
.PermitUncheckedError();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
void ChargedCache::SetCapacity(size_t capacity) {
|
|
109
|
+
cache_->SetCapacity(capacity);
|
|
110
|
+
// SetCapacity can result in evictions when the cache capacity is decreased,
|
|
111
|
+
// so we would want to update the cache reservation here as well.
|
|
112
|
+
assert(cache_res_mgr_);
|
|
113
|
+
cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
|
|
114
|
+
.PermitUncheckedError();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
2
|
+
// This source code is licensed under both the GPLv2 (found in the
|
|
3
|
+
// COPYING file in the root directory) and Apache 2.0 License
|
|
4
|
+
// (found in the LICENSE.Apache file in the root directory).
|
|
5
|
+
|
|
6
|
+
#pragma once
|
|
7
|
+
|
|
8
|
+
#include <string>
|
|
9
|
+
|
|
10
|
+
#include "port/port.h"
|
|
11
|
+
#include "rocksdb/cache.h"
|
|
12
|
+
|
|
13
|
+
namespace ROCKSDB_NAMESPACE {
|
|
14
|
+
|
|
15
|
+
class ConcurrentCacheReservationManager;
|
|
16
|
+
|
|
17
|
+
// A cache interface which wraps around another cache and takes care of
|
|
18
|
+
// reserving space in block cache towards a single global memory limit, and
|
|
19
|
+
// forwards all the calls to the underlying cache.
|
|
20
|
+
class ChargedCache : public Cache {
|
|
21
|
+
public:
|
|
22
|
+
ChargedCache(std::shared_ptr<Cache> cache,
|
|
23
|
+
std::shared_ptr<Cache> block_cache);
|
|
24
|
+
~ChargedCache() override = default;
|
|
25
|
+
|
|
26
|
+
Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter,
|
|
27
|
+
Handle** handle, Priority priority) override;
|
|
28
|
+
Status Insert(const Slice& key, void* value, const CacheItemHelper* helper,
|
|
29
|
+
size_t charge, Handle** handle = nullptr,
|
|
30
|
+
Priority priority = Priority::LOW) override;
|
|
31
|
+
|
|
32
|
+
Cache::Handle* Lookup(const Slice& key, Statistics* stats) override;
|
|
33
|
+
Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
|
|
34
|
+
const CreateCallback& create_cb, Priority priority,
|
|
35
|
+
bool wait, Statistics* stats = nullptr) override;
|
|
36
|
+
|
|
37
|
+
bool Release(Cache::Handle* handle, bool useful,
|
|
38
|
+
bool erase_if_last_ref = false) override;
|
|
39
|
+
bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
|
|
40
|
+
|
|
41
|
+
void Erase(const Slice& key) override;
|
|
42
|
+
void EraseUnRefEntries() override;
|
|
43
|
+
|
|
44
|
+
static const char* kClassName() { return "ChargedCache"; }
|
|
45
|
+
const char* Name() const override { return kClassName(); }
|
|
46
|
+
|
|
47
|
+
uint64_t NewId() override { return cache_->NewId(); }
|
|
48
|
+
|
|
49
|
+
void SetCapacity(size_t capacity) override;
|
|
50
|
+
|
|
51
|
+
void SetStrictCapacityLimit(bool strict_capacity_limit) override {
|
|
52
|
+
cache_->SetStrictCapacityLimit(strict_capacity_limit);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
bool HasStrictCapacityLimit() const override {
|
|
56
|
+
return cache_->HasStrictCapacityLimit();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
void* Value(Cache::Handle* handle) override { return cache_->Value(handle); }
|
|
60
|
+
|
|
61
|
+
bool IsReady(Cache::Handle* handle) override {
|
|
62
|
+
return cache_->IsReady(handle);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
void Wait(Cache::Handle* handle) override { cache_->Wait(handle); }
|
|
66
|
+
|
|
67
|
+
void WaitAll(std::vector<Handle*>& handles) override {
|
|
68
|
+
cache_->WaitAll(handles);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
bool Ref(Cache::Handle* handle) override { return cache_->Ref(handle); }
|
|
72
|
+
|
|
73
|
+
size_t GetCapacity() const override { return cache_->GetCapacity(); }
|
|
74
|
+
|
|
75
|
+
size_t GetUsage() const override { return cache_->GetUsage(); }
|
|
76
|
+
|
|
77
|
+
size_t GetUsage(Cache::Handle* handle) const override {
|
|
78
|
+
return cache_->GetUsage(handle);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
size_t GetPinnedUsage() const override { return cache_->GetPinnedUsage(); }
|
|
82
|
+
|
|
83
|
+
size_t GetCharge(Cache::Handle* handle) const override {
|
|
84
|
+
return cache_->GetCharge(handle);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
Cache::DeleterFn GetDeleter(Cache::Handle* handle) const override {
|
|
88
|
+
return cache_->GetDeleter(handle);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
void ApplyToAllEntries(
|
|
92
|
+
const std::function<void(const Slice& key, void* value, size_t charge,
|
|
93
|
+
Cache::DeleterFn deleter)>& callback,
|
|
94
|
+
const Cache::ApplyToAllEntriesOptions& opts) override {
|
|
95
|
+
cache_->ApplyToAllEntries(callback, opts);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
void ApplyToAllCacheEntries(void (*callback)(void* value, size_t charge),
|
|
99
|
+
bool thread_safe) override {
|
|
100
|
+
cache_->ApplyToAllCacheEntries(callback, thread_safe);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
std::string GetPrintableOptions() const override {
|
|
104
|
+
return cache_->GetPrintableOptions();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
void DisownData() override { return cache_->DisownData(); }
|
|
108
|
+
|
|
109
|
+
inline Cache* GetCache() const { return cache_.get(); }
|
|
110
|
+
|
|
111
|
+
inline ConcurrentCacheReservationManager* TEST_GetCacheReservationManager()
|
|
112
|
+
const {
|
|
113
|
+
return cache_res_mgr_.get();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private:
|
|
117
|
+
std::shared_ptr<Cache> cache_;
|
|
118
|
+
std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
} // namespace ROCKSDB_NAMESPACE
|