@nxtedition/rocksdb 8.2.0-alpha.1 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +11 -74
- package/binding.gyp +7 -5
- package/deps/rocksdb/rocksdb/CMakeLists.txt +4 -0
- package/deps/rocksdb/rocksdb/TARGETS +7 -0
- package/deps/rocksdb/rocksdb/cache/cache.cc +43 -0
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +8 -5
- package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +1 -1
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +12 -48
- package/deps/rocksdb/rocksdb/cache/charged_cache.cc +26 -18
- package/deps/rocksdb/rocksdb/cache/charged_cache.h +5 -62
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +119 -44
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +34 -29
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +3 -3
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +2 -2
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +148 -209
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +118 -284
- package/deps/rocksdb/rocksdb/cache/lru_cache.h +23 -71
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +351 -392
- package/deps/rocksdb/rocksdb/cache/secondary_cache.cc +5 -2
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +296 -0
- package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.h +52 -0
- package/deps/rocksdb/rocksdb/cache/sharded_cache.h +22 -19
- package/deps/rocksdb/rocksdb/cache/typed_cache.h +56 -20
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +3 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +4 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +3 -3
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +19 -25
- package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +216 -0
- package/deps/rocksdb/rocksdb/db/c.cc +90 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +8 -7
- package/deps/rocksdb/rocksdb/db/column_family.h +0 -6
- package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +5 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +24 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +17 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +18 -12
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +3 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +245 -302
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +13 -2
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +5 -0
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +75 -15
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -5
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +91 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +5 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +16 -4
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +47 -24
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +4 -2
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +32 -3
- package/deps/rocksdb/rocksdb/db/db_iter.cc +28 -29
- package/deps/rocksdb/rocksdb/db/db_iter.h +0 -3
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +176 -0
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +391 -2
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +26 -0
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +13 -5
- package/deps/rocksdb/rocksdb/db/dbformat.h +3 -1
- package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +0 -1
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +0 -6
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +3 -0
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +1 -1
- package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +4 -0
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +68 -40
- package/deps/rocksdb/rocksdb/db/import_column_family_job.h +3 -3
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +115 -0
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +169 -72
- package/deps/rocksdb/rocksdb/db/internal_stats.h +36 -7
- package/deps/rocksdb/rocksdb/db/memtable.cc +6 -4
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +4 -0
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +151 -0
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +47 -16
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.h +10 -8
- package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +91 -93
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +1 -2
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +1 -1
- package/deps/rocksdb/rocksdb/db/version_set.cc +30 -14
- package/deps/rocksdb/rocksdb/db/version_set.h +1 -0
- package/deps/rocksdb/rocksdb/db/write_stall_stats.cc +179 -0
- package/deps/rocksdb/rocksdb/db/write_stall_stats.h +47 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +109 -7
- package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +147 -12
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +31 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +22 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +4 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +42 -59
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +7 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +7 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +6 -10
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +6 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +4 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +127 -36
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +8 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +35 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +29 -8
- package/deps/rocksdb/rocksdb/file/file_util.cc +14 -10
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +183 -63
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +159 -66
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +3 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +52 -5
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +134 -73
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +46 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +6 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +0 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +7 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +6 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +3 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +18 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/types.h +28 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h +39 -0
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +5 -0
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +9 -1
- package/deps/rocksdb/rocksdb/options/customizable_test.cc +2 -2
- package/deps/rocksdb/rocksdb/port/stack_trace.cc +17 -7
- package/deps/rocksdb/rocksdb/port/win/env_win.h +1 -0
- package/deps/rocksdb/rocksdb/src.mk +4 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +38 -34
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +11 -12
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +5 -5
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +126 -132
- package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +16 -16
- package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h +0 -16
- package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +3 -4
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +370 -0
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +44 -0
- package/deps/rocksdb/rocksdb/table/get_context.cc +4 -2
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +555 -267
- package/deps/rocksdb/rocksdb/table/merging_iterator.h +10 -5
- package/deps/rocksdb/rocksdb/table/table_test.cc +113 -70
- package/deps/rocksdb/rocksdb/test_util/secondary_cache_test_util.cc +96 -0
- package/deps/rocksdb/rocksdb/test_util/secondary_cache_test_util.h +117 -0
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +5 -3
- package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.cc +3 -3
- package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.h +1 -1
- package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache.cc +9 -2
- package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +5 -1
- package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +11 -0
- package/deps/rocksdb/rocksdb.gyp +6 -7
- package/index.js +0 -6
- package/package.json +1 -1
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/deps/liburing/liburing.gyp +0 -20
- package/tmp/test.js +0 -7
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
#include "cache/lru_cache.h"
|
|
7
7
|
|
|
8
|
+
#include <memory>
|
|
8
9
|
#include <string>
|
|
9
10
|
#include <vector>
|
|
10
11
|
|
|
@@ -19,6 +20,7 @@
|
|
|
19
20
|
#include "rocksdb/io_status.h"
|
|
20
21
|
#include "rocksdb/sst_file_manager.h"
|
|
21
22
|
#include "rocksdb/utilities/cache_dump_load.h"
|
|
23
|
+
#include "test_util/secondary_cache_test_util.h"
|
|
22
24
|
#include "test_util/testharness.h"
|
|
23
25
|
#include "typed_cache.h"
|
|
24
26
|
#include "util/coding.h"
|
|
@@ -51,8 +53,7 @@ class LRUCacheTest : public testing::Test {
|
|
|
51
53
|
high_pri_pool_ratio, low_pri_pool_ratio,
|
|
52
54
|
use_adaptive_mutex, kDontChargeCacheMetadata,
|
|
53
55
|
/*max_upper_hash_bits=*/24,
|
|
54
|
-
/*allocator*/ nullptr,
|
|
55
|
-
/*secondary_cache=*/nullptr);
|
|
56
|
+
/*allocator*/ nullptr, &eviction_callback_);
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
void Insert(const std::string& key,
|
|
@@ -68,7 +69,7 @@ class LRUCacheTest : public testing::Test {
|
|
|
68
69
|
|
|
69
70
|
bool Lookup(const std::string& key) {
|
|
70
71
|
auto handle = cache_->Lookup(key, 0 /*hash*/, nullptr, nullptr,
|
|
71
|
-
Cache::Priority::LOW,
|
|
72
|
+
Cache::Priority::LOW, nullptr);
|
|
72
73
|
if (handle) {
|
|
73
74
|
cache_->Release(handle, true /*useful*/, false /*erase*/);
|
|
74
75
|
return true;
|
|
@@ -145,6 +146,7 @@ class LRUCacheTest : public testing::Test {
|
|
|
145
146
|
|
|
146
147
|
private:
|
|
147
148
|
LRUCacheShard* cache_ = nullptr;
|
|
149
|
+
Cache::EvictionCallback eviction_callback_;
|
|
148
150
|
};
|
|
149
151
|
|
|
150
152
|
TEST_F(LRUCacheTest, BasicLRU) {
|
|
@@ -393,8 +395,9 @@ class ClockCacheTest : public testing::Test {
|
|
|
393
395
|
|
|
394
396
|
Table::Opts opts;
|
|
395
397
|
opts.estimated_value_size = 1;
|
|
396
|
-
new (shard_)
|
|
397
|
-
|
|
398
|
+
new (shard_)
|
|
399
|
+
Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata,
|
|
400
|
+
/*allocator*/ nullptr, &eviction_callback_, opts);
|
|
398
401
|
}
|
|
399
402
|
|
|
400
403
|
Status Insert(const UniqueId64x2& hashed_key,
|
|
@@ -449,6 +452,9 @@ class ClockCacheTest : public testing::Test {
|
|
|
449
452
|
}
|
|
450
453
|
|
|
451
454
|
Shard* shard_ = nullptr;
|
|
455
|
+
|
|
456
|
+
private:
|
|
457
|
+
Cache::EvictionCallback eviction_callback_;
|
|
452
458
|
};
|
|
453
459
|
|
|
454
460
|
TEST_F(ClockCacheTest, Misc) {
|
|
@@ -934,12 +940,12 @@ class TestSecondaryCache : public SecondaryCache {
|
|
|
934
940
|
std::unique_ptr<SecondaryCacheResultHandle> Lookup(
|
|
935
941
|
const Slice& key, const Cache::CacheItemHelper* helper,
|
|
936
942
|
Cache::CreateContext* create_context, bool /*wait*/,
|
|
937
|
-
bool /*advise_erase*/, bool&
|
|
943
|
+
bool /*advise_erase*/, bool& kept_in_sec_cache) override {
|
|
938
944
|
std::string key_str = key.ToString();
|
|
939
945
|
TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str);
|
|
940
946
|
|
|
941
947
|
std::unique_ptr<SecondaryCacheResultHandle> secondary_handle;
|
|
942
|
-
|
|
948
|
+
kept_in_sec_cache = false;
|
|
943
949
|
ResultType type = ResultType::SUCCESS;
|
|
944
950
|
auto iter = result_map_.find(key.ToString());
|
|
945
951
|
if (iter != result_map_.end()) {
|
|
@@ -965,7 +971,7 @@ class TestSecondaryCache : public SecondaryCache {
|
|
|
965
971
|
if (s.ok()) {
|
|
966
972
|
secondary_handle.reset(new TestSecondaryCacheResultHandle(
|
|
967
973
|
cache_.get(), handle, value, charge, type));
|
|
968
|
-
|
|
974
|
+
kept_in_sec_cache = true;
|
|
969
975
|
} else {
|
|
970
976
|
cache_.Release(handle);
|
|
971
977
|
}
|
|
@@ -1052,7 +1058,16 @@ class TestSecondaryCache : public SecondaryCache {
|
|
|
1052
1058
|
ResultMap result_map_;
|
|
1053
1059
|
};
|
|
1054
1060
|
|
|
1055
|
-
|
|
1061
|
+
using secondary_cache_test_util::GetTestingCacheTypes;
|
|
1062
|
+
using secondary_cache_test_util::WithCacheTypeParam;
|
|
1063
|
+
|
|
1064
|
+
class BasicSecondaryCacheTest : public testing::Test,
|
|
1065
|
+
public WithCacheTypeParam {};
|
|
1066
|
+
|
|
1067
|
+
INSTANTIATE_TEST_CASE_P(BasicSecondaryCacheTest, BasicSecondaryCacheTest,
|
|
1068
|
+
GetTestingCacheTypes());
|
|
1069
|
+
|
|
1070
|
+
class DBSecondaryCacheTest : public DBTestBase, public WithCacheTypeParam {
|
|
1056
1071
|
public:
|
|
1057
1072
|
DBSecondaryCacheTest()
|
|
1058
1073
|
: DBTestBase("db_secondary_cache_test", /*env_do_fsync=*/true) {
|
|
@@ -1064,98 +1079,15 @@ class DBSecondaryCacheTest : public DBTestBase {
|
|
|
1064
1079
|
std::unique_ptr<Env> fault_env_;
|
|
1065
1080
|
};
|
|
1066
1081
|
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
public:
|
|
1070
|
-
LRUCacheSecondaryCacheTest() : fail_create_(false) {}
|
|
1071
|
-
~LRUCacheSecondaryCacheTest() {}
|
|
1072
|
-
|
|
1073
|
-
protected:
|
|
1074
|
-
class TestItem {
|
|
1075
|
-
public:
|
|
1076
|
-
TestItem(const char* buf, size_t size) : buf_(new char[size]), size_(size) {
|
|
1077
|
-
memcpy(buf_.get(), buf, size);
|
|
1078
|
-
}
|
|
1079
|
-
~TestItem() {}
|
|
1080
|
-
|
|
1081
|
-
char* Buf() { return buf_.get(); }
|
|
1082
|
-
size_t Size() { return size_; }
|
|
1083
|
-
std::string ToString() { return std::string(Buf(), Size()); }
|
|
1084
|
-
|
|
1085
|
-
private:
|
|
1086
|
-
std::unique_ptr<char[]> buf_;
|
|
1087
|
-
size_t size_;
|
|
1088
|
-
};
|
|
1089
|
-
|
|
1090
|
-
static size_t SizeCallback(Cache::ObjectPtr obj) {
|
|
1091
|
-
return static_cast<TestItem*>(obj)->Size();
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
static Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset,
|
|
1095
|
-
size_t length, char* out) {
|
|
1096
|
-
TestItem* item = static_cast<TestItem*>(from_obj);
|
|
1097
|
-
char* buf = item->Buf();
|
|
1098
|
-
EXPECT_EQ(length, item->Size());
|
|
1099
|
-
EXPECT_EQ(from_offset, 0);
|
|
1100
|
-
memcpy(out, buf, length);
|
|
1101
|
-
return Status::OK();
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1104
|
-
static void DeletionCallback(Cache::ObjectPtr obj,
|
|
1105
|
-
MemoryAllocator* /*alloc*/) {
|
|
1106
|
-
delete static_cast<TestItem*>(obj);
|
|
1107
|
-
}
|
|
1108
|
-
|
|
1109
|
-
static Cache::CacheItemHelper helper_;
|
|
1110
|
-
|
|
1111
|
-
static Status SaveToCallbackFail(Cache::ObjectPtr /*from_obj*/,
|
|
1112
|
-
size_t /*from_offset*/, size_t /*length*/,
|
|
1113
|
-
char* /*out*/) {
|
|
1114
|
-
return Status::NotSupported();
|
|
1115
|
-
}
|
|
1116
|
-
|
|
1117
|
-
static Cache::CacheItemHelper helper_fail_;
|
|
1118
|
-
|
|
1119
|
-
static Status CreateCallback(const Slice& data, Cache::CreateContext* context,
|
|
1120
|
-
MemoryAllocator* /*allocator*/,
|
|
1121
|
-
Cache::ObjectPtr* out_obj, size_t* out_charge) {
|
|
1122
|
-
auto t = static_cast<LRUCacheSecondaryCacheTest*>(context);
|
|
1123
|
-
if (t->fail_create_) {
|
|
1124
|
-
return Status::NotSupported();
|
|
1125
|
-
}
|
|
1126
|
-
*out_obj = new TestItem(data.data(), data.size());
|
|
1127
|
-
*out_charge = data.size();
|
|
1128
|
-
return Status::OK();
|
|
1129
|
-
};
|
|
1130
|
-
|
|
1131
|
-
void SetFailCreate(bool fail) { fail_create_ = fail; }
|
|
1132
|
-
|
|
1133
|
-
private:
|
|
1134
|
-
bool fail_create_;
|
|
1135
|
-
};
|
|
1082
|
+
INSTANTIATE_TEST_CASE_P(DBSecondaryCacheTest, DBSecondaryCacheTest,
|
|
1083
|
+
GetTestingCacheTypes());
|
|
1136
1084
|
|
|
1137
|
-
|
|
1138
|
-
CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback,
|
|
1139
|
-
LRUCacheSecondaryCacheTest::SizeCallback,
|
|
1140
|
-
LRUCacheSecondaryCacheTest::SaveToCallback,
|
|
1141
|
-
LRUCacheSecondaryCacheTest::CreateCallback};
|
|
1142
|
-
|
|
1143
|
-
Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_fail_{
|
|
1144
|
-
CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback,
|
|
1145
|
-
LRUCacheSecondaryCacheTest::SizeCallback,
|
|
1146
|
-
LRUCacheSecondaryCacheTest::SaveToCallbackFail,
|
|
1147
|
-
LRUCacheSecondaryCacheTest::CreateCallback};
|
|
1148
|
-
|
|
1149
|
-
TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
|
|
1150
|
-
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1151
|
-
false /* strict_capacity_limit */,
|
|
1152
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1153
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1154
|
-
kDontChargeCacheMetadata);
|
|
1085
|
+
TEST_P(BasicSecondaryCacheTest, BasicTest) {
|
|
1155
1086
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1156
1087
|
std::make_shared<TestSecondaryCache>(4096);
|
|
1157
|
-
|
|
1158
|
-
|
|
1088
|
+
std::shared_ptr<Cache> cache =
|
|
1089
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1090
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1159
1091
|
std::shared_ptr<Statistics> stats = CreateDBStatistics();
|
|
1160
1092
|
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1161
1093
|
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
@@ -1166,37 +1098,32 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
|
|
|
1166
1098
|
std::string str3 = rnd.RandomString(1021);
|
|
1167
1099
|
ASSERT_OK(secondary_cache->InsertSaved(k3.AsSlice(), str3));
|
|
1168
1100
|
|
|
1169
|
-
std::string str1 = rnd.RandomString(
|
|
1101
|
+
std::string str1 = rnd.RandomString(1021);
|
|
1170
1102
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
|
1171
|
-
ASSERT_OK(cache->Insert(k1.AsSlice(), item1,
|
|
1172
|
-
&LRUCacheSecondaryCacheTest::helper_, str1.length()));
|
|
1103
|
+
ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length()));
|
|
1173
1104
|
std::string str2 = rnd.RandomString(1021);
|
|
1174
1105
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
|
1175
1106
|
// k1 should be demoted to NVM
|
|
1176
|
-
ASSERT_OK(cache->Insert(k2.AsSlice(), item2,
|
|
1177
|
-
&LRUCacheSecondaryCacheTest::helper_, str2.length()));
|
|
1107
|
+
ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length()));
|
|
1178
1108
|
|
|
1179
1109
|
get_perf_context()->Reset();
|
|
1180
1110
|
Cache::Handle* handle;
|
|
1181
|
-
handle =
|
|
1182
|
-
|
|
1183
|
-
/*context*/ this, Cache::Priority::LOW, true, stats.get());
|
|
1111
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelper(),
|
|
1112
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1184
1113
|
ASSERT_NE(handle, nullptr);
|
|
1185
1114
|
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str2.size());
|
|
1186
1115
|
cache->Release(handle);
|
|
1187
1116
|
|
|
1188
1117
|
// This lookup should promote k1 and demote k2
|
|
1189
|
-
handle =
|
|
1190
|
-
|
|
1191
|
-
/*context*/ this, Cache::Priority::LOW, true, stats.get());
|
|
1118
|
+
handle = cache->Lookup(k1.AsSlice(), GetHelper(),
|
|
1119
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1192
1120
|
ASSERT_NE(handle, nullptr);
|
|
1193
1121
|
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str1.size());
|
|
1194
1122
|
cache->Release(handle);
|
|
1195
1123
|
|
|
1196
1124
|
// This lookup should promote k3 and demote k1
|
|
1197
|
-
handle =
|
|
1198
|
-
|
|
1199
|
-
/*context*/ this, Cache::Priority::LOW, true, stats.get());
|
|
1125
|
+
handle = cache->Lookup(k3.AsSlice(), GetHelper(),
|
|
1126
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1200
1127
|
ASSERT_NE(handle, nullptr);
|
|
1201
1128
|
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str3.size());
|
|
1202
1129
|
cache->Release(handle);
|
|
@@ -1212,16 +1139,66 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
|
|
|
1212
1139
|
secondary_cache.reset();
|
|
1213
1140
|
}
|
|
1214
1141
|
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1142
|
+
TEST_P(BasicSecondaryCacheTest, StatsTest) {
|
|
1143
|
+
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1144
|
+
std::make_shared<TestSecondaryCache>(4096);
|
|
1145
|
+
std::shared_ptr<Cache> cache =
|
|
1146
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1147
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1148
|
+
std::shared_ptr<Statistics> stats = CreateDBStatistics();
|
|
1149
|
+
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1150
|
+
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1151
|
+
CacheKey k3 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1152
|
+
|
|
1153
|
+
Random rnd(301);
|
|
1154
|
+
// Start with warming secondary cache
|
|
1155
|
+
std::string str1 = rnd.RandomString(1020);
|
|
1156
|
+
std::string str2 = rnd.RandomString(1020);
|
|
1157
|
+
std::string str3 = rnd.RandomString(1020);
|
|
1158
|
+
ASSERT_OK(secondary_cache->InsertSaved(k1.AsSlice(), str1));
|
|
1159
|
+
ASSERT_OK(secondary_cache->InsertSaved(k2.AsSlice(), str2));
|
|
1160
|
+
ASSERT_OK(secondary_cache->InsertSaved(k3.AsSlice(), str3));
|
|
1161
|
+
|
|
1162
|
+
get_perf_context()->Reset();
|
|
1163
|
+
Cache::Handle* handle;
|
|
1164
|
+
handle = cache->Lookup(k1.AsSlice(), GetHelper(CacheEntryRole::kFilterBlock),
|
|
1165
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1166
|
+
ASSERT_NE(handle, nullptr);
|
|
1167
|
+
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str1.size());
|
|
1168
|
+
cache->Release(handle);
|
|
1169
|
+
|
|
1170
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelper(CacheEntryRole::kIndexBlock),
|
|
1171
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1172
|
+
ASSERT_NE(handle, nullptr);
|
|
1173
|
+
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str2.size());
|
|
1174
|
+
cache->Release(handle);
|
|
1175
|
+
|
|
1176
|
+
handle = cache->Lookup(k3.AsSlice(), GetHelper(CacheEntryRole::kDataBlock),
|
|
1177
|
+
/*context*/ this, Cache::Priority::LOW, stats.get());
|
|
1178
|
+
ASSERT_NE(handle, nullptr);
|
|
1179
|
+
ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str3.size());
|
|
1180
|
+
cache->Release(handle);
|
|
1181
|
+
|
|
1182
|
+
ASSERT_EQ(secondary_cache->num_inserts(), 3u);
|
|
1183
|
+
ASSERT_EQ(secondary_cache->num_lookups(), 3u);
|
|
1184
|
+
ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_HITS),
|
|
1185
|
+
secondary_cache->num_lookups());
|
|
1186
|
+
ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_FILTER_HITS), 1);
|
|
1187
|
+
ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_INDEX_HITS), 1);
|
|
1188
|
+
ASSERT_EQ(stats->getTickerCount(SECONDARY_CACHE_DATA_HITS), 1);
|
|
1189
|
+
PerfContext perf_ctx = *get_perf_context();
|
|
1190
|
+
ASSERT_EQ(perf_ctx.secondary_cache_hit_count, secondary_cache->num_lookups());
|
|
1191
|
+
|
|
1192
|
+
cache.reset();
|
|
1193
|
+
secondary_cache.reset();
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
TEST_P(BasicSecondaryCacheTest, BasicFailTest) {
|
|
1221
1197
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1222
1198
|
std::make_shared<TestSecondaryCache>(2048);
|
|
1223
|
-
|
|
1224
|
-
|
|
1199
|
+
std::shared_ptr<Cache> cache =
|
|
1200
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1201
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1225
1202
|
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1226
1203
|
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1227
1204
|
|
|
@@ -1231,65 +1208,65 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicFailTest) {
|
|
|
1231
1208
|
// NOTE: changed to assert helper != nullptr for efficiency / code size
|
|
1232
1209
|
// ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr,
|
|
1233
1210
|
// str1.length()).IsInvalidArgument());
|
|
1234
|
-
ASSERT_OK(
|
|
1235
|
-
|
|
1211
|
+
ASSERT_OK(
|
|
1212
|
+
cache->Insert(k1.AsSlice(), item1.get(), GetHelper(), str1.length()));
|
|
1236
1213
|
item1.release(); // Appease clang-analyze "potential memory leak"
|
|
1237
1214
|
|
|
1238
1215
|
Cache::Handle* handle;
|
|
1239
1216
|
handle = cache->Lookup(k2.AsSlice(), nullptr, /*context*/ this,
|
|
1240
|
-
Cache::Priority::LOW
|
|
1217
|
+
Cache::Priority::LOW);
|
|
1241
1218
|
ASSERT_EQ(handle, nullptr);
|
|
1242
|
-
|
|
1243
|
-
|
|
1219
|
+
|
|
1220
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelper(),
|
|
1221
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1222
|
+
ASSERT_EQ(handle, nullptr);
|
|
1223
|
+
|
|
1224
|
+
Cache::AsyncLookupHandle async_handle;
|
|
1225
|
+
async_handle.key = k2.AsSlice();
|
|
1226
|
+
async_handle.helper = GetHelper();
|
|
1227
|
+
async_handle.create_context = this;
|
|
1228
|
+
async_handle.priority = Cache::Priority::LOW;
|
|
1229
|
+
cache->StartAsyncLookup(async_handle);
|
|
1230
|
+
cache->Wait(async_handle);
|
|
1231
|
+
handle = async_handle.Result();
|
|
1244
1232
|
ASSERT_EQ(handle, nullptr);
|
|
1245
1233
|
|
|
1246
1234
|
cache.reset();
|
|
1247
1235
|
secondary_cache.reset();
|
|
1248
1236
|
}
|
|
1249
1237
|
|
|
1250
|
-
|
|
1251
|
-
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1252
|
-
false /* strict_capacity_limit */,
|
|
1253
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1254
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1255
|
-
kDontChargeCacheMetadata);
|
|
1238
|
+
TEST_P(BasicSecondaryCacheTest, SaveFailTest) {
|
|
1256
1239
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1257
1240
|
std::make_shared<TestSecondaryCache>(2048);
|
|
1258
|
-
|
|
1259
|
-
|
|
1241
|
+
std::shared_ptr<Cache> cache =
|
|
1242
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1243
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1260
1244
|
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1261
1245
|
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1262
1246
|
|
|
1263
1247
|
Random rnd(301);
|
|
1264
1248
|
std::string str1 = rnd.RandomString(1020);
|
|
1265
1249
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
|
1266
|
-
ASSERT_OK(cache->Insert(k1.AsSlice(), item1,
|
|
1267
|
-
&LRUCacheSecondaryCacheTest::helper_fail_,
|
|
1268
|
-
str1.length()));
|
|
1250
|
+
ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelperFail(), str1.length()));
|
|
1269
1251
|
std::string str2 = rnd.RandomString(1020);
|
|
1270
1252
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
|
1271
1253
|
// k1 should be demoted to NVM
|
|
1272
1254
|
ASSERT_EQ(secondary_cache->num_inserts(), 0u);
|
|
1273
|
-
ASSERT_OK(cache->Insert(k2.AsSlice(), item2,
|
|
1274
|
-
&LRUCacheSecondaryCacheTest::helper_fail_,
|
|
1275
|
-
str2.length()));
|
|
1255
|
+
ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelperFail(), str2.length()));
|
|
1276
1256
|
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
1277
1257
|
|
|
1278
1258
|
Cache::Handle* handle;
|
|
1279
|
-
handle =
|
|
1280
|
-
|
|
1281
|
-
/*context*/ this, Cache::Priority::LOW, true);
|
|
1259
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelperFail(),
|
|
1260
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1282
1261
|
ASSERT_NE(handle, nullptr);
|
|
1283
1262
|
cache->Release(handle);
|
|
1284
1263
|
// This lookup should fail, since k1 demotion would have failed
|
|
1285
|
-
handle =
|
|
1286
|
-
|
|
1287
|
-
/*context*/ this, Cache::Priority::LOW, true);
|
|
1264
|
+
handle = cache->Lookup(k1.AsSlice(), GetHelperFail(),
|
|
1265
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1288
1266
|
ASSERT_EQ(handle, nullptr);
|
|
1289
1267
|
// Since k1 didn't get promoted, k2 should still be in cache
|
|
1290
|
-
handle =
|
|
1291
|
-
|
|
1292
|
-
/*context*/ this, Cache::Priority::LOW, true);
|
|
1268
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelperFail(),
|
|
1269
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1293
1270
|
ASSERT_NE(handle, nullptr);
|
|
1294
1271
|
cache->Release(handle);
|
|
1295
1272
|
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
@@ -1299,43 +1276,37 @@ TEST_F(LRUCacheSecondaryCacheTest, SaveFailTest) {
|
|
|
1299
1276
|
secondary_cache.reset();
|
|
1300
1277
|
}
|
|
1301
1278
|
|
|
1302
|
-
|
|
1303
|
-
LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1304
|
-
false /* strict_capacity_limit */,
|
|
1305
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1306
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1307
|
-
kDontChargeCacheMetadata);
|
|
1279
|
+
TEST_P(BasicSecondaryCacheTest, CreateFailTest) {
|
|
1308
1280
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1309
1281
|
std::make_shared<TestSecondaryCache>(2048);
|
|
1310
|
-
|
|
1311
|
-
|
|
1282
|
+
std::shared_ptr<Cache> cache =
|
|
1283
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1284
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1312
1285
|
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1313
1286
|
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1314
1287
|
|
|
1315
1288
|
Random rnd(301);
|
|
1316
1289
|
std::string str1 = rnd.RandomString(1020);
|
|
1317
1290
|
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
|
1318
|
-
ASSERT_OK(cache->Insert(k1.AsSlice(), item1,
|
|
1319
|
-
&LRUCacheSecondaryCacheTest::helper_, str1.length()));
|
|
1291
|
+
ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length()));
|
|
1320
1292
|
std::string str2 = rnd.RandomString(1020);
|
|
1321
1293
|
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
|
1322
1294
|
// k1 should be demoted to NVM
|
|
1323
|
-
ASSERT_OK(cache->Insert(k2.AsSlice(), item2,
|
|
1324
|
-
&LRUCacheSecondaryCacheTest::helper_, str2.length()));
|
|
1295
|
+
ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length()));
|
|
1325
1296
|
|
|
1326
1297
|
Cache::Handle* handle;
|
|
1327
1298
|
SetFailCreate(true);
|
|
1328
|
-
handle = cache->Lookup(k2.AsSlice(),
|
|
1329
|
-
/*context*/ this, Cache::Priority::LOW
|
|
1299
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelper(),
|
|
1300
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1330
1301
|
ASSERT_NE(handle, nullptr);
|
|
1331
1302
|
cache->Release(handle);
|
|
1332
1303
|
// This lookup should fail, since k1 creation would have failed
|
|
1333
|
-
handle = cache->Lookup(k1.AsSlice(),
|
|
1334
|
-
/*context*/ this, Cache::Priority::LOW
|
|
1304
|
+
handle = cache->Lookup(k1.AsSlice(), GetHelper(),
|
|
1305
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1335
1306
|
ASSERT_EQ(handle, nullptr);
|
|
1336
1307
|
// Since k1 didn't get promoted, k2 should still be in cache
|
|
1337
|
-
handle = cache->Lookup(k2.AsSlice(),
|
|
1338
|
-
/*context*/ this, Cache::Priority::LOW
|
|
1308
|
+
handle = cache->Lookup(k2.AsSlice(), GetHelper(),
|
|
1309
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1339
1310
|
ASSERT_NE(handle, nullptr);
|
|
1340
1311
|
cache->Release(handle);
|
|
1341
1312
|
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
@@ -1345,52 +1316,70 @@ TEST_F(LRUCacheSecondaryCacheTest, CreateFailTest) {
|
|
|
1345
1316
|
secondary_cache.reset();
|
|
1346
1317
|
}
|
|
1347
1318
|
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1319
|
+
TEST_P(BasicSecondaryCacheTest, FullCapacityTest) {
|
|
1320
|
+
for (bool strict_capacity_limit : {false, true}) {
|
|
1321
|
+
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1322
|
+
std::make_shared<TestSecondaryCache>(2048);
|
|
1323
|
+
std::shared_ptr<Cache> cache =
|
|
1324
|
+
NewCache(1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1325
|
+
strict_capacity_limit, secondary_cache);
|
|
1326
|
+
CacheKey k1 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1327
|
+
CacheKey k2 = CacheKey::CreateUniqueForCacheLifetime(cache.get());
|
|
1328
|
+
|
|
1329
|
+
Random rnd(301);
|
|
1330
|
+
std::string str1 = rnd.RandomString(1020);
|
|
1331
|
+
TestItem* item1 = new TestItem(str1.data(), str1.length());
|
|
1332
|
+
ASSERT_OK(cache->Insert(k1.AsSlice(), item1, GetHelper(), str1.length()));
|
|
1333
|
+
std::string str2 = rnd.RandomString(1020);
|
|
1334
|
+
TestItem* item2 = new TestItem(str2.data(), str2.length());
|
|
1335
|
+
// k1 should be demoted to NVM
|
|
1336
|
+
ASSERT_OK(cache->Insert(k2.AsSlice(), item2, GetHelper(), str2.length()));
|
|
1337
|
+
|
|
1338
|
+
Cache::Handle* handle2;
|
|
1339
|
+
handle2 = cache->Lookup(k2.AsSlice(), GetHelper(),
|
|
1340
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1341
|
+
ASSERT_NE(handle2, nullptr);
|
|
1342
|
+
// k1 lookup fails without secondary cache support
|
|
1343
|
+
Cache::Handle* handle1;
|
|
1344
|
+
handle1 = cache->Lookup(
|
|
1345
|
+
k1.AsSlice(),
|
|
1346
|
+
GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false),
|
|
1347
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1348
|
+
ASSERT_EQ(handle1, nullptr);
|
|
1349
|
+
|
|
1350
|
+
// k1 promotion can fail with strict_capacit_limit=true, but Lookup still
|
|
1351
|
+
// succeeds using a standalone handle
|
|
1352
|
+
handle1 = cache->Lookup(k1.AsSlice(), GetHelper(),
|
|
1353
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1354
|
+
ASSERT_NE(handle1, nullptr);
|
|
1355
|
+
|
|
1356
|
+
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
1357
|
+
ASSERT_EQ(secondary_cache->num_lookups(), 1u);
|
|
1358
|
+
|
|
1359
|
+
// Releasing k2's handle first, k2 is evicted from primary iff k1 promotion
|
|
1360
|
+
// was charged to the cache (except HCC doesn't erase in Release() over
|
|
1361
|
+
// capacity)
|
|
1362
|
+
// FIXME: Insert to secondary from Release disabled
|
|
1363
|
+
cache->Release(handle2);
|
|
1364
|
+
cache->Release(handle1);
|
|
1365
|
+
handle2 = cache->Lookup(
|
|
1366
|
+
k2.AsSlice(),
|
|
1367
|
+
GetHelper(CacheEntryRole::kDataBlock, /*secondary_compatible=*/false),
|
|
1368
|
+
/*context*/ this, Cache::Priority::LOW);
|
|
1369
|
+
if (strict_capacity_limit || GetParam() == kHyperClock) {
|
|
1370
|
+
ASSERT_NE(handle2, nullptr);
|
|
1371
|
+
cache->Release(handle2);
|
|
1372
|
+
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
1373
|
+
} else {
|
|
1374
|
+
ASSERT_EQ(handle2, nullptr);
|
|
1375
|
+
// FIXME: Insert to secondary from Release disabled
|
|
1376
|
+
// ASSERT_EQ(secondary_cache->num_inserts(), 2u);
|
|
1377
|
+
ASSERT_EQ(secondary_cache->num_inserts(), 1u);
|
|
1378
|
+
}
|
|
1391
1379
|
|
|
1392
|
-
|
|
1393
|
-
|
|
1380
|
+
cache.reset();
|
|
1381
|
+
secondary_cache.reset();
|
|
1382
|
+
}
|
|
1394
1383
|
}
|
|
1395
1384
|
|
|
1396
1385
|
// In this test, the block cache size is set to 4096, after insert 6 KV-pairs
|
|
@@ -1399,16 +1388,24 @@ TEST_F(LRUCacheSecondaryCacheTest, FullCapacityTest) {
|
|
|
1399
1388
|
// of the meta blocks are about 900 to 1000. Therefore, in any situation,
|
|
1400
1389
|
// if we try to insert block_1 to the block cache, it will always fails. Only
|
|
1401
1390
|
// block_2 will be successfully inserted into the block cache.
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1391
|
+
// CORRECTION: this is not quite right. block_1 can be inserted into the block
|
|
1392
|
+
// cache because strict_capacity_limit=false, but it is removed from the cache
|
|
1393
|
+
// in Release() because of being over-capacity, without demoting to secondary
|
|
1394
|
+
// cache. HyperClockCache doesn't check capacity on release (for efficiency)
|
|
1395
|
+
// so can demote the over-capacity item to secondary cache. Also, we intend to
|
|
1396
|
+
// add support for demotion in Release, but that currently causes too much
|
|
1397
|
+
// unit test churn.
|
|
1398
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) {
|
|
1399
|
+
if (GetParam() == kHyperClock) {
|
|
1400
|
+
// See CORRECTION above
|
|
1401
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1408
1404
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1409
1405
|
new TestSecondaryCache(2048 * 1024));
|
|
1410
|
-
|
|
1411
|
-
|
|
1406
|
+
std::shared_ptr<Cache> cache =
|
|
1407
|
+
NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1408
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1412
1409
|
BlockBasedTableOptions table_options;
|
|
1413
1410
|
table_options.block_cache = cache;
|
|
1414
1411
|
table_options.block_size = 4 * 1024;
|
|
@@ -1496,16 +1493,16 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheCorrectness1) {
|
|
|
1496
1493
|
// of the meta blocks are about 900 to 1000. Therefore, we can successfully
|
|
1497
1494
|
// insert and cache block_1 in the block cache (this is the different place
|
|
1498
1495
|
// from TestSecondaryCacheCorrectness1)
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
kDontChargeCacheMetadata);
|
|
1496
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) {
|
|
1497
|
+
if (GetParam() == kHyperClock) {
|
|
1498
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
1499
|
+
return;
|
|
1500
|
+
}
|
|
1505
1501
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1506
1502
|
new TestSecondaryCache(2048 * 1024));
|
|
1507
|
-
|
|
1508
|
-
|
|
1503
|
+
std::shared_ptr<Cache> cache =
|
|
1504
|
+
NewCache(6100 /* capacity */, 0 /* num_shard_bits */,
|
|
1505
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1509
1506
|
BlockBasedTableOptions table_options;
|
|
1510
1507
|
table_options.block_cache = cache;
|
|
1511
1508
|
table_options.block_size = 4 * 1024;
|
|
@@ -1589,16 +1586,12 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheCorrectness2) {
|
|
|
1589
1586
|
// of the meta blocks are about 900 to 1000. Therefore, we can successfully
|
|
1590
1587
|
// cache all the blocks in the block cache and there is not secondary cache
|
|
1591
1588
|
// insertion. 2 lookup is needed for the blocks.
|
|
1592
|
-
|
|
1593
|
-
LRUCacheOptions opts(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1594
|
-
false /* strict_capacity_limit */,
|
|
1595
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1596
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1597
|
-
kDontChargeCacheMetadata);
|
|
1589
|
+
TEST_P(DBSecondaryCacheTest, NoSecondaryCacheInsertion) {
|
|
1598
1590
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1599
1591
|
new TestSecondaryCache(2048 * 1024));
|
|
1600
|
-
|
|
1601
|
-
|
|
1592
|
+
std::shared_ptr<Cache> cache =
|
|
1593
|
+
NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1594
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1602
1595
|
BlockBasedTableOptions table_options;
|
|
1603
1596
|
table_options.block_cache = cache;
|
|
1604
1597
|
table_options.block_size = 4 * 1024;
|
|
@@ -1643,16 +1636,12 @@ TEST_F(DBSecondaryCacheTest, NoSecondaryCacheInsertion) {
|
|
|
1643
1636
|
Destroy(options);
|
|
1644
1637
|
}
|
|
1645
1638
|
|
|
1646
|
-
|
|
1647
|
-
LRUCacheOptions opts(8 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1648
|
-
false /* strict_capacity_limit */,
|
|
1649
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1650
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1651
|
-
kDontChargeCacheMetadata);
|
|
1639
|
+
TEST_P(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) {
|
|
1652
1640
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1653
1641
|
new TestSecondaryCache(2048 * 1024));
|
|
1654
|
-
|
|
1655
|
-
|
|
1642
|
+
std::shared_ptr<Cache> cache =
|
|
1643
|
+
NewCache(8 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1644
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1656
1645
|
BlockBasedTableOptions table_options;
|
|
1657
1646
|
table_options.block_cache = cache;
|
|
1658
1647
|
table_options.block_size = 4 * 1024;
|
|
@@ -1692,16 +1681,16 @@ TEST_F(DBSecondaryCacheTest, SecondaryCacheIntensiveTesting) {
|
|
|
1692
1681
|
// of the meta blocks are about 900 to 1000. Therefore, in any situation,
|
|
1693
1682
|
// if we try to insert block_1 to the block cache, it will always fails. Only
|
|
1694
1683
|
// block_2 will be successfully inserted into the block cache.
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
kDontChargeCacheMetadata);
|
|
1684
|
+
TEST_P(DBSecondaryCacheTest, SecondaryCacheFailureTest) {
|
|
1685
|
+
if (GetParam() == kHyperClock) {
|
|
1686
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
1687
|
+
return;
|
|
1688
|
+
}
|
|
1701
1689
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1702
1690
|
new TestSecondaryCache(2048 * 1024));
|
|
1703
|
-
|
|
1704
|
-
|
|
1691
|
+
std::shared_ptr<Cache> cache =
|
|
1692
|
+
NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1693
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1705
1694
|
BlockBasedTableOptions table_options;
|
|
1706
1695
|
table_options.block_cache = cache;
|
|
1707
1696
|
table_options.block_size = 4 * 1024;
|
|
@@ -1784,16 +1773,12 @@ TEST_F(DBSecondaryCacheTest, SecondaryCacheFailureTest) {
|
|
|
1784
1773
|
Destroy(options);
|
|
1785
1774
|
}
|
|
1786
1775
|
|
|
1787
|
-
|
|
1788
|
-
LRUCacheOptions opts(1024 /* capacity */, 2 /* num_shard_bits */,
|
|
1789
|
-
false /* strict_capacity_limit */,
|
|
1790
|
-
0.5 /* high_pri_pool_ratio */,
|
|
1791
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
1792
|
-
kDontChargeCacheMetadata);
|
|
1776
|
+
TEST_P(BasicSecondaryCacheTest, BasicWaitAllTest) {
|
|
1793
1777
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
1794
1778
|
std::make_shared<TestSecondaryCache>(32 * 1024);
|
|
1795
|
-
|
|
1796
|
-
|
|
1779
|
+
std::shared_ptr<Cache> cache =
|
|
1780
|
+
NewCache(1024 /* capacity */, 2 /* num_shard_bits */,
|
|
1781
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1797
1782
|
const int num_keys = 32;
|
|
1798
1783
|
OffsetableCacheKey ock{"foo", "bar", 1};
|
|
1799
1784
|
|
|
@@ -1803,12 +1788,19 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
|
|
|
1803
1788
|
std::string str = rnd.RandomString(1020);
|
|
1804
1789
|
values.emplace_back(str);
|
|
1805
1790
|
TestItem* item = new TestItem(str.data(), str.length());
|
|
1806
|
-
ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), item,
|
|
1807
|
-
&LRUCacheSecondaryCacheTest::helper_,
|
|
1791
|
+
ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), item, GetHelper(),
|
|
1808
1792
|
str.length()));
|
|
1809
1793
|
}
|
|
1810
1794
|
// Force all entries to be evicted to the secondary cache
|
|
1811
|
-
|
|
1795
|
+
if (GetParam() == kHyperClock) {
|
|
1796
|
+
// HCC doesn't respond immediately to SetCapacity
|
|
1797
|
+
for (int i = 9000; i < 9030; ++i) {
|
|
1798
|
+
ASSERT_OK(cache->Insert(ock.WithOffset(i).AsSlice(), nullptr,
|
|
1799
|
+
&kNoopCacheItemHelper, 256));
|
|
1800
|
+
}
|
|
1801
|
+
} else {
|
|
1802
|
+
cache->SetCapacity(0);
|
|
1803
|
+
}
|
|
1812
1804
|
ASSERT_EQ(secondary_cache->num_inserts(), 32u);
|
|
1813
1805
|
cache->SetCapacity(32 * 1024);
|
|
1814
1806
|
|
|
@@ -1819,24 +1811,31 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
|
|
|
1819
1811
|
TestSecondaryCache::ResultType::DEFER_AND_FAIL},
|
|
1820
1812
|
{ock.WithOffset(5).AsSlice().ToString(),
|
|
1821
1813
|
TestSecondaryCache::ResultType::FAIL}});
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1814
|
+
|
|
1815
|
+
std::array<Cache::AsyncLookupHandle, 6> async_handles;
|
|
1816
|
+
std::array<CacheKey, 6> cache_keys;
|
|
1817
|
+
for (size_t i = 0; i < async_handles.size(); ++i) {
|
|
1818
|
+
auto& ah = async_handles[i];
|
|
1819
|
+
cache_keys[i] = ock.WithOffset(i);
|
|
1820
|
+
ah.key = cache_keys[i].AsSlice();
|
|
1821
|
+
ah.helper = GetHelper();
|
|
1822
|
+
ah.create_context = this;
|
|
1823
|
+
ah.priority = Cache::Priority::LOW;
|
|
1824
|
+
cache->StartAsyncLookup(ah);
|
|
1825
|
+
}
|
|
1826
|
+
cache->WaitAll(&async_handles[0], async_handles.size());
|
|
1827
|
+
for (size_t i = 0; i < async_handles.size(); ++i) {
|
|
1828
|
+
SCOPED_TRACE("i = " + std::to_string(i));
|
|
1829
|
+
Cache::Handle* result = async_handles[i].Result();
|
|
1830
|
+
if (i == 4 || i == 5) {
|
|
1831
|
+
ASSERT_EQ(result, nullptr);
|
|
1834
1832
|
continue;
|
|
1835
1833
|
} else {
|
|
1836
|
-
|
|
1834
|
+
ASSERT_NE(result, nullptr);
|
|
1835
|
+
TestItem* item = static_cast<TestItem*>(cache->Value(result));
|
|
1837
1836
|
ASSERT_EQ(item->ToString(), values[i]);
|
|
1838
1837
|
}
|
|
1839
|
-
cache->Release(
|
|
1838
|
+
cache->Release(result);
|
|
1840
1839
|
}
|
|
1841
1840
|
|
|
1842
1841
|
cache.reset();
|
|
@@ -1847,16 +1846,16 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
|
|
|
1847
1846
|
// the cache key associated with each data block (and thus each KV) by using
|
|
1848
1847
|
// a sync point callback in TestSecondaryCache::Lookup. We then control the
|
|
1849
1848
|
// lookup result by setting the ResultMap.
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
kDontChargeCacheMetadata);
|
|
1849
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) {
|
|
1850
|
+
if (GetParam() == kHyperClock) {
|
|
1851
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
1852
|
+
return;
|
|
1853
|
+
}
|
|
1856
1854
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
1857
1855
|
new TestSecondaryCache(2048 * 1024));
|
|
1858
|
-
|
|
1859
|
-
|
|
1856
|
+
std::shared_ptr<Cache> cache =
|
|
1857
|
+
NewCache(1 << 20 /* capacity */, 0 /* num_shard_bits */,
|
|
1858
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
1860
1859
|
BlockBasedTableOptions table_options;
|
|
1861
1860
|
table_options.block_cache = cache;
|
|
1862
1861
|
table_options.block_size = 4 * 1024;
|
|
@@ -1930,37 +1929,25 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheMultiGet) {
|
|
|
1930
1929
|
Destroy(options);
|
|
1931
1930
|
}
|
|
1932
1931
|
|
|
1933
|
-
class
|
|
1932
|
+
class CacheWithStats : public CacheWrapper {
|
|
1934
1933
|
public:
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
bool _use_adaptive_mutex = kDefaultToAdaptiveMutex,
|
|
1940
|
-
CacheMetadataChargePolicy _metadata_charge_policy =
|
|
1941
|
-
kDontChargeCacheMetadata,
|
|
1942
|
-
const std::shared_ptr<SecondaryCache>& _secondary_cache = nullptr)
|
|
1943
|
-
: LRUCache(_capacity, _num_shard_bits, _strict_capacity_limit,
|
|
1944
|
-
_high_pri_pool_ratio, _low_pri_pool_ratio, _memory_allocator,
|
|
1945
|
-
_use_adaptive_mutex, _metadata_charge_policy,
|
|
1946
|
-
_secondary_cache) {
|
|
1947
|
-
insert_count_ = 0;
|
|
1948
|
-
lookup_count_ = 0;
|
|
1949
|
-
}
|
|
1950
|
-
~LRUCacheWithStat() {}
|
|
1934
|
+
using CacheWrapper::CacheWrapper;
|
|
1935
|
+
|
|
1936
|
+
static const char* kClassName() { return "CacheWithStats"; }
|
|
1937
|
+
const char* Name() const override { return kClassName(); }
|
|
1951
1938
|
|
|
1952
1939
|
Status Insert(const Slice& key, Cache::ObjectPtr value,
|
|
1953
1940
|
const CacheItemHelper* helper, size_t charge,
|
|
1954
1941
|
Handle** handle = nullptr,
|
|
1955
1942
|
Priority priority = Priority::LOW) override {
|
|
1956
1943
|
insert_count_++;
|
|
1957
|
-
return
|
|
1944
|
+
return target_->Insert(key, value, helper, charge, handle, priority);
|
|
1958
1945
|
}
|
|
1959
1946
|
Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
|
|
1960
|
-
CreateContext* create_context, Priority priority,
|
|
1947
|
+
CreateContext* create_context, Priority priority,
|
|
1961
1948
|
Statistics* stats = nullptr) override {
|
|
1962
1949
|
lookup_count_++;
|
|
1963
|
-
return
|
|
1950
|
+
return target_->Lookup(key, helper, create_context, priority, stats);
|
|
1964
1951
|
}
|
|
1965
1952
|
|
|
1966
1953
|
uint32_t GetInsertCount() { return insert_count_; }
|
|
@@ -1971,24 +1958,16 @@ class LRUCacheWithStat : public LRUCache {
|
|
|
1971
1958
|
}
|
|
1972
1959
|
|
|
1973
1960
|
private:
|
|
1974
|
-
uint32_t insert_count_;
|
|
1975
|
-
uint32_t lookup_count_;
|
|
1961
|
+
uint32_t insert_count_ = 0;
|
|
1962
|
+
uint32_t lookup_count_ = 0;
|
|
1976
1963
|
};
|
|
1977
1964
|
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
|
|
1985
|
-
LRUCacheWithStat* tmp_cache = new LRUCacheWithStat(
|
|
1986
|
-
cache_opts.capacity, cache_opts.num_shard_bits,
|
|
1987
|
-
cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
|
|
1988
|
-
cache_opts.low_pri_pool_ratio, cache_opts.memory_allocator,
|
|
1989
|
-
cache_opts.use_adaptive_mutex, cache_opts.metadata_charge_policy,
|
|
1990
|
-
cache_opts.secondary_cache);
|
|
1991
|
-
std::shared_ptr<Cache> cache(tmp_cache);
|
|
1965
|
+
TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
1966
|
+
std::shared_ptr<Cache> base_cache =
|
|
1967
|
+
NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
1968
|
+
false /* strict_capacity_limit */);
|
|
1969
|
+
std::shared_ptr<CacheWithStats> cache =
|
|
1970
|
+
std::make_shared<CacheWithStats>(base_cache);
|
|
1992
1971
|
BlockBasedTableOptions table_options;
|
|
1993
1972
|
table_options.block_cache = cache;
|
|
1994
1973
|
table_options.block_size = 4 * 1024;
|
|
@@ -2016,15 +1995,15 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
|
2016
1995
|
|
|
2017
1996
|
// do th eread for all the key value pairs, so all the blocks should be in
|
|
2018
1997
|
// cache
|
|
2019
|
-
uint32_t start_insert =
|
|
2020
|
-
uint32_t start_lookup =
|
|
1998
|
+
uint32_t start_insert = cache->GetInsertCount();
|
|
1999
|
+
uint32_t start_lookup = cache->GetLookupcount();
|
|
2021
2000
|
std::string v;
|
|
2022
2001
|
for (int i = 0; i < N; i++) {
|
|
2023
2002
|
v = Get(Key(i));
|
|
2024
2003
|
ASSERT_EQ(v, value[i]);
|
|
2025
2004
|
}
|
|
2026
|
-
uint32_t dump_insert =
|
|
2027
|
-
uint32_t dump_lookup =
|
|
2005
|
+
uint32_t dump_insert = cache->GetInsertCount() - start_insert;
|
|
2006
|
+
uint32_t dump_lookup = cache->GetLookupcount() - start_lookup;
|
|
2028
2007
|
ASSERT_EQ(63,
|
|
2029
2008
|
static_cast<int>(dump_insert)); // the insert in the block cache
|
|
2030
2009
|
ASSERT_EQ(256,
|
|
@@ -2054,15 +2033,11 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
|
2054
2033
|
// dumpload
|
|
2055
2034
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
2056
2035
|
std::make_shared<TestSecondaryCache>(2048 * 1024);
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
cache_opts.use_adaptive_mutex, cache_opts.metadata_charge_policy,
|
|
2063
|
-
cache_opts.secondary_cache);
|
|
2064
|
-
std::shared_ptr<Cache> cache_new(tmp_cache);
|
|
2065
|
-
table_options.block_cache = cache_new;
|
|
2036
|
+
// This time with secondary cache
|
|
2037
|
+
base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2038
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
2039
|
+
cache = std::make_shared<CacheWithStats>(base_cache);
|
|
2040
|
+
table_options.block_cache = cache;
|
|
2066
2041
|
table_options.block_size = 4 * 1024;
|
|
2067
2042
|
options.create_if_missing = true;
|
|
2068
2043
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
@@ -2093,8 +2068,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
|
2093
2068
|
// After load, we do the Get again
|
|
2094
2069
|
start_insert = secondary_cache->num_inserts();
|
|
2095
2070
|
start_lookup = secondary_cache->num_lookups();
|
|
2096
|
-
uint32_t cache_insert =
|
|
2097
|
-
uint32_t cache_lookup =
|
|
2071
|
+
uint32_t cache_insert = cache->GetInsertCount();
|
|
2072
|
+
uint32_t cache_lookup = cache->GetLookupcount();
|
|
2098
2073
|
for (int i = 0; i < N; i++) {
|
|
2099
2074
|
v = Get(Key(i));
|
|
2100
2075
|
ASSERT_EQ(v, value[i]);
|
|
@@ -2105,8 +2080,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
|
2105
2080
|
ASSERT_EQ(0, static_cast<int>(final_insert));
|
|
2106
2081
|
// lookup the secondary to get all blocks
|
|
2107
2082
|
ASSERT_EQ(64, static_cast<int>(final_lookup));
|
|
2108
|
-
uint32_t block_insert =
|
|
2109
|
-
uint32_t block_lookup =
|
|
2083
|
+
uint32_t block_insert = cache->GetInsertCount() - cache_insert;
|
|
2084
|
+
uint32_t block_lookup = cache->GetLookupcount() - cache_lookup;
|
|
2110
2085
|
// Check the new block cache insert and lookup, should be no insert since all
|
|
2111
2086
|
// blocks are from the secondary cache.
|
|
2112
2087
|
ASSERT_EQ(0, static_cast<int>(block_insert));
|
|
@@ -2116,19 +2091,12 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadBasic) {
|
|
|
2116
2091
|
Destroy(options);
|
|
2117
2092
|
}
|
|
2118
2093
|
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
LRUCacheWithStat* tmp_cache = new LRUCacheWithStat(
|
|
2126
|
-
cache_opts.capacity, cache_opts.num_shard_bits,
|
|
2127
|
-
cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
|
|
2128
|
-
cache_opts.low_pri_pool_ratio, cache_opts.memory_allocator,
|
|
2129
|
-
cache_opts.use_adaptive_mutex, cache_opts.metadata_charge_policy,
|
|
2130
|
-
cache_opts.secondary_cache);
|
|
2131
|
-
std::shared_ptr<Cache> cache(tmp_cache);
|
|
2094
|
+
TEST_P(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
2095
|
+
std::shared_ptr<Cache> base_cache =
|
|
2096
|
+
NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2097
|
+
false /* strict_capacity_limit */);
|
|
2098
|
+
std::shared_ptr<CacheWithStats> cache =
|
|
2099
|
+
std::make_shared<CacheWithStats>(base_cache);
|
|
2132
2100
|
BlockBasedTableOptions table_options;
|
|
2133
2101
|
table_options.block_cache = cache;
|
|
2134
2102
|
table_options.block_size = 4 * 1024;
|
|
@@ -2178,8 +2146,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2178
2146
|
|
|
2179
2147
|
// do th eread for all the key value pairs, so all the blocks should be in
|
|
2180
2148
|
// cache
|
|
2181
|
-
uint32_t start_insert =
|
|
2182
|
-
uint32_t start_lookup =
|
|
2149
|
+
uint32_t start_insert = cache->GetInsertCount();
|
|
2150
|
+
uint32_t start_lookup = cache->GetLookupcount();
|
|
2183
2151
|
ReadOptions ro;
|
|
2184
2152
|
std::string v;
|
|
2185
2153
|
for (int i = 0; i < N; i++) {
|
|
@@ -2190,8 +2158,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2190
2158
|
ASSERT_OK(db2->Get(ro, Key(i), &v));
|
|
2191
2159
|
ASSERT_EQ(v, value2[i]);
|
|
2192
2160
|
}
|
|
2193
|
-
uint32_t dump_insert =
|
|
2194
|
-
uint32_t dump_lookup =
|
|
2161
|
+
uint32_t dump_insert = cache->GetInsertCount() - start_insert;
|
|
2162
|
+
uint32_t dump_lookup = cache->GetLookupcount() - start_lookup;
|
|
2195
2163
|
ASSERT_EQ(128,
|
|
2196
2164
|
static_cast<int>(dump_insert)); // the insert in the block cache
|
|
2197
2165
|
ASSERT_EQ(512,
|
|
@@ -2221,15 +2189,11 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2221
2189
|
// dumpload
|
|
2222
2190
|
std::shared_ptr<TestSecondaryCache> secondary_cache =
|
|
2223
2191
|
std::make_shared<TestSecondaryCache>(2048 * 1024);
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
cache_opts.use_adaptive_mutex, cache_opts.metadata_charge_policy,
|
|
2230
|
-
cache_opts.secondary_cache);
|
|
2231
|
-
std::shared_ptr<Cache> cache_new(tmp_cache);
|
|
2232
|
-
table_options.block_cache = cache_new;
|
|
2192
|
+
// This time with secondary_cache
|
|
2193
|
+
base_cache = NewCache(1024 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2194
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
2195
|
+
cache = std::make_shared<CacheWithStats>(base_cache);
|
|
2196
|
+
table_options.block_cache = cache;
|
|
2233
2197
|
table_options.block_size = 4 * 1024;
|
|
2234
2198
|
options.create_if_missing = true;
|
|
2235
2199
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
@@ -2265,8 +2229,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2265
2229
|
fault_fs_->SetFilesystemActive(false, error_msg);
|
|
2266
2230
|
start_insert = secondary_cache->num_inserts();
|
|
2267
2231
|
start_lookup = secondary_cache->num_lookups();
|
|
2268
|
-
uint32_t cache_insert =
|
|
2269
|
-
uint32_t cache_lookup =
|
|
2232
|
+
uint32_t cache_insert = cache->GetInsertCount();
|
|
2233
|
+
uint32_t cache_lookup = cache->GetLookupcount();
|
|
2270
2234
|
for (int i = 0; i < N; i++) {
|
|
2271
2235
|
ASSERT_OK(db1->Get(ro, Key(i), &v));
|
|
2272
2236
|
ASSERT_EQ(v, value1[i]);
|
|
@@ -2277,8 +2241,8 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2277
2241
|
ASSERT_EQ(0, static_cast<int>(final_insert));
|
|
2278
2242
|
// lookup the secondary to get all blocks
|
|
2279
2243
|
ASSERT_EQ(64, static_cast<int>(final_lookup));
|
|
2280
|
-
uint32_t block_insert =
|
|
2281
|
-
uint32_t block_lookup =
|
|
2244
|
+
uint32_t block_insert = cache->GetInsertCount() - cache_insert;
|
|
2245
|
+
uint32_t block_lookup = cache->GetLookupcount() - cache_lookup;
|
|
2282
2246
|
// Check the new block cache insert and lookup, should be no insert since all
|
|
2283
2247
|
// blocks are from the secondary cache.
|
|
2284
2248
|
ASSERT_EQ(0, static_cast<int>(block_insert));
|
|
@@ -2292,16 +2256,12 @@ TEST_F(DBSecondaryCacheTest, LRUCacheDumpLoadWithFilter) {
|
|
|
2292
2256
|
}
|
|
2293
2257
|
|
|
2294
2258
|
// Test the option not to use the secondary cache in a certain DB.
|
|
2295
|
-
|
|
2296
|
-
LRUCacheOptions opts(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2297
|
-
false /* strict_capacity_limit */,
|
|
2298
|
-
0.5 /* high_pri_pool_ratio */,
|
|
2299
|
-
nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
|
|
2300
|
-
kDontChargeCacheMetadata);
|
|
2259
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) {
|
|
2301
2260
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
2302
2261
|
new TestSecondaryCache(2048 * 1024));
|
|
2303
|
-
|
|
2304
|
-
|
|
2262
|
+
std::shared_ptr<Cache> cache =
|
|
2263
|
+
NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2264
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
2305
2265
|
BlockBasedTableOptions table_options;
|
|
2306
2266
|
table_options.block_cache = cache;
|
|
2307
2267
|
table_options.block_size = 4 * 1024;
|
|
@@ -2387,16 +2347,16 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionBasic) {
|
|
|
2387
2347
|
// We disable the secondary cache in DBOptions at first. Close and reopen the DB
|
|
2388
2348
|
// with new options, which set the lowest_used_cache_tier to
|
|
2389
2349
|
// kNonVolatileBlockTier. So secondary cache will be used.
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
kDontChargeCacheMetadata);
|
|
2350
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) {
|
|
2351
|
+
if (GetParam() == kHyperClock) {
|
|
2352
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
2353
|
+
return;
|
|
2354
|
+
}
|
|
2396
2355
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
2397
2356
|
new TestSecondaryCache(2048 * 1024));
|
|
2398
|
-
|
|
2399
|
-
|
|
2357
|
+
std::shared_ptr<Cache> cache =
|
|
2358
|
+
NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2359
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
2400
2360
|
BlockBasedTableOptions table_options;
|
|
2401
2361
|
table_options.block_cache = cache;
|
|
2402
2362
|
table_options.block_size = 4 * 1024;
|
|
@@ -2482,16 +2442,16 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionChange) {
|
|
|
2482
2442
|
|
|
2483
2443
|
// Two DB test. We create 2 DBs sharing the same block cache and secondary
|
|
2484
2444
|
// cache. We diable the secondary cache option for DB2.
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
kDontChargeCacheMetadata);
|
|
2445
|
+
TEST_P(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
|
|
2446
|
+
if (GetParam() == kHyperClock) {
|
|
2447
|
+
ROCKSDB_GTEST_BYPASS("Test depends on LRUCache-specific behaviors");
|
|
2448
|
+
return;
|
|
2449
|
+
}
|
|
2491
2450
|
std::shared_ptr<TestSecondaryCache> secondary_cache(
|
|
2492
2451
|
new TestSecondaryCache(2048 * 1024));
|
|
2493
|
-
|
|
2494
|
-
|
|
2452
|
+
std::shared_ptr<Cache> cache =
|
|
2453
|
+
NewCache(4 * 1024 /* capacity */, 0 /* num_shard_bits */,
|
|
2454
|
+
false /* strict_capacity_limit */, secondary_cache);
|
|
2495
2455
|
BlockBasedTableOptions table_options;
|
|
2496
2456
|
table_options.block_cache = cache;
|
|
2497
2457
|
table_options.block_size = 4 * 1024;
|
|
@@ -2589,7 +2549,6 @@ TEST_F(DBSecondaryCacheTest, TestSecondaryCacheOptionTwoDB) {
|
|
|
2589
2549
|
ASSERT_OK(DestroyDB(dbname2, options));
|
|
2590
2550
|
}
|
|
2591
2551
|
|
|
2592
|
-
|
|
2593
2552
|
} // namespace ROCKSDB_NAMESPACE
|
|
2594
2553
|
|
|
2595
2554
|
int main(int argc, char** argv) {
|