@nxtedition/rocksdb 7.0.24 → 7.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +12 -3
- package/deps/rocksdb/rocksdb/CMakeLists.txt +5 -0
- package/deps/rocksdb/rocksdb/Makefile +6 -2
- package/deps/rocksdb/rocksdb/TARGETS +14 -0
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +4 -1
- package/deps/rocksdb/rocksdb/cache/cache_helpers.h +20 -0
- package/deps/rocksdb/rocksdb/cache/cache_reservation_manager_test.cc +2 -2
- package/deps/rocksdb/rocksdb/cache/cache_test.cc +44 -31
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +491 -722
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +468 -2
- package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -1
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +51 -52
- package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h +28 -16
- package/deps/rocksdb/rocksdb/cache/lru_cache.cc +12 -1
- package/deps/rocksdb/rocksdb/cache/lru_cache.h +1 -0
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +170 -36
- package/deps/rocksdb/rocksdb/db/blob/blob_file_cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +63 -36
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +4 -6
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +57 -38
- package/deps/rocksdb/rocksdb/db/blob/blob_read_request.h +58 -0
- package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +164 -74
- package/deps/rocksdb/rocksdb/db/blob/blob_source.h +42 -29
- package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +419 -62
- package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +208 -8
- package/deps/rocksdb/rocksdb/db/c.cc +68 -0
- package/deps/rocksdb/rocksdb/db/c_test.c +95 -2
- package/deps/rocksdb/rocksdb/db/column_family.cc +12 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +92 -15
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +76 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +52 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +30 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +126 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +203 -1584
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +93 -26
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +87 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +314 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +328 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +32 -6
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +4 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +7 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +174 -33
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +474 -7
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +5 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +825 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.cc +46 -0
- package/deps/rocksdb/rocksdb/db/compaction/compaction_state.h +42 -0
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +223 -0
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +255 -0
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +1253 -0
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +32 -8
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +3 -1
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +13 -8
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +376 -0
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +103 -78
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +4 -6
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +0 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +10 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +21 -6
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +19 -1
- package/deps/rocksdb/rocksdb/db/db_iter.cc +91 -14
- package/deps/rocksdb/rocksdb/db/db_iter.h +5 -0
- package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +33 -0
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +79 -0
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +2 -0
- package/deps/rocksdb/rocksdb/db/db_test2.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +5 -2
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +185 -0
- package/deps/rocksdb/rocksdb/db/dbformat.cc +1 -4
- package/deps/rocksdb/rocksdb/db/dbformat.h +2 -8
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +71 -29
- package/deps/rocksdb/rocksdb/db/internal_stats.h +160 -5
- package/deps/rocksdb/rocksdb/db/log_reader.cc +29 -3
- package/deps/rocksdb/rocksdb/db/log_reader.h +12 -3
- package/deps/rocksdb/rocksdb/db/repair_test.cc +1 -3
- package/deps/rocksdb/rocksdb/db/version_edit.cc +6 -0
- package/deps/rocksdb/rocksdb/db/version_set.cc +93 -129
- package/deps/rocksdb/rocksdb/db/version_set.h +4 -4
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +2 -2
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +42 -35
- package/deps/rocksdb/rocksdb/db/write_batch.cc +10 -2
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +4 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +10 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +3 -3
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +3 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +4 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +5 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +140 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +12 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +46 -7
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +7 -0
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +27 -7
- package/deps/rocksdb/rocksdb/env/composite_env_wrapper.h +8 -0
- package/deps/rocksdb/rocksdb/env/env_posix.cc +14 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +130 -1
- package/deps/rocksdb/rocksdb/env/fs_posix.cc +7 -1
- package/deps/rocksdb/rocksdb/env/io_posix.cc +18 -50
- package/deps/rocksdb/rocksdb/env/io_posix.h +53 -6
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +8 -10
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +3 -7
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +239 -259
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +84 -19
- package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +24 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +1 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +31 -1
- package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +11 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +2 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +14 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/env.h +20 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +37 -13
- package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +7 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +14 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/threadpool.h +9 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +13 -13
- package/deps/rocksdb/rocksdb/logging/auto_roll_logger.cc +12 -2
- package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +38 -0
- package/deps/rocksdb/rocksdb/monitoring/statistics.cc +7 -1
- package/deps/rocksdb/rocksdb/port/win/env_win.cc +17 -0
- package/deps/rocksdb/rocksdb/port/win/env_win.h +8 -0
- package/deps/rocksdb/rocksdb/port/win/io_win.cc +6 -3
- package/deps/rocksdb/rocksdb/src.mk +5 -0
- package/deps/rocksdb/rocksdb/table/block_based/block.h +1 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +5 -2
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +15 -12
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +5 -4
- package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +2 -1
- package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +4 -4
- package/deps/rocksdb/rocksdb/table/block_fetcher.cc +1 -2
- package/deps/rocksdb/rocksdb/table/get_context.cc +1 -0
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +24 -4
- package/deps/rocksdb/rocksdb/util/async_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/util/compression.h +2 -0
- package/deps/rocksdb/rocksdb/util/thread_list_test.cc +18 -1
- package/deps/rocksdb/rocksdb/util/threadpool_imp.cc +67 -4
- package/deps/rocksdb/rocksdb/util/threadpool_imp.h +8 -0
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +15 -12
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -2
- package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc +1 -1
- package/deps/rocksdb/rocksdb.gyp +5 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
|
@@ -115,7 +115,7 @@ class BlobSourceTest : public DBTestBase {
|
|
|
115
115
|
options_.create_if_missing = true;
|
|
116
116
|
|
|
117
117
|
LRUCacheOptions co;
|
|
118
|
-
co.capacity =
|
|
118
|
+
co.capacity = 8 << 20;
|
|
119
119
|
co.num_shard_bits = 2;
|
|
120
120
|
co.metadata_charge_policy = kDontChargeCacheMetadata;
|
|
121
121
|
options_.blob_cache = NewLRUCache(co);
|
|
@@ -134,6 +134,10 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
134
134
|
options_.cf_paths.emplace_back(
|
|
135
135
|
test::PerThreadDBPath(env_, "BlobSourceTest_GetBlobsFromCache"), 0);
|
|
136
136
|
|
|
137
|
+
options_.statistics = CreateDBStatistics();
|
|
138
|
+
Statistics* statistics = options_.statistics.get();
|
|
139
|
+
assert(statistics);
|
|
140
|
+
|
|
137
141
|
DestroyAndReopen(options_);
|
|
138
142
|
|
|
139
143
|
ImmutableOptions immutable_options(options_);
|
|
@@ -193,8 +197,11 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
193
197
|
// GetBlob
|
|
194
198
|
std::vector<PinnableSlice> values(keys.size());
|
|
195
199
|
uint64_t bytes_read = 0;
|
|
200
|
+
uint64_t blob_bytes = 0;
|
|
201
|
+
uint64_t total_bytes = 0;
|
|
196
202
|
|
|
197
203
|
read_options.fill_cache = false;
|
|
204
|
+
get_perf_context()->Reset();
|
|
198
205
|
|
|
199
206
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
200
207
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -210,9 +217,28 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
210
217
|
|
|
211
218
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
212
219
|
blob_offsets[i]));
|
|
220
|
+
total_bytes += bytes_read;
|
|
213
221
|
}
|
|
214
222
|
|
|
223
|
+
// Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
|
|
224
|
+
// GetBlob, and TEST_BlobInCache.
|
|
225
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
|
|
226
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
|
|
227
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
|
|
228
|
+
ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
|
|
229
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
230
|
+
|
|
231
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3);
|
|
232
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0);
|
|
233
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
234
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0);
|
|
235
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
236
|
+
|
|
215
237
|
read_options.fill_cache = true;
|
|
238
|
+
blob_bytes = 0;
|
|
239
|
+
total_bytes = 0;
|
|
240
|
+
get_perf_context()->Reset();
|
|
241
|
+
statistics->Reset().PermitUncheckedError();
|
|
216
242
|
|
|
217
243
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
218
244
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -226,11 +252,36 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
226
252
|
ASSERT_EQ(bytes_read,
|
|
227
253
|
BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
|
|
228
254
|
|
|
255
|
+
blob_bytes += blob_sizes[i];
|
|
256
|
+
total_bytes += bytes_read;
|
|
257
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i);
|
|
258
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
|
|
259
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
|
|
260
|
+
|
|
229
261
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
230
262
|
blob_offsets[i]));
|
|
263
|
+
|
|
264
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, i + 1);
|
|
265
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, i + 1);
|
|
266
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
|
|
231
267
|
}
|
|
232
268
|
|
|
269
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs);
|
|
270
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, num_blobs);
|
|
271
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, total_bytes);
|
|
272
|
+
|
|
273
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2);
|
|
274
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs);
|
|
275
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), num_blobs);
|
|
276
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), blob_bytes);
|
|
277
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE),
|
|
278
|
+
blob_bytes);
|
|
279
|
+
|
|
233
280
|
read_options.fill_cache = true;
|
|
281
|
+
total_bytes = 0;
|
|
282
|
+
blob_bytes = 0;
|
|
283
|
+
get_perf_context()->Reset();
|
|
284
|
+
statistics->Reset().PermitUncheckedError();
|
|
234
285
|
|
|
235
286
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
236
287
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -246,10 +297,29 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
246
297
|
|
|
247
298
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
248
299
|
blob_offsets[i]));
|
|
300
|
+
total_bytes += bytes_read; // on-disk blob record size
|
|
301
|
+
blob_bytes += blob_sizes[i]; // cached blob value size
|
|
249
302
|
}
|
|
250
303
|
|
|
304
|
+
// Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
|
|
305
|
+
// GetBlob, and TEST_BlobInCache.
|
|
306
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3);
|
|
307
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // without i/o
|
|
308
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // without i/o
|
|
309
|
+
|
|
310
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0);
|
|
311
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 3);
|
|
312
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
313
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
314
|
+
blob_bytes * 3);
|
|
315
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
316
|
+
|
|
251
317
|
// Cache-only GetBlob
|
|
252
318
|
read_options.read_tier = ReadTier::kBlockCacheTier;
|
|
319
|
+
total_bytes = 0;
|
|
320
|
+
blob_bytes = 0;
|
|
321
|
+
get_perf_context()->Reset();
|
|
322
|
+
statistics->Reset().PermitUncheckedError();
|
|
253
323
|
|
|
254
324
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
255
325
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -265,7 +335,22 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
265
335
|
|
|
266
336
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
267
337
|
blob_offsets[i]));
|
|
338
|
+
total_bytes += bytes_read;
|
|
339
|
+
blob_bytes += blob_sizes[i];
|
|
268
340
|
}
|
|
341
|
+
|
|
342
|
+
// Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
|
|
343
|
+
// GetBlob, and TEST_BlobInCache.
|
|
344
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 3);
|
|
345
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // without i/o
|
|
346
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // without i/o
|
|
347
|
+
|
|
348
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0);
|
|
349
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 3);
|
|
350
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
351
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
352
|
+
blob_bytes * 3);
|
|
353
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
269
354
|
}
|
|
270
355
|
|
|
271
356
|
options_.blob_cache->EraseUnRefEntries();
|
|
@@ -277,6 +362,8 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
277
362
|
|
|
278
363
|
read_options.read_tier = ReadTier::kBlockCacheTier;
|
|
279
364
|
read_options.fill_cache = true;
|
|
365
|
+
get_perf_context()->Reset();
|
|
366
|
+
statistics->Reset().PermitUncheckedError();
|
|
280
367
|
|
|
281
368
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
282
369
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -294,6 +381,18 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
294
381
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
295
382
|
blob_offsets[i]));
|
|
296
383
|
}
|
|
384
|
+
|
|
385
|
+
// Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
|
|
386
|
+
// GetBlob, and TEST_BlobInCache.
|
|
387
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
|
|
388
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0);
|
|
389
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0);
|
|
390
|
+
|
|
391
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3);
|
|
392
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0);
|
|
393
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
394
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0);
|
|
395
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
297
396
|
}
|
|
298
397
|
|
|
299
398
|
{
|
|
@@ -304,6 +403,8 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
304
403
|
|
|
305
404
|
read_options.read_tier = ReadTier::kReadAllTier;
|
|
306
405
|
read_options.fill_cache = true;
|
|
406
|
+
get_perf_context()->Reset();
|
|
407
|
+
statistics->Reset().PermitUncheckedError();
|
|
307
408
|
|
|
308
409
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
309
410
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
@@ -321,6 +422,18 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
|
|
|
321
422
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
322
423
|
blob_offsets[i]));
|
|
323
424
|
}
|
|
425
|
+
|
|
426
|
+
// Retrieved the blob cache num_blobs * 3 times via TEST_BlobInCache,
|
|
427
|
+
// GetBlob, and TEST_BlobInCache.
|
|
428
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
|
|
429
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0);
|
|
430
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0);
|
|
431
|
+
|
|
432
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 3);
|
|
433
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0);
|
|
434
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
435
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0);
|
|
436
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
324
437
|
}
|
|
325
438
|
}
|
|
326
439
|
|
|
@@ -403,6 +516,7 @@ TEST_F(BlobSourceTest, GetCompressedBlobs) {
|
|
|
403
516
|
|
|
404
517
|
read_options.fill_cache = true;
|
|
405
518
|
read_options.read_tier = ReadTier::kReadAllTier;
|
|
519
|
+
get_perf_context()->Reset();
|
|
406
520
|
|
|
407
521
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
408
522
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
@@ -420,7 +534,10 @@ TEST_F(BlobSourceTest, GetCompressedBlobs) {
|
|
|
420
534
|
blob_offsets[i]));
|
|
421
535
|
}
|
|
422
536
|
|
|
537
|
+
ASSERT_GE((int)get_perf_context()->blob_decompress_time, 0);
|
|
538
|
+
|
|
423
539
|
read_options.read_tier = ReadTier::kBlockCacheTier;
|
|
540
|
+
get_perf_context()->Reset();
|
|
424
541
|
|
|
425
542
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
426
543
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
@@ -439,6 +556,201 @@ TEST_F(BlobSourceTest, GetCompressedBlobs) {
|
|
|
439
556
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
440
557
|
blob_offsets[i]));
|
|
441
558
|
}
|
|
559
|
+
|
|
560
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
|
|
565
|
+
options_.cf_paths.emplace_back(
|
|
566
|
+
test::PerThreadDBPath(env_, "BlobSourceTest_MultiGetBlobsFromMultiFiles"),
|
|
567
|
+
0);
|
|
568
|
+
|
|
569
|
+
options_.statistics = CreateDBStatistics();
|
|
570
|
+
Statistics* statistics = options_.statistics.get();
|
|
571
|
+
assert(statistics);
|
|
572
|
+
|
|
573
|
+
DestroyAndReopen(options_);
|
|
574
|
+
|
|
575
|
+
ImmutableOptions immutable_options(options_);
|
|
576
|
+
|
|
577
|
+
constexpr uint32_t column_family_id = 1;
|
|
578
|
+
constexpr bool has_ttl = false;
|
|
579
|
+
constexpr ExpirationRange expiration_range;
|
|
580
|
+
constexpr uint64_t blob_files = 2;
|
|
581
|
+
constexpr size_t num_blobs = 32;
|
|
582
|
+
|
|
583
|
+
std::vector<std::string> key_strs;
|
|
584
|
+
std::vector<std::string> blob_strs;
|
|
585
|
+
|
|
586
|
+
for (size_t i = 0; i < num_blobs; ++i) {
|
|
587
|
+
key_strs.push_back("key" + std::to_string(i));
|
|
588
|
+
blob_strs.push_back("blob" + std::to_string(i));
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
std::vector<Slice> keys;
|
|
592
|
+
std::vector<Slice> blobs;
|
|
593
|
+
|
|
594
|
+
uint64_t file_size = BlobLogHeader::kSize;
|
|
595
|
+
uint64_t blob_value_bytes = 0;
|
|
596
|
+
for (size_t i = 0; i < num_blobs; ++i) {
|
|
597
|
+
keys.push_back({key_strs[i]});
|
|
598
|
+
blobs.push_back({blob_strs[i]});
|
|
599
|
+
blob_value_bytes += blobs[i].size();
|
|
600
|
+
file_size += BlobLogRecord::kHeaderSize + keys[i].size() + blobs[i].size();
|
|
601
|
+
}
|
|
602
|
+
file_size += BlobLogFooter::kSize;
|
|
603
|
+
const uint64_t blob_records_bytes =
|
|
604
|
+
file_size - BlobLogHeader::kSize - BlobLogFooter::kSize;
|
|
605
|
+
|
|
606
|
+
std::vector<uint64_t> blob_offsets(keys.size());
|
|
607
|
+
std::vector<uint64_t> blob_sizes(keys.size());
|
|
608
|
+
|
|
609
|
+
{
|
|
610
|
+
// Write key/blob pairs to multiple blob files.
|
|
611
|
+
for (size_t i = 0; i < blob_files; ++i) {
|
|
612
|
+
const uint64_t file_number = i + 1;
|
|
613
|
+
WriteBlobFile(immutable_options, column_family_id, has_ttl,
|
|
614
|
+
expiration_range, expiration_range, file_number, keys,
|
|
615
|
+
blobs, kNoCompression, blob_offsets, blob_sizes);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
constexpr size_t capacity = 10;
|
|
620
|
+
std::shared_ptr<Cache> backing_cache =
|
|
621
|
+
NewLRUCache(capacity); // Blob file cache
|
|
622
|
+
|
|
623
|
+
FileOptions file_options;
|
|
624
|
+
constexpr HistogramImpl* blob_file_read_hist = nullptr;
|
|
625
|
+
|
|
626
|
+
std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
|
|
627
|
+
backing_cache.get(), &immutable_options, &file_options, column_family_id,
|
|
628
|
+
blob_file_read_hist, nullptr /*IOTracer*/));
|
|
629
|
+
|
|
630
|
+
BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
|
|
631
|
+
blob_file_cache.get());
|
|
632
|
+
|
|
633
|
+
ReadOptions read_options;
|
|
634
|
+
read_options.verify_checksums = true;
|
|
635
|
+
|
|
636
|
+
uint64_t bytes_read = 0;
|
|
637
|
+
|
|
638
|
+
{
|
|
639
|
+
// MultiGetBlob
|
|
640
|
+
read_options.fill_cache = true;
|
|
641
|
+
read_options.read_tier = ReadTier::kReadAllTier;
|
|
642
|
+
|
|
643
|
+
autovector<BlobFileReadRequests> blob_reqs;
|
|
644
|
+
std::array<autovector<BlobReadRequest>, blob_files> blob_reqs_in_file;
|
|
645
|
+
std::array<PinnableSlice, num_blobs * blob_files> value_buf;
|
|
646
|
+
std::array<Status, num_blobs * blob_files> statuses_buf;
|
|
647
|
+
|
|
648
|
+
for (size_t i = 0; i < blob_files; ++i) {
|
|
649
|
+
const uint64_t file_number = i + 1;
|
|
650
|
+
for (size_t j = 0; j < num_blobs; ++j) {
|
|
651
|
+
blob_reqs_in_file[i].emplace_back(
|
|
652
|
+
keys[j], blob_offsets[j], blob_sizes[j], kNoCompression,
|
|
653
|
+
&value_buf[i * num_blobs + j], &statuses_buf[i * num_blobs + j]);
|
|
654
|
+
}
|
|
655
|
+
blob_reqs.emplace_back(file_number, file_size, blob_reqs_in_file[i]);
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
get_perf_context()->Reset();
|
|
659
|
+
statistics->Reset().PermitUncheckedError();
|
|
660
|
+
|
|
661
|
+
blob_source.MultiGetBlob(read_options, blob_reqs, &bytes_read);
|
|
662
|
+
|
|
663
|
+
for (size_t i = 0; i < blob_files; ++i) {
|
|
664
|
+
const uint64_t file_number = i + 1;
|
|
665
|
+
for (size_t j = 0; j < num_blobs; ++j) {
|
|
666
|
+
ASSERT_OK(statuses_buf[i * num_blobs + j]);
|
|
667
|
+
ASSERT_EQ(value_buf[i * num_blobs + j], blobs[j]);
|
|
668
|
+
ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
669
|
+
blob_offsets[j]));
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// Retrieved all blobs from 2 blob files twice via MultiGetBlob and
|
|
674
|
+
// TEST_BlobInCache.
|
|
675
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
|
|
676
|
+
num_blobs * blob_files);
|
|
677
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count,
|
|
678
|
+
num_blobs * blob_files); // blocking i/o
|
|
679
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte,
|
|
680
|
+
blob_records_bytes * blob_files); // blocking i/o
|
|
681
|
+
ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
|
|
682
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
683
|
+
|
|
684
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS),
|
|
685
|
+
num_blobs * blob_files); // MultiGetBlob
|
|
686
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT),
|
|
687
|
+
num_blobs * blob_files); // TEST_BlobInCache
|
|
688
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD),
|
|
689
|
+
num_blobs * blob_files); // MultiGetBlob
|
|
690
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
691
|
+
blob_value_bytes * blob_files); // TEST_BlobInCache
|
|
692
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE),
|
|
693
|
+
blob_value_bytes * blob_files); // MultiGetBlob
|
|
694
|
+
|
|
695
|
+
get_perf_context()->Reset();
|
|
696
|
+
statistics->Reset().PermitUncheckedError();
|
|
697
|
+
|
|
698
|
+
autovector<BlobReadRequest> fake_blob_reqs_in_file;
|
|
699
|
+
std::array<PinnableSlice, num_blobs> fake_value_buf;
|
|
700
|
+
std::array<Status, num_blobs> fake_statuses_buf;
|
|
701
|
+
|
|
702
|
+
const uint64_t fake_file_number = 100;
|
|
703
|
+
for (size_t i = 0; i < num_blobs; ++i) {
|
|
704
|
+
fake_blob_reqs_in_file.emplace_back(
|
|
705
|
+
keys[i], blob_offsets[i], blob_sizes[i], kNoCompression,
|
|
706
|
+
&fake_value_buf[i], &fake_statuses_buf[i]);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// Add a fake multi-get blob request.
|
|
710
|
+
blob_reqs.emplace_back(fake_file_number, file_size, fake_blob_reqs_in_file);
|
|
711
|
+
|
|
712
|
+
blob_source.MultiGetBlob(read_options, blob_reqs, &bytes_read);
|
|
713
|
+
|
|
714
|
+
// Check the real blob read requests.
|
|
715
|
+
for (size_t i = 0; i < blob_files; ++i) {
|
|
716
|
+
const uint64_t file_number = i + 1;
|
|
717
|
+
for (size_t j = 0; j < num_blobs; ++j) {
|
|
718
|
+
ASSERT_OK(statuses_buf[i * num_blobs + j]);
|
|
719
|
+
ASSERT_EQ(value_buf[i * num_blobs + j], blobs[j]);
|
|
720
|
+
ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
721
|
+
blob_offsets[j]));
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// Check the fake blob request.
|
|
726
|
+
for (size_t i = 0; i < num_blobs; ++i) {
|
|
727
|
+
ASSERT_TRUE(fake_statuses_buf[i].IsIOError());
|
|
728
|
+
ASSERT_TRUE(fake_value_buf[i].empty());
|
|
729
|
+
ASSERT_FALSE(blob_source.TEST_BlobInCache(fake_file_number, file_size,
|
|
730
|
+
blob_offsets[i]));
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
// Retrieved all blobs from 3 blob files (including the fake one) twice
|
|
734
|
+
// via MultiGetBlob and TEST_BlobInCache.
|
|
735
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count,
|
|
736
|
+
num_blobs * blob_files * 2);
|
|
737
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count,
|
|
738
|
+
0); // blocking i/o
|
|
739
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte,
|
|
740
|
+
0); // blocking i/o
|
|
741
|
+
ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
|
|
742
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
743
|
+
|
|
744
|
+
// Fake blob requests: MultiGetBlob and TEST_BlobInCache
|
|
745
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2);
|
|
746
|
+
// Real blob requests: MultiGetBlob and TEST_BlobInCache
|
|
747
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT),
|
|
748
|
+
num_blobs * blob_files * 2);
|
|
749
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
750
|
+
// Real blob requests: MultiGetBlob and TEST_BlobInCache
|
|
751
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
752
|
+
blob_value_bytes * blob_files * 2);
|
|
753
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
442
754
|
}
|
|
443
755
|
}
|
|
444
756
|
|
|
@@ -446,6 +758,10 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
|
|
|
446
758
|
options_.cf_paths.emplace_back(
|
|
447
759
|
test::PerThreadDBPath(env_, "BlobSourceTest_MultiGetBlobsFromCache"), 0);
|
|
448
760
|
|
|
761
|
+
options_.statistics = CreateDBStatistics();
|
|
762
|
+
Statistics* statistics = options_.statistics.get();
|
|
763
|
+
assert(statistics);
|
|
764
|
+
|
|
449
765
|
DestroyAndReopen(options_);
|
|
450
766
|
|
|
451
767
|
ImmutableOptions immutable_options(options_);
|
|
@@ -502,41 +818,39 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
|
|
|
502
818
|
constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
|
|
503
819
|
|
|
504
820
|
{
|
|
505
|
-
//
|
|
821
|
+
// MultiGetBlobFromOneFile
|
|
506
822
|
uint64_t bytes_read = 0;
|
|
507
|
-
|
|
508
|
-
autovector<std::reference_wrapper<const Slice>> key_refs;
|
|
509
|
-
autovector<uint64_t> offsets;
|
|
510
|
-
autovector<uint64_t> sizes;
|
|
511
823
|
std::array<Status, num_blobs> statuses_buf;
|
|
512
|
-
autovector<Status*> statuses;
|
|
513
824
|
std::array<PinnableSlice, num_blobs> value_buf;
|
|
514
|
-
autovector<
|
|
825
|
+
autovector<BlobReadRequest> blob_reqs;
|
|
515
826
|
|
|
516
827
|
for (size_t i = 0; i < num_blobs; i += 2) { // even index
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
sizes.push_back(blob_sizes[i]);
|
|
520
|
-
statuses.push_back(&statuses_buf[i]);
|
|
521
|
-
values.push_back(&value_buf[i]);
|
|
828
|
+
blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i],
|
|
829
|
+
kNoCompression, &value_buf[i], &statuses_buf[i]);
|
|
522
830
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
523
831
|
blob_offsets[i]));
|
|
524
832
|
}
|
|
525
833
|
|
|
526
834
|
read_options.fill_cache = true;
|
|
527
835
|
read_options.read_tier = ReadTier::kReadAllTier;
|
|
836
|
+
get_perf_context()->Reset();
|
|
837
|
+
statistics->Reset().PermitUncheckedError();
|
|
528
838
|
|
|
529
839
|
// Get half of blobs
|
|
530
|
-
blob_source.
|
|
531
|
-
|
|
532
|
-
&bytes_read);
|
|
840
|
+
blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number,
|
|
841
|
+
file_size, blob_reqs, &bytes_read);
|
|
533
842
|
|
|
843
|
+
uint64_t fs_read_bytes = 0;
|
|
844
|
+
uint64_t ca_read_bytes = 0;
|
|
534
845
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
535
846
|
if (i % 2 == 0) {
|
|
536
847
|
ASSERT_OK(statuses_buf[i]);
|
|
537
848
|
ASSERT_EQ(value_buf[i], blobs[i]);
|
|
849
|
+
fs_read_bytes +=
|
|
850
|
+
blob_sizes[i] + keys[i].size() + BlobLogRecord::kHeaderSize;
|
|
538
851
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
539
852
|
blob_offsets[i]));
|
|
853
|
+
ca_read_bytes += blob_sizes[i];
|
|
540
854
|
} else {
|
|
541
855
|
statuses_buf[i].PermitUncheckedError();
|
|
542
856
|
ASSERT_TRUE(value_buf[i].empty());
|
|
@@ -545,6 +859,23 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
|
|
|
545
859
|
}
|
|
546
860
|
}
|
|
547
861
|
|
|
862
|
+
constexpr int num_even_blobs = num_blobs / 2;
|
|
863
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_even_blobs);
|
|
864
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count,
|
|
865
|
+
num_even_blobs); // blocking i/o
|
|
866
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte,
|
|
867
|
+
fs_read_bytes); // blocking i/o
|
|
868
|
+
ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
|
|
869
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
870
|
+
|
|
871
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs);
|
|
872
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_even_blobs);
|
|
873
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), num_even_blobs);
|
|
874
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
875
|
+
ca_read_bytes);
|
|
876
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE),
|
|
877
|
+
ca_read_bytes);
|
|
878
|
+
|
|
548
879
|
// Get the rest of blobs
|
|
549
880
|
for (size_t i = 1; i < num_blobs; i += 2) { // odd index
|
|
550
881
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
@@ -562,62 +893,68 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
|
|
|
562
893
|
blob_offsets[i]));
|
|
563
894
|
}
|
|
564
895
|
|
|
565
|
-
// Cache-only
|
|
896
|
+
// Cache-only MultiGetBlobFromOneFile
|
|
566
897
|
read_options.read_tier = ReadTier::kBlockCacheTier;
|
|
898
|
+
get_perf_context()->Reset();
|
|
899
|
+
statistics->Reset().PermitUncheckedError();
|
|
567
900
|
|
|
568
|
-
|
|
569
|
-
offsets.clear();
|
|
570
|
-
sizes.clear();
|
|
571
|
-
statuses.clear();
|
|
572
|
-
values.clear();
|
|
901
|
+
blob_reqs.clear();
|
|
573
902
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
sizes.push_back(blob_sizes[i]);
|
|
577
|
-
statuses.push_back(&statuses_buf[i]);
|
|
578
|
-
values.push_back(&value_buf[i]);
|
|
903
|
+
blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i],
|
|
904
|
+
kNoCompression, &value_buf[i], &statuses_buf[i]);
|
|
579
905
|
}
|
|
580
906
|
|
|
581
|
-
blob_source.
|
|
582
|
-
|
|
583
|
-
&bytes_read);
|
|
907
|
+
blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number,
|
|
908
|
+
file_size, blob_reqs, &bytes_read);
|
|
584
909
|
|
|
910
|
+
uint64_t blob_bytes = 0;
|
|
585
911
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
586
912
|
ASSERT_OK(statuses_buf[i]);
|
|
587
913
|
ASSERT_EQ(value_buf[i], blobs[i]);
|
|
588
914
|
ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
589
915
|
blob_offsets[i]));
|
|
916
|
+
blob_bytes += blob_sizes[i];
|
|
590
917
|
}
|
|
918
|
+
|
|
919
|
+
// Retrieved the blob cache num_blobs * 2 times via GetBlob and
|
|
920
|
+
// TEST_BlobInCache.
|
|
921
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, num_blobs * 2);
|
|
922
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o
|
|
923
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o
|
|
924
|
+
ASSERT_GE((int)get_perf_context()->blob_checksum_time, 0);
|
|
925
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
926
|
+
|
|
927
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0);
|
|
928
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), num_blobs * 2);
|
|
929
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
930
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ),
|
|
931
|
+
blob_bytes * 2);
|
|
932
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
591
933
|
}
|
|
592
934
|
|
|
593
935
|
options_.blob_cache->EraseUnRefEntries();
|
|
594
936
|
|
|
595
937
|
{
|
|
596
|
-
// Cache-only
|
|
938
|
+
// Cache-only MultiGetBlobFromOneFile
|
|
597
939
|
uint64_t bytes_read = 0;
|
|
598
940
|
read_options.read_tier = ReadTier::kBlockCacheTier;
|
|
599
941
|
|
|
600
|
-
autovector<std::reference_wrapper<const Slice>> key_refs;
|
|
601
|
-
autovector<uint64_t> offsets;
|
|
602
|
-
autovector<uint64_t> sizes;
|
|
603
942
|
std::array<Status, num_blobs> statuses_buf;
|
|
604
|
-
autovector<Status*> statuses;
|
|
605
943
|
std::array<PinnableSlice, num_blobs> value_buf;
|
|
606
|
-
autovector<
|
|
944
|
+
autovector<BlobReadRequest> blob_reqs;
|
|
607
945
|
|
|
608
946
|
for (size_t i = 0; i < num_blobs; i++) {
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
sizes.push_back(blob_sizes[i]);
|
|
612
|
-
statuses.push_back(&statuses_buf[i]);
|
|
613
|
-
values.push_back(&value_buf[i]);
|
|
947
|
+
blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i],
|
|
948
|
+
kNoCompression, &value_buf[i], &statuses_buf[i]);
|
|
614
949
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
615
950
|
blob_offsets[i]));
|
|
616
951
|
}
|
|
617
952
|
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
953
|
+
get_perf_context()->Reset();
|
|
954
|
+
statistics->Reset().PermitUncheckedError();
|
|
955
|
+
|
|
956
|
+
blob_source.MultiGetBlobFromOneFile(read_options, blob_file_number,
|
|
957
|
+
file_size, blob_reqs, &bytes_read);
|
|
621
958
|
|
|
622
959
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
623
960
|
ASSERT_TRUE(statuses_buf[i].IsIncomplete());
|
|
@@ -625,41 +962,61 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
|
|
|
625
962
|
ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
|
|
626
963
|
blob_offsets[i]));
|
|
627
964
|
}
|
|
965
|
+
|
|
966
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
|
|
967
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o
|
|
968
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o
|
|
969
|
+
ASSERT_EQ((int)get_perf_context()->blob_checksum_time, 0);
|
|
970
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
971
|
+
|
|
972
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2);
|
|
973
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0);
|
|
974
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
975
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0);
|
|
976
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
628
977
|
}
|
|
629
978
|
|
|
630
979
|
{
|
|
631
|
-
//
|
|
980
|
+
// MultiGetBlobFromOneFile from non-existing file
|
|
632
981
|
uint64_t bytes_read = 0;
|
|
633
|
-
uint64_t
|
|
982
|
+
uint64_t non_existing_file_number = 100;
|
|
634
983
|
read_options.read_tier = ReadTier::kReadAllTier;
|
|
635
984
|
|
|
636
|
-
autovector<std::reference_wrapper<const Slice>> key_refs;
|
|
637
|
-
autovector<uint64_t> offsets;
|
|
638
|
-
autovector<uint64_t> sizes;
|
|
639
985
|
std::array<Status, num_blobs> statuses_buf;
|
|
640
|
-
autovector<Status*> statuses;
|
|
641
986
|
std::array<PinnableSlice, num_blobs> value_buf;
|
|
642
|
-
autovector<
|
|
987
|
+
autovector<BlobReadRequest> blob_reqs;
|
|
643
988
|
|
|
644
989
|
for (size_t i = 0; i < num_blobs; i++) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
values.push_back(&value_buf[i]);
|
|
650
|
-
ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
|
|
651
|
-
blob_offsets[i]));
|
|
990
|
+
blob_reqs.emplace_back(keys[i], blob_offsets[i], blob_sizes[i],
|
|
991
|
+
kNoCompression, &value_buf[i], &statuses_buf[i]);
|
|
992
|
+
ASSERT_FALSE(blob_source.TEST_BlobInCache(non_existing_file_number,
|
|
993
|
+
file_size, blob_offsets[i]));
|
|
652
994
|
}
|
|
653
995
|
|
|
654
|
-
|
|
655
|
-
|
|
996
|
+
get_perf_context()->Reset();
|
|
997
|
+
statistics->Reset().PermitUncheckedError();
|
|
998
|
+
|
|
999
|
+
blob_source.MultiGetBlobFromOneFile(read_options, non_existing_file_number,
|
|
1000
|
+
file_size, blob_reqs, &bytes_read);
|
|
656
1001
|
|
|
657
1002
|
for (size_t i = 0; i < num_blobs; ++i) {
|
|
658
1003
|
ASSERT_TRUE(statuses_buf[i].IsIOError());
|
|
659
1004
|
ASSERT_TRUE(value_buf[i].empty());
|
|
660
|
-
ASSERT_FALSE(blob_source.TEST_BlobInCache(
|
|
661
|
-
blob_offsets[i]));
|
|
1005
|
+
ASSERT_FALSE(blob_source.TEST_BlobInCache(non_existing_file_number,
|
|
1006
|
+
file_size, blob_offsets[i]));
|
|
662
1007
|
}
|
|
1008
|
+
|
|
1009
|
+
ASSERT_EQ((int)get_perf_context()->blob_cache_hit_count, 0);
|
|
1010
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_count, 0); // blocking i/o
|
|
1011
|
+
ASSERT_EQ((int)get_perf_context()->blob_read_byte, 0); // blocking i/o
|
|
1012
|
+
ASSERT_EQ((int)get_perf_context()->blob_checksum_time, 0);
|
|
1013
|
+
ASSERT_EQ((int)get_perf_context()->blob_decompress_time, 0);
|
|
1014
|
+
|
|
1015
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), num_blobs * 2);
|
|
1016
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), 0);
|
|
1017
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
|
|
1018
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_READ), 0);
|
|
1019
|
+
ASSERT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_BYTES_WRITE), 0);
|
|
663
1020
|
}
|
|
664
1021
|
}
|
|
665
1022
|
|