@nxtedition/rocksdb 7.0.23 → 7.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/binding.cc +3 -1
  2. package/deps/rocksdb/rocksdb/CMakeLists.txt +5 -0
  3. package/deps/rocksdb/rocksdb/Makefile +6 -2
  4. package/deps/rocksdb/rocksdb/TARGETS +14 -0
  5. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +4 -1
  6. package/deps/rocksdb/rocksdb/cache/cache_helpers.h +20 -0
  7. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager_test.cc +2 -2
  8. package/deps/rocksdb/rocksdb/cache/cache_test.cc +44 -31
  9. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +491 -722
  10. package/deps/rocksdb/rocksdb/cache/clock_cache.h +468 -2
  11. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -1
  12. package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +51 -52
  13. package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h +28 -16
  14. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +12 -1
  15. package/deps/rocksdb/rocksdb/cache/lru_cache.h +1 -0
  16. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +170 -36
  17. package/deps/rocksdb/rocksdb/db/blob/blob_file_cache_test.cc +1 -1
  18. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +63 -36
  19. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +4 -6
  20. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +57 -38
  21. package/deps/rocksdb/rocksdb/db/blob/blob_read_request.h +58 -0
  22. package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +164 -74
  23. package/deps/rocksdb/rocksdb/db/blob/blob_source.h +42 -29
  24. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +419 -62
  25. package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +208 -8
  26. package/deps/rocksdb/rocksdb/db/c.cc +68 -0
  27. package/deps/rocksdb/rocksdb/db/c_test.c +95 -2
  28. package/deps/rocksdb/rocksdb/db/column_family.cc +12 -3
  29. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +92 -15
  30. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +76 -4
  31. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +52 -1
  32. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +30 -1
  33. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +126 -0
  34. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +203 -1584
  35. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +93 -26
  36. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +87 -1
  37. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +314 -0
  38. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +328 -0
  39. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +32 -6
  40. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +4 -1
  41. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +7 -3
  42. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +174 -33
  43. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +474 -7
  44. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +5 -2
  45. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +825 -0
  46. package/deps/rocksdb/rocksdb/db/compaction/compaction_state.cc +46 -0
  47. package/deps/rocksdb/rocksdb/db/compaction/compaction_state.h +42 -0
  48. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +223 -0
  49. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +255 -0
  50. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +1253 -0
  51. package/deps/rocksdb/rocksdb/db/corruption_test.cc +32 -8
  52. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +3 -1
  53. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +13 -8
  54. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +376 -0
  55. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +103 -78
  56. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +4 -6
  57. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +0 -8
  58. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +10 -3
  59. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +21 -6
  60. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +19 -1
  61. package/deps/rocksdb/rocksdb/db/db_iter.cc +91 -14
  62. package/deps/rocksdb/rocksdb/db/db_iter.h +5 -0
  63. package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +33 -0
  64. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +79 -0
  65. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +2 -0
  66. package/deps/rocksdb/rocksdb/db/db_test2.cc +1 -1
  67. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +5 -2
  68. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +185 -0
  69. package/deps/rocksdb/rocksdb/db/dbformat.cc +1 -4
  70. package/deps/rocksdb/rocksdb/db/dbformat.h +2 -8
  71. package/deps/rocksdb/rocksdb/db/internal_stats.cc +71 -29
  72. package/deps/rocksdb/rocksdb/db/internal_stats.h +160 -5
  73. package/deps/rocksdb/rocksdb/db/log_reader.cc +29 -3
  74. package/deps/rocksdb/rocksdb/db/log_reader.h +12 -3
  75. package/deps/rocksdb/rocksdb/db/repair_test.cc +1 -3
  76. package/deps/rocksdb/rocksdb/db/version_edit.cc +6 -0
  77. package/deps/rocksdb/rocksdb/db/version_set.cc +93 -129
  78. package/deps/rocksdb/rocksdb/db/version_set.h +4 -4
  79. package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +2 -2
  80. package/deps/rocksdb/rocksdb/db/version_set_test.cc +42 -35
  81. package/deps/rocksdb/rocksdb/db/write_batch.cc +10 -2
  82. package/deps/rocksdb/rocksdb/db/write_batch_internal.h +4 -1
  83. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +10 -4
  84. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +3 -3
  85. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +3 -2
  86. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +4 -0
  87. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +5 -1
  88. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +140 -8
  89. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +12 -0
  90. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +46 -7
  91. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +7 -0
  92. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +27 -7
  93. package/deps/rocksdb/rocksdb/env/composite_env_wrapper.h +8 -0
  94. package/deps/rocksdb/rocksdb/env/env_posix.cc +14 -0
  95. package/deps/rocksdb/rocksdb/env/env_test.cc +130 -1
  96. package/deps/rocksdb/rocksdb/env/fs_posix.cc +7 -1
  97. package/deps/rocksdb/rocksdb/env/io_posix.cc +18 -50
  98. package/deps/rocksdb/rocksdb/env/io_posix.h +53 -6
  99. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +8 -10
  100. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +3 -7
  101. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +239 -259
  102. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +84 -19
  103. package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +24 -4
  104. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +1 -1
  105. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +31 -1
  106. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +11 -7
  107. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +2 -0
  108. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +14 -0
  109. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +20 -0
  110. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +37 -13
  111. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +7 -0
  112. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +14 -0
  113. package/deps/rocksdb/rocksdb/include/rocksdb/threadpool.h +9 -0
  114. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +13 -13
  115. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.cc +12 -2
  116. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +38 -0
  117. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +7 -1
  118. package/deps/rocksdb/rocksdb/port/win/env_win.cc +17 -0
  119. package/deps/rocksdb/rocksdb/port/win/env_win.h +8 -0
  120. package/deps/rocksdb/rocksdb/port/win/io_win.cc +6 -3
  121. package/deps/rocksdb/rocksdb/src.mk +5 -0
  122. package/deps/rocksdb/rocksdb/table/block_based/block.h +1 -2
  123. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1 -1
  124. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +5 -2
  125. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +1 -1
  126. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +15 -12
  127. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +5 -4
  128. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +2 -1
  129. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +1 -1
  130. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +4 -4
  131. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +1 -2
  132. package/deps/rocksdb/rocksdb/table/get_context.cc +1 -0
  133. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -2
  134. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +24 -4
  135. package/deps/rocksdb/rocksdb/util/async_file_reader.cc +1 -1
  136. package/deps/rocksdb/rocksdb/util/compression.h +2 -0
  137. package/deps/rocksdb/rocksdb/util/thread_list_test.cc +18 -1
  138. package/deps/rocksdb/rocksdb/util/threadpool_imp.cc +67 -4
  139. package/deps/rocksdb/rocksdb/util/threadpool_imp.h +8 -0
  140. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +15 -12
  141. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -2
  142. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc +1 -1
  143. package/deps/rocksdb/rocksdb.gyp +5 -1
  144. package/package.json +1 -1
  145. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  146. package/prebuilds/linux-x64/node.napi.node +0 -0
@@ -7,6 +7,7 @@
7
7
  // Use of this source code is governed by a BSD-style license that can be
8
8
  // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
9
 
10
+ #include "rocksdb/options.h"
10
11
  #ifndef ROCKSDB_LITE
11
12
 
12
13
  #include <fcntl.h>
@@ -553,7 +554,10 @@ TEST_F(CorruptionTest, CorruptedDescriptor) {
553
554
  ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
554
555
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
555
556
  ASSERT_OK(dbi->TEST_FlushMemTable());
556
- ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr));
557
+ CompactRangeOptions cro;
558
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
559
+ ASSERT_OK(
560
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
557
561
 
558
562
  Corrupt(kDescriptorFile, 0, 1000);
559
563
  Status s = TryReopen();
@@ -770,7 +774,9 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
770
774
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
771
775
  ASSERT_OK(dbi->TEST_FlushMemTable());
772
776
  mock->SetCorruptionMode(mode);
773
- s = dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true);
777
+ CompactRangeOptions cro;
778
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
779
+ s = dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr);
774
780
  if (mode == mock::MockTableFactory::kCorruptNone) {
775
781
  ASSERT_OK(s);
776
782
  } else {
@@ -806,7 +812,10 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeFirst) {
806
812
  } else {
807
813
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
808
814
  ASSERT_OK(dbi->TEST_FlushMemTable());
809
- ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
815
+ CompactRangeOptions cro;
816
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
817
+ ASSERT_OK(
818
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
810
819
  }
811
820
  db_->ReleaseSnapshot(snap);
812
821
  }
@@ -842,7 +851,10 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRange) {
842
851
  } else {
843
852
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
844
853
  ASSERT_OK(dbi->TEST_FlushMemTable());
845
- ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
854
+ CompactRangeOptions cro;
855
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
856
+ ASSERT_OK(
857
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
846
858
  }
847
859
  db_->ReleaseSnapshot(snap);
848
860
  }
@@ -875,7 +887,10 @@ TEST_F(CorruptionTest, ParanoidFileChecksWithDeleteRangeLast) {
875
887
  } else {
876
888
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
877
889
  ASSERT_OK(dbi->TEST_FlushMemTable());
878
- ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
890
+ CompactRangeOptions cro;
891
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
892
+ ASSERT_OK(
893
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
879
894
  }
880
895
  db_->ReleaseSnapshot(snap);
881
896
  }
@@ -902,7 +917,10 @@ TEST_F(CorruptionTest, LogCorruptionErrorsInCompactionIterator) {
902
917
 
903
918
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
904
919
  ASSERT_OK(dbi->TEST_FlushMemTable());
905
- Status s = dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true);
920
+ CompactRangeOptions cro;
921
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
922
+ Status s =
923
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr);
906
924
  ASSERT_NOK(s);
907
925
  ASSERT_TRUE(s.IsCorruption());
908
926
  }
@@ -928,7 +946,10 @@ TEST_F(CorruptionTest, CompactionKeyOrderCheck) {
928
946
 
929
947
  mock->SetCorruptionMode(mock::MockTableFactory::kCorruptNone);
930
948
  ASSERT_OK(db_->SetOptions({{"check_flush_compaction_key_order", "true"}}));
931
- ASSERT_NOK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
949
+ CompactRangeOptions cro;
950
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
951
+ ASSERT_NOK(
952
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
932
953
  }
933
954
 
934
955
  TEST_F(CorruptionTest, FlushKeyOrderCheck) {
@@ -975,7 +996,10 @@ TEST_F(CorruptionTest, DisableKeyOrderCheck) {
975
996
  ASSERT_OK(db_->Put(WriteOptions(), "foo2", "v1"));
976
997
  ASSERT_OK(db_->Put(WriteOptions(), "foo4", "v1"));
977
998
  ASSERT_OK(dbi->TEST_FlushMemTable());
978
- ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr, nullptr, true));
999
+ CompactRangeOptions cro;
1000
+ cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
1001
+ ASSERT_OK(
1002
+ dbi->CompactRange(cro, dbi->DefaultColumnFamily(), nullptr, nullptr));
979
1003
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
980
1004
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
981
1005
  }
@@ -4195,7 +4195,9 @@ TEST_F(DBBasicTest, VerifyFileChecksums) {
4195
4195
  ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsInvalidArgument());
4196
4196
  }
4197
4197
 
4198
- TEST_F(DBBasicTest, ManualWalSync) {
4198
+ // TODO: re-enable after we provide finer-grained control for WAL tracking to
4199
+ // meet the needs of different use cases, durability levels and recovery modes.
4200
+ TEST_F(DBBasicTest, DISABLED_ManualWalSync) {
4199
4201
  Options options = CurrentOptions();
4200
4202
  options.track_and_verify_wals_in_manifest = true;
4201
4203
  options.wal_recovery_mode = WALRecoveryMode::kAbsoluteConsistency;
@@ -13,6 +13,7 @@
13
13
 
14
14
  #include "cache/cache_entry_roles.h"
15
15
  #include "cache/cache_key.h"
16
+ #include "cache/clock_cache.h"
16
17
  #include "cache/fast_lru_cache.h"
17
18
  #include "cache/lru_cache.h"
18
19
  #include "db/column_family.h"
@@ -250,7 +251,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
250
251
  cache->SetStrictCapacityLimit(true);
251
252
  iter = db_->NewIterator(read_options);
252
253
  iter->Seek(std::to_string(kNumBlocks - 1));
253
- ASSERT_TRUE(iter->status().IsIncomplete());
254
+ ASSERT_TRUE(iter->status().IsMemoryLimit());
254
255
  CheckCacheCounters(options, 1, 0, 0, 1);
255
256
  delete iter;
256
257
  iter = nullptr;
@@ -333,8 +334,10 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
333
334
  ASSERT_EQ(usage, cache->GetPinnedUsage());
334
335
 
335
336
  // Load last key block.
336
- ASSERT_EQ("Result incomplete: Insert failed due to LRU cache being full.",
337
- Get(std::to_string(kNumBlocks - 1)));
337
+ ASSERT_EQ(
338
+ "Operation aborted: Memory limit reached: Insert failed due to LRU cache "
339
+ "being full.",
340
+ Get(std::to_string(kNumBlocks - 1)));
338
341
  // Failure will also record the miss counter.
339
342
  CheckCacheCounters(options, 1, 0, 0, 1);
340
343
  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
@@ -676,6 +679,7 @@ TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) {
676
679
  Options options = CurrentOptions();
677
680
  options.create_if_missing = true;
678
681
  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
682
+ options.max_compaction_bytes = 2000;
679
683
 
680
684
  BlockBasedTableOptions table_options;
681
685
  table_options.block_cache = NewLRUCache(1 << 25, 0, false);
@@ -932,7 +936,9 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
932
936
  int iterations_tested = 0;
933
937
  for (std::shared_ptr<Cache> base_cache :
934
938
  {NewLRUCache(capacity, num_shard_bits),
935
- NewClockCache(capacity, num_shard_bits),
939
+ ExperimentalNewClockCache(
940
+ capacity, 1 /*estimated_value_size*/, num_shard_bits,
941
+ false /*strict_capacity_limit*/, kDefaultCacheMetadataChargePolicy),
936
942
  NewFastLRUCache(capacity, 1 /*estimated_value_size*/, num_shard_bits,
937
943
  false /*strict_capacity_limit*/,
938
944
  kDefaultCacheMetadataChargePolicy)}) {
@@ -1288,11 +1294,10 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
1288
1294
  const size_t capacity = size_t{1} << 25;
1289
1295
  int iterations_tested = 0;
1290
1296
  for (bool partition : {false, true}) {
1291
- for (std::shared_ptr<Cache> cache :
1292
- {NewLRUCache(capacity), NewClockCache(capacity)}) {
1293
- // This test doesn't support FastLRUCache because the
1297
+ for (std::shared_ptr<Cache> cache : {NewLRUCache(capacity)}) {
1298
+ // This test doesn't support FastLRUCache nor ClockCache because the
1294
1299
  // keys used are not 16B long.
1295
- // TODO(guido) Add support for FastLRUCache.
1300
+ // TODO(guido) Add support for FastLRUCache and ClockCache.
1296
1301
  if (!cache) {
1297
1302
  // Skip clock cache when not supported
1298
1303
  continue;
@@ -3113,6 +3113,382 @@ TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) {
3113
3113
  it.reset();
3114
3114
  }
3115
3115
 
3116
+ namespace {
3117
+ class BackwardBytewiseComparator : public Comparator {
3118
+ public:
3119
+ const char* Name() const override { return "BackwardBytewiseComparator"; }
3120
+
3121
+ int Compare(const Slice& a, const Slice& b) const override {
3122
+ int min_size_neg = -static_cast<int>(std::min(a.size(), b.size()));
3123
+ const char* a_end = a.data() + a.size();
3124
+ const char* b_end = b.data() + b.size();
3125
+ for (int i = -1; i >= min_size_neg; --i) {
3126
+ if (a_end[i] != b_end[i]) {
3127
+ if (static_cast<unsigned char>(a_end[i]) <
3128
+ static_cast<unsigned char>(b_end[i])) {
3129
+ return -1;
3130
+ } else {
3131
+ return 1;
3132
+ }
3133
+ }
3134
+ }
3135
+ return static_cast<int>(a.size()) - static_cast<int>(b.size());
3136
+ }
3137
+
3138
+ void FindShortestSeparator(std::string* /*start*/,
3139
+ const Slice& /*limit*/) const override {}
3140
+
3141
+ void FindShortSuccessor(std::string* /*key*/) const override {}
3142
+ };
3143
+
3144
+ const BackwardBytewiseComparator kBackwardBytewiseComparator{};
3145
+
3146
+ class FixedSuffix4Transform : public SliceTransform {
3147
+ const char* Name() const override { return "FixedSuffixTransform"; }
3148
+
3149
+ Slice Transform(const Slice& src) const override {
3150
+ return Slice(src.data() + src.size() - 4, 4);
3151
+ }
3152
+
3153
+ bool InDomain(const Slice& src) const override { return src.size() >= 4; }
3154
+ };
3155
+
3156
+ std::pair<uint64_t, uint64_t> GetBloomStat(const Options& options, bool sst) {
3157
+ if (sst) {
3158
+ return {
3159
+ options.statistics->getAndResetTickerCount(BLOOM_FILTER_PREFIX_CHECKED),
3160
+ options.statistics->getAndResetTickerCount(BLOOM_FILTER_PREFIX_USEFUL)};
3161
+ } else {
3162
+ auto hit = std::exchange(get_perf_context()->bloom_memtable_hit_count, 0);
3163
+ auto miss = std::exchange(get_perf_context()->bloom_memtable_miss_count, 0);
3164
+ return {hit + miss, miss};
3165
+ }
3166
+ }
3167
+
3168
+ std::pair<uint64_t, uint64_t> CheckedAndUseful(uint64_t checked,
3169
+ uint64_t useful) {
3170
+ return {checked, useful};
3171
+ }
3172
+ } // namespace
3173
+
3174
+ // This uses a prefix_extractor + comparator combination that violates
3175
+ // one of the old obsolete, unnecessary axioms of prefix extraction:
3176
+ // * key.starts_with(prefix(key))
3177
+ // This axiom is not really needed, and we validate that here.
3178
+ TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter1) {
3179
+ BlockBasedTableOptions bbto;
3180
+ bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10));
3181
+ bbto.whole_key_filtering = false;
3182
+
3183
+ Options options = CurrentOptions();
3184
+ options.comparator = &kBackwardBytewiseComparator;
3185
+ options.prefix_extractor = std::make_shared<FixedSuffix4Transform>();
3186
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
3187
+ options.memtable_prefix_bloom_size_ratio = 0.1;
3188
+ options.statistics = CreateDBStatistics();
3189
+
3190
+ DestroyAndReopen(options);
3191
+
3192
+ ASSERT_OK(Put("321aaaa", "val1"));
3193
+ ASSERT_OK(Put("112aaaa", "val2"));
3194
+ ASSERT_OK(Put("009aaaa", "val3"));
3195
+ ASSERT_OK(Put("baa", "val4")); // out of domain
3196
+ ASSERT_OK(Put("321abaa", "val5"));
3197
+ ASSERT_OK(Put("zzz", "val6")); // out of domain
3198
+
3199
+ for (auto flushed : {false, true}) {
3200
+ SCOPED_TRACE("flushed=" + std::to_string(flushed));
3201
+ if (flushed) {
3202
+ ASSERT_OK(Flush());
3203
+ }
3204
+ ReadOptions read_options;
3205
+ if (flushed) { // TODO: support auto_prefix_mode in memtable?
3206
+ read_options.auto_prefix_mode = true;
3207
+ }
3208
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3209
+ {
3210
+ Slice ub("999aaaa");
3211
+ read_options.iterate_upper_bound = &ub;
3212
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3213
+ EXPECT_EQ(CountIter(iter, "aaaa"), 3);
3214
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3215
+ }
3216
+ {
3217
+ Slice ub("999abaa");
3218
+ read_options.iterate_upper_bound = &ub;
3219
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3220
+ EXPECT_EQ(CountIter(iter, "abaa"), 1);
3221
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3222
+ }
3223
+ {
3224
+ Slice ub("999acaa");
3225
+ read_options.iterate_upper_bound = &ub;
3226
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3227
+ EXPECT_EQ(CountIter(iter, "acaa"), 0);
3228
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 1));
3229
+ }
3230
+ {
3231
+ Slice ub("zzzz");
3232
+ read_options.iterate_upper_bound = &ub;
3233
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3234
+ EXPECT_EQ(CountIter(iter, "baa"), 3);
3235
+ if (flushed) { // TODO: fix memtable case
3236
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3237
+ }
3238
+ }
3239
+ }
3240
+ }
3241
+
3242
+ // This uses a prefix_extractor + comparator combination that violates
3243
+ // one of the old obsolete, unnecessary axioms of prefix extraction:
3244
+ // * Compare(prefix(key), key) <= 0
3245
+ // This axiom is not really needed, and we validate that here.
3246
+ TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter2) {
3247
+ BlockBasedTableOptions bbto;
3248
+ bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10));
3249
+ bbto.whole_key_filtering = false;
3250
+
3251
+ Options options = CurrentOptions();
3252
+ options.comparator = ReverseBytewiseComparator();
3253
+ options.prefix_extractor.reset(NewFixedPrefixTransform(4));
3254
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
3255
+ options.memtable_prefix_bloom_size_ratio = 0.1;
3256
+ options.statistics = CreateDBStatistics();
3257
+
3258
+ DestroyAndReopen(options);
3259
+
3260
+ ASSERT_OK(Put("aaaa123", "val1"));
3261
+ ASSERT_OK(Put("aaaa211", "val2"));
3262
+ ASSERT_OK(Put("aaaa900", "val3"));
3263
+ ASSERT_OK(Put("aab", "val4")); // out of domain
3264
+ ASSERT_OK(Put("aaba123", "val5"));
3265
+ ASSERT_OK(Put("qqqq123", "val7"));
3266
+ ASSERT_OK(Put("qqqq", "val8"));
3267
+ ASSERT_OK(Put("zzz", "val8")); // out of domain
3268
+
3269
+ for (auto flushed : {false, true}) {
3270
+ SCOPED_TRACE("flushed=" + std::to_string(flushed));
3271
+ if (flushed) {
3272
+ ASSERT_OK(Flush());
3273
+ }
3274
+ ReadOptions read_options;
3275
+ if (flushed) { // TODO: support auto_prefix_mode in memtable?
3276
+ read_options.auto_prefix_mode = true;
3277
+ } else {
3278
+ // TODO: why needed?
3279
+ get_perf_context()->bloom_memtable_hit_count = 0;
3280
+ get_perf_context()->bloom_memtable_miss_count = 0;
3281
+ }
3282
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3283
+ {
3284
+ Slice ub("aaaa000");
3285
+ read_options.iterate_upper_bound = &ub;
3286
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3287
+ EXPECT_EQ(CountIter(iter, "aaaa999"), 3);
3288
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3289
+ }
3290
+ {
3291
+ // Note: prefix does work as upper bound
3292
+ Slice ub("aaaa");
3293
+ read_options.iterate_upper_bound = &ub;
3294
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3295
+ EXPECT_EQ(CountIter(iter, "aaaa999"), 3);
3296
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3297
+ }
3298
+ {
3299
+ // Note: prefix does not work here as seek key
3300
+ Slice ub("aaaa500");
3301
+ read_options.iterate_upper_bound = &ub;
3302
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3303
+ EXPECT_EQ(CountIter(iter, "aaaa"), 0);
3304
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3305
+ }
3306
+ {
3307
+ Slice ub("aaba000");
3308
+ read_options.iterate_upper_bound = &ub;
3309
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3310
+ EXPECT_EQ(CountIter(iter, "aaba999"), 1);
3311
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3312
+ }
3313
+ {
3314
+ Slice ub("aaca000");
3315
+ read_options.iterate_upper_bound = &ub;
3316
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3317
+ EXPECT_EQ(CountIter(iter, "aaca999"), 0);
3318
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 1));
3319
+ }
3320
+ {
3321
+ Slice ub("aaaz");
3322
+ read_options.iterate_upper_bound = &ub;
3323
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3324
+ EXPECT_EQ(CountIter(iter, "zzz"), 5);
3325
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3326
+ }
3327
+ {
3328
+ // Note: prefix does work here as seek key, but only finds key equal
3329
+ // to prefix (others with same prefix are less)
3330
+ read_options.auto_prefix_mode = false;
3331
+ read_options.iterate_upper_bound = nullptr;
3332
+ read_options.prefix_same_as_start = true;
3333
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3334
+ EXPECT_EQ(CountIter(iter, "qqqq"), 1);
3335
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3336
+ }
3337
+ }
3338
+ }
3339
+
3340
+ namespace {
3341
+ // A weird comparator that in combination with NonIdempotentFixed4Transform
3342
+ // breaks an old axiom of prefix filtering.
3343
+ class WeirdComparator : public Comparator {
3344
+ public:
3345
+ const char* Name() const override { return "WeirdComparator"; }
3346
+
3347
+ int Compare(const Slice& a, const Slice& b) const override {
3348
+ bool a_in = a.size() >= 5;
3349
+ bool b_in = b.size() >= 5;
3350
+ if (a_in != b_in) {
3351
+ // Order keys after prefixes
3352
+ return a_in - b_in;
3353
+ }
3354
+ if (a_in) {
3355
+ return BytewiseComparator()->Compare(a, b);
3356
+ } else {
3357
+ // Different ordering on the prefixes
3358
+ return ReverseBytewiseComparator()->Compare(a, b);
3359
+ }
3360
+ }
3361
+
3362
+ void FindShortestSeparator(std::string* /*start*/,
3363
+ const Slice& /*limit*/) const override {}
3364
+
3365
+ void FindShortSuccessor(std::string* /*key*/) const override {}
3366
+ };
3367
+ const WeirdComparator kWeirdComparator{};
3368
+
3369
+ // Non-idempotentent because prefix is always 4 bytes, but this is
3370
+ // out-of-domain for keys to be assigned prefixes (>= 5 bytes)
3371
+ class NonIdempotentFixed4Transform : public SliceTransform {
3372
+ const char* Name() const override { return "NonIdempotentFixed4Transform"; }
3373
+
3374
+ Slice Transform(const Slice& src) const override {
3375
+ return Slice(src.data(), 4);
3376
+ }
3377
+
3378
+ bool InDomain(const Slice& src) const override { return src.size() >= 5; }
3379
+ };
3380
+ } // namespace
3381
+
3382
+ // This uses a prefix_extractor + comparator combination that violates
3383
+ // two of the old obsolete, unnecessary axioms of prefix extraction:
3384
+ // * prefix(prefix(key)) == prefix(key)
3385
+ // * If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0
3386
+ // This axiom is not really needed, and we validate that here.
3387
+ TEST_F(DBBloomFilterTest, WeirdPrefixExtractorWithFilter3) {
3388
+ BlockBasedTableOptions bbto;
3389
+ bbto.filter_policy.reset(ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10));
3390
+ bbto.whole_key_filtering = false;
3391
+
3392
+ Options options = CurrentOptions();
3393
+ options.prefix_extractor = std::make_shared<NonIdempotentFixed4Transform>();
3394
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
3395
+ options.memtable_prefix_bloom_size_ratio = 0.1;
3396
+ options.statistics = CreateDBStatistics();
3397
+
3398
+ for (auto weird_comparator : {false, true}) {
3399
+ if (weird_comparator) {
3400
+ options.comparator = &kWeirdComparator;
3401
+ }
3402
+ DestroyAndReopen(options);
3403
+
3404
+ ASSERT_OK(Put("aaaa123", "val1"));
3405
+ ASSERT_OK(Put("aaaa211", "val2"));
3406
+ ASSERT_OK(Put("aaaa900", "val3"));
3407
+ ASSERT_OK(Put("aab", "val4")); // out of domain
3408
+ ASSERT_OK(Put("aaba123", "val5"));
3409
+ ASSERT_OK(Put("qqqq123", "val7"));
3410
+ ASSERT_OK(Put("qqqq", "val8")); // out of domain
3411
+ ASSERT_OK(Put("zzzz", "val8")); // out of domain
3412
+
3413
+ for (auto flushed : {false, true}) {
3414
+ SCOPED_TRACE("flushed=" + std::to_string(flushed));
3415
+ if (flushed) {
3416
+ ASSERT_OK(Flush());
3417
+ }
3418
+ ReadOptions read_options;
3419
+ if (flushed) { // TODO: support auto_prefix_mode in memtable?
3420
+ read_options.auto_prefix_mode = true;
3421
+ } else {
3422
+ // TODO: why needed?
3423
+ get_perf_context()->bloom_memtable_hit_count = 0;
3424
+ get_perf_context()->bloom_memtable_miss_count = 0;
3425
+ }
3426
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3427
+ {
3428
+ Slice ub("aaaa999");
3429
+ read_options.iterate_upper_bound = &ub;
3430
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3431
+ EXPECT_EQ(CountIter(iter, "aaaa000"), 3);
3432
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3433
+ }
3434
+ {
3435
+ // Note: prefix as seek key is not bloom-optimized
3436
+ // Note: the count works with weird_comparator because "aaaa" is
3437
+ // ordered as the last of the prefixes
3438
+ Slice ub("aaaa999");
3439
+ read_options.iterate_upper_bound = &ub;
3440
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3441
+ EXPECT_EQ(CountIter(iter, "aaaa"), 3);
3442
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3443
+ }
3444
+ {
3445
+ Slice ub("aaba9");
3446
+ read_options.iterate_upper_bound = &ub;
3447
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3448
+ EXPECT_EQ(CountIter(iter, "aaba0"), 1);
3449
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3450
+ }
3451
+ {
3452
+ Slice ub("aaca9");
3453
+ read_options.iterate_upper_bound = &ub;
3454
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3455
+ EXPECT_EQ(CountIter(iter, "aaca0"), 0);
3456
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 1));
3457
+ }
3458
+ {
3459
+ Slice ub("qqqq9");
3460
+ read_options.iterate_upper_bound = &ub;
3461
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3462
+ EXPECT_EQ(CountIter(iter, "qqqq0"), 1);
3463
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(1, 0));
3464
+ }
3465
+ {
3466
+ // Note: prefix as seek key is not bloom-optimized
3467
+ Slice ub("qqqq9");
3468
+ read_options.iterate_upper_bound = &ub;
3469
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3470
+ EXPECT_EQ(CountIter(iter, "qqqq"), weird_comparator ? 7 : 2);
3471
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3472
+ }
3473
+ {
3474
+ // Note: prefix as seek key is not bloom-optimized
3475
+ Slice ub("zzzz9");
3476
+ read_options.iterate_upper_bound = &ub;
3477
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3478
+ EXPECT_EQ(CountIter(iter, "zzzz"), weird_comparator ? 8 : 1);
3479
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3480
+ }
3481
+ {
3482
+ Slice ub("zzzz9");
3483
+ read_options.iterate_upper_bound = &ub;
3484
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
3485
+ EXPECT_EQ(CountIter(iter, "aab"), weird_comparator ? 6 : 5);
3486
+ EXPECT_EQ(GetBloomStat(options, flushed), CheckedAndUseful(0, 0));
3487
+ }
3488
+ }
3489
+ }
3490
+ }
3491
+
3116
3492
  #endif // ROCKSDB_LITE
3117
3493
 
3118
3494
  } // namespace ROCKSDB_NAMESPACE