@nxtedition/rocksdb 10.1.4 → 10.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. package/binding.cc +16 -12
  2. package/deps/rocksdb/rocksdb/CMakeLists.txt +16 -5
  3. package/deps/rocksdb/rocksdb/Makefile +38 -15
  4. package/deps/rocksdb/rocksdb/TARGETS +10 -0
  5. package/deps/rocksdb/rocksdb/cache/cache_test.cc +58 -0
  6. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +4 -4
  7. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +4 -2
  8. package/deps/rocksdb/rocksdb/db/builder.cc +2 -2
  9. package/deps/rocksdb/rocksdb/db/builder.h +1 -1
  10. package/deps/rocksdb/rocksdb/db/c.cc +205 -6
  11. package/deps/rocksdb/rocksdb/db/c_test.c +189 -1
  12. package/deps/rocksdb/rocksdb/db/column_family.cc +28 -0
  13. package/deps/rocksdb/rocksdb/db/column_family.h +17 -0
  14. package/deps/rocksdb/rocksdb/db/column_family_test.cc +234 -60
  15. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +8 -1
  16. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +11 -9
  17. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +4 -4
  18. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +2 -0
  19. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +1 -0
  20. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +22 -25
  21. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -0
  22. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +112 -0
  23. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +72 -21
  24. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -0
  25. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +77 -0
  26. package/deps/rocksdb/rocksdb/db/convenience.cc +3 -0
  27. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +269 -112
  28. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +107 -43
  29. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +93 -24
  30. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +5 -5
  31. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +157 -68
  32. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +56 -15
  33. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +78 -105
  34. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +39 -9
  35. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
  36. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +21 -14
  37. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +107 -63
  38. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +43 -2
  39. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -0
  40. package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +6 -0
  41. package/deps/rocksdb/rocksdb/db/db_test.cc +10 -2
  42. package/deps/rocksdb/rocksdb/db/db_test2.cc +1 -1
  43. package/deps/rocksdb/rocksdb/db/db_test_util.cc +5 -0
  44. package/deps/rocksdb/rocksdb/db/db_test_util.h +7 -6
  45. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +92 -2
  46. package/deps/rocksdb/rocksdb/db/error_handler.cc +34 -39
  47. package/deps/rocksdb/rocksdb/db/error_handler.h +3 -4
  48. package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +8 -4
  49. package/deps/rocksdb/rocksdb/db/event_helpers.cc +6 -3
  50. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +71 -15
  51. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +11 -0
  52. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +383 -4
  53. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +88 -72
  54. package/deps/rocksdb/rocksdb/db/flush_job.cc +30 -3
  55. package/deps/rocksdb/rocksdb/db/flush_job.h +14 -0
  56. package/deps/rocksdb/rocksdb/db/internal_stats.cc +60 -1
  57. package/deps/rocksdb/rocksdb/db/internal_stats.h +20 -1
  58. package/deps/rocksdb/rocksdb/db/log_writer.cc +24 -0
  59. package/deps/rocksdb/rocksdb/db/log_writer.h +5 -0
  60. package/deps/rocksdb/rocksdb/db/memtable.cc +6 -4
  61. package/deps/rocksdb/rocksdb/db/memtable.h +10 -10
  62. package/deps/rocksdb/rocksdb/db/memtable_list.cc +4 -4
  63. package/deps/rocksdb/rocksdb/db/multi_cf_iterator_impl.h +10 -3
  64. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +8 -10
  65. package/deps/rocksdb/rocksdb/db/repair.cc +4 -3
  66. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +30 -0
  67. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +9 -0
  68. package/deps/rocksdb/rocksdb/db/table_cache.cc +17 -2
  69. package/deps/rocksdb/rocksdb/db/table_cache.h +9 -1
  70. package/deps/rocksdb/rocksdb/db/table_properties_collector.h +9 -2
  71. package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +3 -1
  72. package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +3 -3
  73. package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +7 -7
  74. package/deps/rocksdb/rocksdb/db/version_edit.cc +0 -1
  75. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -6
  76. package/deps/rocksdb/rocksdb/db/version_set.cc +54 -31
  77. package/deps/rocksdb/rocksdb/db/version_set.h +14 -7
  78. package/deps/rocksdb/rocksdb/db/wal_manager.cc +37 -29
  79. package/deps/rocksdb/rocksdb/db/wal_manager.h +6 -5
  80. package/deps/rocksdb/rocksdb/db/wide/wide_columns_helper.cc +6 -0
  81. package/deps/rocksdb/rocksdb/db/write_batch.cc +54 -23
  82. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +46 -5
  83. package/deps/rocksdb/rocksdb/db/write_thread.cc +53 -5
  84. package/deps/rocksdb/rocksdb/db/write_thread.h +36 -4
  85. package/deps/rocksdb/rocksdb/db_stress_tool/CMakeLists.txt +1 -0
  86. package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +5 -0
  87. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +57 -17
  88. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +11 -3
  89. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +8 -4
  90. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +10 -25
  91. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +25 -88
  92. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_filters.cc +93 -0
  93. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_filters.h +16 -0
  94. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +43 -0
  95. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +109 -21
  96. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +8 -0
  97. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +666 -205
  98. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +55 -10
  99. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +18 -16
  100. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +19 -0
  101. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +5 -0
  102. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +782 -494
  103. package/deps/rocksdb/rocksdb/env/composite_env_wrapper.h +21 -0
  104. package/deps/rocksdb/rocksdb/env/env.cc +6 -0
  105. package/deps/rocksdb/rocksdb/env/io_posix.cc +0 -1
  106. package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
  107. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +34 -19
  108. package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +29 -32
  109. package/deps/rocksdb/rocksdb/file/writable_file_writer.h +41 -15
  110. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +4 -2
  111. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +63 -0
  112. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +16 -5
  113. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +5 -0
  114. package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +0 -16
  115. package/deps/rocksdb/rocksdb/include/rocksdb/iterator_base.h +16 -0
  116. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +21 -0
  117. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +76 -3
  118. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +17 -0
  119. package/deps/rocksdb/rocksdb/include/rocksdb/transaction_log.h +12 -6
  120. package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +31 -0
  121. package/deps/rocksdb/rocksdb/include/rocksdb/user_write_callback.h +29 -0
  122. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/checkpoint.h +4 -2
  123. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/customizable_util.h +0 -1
  124. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +17 -8
  125. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +2 -2
  126. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +46 -0
  127. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +7 -0
  128. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  129. package/deps/rocksdb/rocksdb/options/cf_options.cc +13 -2
  130. package/deps/rocksdb/rocksdb/options/cf_options.h +6 -2
  131. package/deps/rocksdb/rocksdb/options/db_options.cc +8 -0
  132. package/deps/rocksdb/rocksdb/options/db_options.h +9 -5
  133. package/deps/rocksdb/rocksdb/options/options.cc +3 -0
  134. package/deps/rocksdb/rocksdb/options/options_helper.cc +1 -0
  135. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +3 -1
  136. package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +2 -2
  137. package/deps/rocksdb/rocksdb/port/stack_trace.cc +1 -0
  138. package/deps/rocksdb/rocksdb/port/win/port_win.cc +3 -2
  139. package/deps/rocksdb/rocksdb/src.mk +4 -0
  140. package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +1 -2
  141. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +4 -2
  142. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +15 -0
  143. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +102 -41
  144. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +15 -7
  145. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +1 -3
  146. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -6
  147. package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +31 -0
  148. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +6 -0
  149. package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h +10 -5
  150. package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +11 -15
  151. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +17 -11
  152. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.h +5 -2
  153. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +28 -21
  154. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +9 -11
  155. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +16 -16
  156. package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +1 -2
  157. package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +14 -9
  158. package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.h +4 -1
  159. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +82 -41
  160. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +13 -14
  161. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +18 -22
  162. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +51 -13
  163. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.h +2 -0
  164. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +3 -11
  165. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +2 -3
  166. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +9 -10
  167. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +3 -2
  168. package/deps/rocksdb/rocksdb/table/format.cc +1 -2
  169. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +18 -13
  170. package/deps/rocksdb/rocksdb/table/merging_iterator.h +5 -3
  171. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +2 -2
  172. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +1 -1
  173. package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +3 -1
  174. package/deps/rocksdb/rocksdb/table/table_builder.h +8 -7
  175. package/deps/rocksdb/rocksdb/table/table_reader.h +9 -0
  176. package/deps/rocksdb/rocksdb/test_util/testutil.cc +1 -0
  177. package/deps/rocksdb/rocksdb/test_util/testutil.h +6 -0
  178. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +19 -0
  179. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +434 -110
  180. package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +3 -1
  181. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +3 -0
  182. package/deps/rocksdb/rocksdb/util/aligned_storage.h +24 -0
  183. package/deps/rocksdb/rocksdb/util/filter_bench.cc +1 -1
  184. package/deps/rocksdb/rocksdb/util/random.cc +2 -1
  185. package/deps/rocksdb/rocksdb/util/stderr_logger.h +1 -1
  186. package/deps/rocksdb/rocksdb/util/udt_util.cc +33 -0
  187. package/deps/rocksdb/rocksdb/util/udt_util.h +7 -0
  188. package/deps/rocksdb/rocksdb/util/udt_util_test.cc +33 -0
  189. package/deps/rocksdb/rocksdb/util/write_batch_util.h +5 -0
  190. package/deps/rocksdb/rocksdb/util/xxhash.h +10 -3
  191. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +13 -13
  192. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +104 -48
  193. package/deps/rocksdb/rocksdb/utilities/debug.cc +16 -4
  194. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +647 -235
  195. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -157
  196. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.cc +144 -0
  197. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.h +45 -0
  198. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector_test.cc +139 -0
  199. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +12 -0
  200. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +3 -0
  201. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +105 -6
  202. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +64 -8
  203. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -0
  204. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.cc +43 -5
  205. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +5 -0
  206. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +154 -6
  207. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +1 -1
  208. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +158 -2
  209. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +16 -11
  210. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +4 -4
  211. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +9 -8
  212. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +2 -1
  213. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +43 -7
  214. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +2 -0
  215. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +1 -1
  216. package/index.js +1 -2
  217. package/package.json +1 -1
  218. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  219. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
  220. package/util.h +25 -2
  221. package/.tap/test-results/node_modules/abstract-level/test/chained-batch-test.js.tap +0 -0
  222. package/.tap/test-results/node_modules/abstract-level/test/get-test.js.tap +0 -0
  223. package/.tap/test-results/test/abstract-level-test.js.tap +0 -1077
  224. package/.tap/test-results/test/batch-test.js.tap +0 -12
  225. package/.tap/test-results/test/chained-batch-gc-test.js.tap +0 -11
  226. package/.tap/test-results/test/cleanup-hanging-iterators-test.js.tap +0 -135
  227. package/.tap/test-results/test/clear-gc-test.js.tap +0 -13
  228. package/.tap/test-results/test/column-test.js.tap +0 -55
  229. package/.tap/test-results/test/common.js.tap +0 -0
  230. package/.tap/test-results/test/compression-test.js.tap +0 -30
  231. package/.tap/test-results/test/db-identity.js.tap +0 -12
  232. package/.tap/test-results/test/electron.js.tap +0 -0
  233. package/.tap/test-results/test/env-cleanup-hook-test.js.tap +0 -40
  234. package/.tap/test-results/test/env-cleanup-hook.js.tap +0 -0
  235. package/.tap/test-results/test/gc.js.tap +0 -0
  236. package/.tap/test-results/test/getproperty-test.js.tap +0 -29
  237. package/.tap/test-results/test/iterator-gc-test.js.tap +0 -15
  238. package/.tap/test-results/test/iterator-hwm-test.js.tap +0 -131
  239. package/.tap/test-results/test/iterator-recursion-test.js.tap +0 -12
  240. package/.tap/test-results/test/iterator-starvation-test.js.tap +0 -73
  241. package/.tap/test-results/test/iterator-test.js.tap +0 -6
  242. package/.tap/test-results/test/leak-tester-batch.js.tap +0 -0
  243. package/.tap/test-results/test/leak-tester-iterator.js.tap +0 -0
  244. package/.tap/test-results/test/leak-tester.js.tap +0 -0
  245. package/.tap/test-results/test/lock-test.js.tap +0 -18
  246. package/.tap/test-results/test/lock.js.tap +0 -0
  247. package/.tap/test-results/test/make.js.tap +0 -0
  248. package/.tap/test-results/test/max-rev-merge.js.tap +0 -0
  249. package/.tap/test-results/test/merge-operator-test.js.tap +0 -12
  250. package/.tap/test-results/test/mkdir-test.js.tap +0 -15
  251. package/.tap/test-results/test/segfault-test.js.tap +0 -76
  252. package/.tap/test-results/test/stack-blower.js.tap +0 -0
  253. package/deps/rocksdb/rocksdb/README.md +0 -29
  254. package/deps/rocksdb/rocksdb/microbench/README.md +0 -60
  255. package/deps/rocksdb/rocksdb/plugin/README.md +0 -43
  256. package/deps/rocksdb/rocksdb/port/README +0 -10
  257. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/README +0 -13
  258. package/tmp/000099.sst +0 -0
  259. package/tmp/000102.sst +0 -0
  260. package/tmp/000103.log +0 -0
  261. package/tmp/CURRENT +0 -1
  262. package/tmp/IDENTITY +0 -1
  263. package/tmp/LOCK +0 -0
  264. package/tmp/MANIFEST-000104 +0 -0
  265. package/tmp/OPTIONS-000098 +0 -207
  266. package/tmp/OPTIONS-000106 +0 -207
@@ -26,6 +26,7 @@
26
26
  #include "rocksdb/table_properties.h"
27
27
  #include "table/block_based/block_based_table_reader.h"
28
28
  #include "table/unique_id_impl.h"
29
+ #include "test_util/secondary_cache_test_util.h"
29
30
  #include "util/compression.h"
30
31
  #include "util/defer.h"
31
32
  #include "util/hash.h"
@@ -740,118 +741,6 @@ class LookupLiarCache : public CacheWrapper {
740
741
 
741
742
  } // anonymous namespace
742
743
 
743
- TEST_F(DBBlockCacheTest, AddRedundantStats) {
744
- const size_t capacity = size_t{1} << 25;
745
- const int num_shard_bits = 0; // 1 shard
746
- int iterations_tested = 0;
747
- for (const std::shared_ptr<Cache>& base_cache :
748
- {NewLRUCache(capacity, num_shard_bits),
749
- // FixedHyperClockCache
750
- HyperClockCacheOptions(
751
- capacity,
752
- BlockBasedTableOptions().block_size /*estimated_value_size*/,
753
- num_shard_bits)
754
- .MakeSharedCache(),
755
- // AutoHyperClockCache
756
- HyperClockCacheOptions(capacity, 0 /*estimated_value_size*/,
757
- num_shard_bits)
758
- .MakeSharedCache()}) {
759
- if (!base_cache) {
760
- // Skip clock cache when not supported
761
- continue;
762
- }
763
- ++iterations_tested;
764
- Options options = CurrentOptions();
765
- options.create_if_missing = true;
766
- options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
767
-
768
- std::shared_ptr<LookupLiarCache> cache =
769
- std::make_shared<LookupLiarCache>(base_cache);
770
-
771
- BlockBasedTableOptions table_options;
772
- table_options.cache_index_and_filter_blocks = true;
773
- table_options.block_cache = cache;
774
- table_options.filter_policy.reset(NewBloomFilterPolicy(50));
775
- options.table_factory.reset(NewBlockBasedTableFactory(table_options));
776
- DestroyAndReopen(options);
777
-
778
- // Create a new table.
779
- ASSERT_OK(Put("foo", "value"));
780
- ASSERT_OK(Put("bar", "value"));
781
- ASSERT_OK(Flush());
782
- ASSERT_EQ(1, NumTableFilesAtLevel(0));
783
-
784
- // Normal access filter+index+data.
785
- ASSERT_EQ("value", Get("foo"));
786
-
787
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
788
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
789
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
790
- // --------
791
- ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
792
-
793
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
794
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
795
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
796
- // --------
797
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
798
-
799
- // Againt access filter+index+data, but force redundant load+insert on index
800
- cache->SetNthLookupNotFound(2);
801
- ASSERT_EQ("value", Get("bar"));
802
-
803
- ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
804
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
805
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
806
- // --------
807
- ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
808
-
809
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
810
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
811
- ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
812
- // --------
813
- ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
814
-
815
- // Access just filter (with high probability), and force redundant
816
- // load+insert
817
- cache->SetNthLookupNotFound(1);
818
- ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
819
-
820
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
821
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
822
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
823
- // --------
824
- EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
825
-
826
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
827
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
828
- EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
829
- // --------
830
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
831
-
832
- // Access just data, forcing redundant load+insert
833
- ReadOptions read_options;
834
- std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
835
- cache->SetNthLookupNotFound(1);
836
- iter->SeekToFirst();
837
- ASSERT_TRUE(iter->Valid());
838
- ASSERT_EQ(iter->key(), "bar");
839
-
840
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
841
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
842
- EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
843
- // --------
844
- EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
845
-
846
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
847
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
848
- EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
849
- // --------
850
- EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
851
- }
852
- EXPECT_GE(iterations_tested, 1);
853
- }
854
-
855
744
  TEST_F(DBBlockCacheTest, ParanoidFileChecks) {
856
745
  Options options = CurrentOptions();
857
746
  options.create_if_missing = true;
@@ -1347,6 +1236,274 @@ TEST_F(DBBlockCacheTest, HyperClockCacheReportProblems) {
1347
1236
  EXPECT_EQ(logger->PopCounts(), (std::array<int, 3>{{0, 1, 0}}));
1348
1237
  }
1349
1238
 
1239
+ class DBBlockCacheTypeTest
1240
+ : public DBBlockCacheTest,
1241
+ public secondary_cache_test_util::WithCacheTypeParam {};
1242
+
1243
+ INSTANTIATE_TEST_CASE_P(DBBlockCacheTypeTestInstance, DBBlockCacheTypeTest,
1244
+ secondary_cache_test_util::GetTestingCacheTypes());
1245
+
1246
+ TEST_P(DBBlockCacheTypeTest, AddRedundantStats) {
1247
+ BlockBasedTableOptions table_options;
1248
+
1249
+ const size_t capacity = size_t{1} << 25;
1250
+ const int num_shard_bits = 0; // 1 shard
1251
+ estimated_value_size_ = table_options.block_size;
1252
+ std::shared_ptr<Cache> base_cache =
1253
+ NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
1254
+ Options options = CurrentOptions();
1255
+ options.create_if_missing = true;
1256
+ options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
1257
+
1258
+ std::shared_ptr<LookupLiarCache> cache =
1259
+ std::make_shared<LookupLiarCache>(base_cache);
1260
+
1261
+ table_options.cache_index_and_filter_blocks = true;
1262
+ table_options.block_cache = cache;
1263
+ table_options.filter_policy.reset(NewBloomFilterPolicy(50));
1264
+ options.table_factory.reset(NewBlockBasedTableFactory(table_options));
1265
+ DestroyAndReopen(options);
1266
+
1267
+ // Create a new table.
1268
+ ASSERT_OK(Put("foo", "value"));
1269
+ ASSERT_OK(Put("bar", "value"));
1270
+ ASSERT_OK(Flush());
1271
+ ASSERT_EQ(1, NumTableFilesAtLevel(0));
1272
+
1273
+ // Normal access filter+index+data.
1274
+ ASSERT_EQ("value", Get("foo"));
1275
+
1276
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
1277
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
1278
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
1279
+ // --------
1280
+ ASSERT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1281
+
1282
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
1283
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
1284
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
1285
+ // --------
1286
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
1287
+
1288
+ // Againt access filter+index+data, but force redundant load+insert on index
1289
+ cache->SetNthLookupNotFound(2);
1290
+ ASSERT_EQ("value", Get("bar"));
1291
+
1292
+ ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
1293
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
1294
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
1295
+ // --------
1296
+ ASSERT_EQ(4, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1297
+
1298
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
1299
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
1300
+ ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
1301
+ // --------
1302
+ ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
1303
+
1304
+ // Access just filter (with high probability), and force redundant
1305
+ // load+insert
1306
+ cache->SetNthLookupNotFound(1);
1307
+ ASSERT_EQ("NOT_FOUND", Get("this key was not added"));
1308
+
1309
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
1310
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
1311
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
1312
+ // --------
1313
+ EXPECT_EQ(5, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1314
+
1315
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
1316
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
1317
+ EXPECT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
1318
+ // --------
1319
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
1320
+
1321
+ // Access just data, forcing redundant load+insert
1322
+ ReadOptions read_options;
1323
+ std::unique_ptr<Iterator> iter{db_->NewIterator(read_options)};
1324
+ cache->SetNthLookupNotFound(1);
1325
+ iter->SeekToFirst();
1326
+ ASSERT_TRUE(iter->Valid());
1327
+ ASSERT_EQ(iter->key(), "bar");
1328
+
1329
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD));
1330
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD));
1331
+ EXPECT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD));
1332
+ // --------
1333
+ EXPECT_EQ(6, TestGetTickerCount(options, BLOCK_CACHE_ADD));
1334
+
1335
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_ADD_REDUNDANT));
1336
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_ADD_REDUNDANT));
1337
+ EXPECT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_DATA_ADD_REDUNDANT));
1338
+ // --------
1339
+ EXPECT_EQ(3, TestGetTickerCount(options, BLOCK_CACHE_ADD_REDUNDANT));
1340
+ }
1341
+
1342
+ namespace {
1343
+ std::string AltKey(int i) {
1344
+ char buf[100];
1345
+ snprintf(buf, sizeof(buf), "altkey%06d", i);
1346
+ return std::string(buf);
1347
+ }
1348
+ } // namespace
1349
+
1350
+ TEST_P(DBBlockCacheTypeTest, Uncache) {
1351
+ for (bool partitioned : {false, true}) {
1352
+ SCOPED_TRACE("partitioned=" + std::to_string(partitioned));
1353
+ for (uint32_t ua : {0, 1, 2, 10000}) {
1354
+ SCOPED_TRACE("ua=" + std::to_string(ua));
1355
+
1356
+ BlockBasedTableOptions table_options;
1357
+ Options options = CurrentOptions();
1358
+ options.uncache_aggressiveness = ua;
1359
+ options.create_if_missing = true;
1360
+ // Don't allow background operations to keep Versions referenced
1361
+ options.stats_dump_period_sec = 0;
1362
+ options.stats_persist_period_sec = 0;
1363
+ auto stats = ROCKSDB_NAMESPACE::CreateDBStatistics();
1364
+ options.statistics = stats;
1365
+
1366
+ const size_t capacity = size_t{1} << 25;
1367
+ const int num_shard_bits = 0; // 1 shard
1368
+ estimated_value_size_ = table_options.block_size;
1369
+ std::shared_ptr<Cache> cache =
1370
+ NewCache(capacity, num_shard_bits, /*strict_capacity_limit=*/false);
1371
+
1372
+ table_options.cache_index_and_filter_blocks = true;
1373
+ table_options.block_cache = cache;
1374
+ table_options.filter_policy.reset(NewBloomFilterPolicy(10));
1375
+ table_options.partition_filters = partitioned;
1376
+ table_options.index_type =
1377
+ partitioned ? BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch
1378
+ : BlockBasedTableOptions::IndexType::kBinarySearch;
1379
+ options.table_factory.reset(NewBlockBasedTableFactory(table_options));
1380
+ DestroyAndReopen(options);
1381
+
1382
+ size_t kBaselineCount = 1; // Because of entry stats collector
1383
+
1384
+ ASSERT_EQ(kBaselineCount, cache->GetOccupancyCount());
1385
+ ASSERT_EQ(0U, cache->GetUsage());
1386
+
1387
+ constexpr uint8_t kNumDataBlocks = 10;
1388
+ constexpr uint8_t kNumFiles = 3;
1389
+ for (int i = 0; i < kNumDataBlocks; i++) {
1390
+ // Force some overlap with ordering
1391
+ ASSERT_OK(Put(Key((i * 7) % kNumDataBlocks),
1392
+ Random::GetTLSInstance()->RandomBinaryString(
1393
+ static_cast<int>(table_options.block_size))));
1394
+ if (i >= kNumDataBlocks - kNumFiles) {
1395
+ ASSERT_OK(Flush());
1396
+ }
1397
+ }
1398
+ ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
1399
+
1400
+ for (int i = 0; i < kNumDataBlocks; i++) {
1401
+ ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1402
+ }
1403
+
1404
+ size_t meta_blocks_per_file = /*index & filter*/ 2U * (1U + partitioned);
1405
+ ASSERT_EQ(
1406
+ cache->GetOccupancyCount(),
1407
+ kBaselineCount + kNumDataBlocks + meta_blocks_per_file * kNumFiles);
1408
+ ASSERT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
1409
+
1410
+ // Combine into one file, making the originals obsolete
1411
+ ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
1412
+ ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
1413
+
1414
+ ASSERT_EQ(1, NumTableFilesAtLevel(1));
1415
+
1416
+ for (int i = 0; i < kNumDataBlocks; i++) {
1417
+ ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1418
+ }
1419
+
1420
+ if (ua == 0) {
1421
+ // Expect to see cache entries for new file and obsolete files
1422
+ EXPECT_EQ(cache->GetOccupancyCount(),
1423
+ kBaselineCount + kNumDataBlocks * 2U +
1424
+ meta_blocks_per_file * (kNumFiles + 1));
1425
+ EXPECT_GE(cache->GetUsage(),
1426
+ kNumDataBlocks * table_options.block_size * 2U);
1427
+ } else {
1428
+ // Expect only to see cache entries for new file
1429
+ EXPECT_EQ(cache->GetOccupancyCount(),
1430
+ kBaselineCount + kNumDataBlocks + meta_blocks_per_file);
1431
+ EXPECT_GE(cache->GetUsage(), kNumDataBlocks * table_options.block_size);
1432
+ EXPECT_LT(cache->GetUsage(),
1433
+ kNumDataBlocks * table_options.block_size * 2U);
1434
+ }
1435
+
1436
+ size_t alt_baseline_count = cache->GetOccupancyCount();
1437
+ size_t alt_baseline_usage = cache->GetUsage();
1438
+ ASSERT_OK(stats->Reset());
1439
+ // We aren't generally cleaning up cache entries on DB::Close, especially
1440
+ // because someone might just re-open the same DB.
1441
+ Reopen(options);
1442
+ for (int i = 0; i < kNumDataBlocks; i++) {
1443
+ ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1444
+ }
1445
+
1446
+ EXPECT_EQ(cache->GetOccupancyCount(), alt_baseline_count);
1447
+ EXPECT_EQ(cache->GetUsage(), alt_baseline_usage);
1448
+
1449
+ // Check for unnecessary unncessary cache churn
1450
+ ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
1451
+ ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
1452
+ ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
1453
+
1454
+ // And now do a similar test as above except with trivial moves, making
1455
+ // sure that we aren't falsely uncaching in that case, which would cause
1456
+ // unnecessary cache misses. Using AltKey instead of Key to avoid
1457
+ // interference.
1458
+ for (int i = 0; i < kNumDataBlocks; i++) {
1459
+ // No overlap
1460
+ ASSERT_OK(
1461
+ Put(AltKey(i), Random::GetTLSInstance()->RandomBinaryString(
1462
+ static_cast<int>(table_options.block_size))));
1463
+ if (i >= kNumDataBlocks - kNumFiles) {
1464
+ ASSERT_OK(Flush());
1465
+ }
1466
+ }
1467
+ ASSERT_EQ(int{kNumFiles}, NumTableFilesAtLevel(0));
1468
+
1469
+ for (int i = 0; i < kNumDataBlocks; i++) {
1470
+ ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
1471
+ }
1472
+
1473
+ ASSERT_EQ(cache->GetOccupancyCount(),
1474
+ alt_baseline_count + kNumDataBlocks +
1475
+ meta_blocks_per_file * kNumFiles);
1476
+ ASSERT_GE(cache->GetUsage(),
1477
+ alt_baseline_usage + kNumDataBlocks * table_options.block_size);
1478
+
1479
+ ASSERT_OK(stats->Reset());
1480
+
1481
+ // Make trivial move
1482
+ {
1483
+ auto a = AltKey(0);
1484
+ auto b = AltKey(kNumDataBlocks);
1485
+ Slice slice_a{a};
1486
+ Slice slice_b{b};
1487
+ ASSERT_OK(db_->CompactRange({}, &slice_a, &slice_b));
1488
+ }
1489
+ ASSERT_EQ(/*old*/ 1 + /*new*/ int{kNumFiles}, NumTableFilesAtLevel(1));
1490
+
1491
+ for (int i = 0; i < kNumDataBlocks; i++) {
1492
+ ASSERT_NE(Get(AltKey(i)), "NOT_FOUND");
1493
+ }
1494
+
1495
+ // Should be the same if trivial move
1496
+ ASSERT_EQ(cache->GetOccupancyCount(),
1497
+ alt_baseline_count + kNumDataBlocks +
1498
+ meta_blocks_per_file * kNumFiles);
1499
+
1500
+ // Check for unnecessary unncessary cache churn
1501
+ ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_ADD), 0U);
1502
+ ASSERT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), 0U);
1503
+ ASSERT_GT(stats->getTickerCount(BLOCK_CACHE_HIT), 0U);
1504
+ }
1505
+ }
1506
+ }
1350
1507
 
1351
1508
  class DBBlockCacheKeyTest
1352
1509
  : public DBTestBase,
@@ -322,6 +322,67 @@ TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
322
322
  }
323
323
  }
324
324
 
325
+ TEST_F(DBBloomFilterTest, FilterNumEntriesCoalesce) {
326
+ for (bool partition_filters : {true, false}) {
327
+ SCOPED_TRACE("partition_filters=" + std::to_string(partition_filters));
328
+ for (bool prefix : {true, false}) {
329
+ SCOPED_TRACE("prefix=" + std::to_string(prefix));
330
+ for (bool whole : {true, false}) {
331
+ SCOPED_TRACE("whole=" + std::to_string(whole));
332
+ Options options = last_options_;
333
+ options.prefix_extractor.reset();
334
+ if (prefix) {
335
+ options.prefix_extractor.reset(NewFixedPrefixTransform(3));
336
+ }
337
+ BlockBasedTableOptions bbto;
338
+ bbto.filter_policy.reset(NewBloomFilterPolicy(10));
339
+ bbto.whole_key_filtering = whole;
340
+ if (partition_filters) {
341
+ bbto.partition_filters = true;
342
+ bbto.index_type =
343
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
344
+ }
345
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
346
+ DestroyAndReopen(options);
347
+
348
+ // Need a snapshot to allow keeping multiple entries for the same key
349
+ std::vector<const Snapshot*> snapshots;
350
+ for (int i = 1; i <= 3; ++i) {
351
+ std::string val = "val" + std::to_string(i);
352
+ ASSERT_OK(Put("foo1", val));
353
+ ASSERT_OK(Put("foo2", val));
354
+ ASSERT_OK(Put("bar1", val));
355
+ ASSERT_OK(Put("bar2", val));
356
+ ASSERT_OK(Put("bar3", val));
357
+ snapshots.push_back(db_->GetSnapshot());
358
+ }
359
+ ASSERT_OK(Flush());
360
+
361
+ TablePropertiesCollection tpc;
362
+ ASSERT_OK(db_->GetPropertiesOfAllTables(&tpc));
363
+ // sanity checks
364
+ ASSERT_EQ(tpc.size(), 1U);
365
+ auto& tp = *tpc.begin()->second;
366
+ EXPECT_EQ(tp.num_entries, 3U * 5U);
367
+
368
+ // test checks
369
+ unsigned ex_filter_entries = 0;
370
+ if (whole) {
371
+ ex_filter_entries += 5; // unique keys
372
+ }
373
+ if (prefix) {
374
+ ex_filter_entries += 2; // unique prefixes
375
+ }
376
+ EXPECT_EQ(tp.num_filter_entries, ex_filter_entries);
377
+
378
+ for (auto* sn : snapshots) {
379
+ db_->ReleaseSnapshot(sn);
380
+ }
381
+ }
382
+ }
383
+ }
384
+ }
385
+
325
386
  TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
326
387
  for (bool partition_filters : {true, false}) {
327
388
  Options options = last_options_;
@@ -3143,52 +3204,55 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
3143
3204
  }
3144
3205
 
3145
3206
  TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) {
3146
- Options options = CurrentOptions();
3147
- constexpr size_t kNumKeys = 10000;
3148
- static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000");
3149
- options.memtable_factory.reset(
3150
- test::NewSpecialSkipListFactory(kNumKeys + 10));
3151
- options.create_if_missing = true;
3152
- constexpr size_t kPrefixLength = 4;
3153
- options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength));
3154
- options.compression = kNoCompression;
3155
- BlockBasedTableOptions bbto;
3156
- bbto.filter_policy.reset(NewBloomFilterPolicy(50));
3157
- bbto.index_shortening =
3158
- BlockBasedTableOptions::IndexShorteningMode::kNoShortening;
3159
- bbto.block_size = 128;
3160
- bbto.metadata_block_size = 128;
3161
- bbto.partition_filters = true;
3162
- bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
3163
- options.table_factory.reset(NewBlockBasedTableFactory(bbto));
3164
- DestroyAndReopen(options);
3207
+ for (bool wkf : {true, false}) {
3208
+ SCOPED_TRACE("whole_key_filtering=" + std::to_string(wkf));
3209
+ Options options = CurrentOptions();
3210
+ constexpr size_t kNumKeys = 10000;
3211
+ static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000");
3212
+ options.memtable_factory.reset(
3213
+ test::NewSpecialSkipListFactory(kNumKeys + 10));
3214
+ options.create_if_missing = true;
3215
+ constexpr size_t kPrefixLength = 4;
3216
+ options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength));
3217
+ options.compression = kNoCompression;
3218
+ BlockBasedTableOptions bbto;
3219
+ bbto.filter_policy.reset(NewBloomFilterPolicy(50));
3220
+ bbto.index_shortening =
3221
+ BlockBasedTableOptions::IndexShorteningMode::kNoShortening;
3222
+ bbto.block_size = 128;
3223
+ bbto.metadata_block_size = 128;
3224
+ bbto.partition_filters = true;
3225
+ bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
3226
+ bbto.whole_key_filtering = wkf;
3227
+ options.table_factory.reset(NewBlockBasedTableFactory(bbto));
3228
+ DestroyAndReopen(options);
3165
3229
 
3166
- const std::string value(64, '\0');
3230
+ const std::string value(64, '\0');
3167
3231
 
3168
- WriteOptions write_opts;
3169
- write_opts.disableWAL = true;
3170
- for (size_t i = 0; i < kNumKeys; ++i) {
3171
- std::ostringstream oss;
3172
- oss << std::setfill('0') << std::setw(4) << std::fixed << i;
3173
- ASSERT_OK(db_->Put(write_opts, oss.str(), value));
3174
- }
3175
- ASSERT_OK(Flush());
3232
+ WriteOptions write_opts;
3233
+ write_opts.disableWAL = true;
3234
+ for (size_t i = 0; i < kNumKeys; ++i) {
3235
+ std::ostringstream oss;
3236
+ oss << std::setfill('0') << std::setw(4) << std::fixed << i;
3237
+ ASSERT_OK(db_->Put(write_opts, oss.str(), value));
3238
+ }
3239
+ ASSERT_OK(Flush());
3176
3240
 
3177
- ReadOptions read_opts;
3178
- // Use legacy, implicit prefix seek
3179
- read_opts.total_order_seek = false;
3180
- read_opts.auto_prefix_mode = false;
3181
- std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
3182
- for (size_t i = 0; i < kNumKeys; ++i) {
3183
- // Seek with a key after each one added but with same prefix. One will
3184
- // surely cross a partition boundary.
3185
- std::ostringstream oss;
3186
- oss << std::setfill('0') << std::setw(4) << std::fixed << i << "a";
3187
- it->SeekForPrev(oss.str());
3188
- ASSERT_OK(it->status());
3189
- ASSERT_TRUE(it->Valid());
3190
- }
3191
- it.reset();
3241
+ ReadOptions read_opts;
3242
+ // Use legacy, implicit prefix seek
3243
+ read_opts.total_order_seek = false;
3244
+ read_opts.auto_prefix_mode = false;
3245
+ std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
3246
+ for (size_t i = 0; i < kNumKeys; ++i) {
3247
+ // Seek with a key after each one added but with same prefix. One will
3248
+ // surely cross a partition boundary.
3249
+ std::ostringstream oss;
3250
+ oss << std::setfill('0') << std::setw(4) << std::fixed << i << "a";
3251
+ it->SeekForPrev(oss.str());
3252
+ ASSERT_OK(it->status());
3253
+ ASSERT_TRUE(it->Valid());
3254
+ }
3255
+ }
3192
3256
  }
3193
3257
 
3194
3258
  namespace {