@nxtedition/rocksdb 15.4.0 → 15.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (402) hide show
  1. package/binding.cc +24 -19
  2. package/cache.js +1 -1
  3. package/chained-batch.js +12 -3
  4. package/deps/rocksdb/rocksdb/.clang-tidy +86 -0
  5. package/deps/rocksdb/rocksdb/BUCK +42 -0
  6. package/deps/rocksdb/rocksdb/CMakeLists.txt +11 -0
  7. package/deps/rocksdb/rocksdb/Makefile +59 -32
  8. package/deps/rocksdb/rocksdb/cache/cache.cc +0 -5
  9. package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +9 -9
  10. package/deps/rocksdb/rocksdb/cache/cache_key.cc +3 -3
  11. package/deps/rocksdb/rocksdb/cache/cache_key.h +5 -5
  12. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +16 -16
  13. package/deps/rocksdb/rocksdb/cache/cache_test.cc +1 -1
  14. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +258 -294
  15. package/deps/rocksdb/rocksdb/cache/clock_cache.h +98 -49
  16. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -5
  17. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +2 -3
  18. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +18 -18
  19. package/deps/rocksdb/rocksdb/crash_test.mk +5 -1
  20. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -22
  21. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -1
  22. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +14 -16
  23. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +38 -26
  24. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +5 -1
  25. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +101 -18
  26. package/deps/rocksdb/rocksdb/db/blob/blob_index.h +12 -0
  27. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +6 -9
  28. package/deps/rocksdb/rocksdb/db/builder.cc +23 -0
  29. package/deps/rocksdb/rocksdb/db/builder.h +7 -0
  30. package/deps/rocksdb/rocksdb/db/c.cc +373 -57
  31. package/deps/rocksdb/rocksdb/db/c_test.c +101 -1
  32. package/deps/rocksdb/rocksdb/db/column_family.cc +31 -3
  33. package/deps/rocksdb/rocksdb/db/column_family_test.cc +10 -13
  34. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +35 -48
  35. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +13 -5
  36. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +201 -39
  37. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +15 -10
  38. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +7 -7
  39. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -455
  40. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +4 -2
  41. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +19 -0
  42. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +72 -9
  43. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -10
  44. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +405 -83
  45. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +25 -1
  46. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +23 -10
  47. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -0
  48. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +1410 -106
  49. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -5
  50. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +2 -1
  51. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +19 -10
  52. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +505 -45
  53. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
  54. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +9 -1
  55. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +4 -4
  56. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +7 -9
  57. package/deps/rocksdb/rocksdb/db/convenience.cc +4 -4
  58. package/deps/rocksdb/rocksdb/db/convenience_impl.h +2 -1
  59. package/deps/rocksdb/rocksdb/db/corruption_test.cc +60 -88
  60. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +10 -12
  61. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +471 -40
  62. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +116 -2
  63. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +5 -15
  64. package/deps/rocksdb/rocksdb/db/db_compaction_abort_test.cc +993 -0
  65. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +329 -29
  66. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +155 -13
  67. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +54 -31
  68. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -0
  69. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +232 -70
  70. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +57 -9
  71. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +224 -31
  72. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
  73. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +4 -2
  74. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +1 -1
  75. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
  76. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +164 -8
  77. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +6 -0
  78. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +5 -0
  79. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +47 -35
  80. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +22 -9
  81. package/deps/rocksdb/rocksdb/db/db_iter.cc +9 -0
  82. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +371 -6
  83. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -5
  84. package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +22 -23
  85. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +0 -2
  86. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +4 -4
  87. package/deps/rocksdb/rocksdb/db/db_options_test.cc +40 -0
  88. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +32 -13
  89. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +1 -1
  90. package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +4 -4
  91. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +68 -15
  92. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +1 -1
  93. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +2 -3
  94. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -21
  95. package/deps/rocksdb/rocksdb/db/db_test.cc +644 -128
  96. package/deps/rocksdb/rocksdb/db/db_test2.cc +198 -81
  97. package/deps/rocksdb/rocksdb/db/db_test_util.cc +35 -10
  98. package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -2
  99. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +36 -32
  100. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +11 -7
  101. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +499 -0
  102. package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +284 -20
  103. package/deps/rocksdb/rocksdb/db/db_write_test.cc +3 -3
  104. package/deps/rocksdb/rocksdb/db/dbformat.h +0 -5
  105. package/deps/rocksdb/rocksdb/db/error_handler.cc +24 -0
  106. package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +12 -14
  107. package/deps/rocksdb/rocksdb/db/experimental.cc +13 -10
  108. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +1 -1
  109. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +22 -3
  110. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +21 -15
  111. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +4 -6
  112. package/deps/rocksdb/rocksdb/db/flush_job.cc +11 -3
  113. package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +5 -6
  114. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +4 -2
  115. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +17 -17
  116. package/deps/rocksdb/rocksdb/db/internal_stats.cc +13 -0
  117. package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -0
  118. package/deps/rocksdb/rocksdb/db/listener_test.cc +154 -27
  119. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +6 -6
  120. package/deps/rocksdb/rocksdb/db/memtable.cc +197 -51
  121. package/deps/rocksdb/rocksdb/db/memtable.h +6 -0
  122. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +3 -4
  123. package/deps/rocksdb/rocksdb/db/merge_test.cc +37 -35
  124. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +2 -1
  125. package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
  126. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +9 -11
  127. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +10 -1
  128. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +292 -15
  129. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +10 -17
  130. package/deps/rocksdb/rocksdb/db/prefix_test.cc +6 -8
  131. package/deps/rocksdb/rocksdb/db/repair.cc +10 -10
  132. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +5 -5
  133. package/deps/rocksdb/rocksdb/db/table_cache.cc +142 -135
  134. package/deps/rocksdb/rocksdb/db/table_cache.h +30 -6
  135. package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +7 -7
  136. package/deps/rocksdb/rocksdb/db/version_builder.cc +11 -50
  137. package/deps/rocksdb/rocksdb/db/version_builder.h +2 -1
  138. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +2 -1
  139. package/deps/rocksdb/rocksdb/db/version_edit.cc +51 -2
  140. package/deps/rocksdb/rocksdb/db/version_edit.h +91 -29
  141. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -7
  142. package/deps/rocksdb/rocksdb/db/version_set.cc +211 -50
  143. package/deps/rocksdb/rocksdb/db/version_set.h +40 -3
  144. package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +5 -0
  145. package/deps/rocksdb/rocksdb/db/version_set_test.cc +294 -21
  146. package/deps/rocksdb/rocksdb/db/version_util.cc +96 -0
  147. package/deps/rocksdb/rocksdb/db/version_util.h +24 -0
  148. package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +5 -5
  149. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +647 -31
  150. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +219 -1
  151. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +549 -12
  152. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -3
  153. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +1 -1
  154. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -0
  155. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +21 -4
  156. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +32 -0
  157. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +74 -22
  158. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +9 -0
  159. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -61
  160. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +15 -2
  161. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +76 -2
  162. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +92 -72
  163. package/deps/rocksdb/rocksdb/env/env.cc +1 -0
  164. package/deps/rocksdb/rocksdb/env/env_test.cc +365 -2
  165. package/deps/rocksdb/rocksdb/env/fs_posix.cc +31 -30
  166. package/deps/rocksdb/rocksdb/env/io_posix.cc +8 -11
  167. package/deps/rocksdb/rocksdb/env/io_posix.h +30 -1
  168. package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
  169. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +1 -1
  170. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +108 -0
  171. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +32 -4
  172. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +4 -4
  173. package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
  174. package/deps/rocksdb/rocksdb/file/file_util.h +2 -1
  175. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +331 -12
  176. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +52 -35
  177. package/deps/rocksdb/rocksdb/folly.mk +22 -5
  178. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +1 -1
  179. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +100 -54
  180. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +67 -2
  181. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +149 -13
  182. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -12
  183. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +78 -97
  184. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +3 -3
  185. package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +2 -2
  186. package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +5 -0
  187. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +17 -2
  188. package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +1 -1
  189. package/deps/rocksdb/rocksdb/include/rocksdb/io_dispatcher.h +358 -0
  190. package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +13 -0
  191. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +43 -0
  192. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +20 -0
  193. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +63 -21
  194. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +10 -1
  195. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +1 -1
  196. package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +2 -7
  197. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +13 -0
  198. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +3 -14
  199. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +49 -9
  200. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +8 -0
  201. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +77 -6
  202. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +15 -0
  203. package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +16 -10
  204. package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +5 -5
  205. package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +2 -4
  206. package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +106 -46
  207. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +1 -1
  208. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +14 -1
  209. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +5 -1
  210. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +2 -1
  211. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +7 -9
  212. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  213. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +1 -2
  214. package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +2 -2
  215. package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +226 -8
  216. package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +490 -0
  217. package/deps/rocksdb/rocksdb/memtable/skiplist.h +3 -3
  218. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +11 -0
  219. package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +4 -12
  220. package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +5 -5
  221. package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +21 -4
  222. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +9 -3
  223. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +21 -2
  224. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +2 -2
  225. package/deps/rocksdb/rocksdb/options/cf_options.cc +21 -1
  226. package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
  227. package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -2
  228. package/deps/rocksdb/rocksdb/options/db_options.cc +26 -5
  229. package/deps/rocksdb/rocksdb/options/db_options.h +3 -1
  230. package/deps/rocksdb/rocksdb/options/options.cc +5 -1
  231. package/deps/rocksdb/rocksdb/options/options_helper.cc +7 -2
  232. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +109 -103
  233. package/deps/rocksdb/rocksdb/options/options_test.cc +14 -0
  234. package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +15 -17
  235. package/deps/rocksdb/rocksdb/port/lang.h +4 -0
  236. package/deps/rocksdb/rocksdb/port/port_example.h +0 -23
  237. package/deps/rocksdb/rocksdb/port/stack_trace.cc +36 -0
  238. package/deps/rocksdb/rocksdb/port/stack_trace.h +9 -0
  239. package/deps/rocksdb/rocksdb/src.mk +12 -0
  240. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -2
  241. package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
  242. package/deps/rocksdb/rocksdb/table/block_based/block.cc +571 -292
  243. package/deps/rocksdb/rocksdb/table/block_based/block.h +143 -53
  244. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +154 -90
  245. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +5 -1
  246. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +51 -14
  247. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +0 -2
  248. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +147 -734
  249. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +30 -233
  250. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +178 -108
  251. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +13 -0
  252. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +17 -4
  253. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -2
  254. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +70 -0
  255. package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +168 -24
  256. package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +25 -9
  257. package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +7 -4
  258. package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +9 -2
  259. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +548 -169
  260. package/deps/rocksdb/rocksdb/table/block_based/block_type.h +30 -0
  261. package/deps/rocksdb/rocksdb/table/block_based/block_util.h +156 -0
  262. package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.cc +73 -30
  263. package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.h +74 -7
  264. package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index.h +1 -1
  265. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +20 -14
  266. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +22 -12
  267. package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +1 -1
  268. package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.cc +332 -0
  269. package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.h +133 -0
  270. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -2
  271. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
  272. package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +3 -2
  273. package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -1
  274. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +0 -1
  275. package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +126 -46
  276. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +31 -3
  277. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +1 -2
  278. package/deps/rocksdb/rocksdb/table/cleanable_test.cc +3 -1
  279. package/deps/rocksdb/rocksdb/table/external_table.cc +25 -4
  280. package/deps/rocksdb/rocksdb/table/format.cc +27 -15
  281. package/deps/rocksdb/rocksdb/table/format.h +41 -15
  282. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -0
  283. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +22 -12
  284. package/deps/rocksdb/rocksdb/table/meta_blocks.h +0 -1
  285. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +7 -21
  286. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
  287. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +88 -13
  288. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +53 -42
  289. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +3 -12
  290. package/deps/rocksdb/rocksdb/table/table_builder.h +0 -4
  291. package/deps/rocksdb/rocksdb/table/table_properties.cc +18 -0
  292. package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +2 -3
  293. package/deps/rocksdb/rocksdb/table/table_test.cc +848 -172
  294. package/deps/rocksdb/rocksdb/table/unique_id.cc +24 -20
  295. package/deps/rocksdb/rocksdb/table/unique_id_impl.h +8 -8
  296. package/deps/rocksdb/rocksdb/test_util/sync_point.h +5 -4
  297. package/deps/rocksdb/rocksdb/test_util/testutil.cc +2 -1
  298. package/deps/rocksdb/rocksdb/test_util/testutil.h +2 -2
  299. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +2 -1
  300. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +238 -120
  301. package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +2 -2
  302. package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +2 -4
  303. package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +4 -8
  304. package/deps/rocksdb/rocksdb/tools/dump/rocksdb_undump.cc +1 -1
  305. package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +2 -3
  306. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +82 -20
  307. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +41 -47
  308. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -0
  309. package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +5 -6
  310. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +1 -1
  311. package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +6 -5
  312. package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +4 -4
  313. package/deps/rocksdb/rocksdb/tools/write_stress.cc +1 -3
  314. package/deps/rocksdb/rocksdb/util/atomic.h +30 -23
  315. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +6 -7
  316. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +3 -3
  317. package/deps/rocksdb/rocksdb/util/bit_fields.h +68 -46
  318. package/deps/rocksdb/rocksdb/util/bloom_impl.h +16 -16
  319. package/deps/rocksdb/rocksdb/util/coding.h +14 -27
  320. package/deps/rocksdb/rocksdb/util/compression.cc +365 -207
  321. package/deps/rocksdb/rocksdb/util/compression.h +16 -1298
  322. package/deps/rocksdb/rocksdb/util/compression_test.cc +347 -61
  323. package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +8 -9
  324. package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +1 -1
  325. package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -1
  326. package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -3
  327. package/deps/rocksdb/rocksdb/util/filter_bench.cc +18 -18
  328. package/deps/rocksdb/rocksdb/util/gflags_compat.h +3 -3
  329. package/deps/rocksdb/rocksdb/util/hash_test.cc +19 -7
  330. package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.cc +1099 -0
  331. package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.h +36 -0
  332. package/deps/rocksdb/rocksdb/util/io_dispatcher_test.cc +1919 -0
  333. package/deps/rocksdb/rocksdb/util/math.h +3 -1
  334. package/deps/rocksdb/rocksdb/util/mutexlock.h +19 -19
  335. package/deps/rocksdb/rocksdb/util/ribbon_alg.h +25 -25
  336. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +5 -7
  337. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -5
  338. package/deps/rocksdb/rocksdb/util/slice.cc +0 -10
  339. package/deps/rocksdb/rocksdb/util/slice_test.cc +35 -1
  340. package/deps/rocksdb/rocksdb/util/slice_transform_test.cc +5 -7
  341. package/deps/rocksdb/rocksdb/util/status.cc +3 -1
  342. package/deps/rocksdb/rocksdb/util/stop_watch.h +2 -0
  343. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -1
  344. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -78
  345. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +12 -93
  346. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +1 -4
  347. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +0 -21
  348. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +6 -48
  349. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +94 -307
  350. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +12 -58
  351. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +2 -8
  352. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +2 -3
  353. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +205 -811
  354. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +18 -9
  355. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +2 -7
  356. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +1 -9
  357. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +17 -11
  358. package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +1 -1
  359. package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.h +1 -1
  360. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
  361. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +68 -61
  362. package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -1
  363. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +105 -59
  364. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -7
  365. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs_test.cc +94 -0
  366. package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +13 -17
  367. package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +16 -3
  368. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +25 -25
  369. package/deps/rocksdb/rocksdb/utilities/object_registry.cc +40 -40
  370. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +2 -5
  371. package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +17 -19
  372. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +2 -2
  373. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +2 -2
  374. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +1 -1
  375. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -2
  376. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +4 -13
  377. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +3 -3
  378. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
  379. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_seqno_test.cc +431 -0
  380. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +1 -2
  381. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +91 -0
  382. package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.cc +562 -0
  383. package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.h +615 -0
  384. package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.cc +2575 -0
  385. package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.h +685 -0
  386. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_db_test.cc +2843 -0
  387. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.cc +567 -0
  388. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.h +275 -0
  389. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_test.cc +5183 -0
  390. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +4 -3
  391. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +1 -1
  392. package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +2 -2
  393. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +3 -3
  394. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +93 -88
  395. package/deps/rocksdb/rocksdb.gyp +7 -0
  396. package/index.js +11 -2
  397. package/iterator.js +15 -7
  398. package/package.json +1 -1
  399. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  400. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
  401. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h +0 -43
  402. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h +0 -55
@@ -84,6 +84,7 @@ class BlobDBTest : public testing::Test {
84
84
  options.stats_dump_period_sec = 0;
85
85
  options.stats_persist_period_sec = 0;
86
86
  }
87
+ bdb_options_ = bdb_options;
87
88
  return BlobDB::Open(options, bdb_options, dbname_, &blob_db_);
88
89
  }
89
90
 
@@ -109,17 +110,16 @@ class BlobDBTest : public testing::Test {
109
110
  void Destroy() {
110
111
  if (blob_db_) {
111
112
  Options options = blob_db_->GetOptions();
112
- BlobDBOptions bdb_options = blob_db_->GetBlobDBOptions();
113
113
  delete blob_db_;
114
114
  blob_db_ = nullptr;
115
- ASSERT_OK(DestroyBlobDB(dbname_, options, bdb_options));
115
+ ASSERT_OK(DestroyBlobDB(dbname_, options, bdb_options_));
116
116
  }
117
117
  }
118
118
 
119
- BlobDBImpl *blob_db_impl() { return static_cast<BlobDBImpl *>(blob_db_); }
119
+ BlobDBImpl* blob_db_impl() { return static_cast<BlobDBImpl*>(blob_db_); }
120
120
 
121
- Status Put(const Slice &key, const Slice &value,
122
- std::map<std::string, std::string> *data = nullptr) {
121
+ Status Put(const Slice& key, const Slice& value,
122
+ std::map<std::string, std::string>* data = nullptr) {
123
123
  Status s = blob_db_->Put(WriteOptions(), key, value);
124
124
  if (data != nullptr) {
125
125
  (*data)[key.ToString()] = value.ToString();
@@ -127,16 +127,16 @@ class BlobDBTest : public testing::Test {
127
127
  return s;
128
128
  }
129
129
 
130
- void Delete(const std::string &key,
131
- std::map<std::string, std::string> *data = nullptr) {
130
+ void Delete(const std::string& key,
131
+ std::map<std::string, std::string>* data = nullptr) {
132
132
  ASSERT_OK(blob_db_->Delete(WriteOptions(), key));
133
133
  if (data != nullptr) {
134
134
  data->erase(key);
135
135
  }
136
136
  }
137
137
 
138
- Status PutWithTTL(const Slice &key, const Slice &value, uint64_t ttl,
139
- std::map<std::string, std::string> *data = nullptr) {
138
+ Status PutWithTTL(const Slice& key, const Slice& value, uint64_t ttl,
139
+ std::map<std::string, std::string>* data = nullptr) {
140
140
  Status s = blob_db_->PutWithTTL(WriteOptions(), key, value, ttl);
141
141
  if (data != nullptr) {
142
142
  (*data)[key.ToString()] = value.ToString();
@@ -144,12 +144,8 @@ class BlobDBTest : public testing::Test {
144
144
  return s;
145
145
  }
146
146
 
147
- Status PutUntil(const Slice &key, const Slice &value, uint64_t expiration) {
148
- return blob_db_->PutUntil(WriteOptions(), key, value, expiration);
149
- }
150
-
151
- void PutRandomWithTTL(const std::string &key, uint64_t ttl, Random *rnd,
152
- std::map<std::string, std::string> *data = nullptr) {
147
+ void PutRandomWithTTL(const std::string& key, uint64_t ttl, Random* rnd,
148
+ std::map<std::string, std::string>* data = nullptr) {
153
149
  int len = rnd->Next() % kMaxBlobSize + 1;
154
150
  std::string value = rnd->HumanReadableString(len);
155
151
  ASSERT_OK(
@@ -159,24 +155,13 @@ class BlobDBTest : public testing::Test {
159
155
  }
160
156
  }
161
157
 
162
- void PutRandomUntil(const std::string &key, uint64_t expiration, Random *rnd,
163
- std::map<std::string, std::string> *data = nullptr) {
164
- int len = rnd->Next() % kMaxBlobSize + 1;
165
- std::string value = rnd->HumanReadableString(len);
166
- ASSERT_OK(blob_db_->PutUntil(WriteOptions(), Slice(key), Slice(value),
167
- expiration));
168
- if (data != nullptr) {
169
- (*data)[key] = value;
170
- }
171
- }
172
-
173
- void PutRandom(const std::string &key, Random *rnd,
174
- std::map<std::string, std::string> *data = nullptr) {
158
+ void PutRandom(const std::string& key, Random* rnd,
159
+ std::map<std::string, std::string>* data = nullptr) {
175
160
  PutRandom(blob_db_, key, rnd, data);
176
161
  }
177
162
 
178
- void PutRandom(DB *db, const std::string &key, Random *rnd,
179
- std::map<std::string, std::string> *data = nullptr) {
163
+ void PutRandom(DB* db, const std::string& key, Random* rnd,
164
+ std::map<std::string, std::string>* data = nullptr) {
180
165
  int len = rnd->Next() % kMaxBlobSize + 1;
181
166
  std::string value = rnd->HumanReadableString(len);
182
167
  ASSERT_OK(db->Put(WriteOptions(), Slice(key), Slice(value)));
@@ -186,8 +171,8 @@ class BlobDBTest : public testing::Test {
186
171
  }
187
172
 
188
173
  void PutRandomToWriteBatch(
189
- const std::string &key, Random *rnd, WriteBatch *batch,
190
- std::map<std::string, std::string> *data = nullptr) {
174
+ const std::string& key, Random* rnd, WriteBatch* batch,
175
+ std::map<std::string, std::string>* data = nullptr) {
191
176
  int len = rnd->Next() % kMaxBlobSize + 1;
192
177
  std::string value = rnd->HumanReadableString(len);
193
178
  ASSERT_OK(batch->Put(key, value));
@@ -197,14 +182,14 @@ class BlobDBTest : public testing::Test {
197
182
  }
198
183
 
199
184
  // Verify blob db contain expected data and nothing more.
200
- void VerifyDB(const std::map<std::string, std::string> &data) {
185
+ void VerifyDB(const std::map<std::string, std::string>& data) {
201
186
  VerifyDB(blob_db_, data);
202
187
  }
203
188
 
204
- void VerifyDB(DB *db, const std::map<std::string, std::string> &data) {
189
+ void VerifyDB(DB* db, const std::map<std::string, std::string>& data) {
205
190
  // Verify normal Get
206
- auto *cfh = db->DefaultColumnFamily();
207
- for (auto &p : data) {
191
+ auto* cfh = db->DefaultColumnFamily();
192
+ for (auto& p : data) {
208
193
  PinnableSlice value_slice;
209
194
  ASSERT_OK(db->Get(ReadOptions(), cfh, p.first, &value_slice));
210
195
  ASSERT_EQ(p.second, value_slice.ToString());
@@ -214,9 +199,9 @@ class BlobDBTest : public testing::Test {
214
199
  }
215
200
 
216
201
  // Verify iterators
217
- Iterator *iter = db->NewIterator(ReadOptions());
202
+ Iterator* iter = db->NewIterator(ReadOptions());
218
203
  iter->SeekToFirst();
219
- for (auto &p : data) {
204
+ for (auto& p : data) {
220
205
  ASSERT_TRUE(iter->Valid());
221
206
  ASSERT_EQ(p.first, iter->key().ToString());
222
207
  ASSERT_EQ(p.second, iter->value().ToString());
@@ -228,16 +213,16 @@ class BlobDBTest : public testing::Test {
228
213
  }
229
214
 
230
215
  void VerifyBaseDB(
231
- const std::map<std::string, KeyVersion> &expected_versions) {
232
- auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
233
- DB *db = blob_db_->GetRootDB();
216
+ const std::map<std::string, KeyVersion>& expected_versions) {
217
+ auto* bdb_impl = static_cast<BlobDBImpl*>(blob_db_);
218
+ DB* db = blob_db_->GetRootDB();
234
219
  const size_t kMaxKeys = 10000;
235
220
  std::vector<KeyVersion> versions;
236
221
  ASSERT_OK(GetAllKeyVersions(db, {}, {}, kMaxKeys, &versions));
237
222
  ASSERT_EQ(expected_versions.size(), versions.size());
238
223
  size_t i = 0;
239
- for (auto &key_version : expected_versions) {
240
- const KeyVersion &expected_version = key_version.second;
224
+ for (auto& key_version : expected_versions) {
225
+ const KeyVersion& expected_version = key_version.second;
241
226
  ASSERT_EQ(expected_version.user_key, versions[i].user_key);
242
227
  ASSERT_EQ(expected_version.sequence, versions[i].sequence);
243
228
  ASSERT_EQ(expected_version.type, versions[i].type);
@@ -255,7 +240,7 @@ class BlobDBTest : public testing::Test {
255
240
  }
256
241
 
257
242
  void VerifyBaseDBBlobIndex(
258
- const std::map<std::string, BlobIndexVersion> &expected_versions) {
243
+ const std::map<std::string, BlobIndexVersion>& expected_versions) {
259
244
  const size_t kMaxKeys = 10000;
260
245
  std::vector<KeyVersion> versions;
261
246
  ASSERT_OK(
@@ -263,8 +248,8 @@ class BlobDBTest : public testing::Test {
263
248
  ASSERT_EQ(versions.size(), expected_versions.size());
264
249
 
265
250
  size_t i = 0;
266
- for (const auto &expected_pair : expected_versions) {
267
- const BlobIndexVersion &expected_version = expected_pair.second;
251
+ for (const auto& expected_pair : expected_versions) {
252
+ const BlobIndexVersion& expected_version = expected_pair.second;
268
253
 
269
254
  ASSERT_EQ(versions[i].user_key, expected_version.user_key);
270
255
  ASSERT_EQ(versions[i].sequence, expected_version.sequence);
@@ -280,10 +265,7 @@ class BlobDBTest : public testing::Test {
280
265
  BlobIndex blob_index;
281
266
  ASSERT_OK(blob_index.DecodeFrom(versions[i].value));
282
267
 
283
- const uint64_t file_number = !blob_index.IsInlined()
284
- ? blob_index.file_number()
285
- : kInvalidBlobFileNumber;
286
- ASSERT_EQ(file_number, expected_version.file_number);
268
+ ASSERT_EQ(blob_index.file_number(), expected_version.file_number);
287
269
 
288
270
  const uint64_t expiration =
289
271
  blob_index.HasTTL() ? blob_index.expiration() : kNoExpiration;
@@ -312,13 +294,13 @@ class BlobDBTest : public testing::Test {
312
294
  std::shared_ptr<MockSystemClock> mock_clock_;
313
295
  std::unique_ptr<Env> mock_env_;
314
296
  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env_;
315
- BlobDB *blob_db_;
297
+ BlobDB* blob_db_;
298
+ BlobDBOptions bdb_options_;
316
299
  }; // class BlobDBTest
317
300
 
318
301
  TEST_F(BlobDBTest, Put) {
319
302
  Random rnd(301);
320
303
  BlobDBOptions bdb_options;
321
- bdb_options.min_blob_size = 0;
322
304
  bdb_options.disable_background_tasks = true;
323
305
  Open(bdb_options);
324
306
  std::map<std::string, std::string> data;
@@ -334,8 +316,6 @@ TEST_F(BlobDBTest, PutWithTTL) {
334
316
  options.env = mock_env_.get();
335
317
  BlobDBOptions bdb_options;
336
318
  bdb_options.ttl_range_secs = 1000;
337
- bdb_options.min_blob_size = 0;
338
- bdb_options.blob_file_size = 256 * 1000 * 1000;
339
319
  bdb_options.disable_background_tasks = true;
340
320
  Open(bdb_options, options);
341
321
  std::map<std::string, std::string> data;
@@ -346,33 +326,7 @@ TEST_F(BlobDBTest, PutWithTTL) {
346
326
  (ttl <= 50 ? nullptr : &data));
347
327
  }
348
328
  mock_clock_->SetCurrentTime(100);
349
- auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
350
- auto blob_files = bdb_impl->TEST_GetBlobFiles();
351
- ASSERT_EQ(1, blob_files.size());
352
- ASSERT_TRUE(blob_files[0]->HasTTL());
353
- ASSERT_OK(bdb_impl->TEST_CloseBlobFile(blob_files[0]));
354
- VerifyDB(data);
355
- }
356
-
357
- TEST_F(BlobDBTest, PutUntil) {
358
- Random rnd(301);
359
- Options options;
360
- options.env = mock_env_.get();
361
- BlobDBOptions bdb_options;
362
- bdb_options.ttl_range_secs = 1000;
363
- bdb_options.min_blob_size = 0;
364
- bdb_options.blob_file_size = 256 * 1000 * 1000;
365
- bdb_options.disable_background_tasks = true;
366
- Open(bdb_options, options);
367
- std::map<std::string, std::string> data;
368
- mock_clock_->SetCurrentTime(50);
369
- for (size_t i = 0; i < 100; i++) {
370
- uint64_t expiration = rnd.Next() % 100 + 50;
371
- PutRandomUntil("key" + std::to_string(i), expiration, &rnd,
372
- (expiration <= 100 ? nullptr : &data));
373
- }
374
- mock_clock_->SetCurrentTime(100);
375
- auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
329
+ auto* bdb_impl = static_cast<BlobDBImpl*>(blob_db_);
376
330
  auto blob_files = bdb_impl->TEST_GetBlobFiles();
377
331
  ASSERT_EQ(1, blob_files.size());
378
332
  ASSERT_TRUE(blob_files[0]->HasTTL());
@@ -383,7 +337,6 @@ TEST_F(BlobDBTest, PutUntil) {
383
337
  TEST_F(BlobDBTest, StackableDBGet) {
384
338
  Random rnd(301);
385
339
  BlobDBOptions bdb_options;
386
- bdb_options.min_blob_size = 0;
387
340
  bdb_options.disable_background_tasks = true;
388
341
  Open(bdb_options);
389
342
  std::map<std::string, std::string> data;
@@ -391,8 +344,8 @@ TEST_F(BlobDBTest, StackableDBGet) {
391
344
  PutRandom("key" + std::to_string(i), &rnd, &data);
392
345
  }
393
346
  for (size_t i = 0; i < 100; i++) {
394
- StackableDB *db = blob_db_;
395
- ColumnFamilyHandle *column_family = db->DefaultColumnFamily();
347
+ StackableDB* db = blob_db_;
348
+ ColumnFamilyHandle* column_family = db->DefaultColumnFamily();
396
349
  std::string key = "key" + std::to_string(i);
397
350
  PinnableSlice pinnable_value;
398
351
  ASSERT_OK(db->Get(ReadOptions(), column_family, key, &pinnable_value));
@@ -426,10 +379,9 @@ TEST_F(BlobDBTest, GetIOError) {
426
379
  Options options;
427
380
  options.env = fault_injection_env_.get();
428
381
  BlobDBOptions bdb_options;
429
- bdb_options.min_blob_size = 0; // Make sure value write to blob file
430
382
  bdb_options.disable_background_tasks = true;
431
383
  Open(bdb_options, options);
432
- ColumnFamilyHandle *column_family = blob_db_->DefaultColumnFamily();
384
+ ColumnFamilyHandle* column_family = blob_db_->DefaultColumnFamily();
433
385
  PinnableSlice value;
434
386
  ASSERT_OK(Put("foo", "bar"));
435
387
  fault_injection_env_->SetFilesystemActive(false, Status::IOError());
@@ -443,7 +395,6 @@ TEST_F(BlobDBTest, PutIOError) {
443
395
  Options options;
444
396
  options.env = fault_injection_env_.get();
445
397
  BlobDBOptions bdb_options;
446
- bdb_options.min_blob_size = 0; // Make sure value write to blob file
447
398
  bdb_options.disable_background_tasks = true;
448
399
  Open(bdb_options, options);
449
400
  fault_injection_env_->SetFilesystemActive(false, Status::IOError());
@@ -455,7 +406,6 @@ TEST_F(BlobDBTest, PutIOError) {
455
406
  TEST_F(BlobDBTest, WriteBatch) {
456
407
  Random rnd(301);
457
408
  BlobDBOptions bdb_options;
458
- bdb_options.min_blob_size = 0;
459
409
  bdb_options.disable_background_tasks = true;
460
410
  Open(bdb_options);
461
411
  std::map<std::string, std::string> data;
@@ -474,7 +424,6 @@ TEST_F(BlobDBTest, WriteBatch) {
474
424
  TEST_F(BlobDBTest, Delete) {
475
425
  Random rnd(301);
476
426
  BlobDBOptions bdb_options;
477
- bdb_options.min_blob_size = 0;
478
427
  bdb_options.disable_background_tasks = true;
479
428
  Open(bdb_options);
480
429
  std::map<std::string, std::string> data;
@@ -490,7 +439,6 @@ TEST_F(BlobDBTest, Delete) {
490
439
  TEST_F(BlobDBTest, DeleteBatch) {
491
440
  Random rnd(301);
492
441
  BlobDBOptions bdb_options;
493
- bdb_options.min_blob_size = 0;
494
442
  bdb_options.disable_background_tasks = true;
495
443
  Open(bdb_options);
496
444
  for (size_t i = 0; i < 100; i++) {
@@ -508,7 +456,6 @@ TEST_F(BlobDBTest, DeleteBatch) {
508
456
  TEST_F(BlobDBTest, Override) {
509
457
  Random rnd(301);
510
458
  BlobDBOptions bdb_options;
511
- bdb_options.min_blob_size = 0;
512
459
  bdb_options.disable_background_tasks = true;
513
460
  Open(bdb_options);
514
461
  std::map<std::string, std::string> data;
@@ -522,208 +469,6 @@ TEST_F(BlobDBTest, Override) {
522
469
  VerifyDB(data);
523
470
  }
524
471
 
525
- #ifdef SNAPPY
526
- TEST_F(BlobDBTest, Compression) {
527
- Random rnd(301);
528
- BlobDBOptions bdb_options;
529
- bdb_options.min_blob_size = 0;
530
- bdb_options.disable_background_tasks = true;
531
- bdb_options.compression = CompressionType::kSnappyCompression;
532
- Open(bdb_options);
533
- std::map<std::string, std::string> data;
534
- for (size_t i = 0; i < 100; i++) {
535
- PutRandom("put-key" + std::to_string(i), &rnd, &data);
536
- }
537
- for (int i = 0; i < 100; i++) {
538
- WriteBatch batch;
539
- for (size_t j = 0; j < 10; j++) {
540
- PutRandomToWriteBatch("write-batch-key" + std::to_string(j * 100 + i),
541
- &rnd, &batch, &data);
542
- }
543
- ASSERT_OK(blob_db_->Write(WriteOptions(), &batch));
544
- }
545
- VerifyDB(data);
546
- }
547
-
548
- TEST_F(BlobDBTest, DecompressAfterReopen) {
549
- Random rnd(301);
550
- BlobDBOptions bdb_options;
551
- bdb_options.min_blob_size = 0;
552
- bdb_options.disable_background_tasks = true;
553
- bdb_options.compression = CompressionType::kSnappyCompression;
554
- Open(bdb_options);
555
- std::map<std::string, std::string> data;
556
- for (size_t i = 0; i < 100; i++) {
557
- PutRandom("put-key" + std::to_string(i), &rnd, &data);
558
- }
559
- VerifyDB(data);
560
- bdb_options.compression = CompressionType::kNoCompression;
561
- Reopen(bdb_options);
562
- VerifyDB(data);
563
- }
564
-
565
- TEST_F(BlobDBTest, EnableDisableCompressionGC) {
566
- Random rnd(301);
567
- BlobDBOptions bdb_options;
568
- bdb_options.min_blob_size = 0;
569
- bdb_options.garbage_collection_cutoff = 1.0;
570
- bdb_options.disable_background_tasks = true;
571
- bdb_options.compression = kSnappyCompression;
572
- Open(bdb_options);
573
- std::map<std::string, std::string> data;
574
- size_t data_idx = 0;
575
- for (; data_idx < 100; data_idx++) {
576
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
577
- }
578
- VerifyDB(data);
579
- auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
580
- ASSERT_EQ(1, blob_files.size());
581
- ASSERT_EQ(kSnappyCompression, blob_files[0]->GetCompressionType());
582
-
583
- // disable compression
584
- bdb_options.compression = kNoCompression;
585
- Reopen(bdb_options);
586
-
587
- // Add more data with new compression type
588
- for (; data_idx < 200; data_idx++) {
589
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
590
- }
591
- VerifyDB(data);
592
-
593
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
594
- ASSERT_EQ(2, blob_files.size());
595
- ASSERT_EQ(kNoCompression, blob_files[1]->GetCompressionType());
596
-
597
- // Enable GC. If we do it earlier the snapshot release triggered compaction
598
- // may compact files and trigger GC before we can verify there are two files.
599
- bdb_options.enable_garbage_collection = true;
600
- Reopen(bdb_options);
601
-
602
- // Trigger compaction
603
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
604
- blob_db_impl()->TEST_DeleteObsoleteFiles();
605
- VerifyDB(data);
606
-
607
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
608
- for (const auto &bfile : blob_files) {
609
- ASSERT_EQ(kNoCompression, bfile->GetCompressionType());
610
- }
611
-
612
- // enabling the compression again
613
- bdb_options.compression = kSnappyCompression;
614
- Reopen(bdb_options);
615
-
616
- // Add more data with new compression type
617
- for (; data_idx < 300; data_idx++) {
618
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
619
- }
620
- VerifyDB(data);
621
-
622
- // Trigger compaction
623
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
624
- blob_db_impl()->TEST_DeleteObsoleteFiles();
625
- VerifyDB(data);
626
-
627
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
628
- for (const auto &bfile : blob_files) {
629
- ASSERT_EQ(kSnappyCompression, bfile->GetCompressionType());
630
- }
631
- }
632
-
633
- #ifdef LZ4
634
- // Test switch compression types and run GC, it needs both Snappy and LZ4
635
- // support.
636
- TEST_F(BlobDBTest, ChangeCompressionGC) {
637
- Random rnd(301);
638
- BlobDBOptions bdb_options;
639
- bdb_options.min_blob_size = 0;
640
- bdb_options.garbage_collection_cutoff = 1.0;
641
- bdb_options.disable_background_tasks = true;
642
- bdb_options.compression = kLZ4Compression;
643
- Open(bdb_options);
644
- std::map<std::string, std::string> data;
645
- size_t data_idx = 0;
646
- for (; data_idx < 100; data_idx++) {
647
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
648
- }
649
- VerifyDB(data);
650
- auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
651
- ASSERT_EQ(1, blob_files.size());
652
- ASSERT_EQ(kLZ4Compression, blob_files[0]->GetCompressionType());
653
-
654
- // Change compression type
655
- bdb_options.compression = kSnappyCompression;
656
- Reopen(bdb_options);
657
-
658
- // Add more data with Snappy compression type
659
- for (; data_idx < 200; data_idx++) {
660
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
661
- }
662
- VerifyDB(data);
663
-
664
- // Verify blob file compression type
665
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
666
- ASSERT_EQ(2, blob_files.size());
667
- ASSERT_EQ(kSnappyCompression, blob_files[1]->GetCompressionType());
668
-
669
- // Enable GC. If we do it earlier the snapshot release triggered compaction
670
- // may compact files and trigger GC before we can verify there are two files.
671
- bdb_options.enable_garbage_collection = true;
672
- Reopen(bdb_options);
673
-
674
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
675
- VerifyDB(data);
676
-
677
- blob_db_impl()->TEST_DeleteObsoleteFiles();
678
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
679
- for (const auto &bfile : blob_files) {
680
- ASSERT_EQ(kSnappyCompression, bfile->GetCompressionType());
681
- }
682
-
683
- // Disable compression
684
- bdb_options.compression = kNoCompression;
685
- Reopen(bdb_options);
686
- for (; data_idx < 300; data_idx++) {
687
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
688
- }
689
- VerifyDB(data);
690
-
691
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
692
- VerifyDB(data);
693
-
694
- blob_db_impl()->TEST_DeleteObsoleteFiles();
695
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
696
- for (const auto &bfile : blob_files) {
697
- ASSERT_EQ(kNoCompression, bfile->GetCompressionType());
698
- }
699
-
700
- // switching different compression types to generate mixed compression types
701
- bdb_options.compression = kSnappyCompression;
702
- Reopen(bdb_options);
703
- for (; data_idx < 400; data_idx++) {
704
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
705
- }
706
- VerifyDB(data);
707
-
708
- bdb_options.compression = kLZ4Compression;
709
- Reopen(bdb_options);
710
- for (; data_idx < 500; data_idx++) {
711
- PutRandom("put-key" + std::to_string(data_idx), &rnd, &data);
712
- }
713
- VerifyDB(data);
714
-
715
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
716
- VerifyDB(data);
717
-
718
- blob_db_impl()->TEST_DeleteObsoleteFiles();
719
- blob_files = blob_db_impl()->TEST_GetBlobFiles();
720
- for (const auto &bfile : blob_files) {
721
- ASSERT_EQ(kLZ4Compression, bfile->GetCompressionType());
722
- }
723
- }
724
- #endif // LZ4
725
- #endif // SNAPPY
726
-
727
472
  TEST_F(BlobDBTest, MultipleWriters) {
728
473
  Open(BlobDBOptions());
729
474
 
@@ -760,21 +505,19 @@ TEST_F(BlobDBTest, SstFileManager) {
760
505
  std::shared_ptr<SstFileManager> sst_file_manager(
761
506
  NewSstFileManager(mock_env_.get()));
762
507
  sst_file_manager->SetDeleteRateBytesPerSecond(1024 * 1024);
763
- SstFileManagerImpl *sfm =
764
- static_cast<SstFileManagerImpl *>(sst_file_manager.get());
508
+ SstFileManagerImpl* sfm =
509
+ static_cast<SstFileManagerImpl*>(sst_file_manager.get());
765
510
 
766
511
  BlobDBOptions bdb_options;
767
- bdb_options.min_blob_size = 0;
768
512
  bdb_options.enable_garbage_collection = true;
769
- bdb_options.garbage_collection_cutoff = 1.0;
770
513
  Options db_options;
771
514
 
772
515
  int files_scheduled_to_delete = 0;
773
516
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
774
- "SstFileManagerImpl::ScheduleFileDeletion", [&](void *arg) {
517
+ "SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) {
775
518
  assert(arg);
776
- const std::string *const file_path =
777
- static_cast<const std::string *>(arg);
519
+ const std::string* const file_path =
520
+ static_cast<const std::string*>(arg);
778
521
  if (file_path->find(".blob") != std::string::npos) {
779
522
  ++files_scheduled_to_delete;
780
523
  }
@@ -784,12 +527,22 @@ TEST_F(BlobDBTest, SstFileManager) {
784
527
 
785
528
  Open(bdb_options, db_options);
786
529
 
787
- // Create one obselete file and clean it.
530
+ // Create 4 blob files. With GC cutoff of 0.25, the oldest file (file 1)
531
+ // will be in the GC zone: floor(0.25 * 4) = 1.
788
532
  ASSERT_OK(blob_db_->Put(WriteOptions(), "foo", "bar"));
789
533
  auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
790
534
  ASSERT_EQ(1, blob_files.size());
791
535
  std::shared_ptr<BlobFile> bfile = blob_files[0];
792
536
  ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(bfile));
537
+
538
+ // Create 3 more blob files (files 2-4, outside GC zone).
539
+ for (int i = 1; i < 4; i++) {
540
+ ASSERT_OK(blob_db_->Put(WriteOptions(), "key" + std::to_string(i), "val"));
541
+ blob_files = blob_db_impl()->TEST_GetBlobFiles();
542
+ ASSERT_EQ(static_cast<size_t>(i + 1), blob_files.size());
543
+ ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[i]));
544
+ }
545
+
793
546
  ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
794
547
  blob_db_impl()->TEST_DeleteObsoleteFiles();
795
548
 
@@ -797,7 +550,8 @@ TEST_F(BlobDBTest, SstFileManager) {
797
550
  ASSERT_EQ(1, files_scheduled_to_delete);
798
551
  Destroy();
799
552
  // Make sure that DestroyBlobDB() also goes through delete scheduler.
800
- ASSERT_EQ(2, files_scheduled_to_delete);
553
+ // Remaining files: 3 original (files 2-4) + 1 GC output file = 4 files.
554
+ ASSERT_EQ(5, files_scheduled_to_delete);
801
555
  SyncPoint::GetInstance()->DisableProcessing();
802
556
  sfm->WaitForEmptyTrash();
803
557
  }
@@ -805,10 +559,10 @@ TEST_F(BlobDBTest, SstFileManager) {
805
559
  TEST_F(BlobDBTest, SstFileManagerRestart) {
806
560
  int files_scheduled_to_delete = 0;
807
561
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
808
- "SstFileManagerImpl::ScheduleFileDeletion", [&](void *arg) {
562
+ "SstFileManagerImpl::ScheduleFileDeletion", [&](void* arg) {
809
563
  assert(arg);
810
- const std::string *const file_path =
811
- static_cast<const std::string *>(arg);
564
+ const std::string* const file_path =
565
+ static_cast<const std::string*>(arg);
812
566
  if (file_path->find(".blob") != std::string::npos) {
813
567
  ++files_scheduled_to_delete;
814
568
  }
@@ -818,11 +572,10 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
818
572
  std::shared_ptr<SstFileManager> sst_file_manager(
819
573
  NewSstFileManager(mock_env_.get()));
820
574
  sst_file_manager->SetDeleteRateBytesPerSecond(1024 * 1024);
821
- SstFileManagerImpl *sfm =
822
- static_cast<SstFileManagerImpl *>(sst_file_manager.get());
575
+ SstFileManagerImpl* sfm =
576
+ static_cast<SstFileManagerImpl*>(sst_file_manager.get());
823
577
 
824
578
  BlobDBOptions bdb_options;
825
- bdb_options.min_blob_size = 0;
826
579
  Options db_options;
827
580
 
828
581
  SyncPoint::GetInstance()->EnableProcessing();
@@ -834,7 +587,7 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
834
587
  Close();
835
588
 
836
589
  // Create 3 dummy trash files under the blob_dir
837
- const auto &fs = db_options.env->GetFileSystem();
590
+ const auto& fs = db_options.env->GetFileSystem();
838
591
  ASSERT_OK(CreateFile(fs, blob_dir + "/000666.blob.trash", "", false));
839
592
  ASSERT_OK(CreateFile(fs, blob_dir + "/000888.blob.trash", "", true));
840
593
  ASSERT_OK(CreateFile(fs, blob_dir + "/something_not_match.trash", "", false));
@@ -849,7 +602,7 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
849
602
  std::vector<std::string> all_files;
850
603
  ASSERT_OK(db_options.env->GetChildren(blob_dir, &all_files));
851
604
  int nfiles = 0;
852
- for (const auto &f : all_files) {
605
+ for (const auto& f : all_files) {
853
606
  assert(!f.empty());
854
607
  if (f[0] == '.') {
855
608
  continue;
@@ -863,22 +616,28 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
863
616
 
864
617
  TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
865
618
  BlobDBOptions bdb_options;
866
- bdb_options.min_blob_size = 0;
867
619
  bdb_options.enable_garbage_collection = true;
868
- bdb_options.garbage_collection_cutoff = 1.0;
869
620
  bdb_options.disable_background_tasks = true;
870
621
 
871
622
  Options options;
872
623
  options.disable_auto_compactions = true;
873
624
 
874
- // i = when to take snapshot
625
+ // This test verifies that snapshots protect blob files from deletion during
626
+ // garbage collection. With fixed GC cutoff of 0.25 and 8 immutable files,
627
+ // floor(0.25 * 8) = 2 files are in the GC zone (files 1 and 2).
628
+ //
629
+ // We run 4 iterations with different snapshot timing:
630
+ // i=0: snapshot after key1 (before key2) - protects file 1
631
+ // i=1: snapshot after key2 (before key3) - protects files 1 and 2
632
+ // i=2: snapshot after key9 (after all keys) - no protection needed
633
+ // i=3: snapshot after Delete(key2) - no protection needed
875
634
  for (int i = 0; i < 4; i++) {
876
635
  Destroy();
877
636
  Open(bdb_options, options);
878
637
 
879
- const Snapshot *snapshot = nullptr;
638
+ const Snapshot* snapshot = nullptr;
880
639
 
881
- // First file
640
+ // Create first blob file (will be in GC zone).
882
641
  ASSERT_OK(Put("key1", "value"));
883
642
  if (i == 0) {
884
643
  snapshot = blob_db_->GetSnapshot();
@@ -888,7 +647,8 @@ TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
888
647
  ASSERT_EQ(1, blob_files.size());
889
648
  ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[0]));
890
649
 
891
- // Second file
650
+ // Create second blob file (will be in GC zone). We track this file
651
+ // to verify it becomes obsolete after GC relocates its blob.
892
652
  ASSERT_OK(Put("key2", "value"));
893
653
  if (i == 1) {
894
654
  snapshot = blob_db_->GetSnapshot();
@@ -896,39 +656,66 @@ TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
896
656
 
897
657
  blob_files = blob_db_impl()->TEST_GetBlobFiles();
898
658
  ASSERT_EQ(2, blob_files.size());
899
- auto bfile = blob_files[1];
900
- ASSERT_FALSE(bfile->Immutable());
901
- ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(bfile));
659
+ auto gc_target_file = blob_files[1];
660
+ ASSERT_FALSE(gc_target_file->Immutable());
661
+ ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(gc_target_file));
662
+
663
+ // Create files 3-8, all closed (these are outside GC zone).
664
+ for (int j = 3; j <= 8; j++) {
665
+ ASSERT_OK(Put("key" + std::to_string(j), "value"));
666
+ blob_files = blob_db_impl()->TEST_GetBlobFiles();
667
+ ASSERT_EQ(static_cast<size_t>(j), blob_files.size());
668
+ ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[j - 1]));
669
+ }
902
670
 
903
- // Third file
904
- ASSERT_OK(Put("key3", "value"));
671
+ // Create file 9 but leave it open (mutable). Only immutable files are
672
+ // counted for GC cutoff calculation.
673
+ ASSERT_OK(Put("key9", "value"));
905
674
  if (i == 2) {
906
675
  snapshot = blob_db_->GetSnapshot();
907
676
  }
908
677
 
678
+ // Verify we have 9 total files (8 immutable + 1 mutable).
679
+ blob_files = blob_db_impl()->TEST_GetBlobFiles();
680
+ ASSERT_EQ(9, blob_files.size());
681
+
682
+ // Trigger GC via compaction. Blobs in files 1 and 2 will be relocated
683
+ // to a new GC output file.
909
684
  ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
910
- ASSERT_TRUE(bfile->Obsolete());
685
+
686
+ // Verify gc_target_file (file 2) is now obsolete.
687
+ ASSERT_TRUE(gc_target_file->Obsolete());
688
+ // Verify the obsolete sequence matches the latest sequence number.
911
689
  ASSERT_EQ(blob_db_->GetLatestSequenceNumber(),
912
- bfile->GetObsoleteSequence());
690
+ gc_target_file->GetObsoleteSequence());
913
691
 
914
692
  Delete("key2");
915
693
  if (i == 3) {
916
694
  snapshot = blob_db_->GetSnapshot();
917
695
  }
918
696
 
919
- ASSERT_EQ(4, blob_db_impl()->TEST_GetBlobFiles().size());
697
+ // Verify we now have 10 files (9 original + 1 GC output file).
698
+ // Files 1 and 2 are obsolete but not yet deleted.
699
+ ASSERT_EQ(10, blob_db_impl()->TEST_GetBlobFiles().size());
920
700
  blob_db_impl()->TEST_DeleteObsoleteFiles();
921
701
 
922
702
  if (i >= 2) {
923
- // The snapshot shouldn't see data in bfile
924
- ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
703
+ // Snapshot was taken after all keys were written, so it sees the
704
+ // post-compaction state where blob indexes point to the GC output file.
705
+ // Obsolete files 1 and 2 can be deleted immediately.
706
+ // Verify 8 files remain (10 - 2 obsolete files deleted).
707
+ ASSERT_EQ(8, blob_db_impl()->TEST_GetBlobFiles().size());
925
708
  blob_db_->ReleaseSnapshot(snapshot);
926
709
  } else {
927
- // The snapshot will see data in bfile, so the file shouldn't be deleted
928
- ASSERT_EQ(4, blob_db_impl()->TEST_GetBlobFiles().size());
710
+ // Snapshot was taken before compaction completed, so it may still
711
+ // reference blobs in the obsolete files. Files cannot be deleted.
712
+ // Verify all 10 files still exist.
713
+ ASSERT_EQ(10, blob_db_impl()->TEST_GetBlobFiles().size());
929
714
  blob_db_->ReleaseSnapshot(snapshot);
930
715
  blob_db_impl()->TEST_DeleteObsoleteFiles();
931
- ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
716
+ // After releasing the snapshot, obsolete files can be deleted.
717
+ // Verify 8 files remain.
718
+ ASSERT_EQ(8, blob_db_impl()->TEST_GetBlobFiles().size());
932
719
  }
933
720
  }
934
721
  }
@@ -938,8 +725,8 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
938
725
  options.env = mock_env_.get();
939
726
  mock_clock_->SetCurrentTime(0);
940
727
  Open(BlobDBOptions(), options);
941
- ColumnFamilyHandle *default_handle = blob_db_->DefaultColumnFamily();
942
- ColumnFamilyHandle *handle = nullptr;
728
+ ColumnFamilyHandle* default_handle = blob_db_->DefaultColumnFamily();
729
+ ColumnFamilyHandle* handle = nullptr;
943
730
  std::string value;
944
731
  std::vector<std::string> values;
945
732
  // The call simply pass through to base db. It should succeed.
@@ -948,8 +735,6 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
948
735
  ASSERT_TRUE(blob_db_->Put(WriteOptions(), handle, "k", "v").IsNotSupported());
949
736
  ASSERT_TRUE(blob_db_->PutWithTTL(WriteOptions(), handle, "k", "v", 60)
950
737
  .IsNotSupported());
951
- ASSERT_TRUE(blob_db_->PutUntil(WriteOptions(), handle, "k", "v", 100)
952
- .IsNotSupported());
953
738
  WriteBatch batch;
954
739
  ASSERT_OK(batch.Put("k1", "v1"));
955
740
  ASSERT_OK(batch.Put(handle, "k2", "v2"));
@@ -970,10 +755,7 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
970
755
  Random rnd(301);
971
756
 
972
757
  BlobDBOptions bdb_options;
973
- bdb_options.blob_dir = "blob_dir";
974
- bdb_options.path_relative = true;
975
758
  bdb_options.ttl_range_secs = 10;
976
- bdb_options.min_blob_size = 0;
977
759
  bdb_options.disable_background_tasks = true;
978
760
 
979
761
  Options options;
@@ -986,8 +768,9 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
986
768
  PutRandom("key" + std::to_string(i), &rnd, &data);
987
769
  }
988
770
 
989
- constexpr uint64_t expiration = 1000ULL;
990
- PutRandomUntil("key100", expiration, &rnd, &data);
771
+ // At time 0, the stored expiration equals TTL
772
+ constexpr uint64_t ttl = 1000ULL;
773
+ PutRandomWithTTL("key100", ttl, &rnd, &data);
991
774
 
992
775
  std::vector<LiveFileMetaData> metadata;
993
776
  blob_db_->GetLiveFilesMetaData(&metadata);
@@ -1003,7 +786,7 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
1003
786
  const std::string filename2("/blob_dir/000002.blob");
1004
787
  ASSERT_EQ(filename2, metadata[1].name);
1005
788
  ASSERT_EQ(2, metadata[1].file_number);
1006
- ASSERT_EQ(expiration, metadata[1].oldest_ancester_time);
789
+ ASSERT_EQ(ttl, metadata[1].oldest_ancester_time);
1007
790
  ASSERT_EQ(kDefaultColumnFamilyName, metadata[1].column_family_name);
1008
791
 
1009
792
  std::vector<std::string> livefile;
@@ -1046,16 +829,15 @@ TEST_F(BlobDBTest, MigrateFromPlainRocksDB) {
1046
829
  // Write to plain rocksdb.
1047
830
  Options options;
1048
831
  options.create_if_missing = true;
1049
- DB *db = nullptr;
832
+ std::unique_ptr<DB> db;
1050
833
  ASSERT_OK(DB::Open(options, dbname_, &db));
1051
834
  for (size_t i = 0; i < kNumIteration; i++) {
1052
835
  auto key_index = rnd.Next() % kNumKey;
1053
836
  std::string key = "key" + std::to_string(key_index);
1054
- PutRandom(db, key, &rnd, &data);
837
+ PutRandom(db.get(), key, &rnd, &data);
1055
838
  }
1056
- VerifyDB(db, data);
1057
- delete db;
1058
- db = nullptr;
839
+ VerifyDB(db.get(), data);
840
+ db.reset();
1059
841
 
1060
842
  // Open as blob db. Verify it can read existing data.
1061
843
  Open();
@@ -1085,7 +867,6 @@ TEST_F(BlobDBTest, MigrateFromPlainRocksDB) {
1085
867
  ASSERT_EQ(data[key], value);
1086
868
  }
1087
869
  }
1088
- delete db;
1089
870
  }
1090
871
 
1091
872
  // Test to verify that a NoSpace IOError Status is returned on reaching
@@ -1096,7 +877,6 @@ TEST_F(BlobDBTest, OutOfSpace) {
1096
877
  options.env = mock_env_.get();
1097
878
  BlobDBOptions bdb_options;
1098
879
  bdb_options.max_db_size = 200;
1099
- bdb_options.is_fifo = false;
1100
880
  bdb_options.disable_background_tasks = true;
1101
881
  Open(bdb_options);
1102
882
 
@@ -1112,256 +892,13 @@ TEST_F(BlobDBTest, OutOfSpace) {
1112
892
  ASSERT_TRUE(s.IsNoSpace());
1113
893
  }
1114
894
 
1115
- TEST_F(BlobDBTest, FIFOEviction) {
1116
- BlobDBOptions bdb_options;
1117
- bdb_options.max_db_size = 200;
1118
- bdb_options.blob_file_size = 100;
1119
- bdb_options.is_fifo = true;
1120
- bdb_options.disable_background_tasks = true;
1121
- Open(bdb_options);
1122
-
1123
- std::atomic<int> evict_count{0};
1124
- SyncPoint::GetInstance()->SetCallBack(
1125
- "BlobDBImpl::EvictOldestBlobFile:Evicted",
1126
- [&](void *) { evict_count++; });
1127
- SyncPoint::GetInstance()->EnableProcessing();
1128
-
1129
- // Each stored blob has an overhead of 32 bytes currently.
1130
- // So a 100 byte blob should take up 132 bytes.
1131
- std::string value(100, 'v');
1132
- ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), "key1", value, 10));
1133
- VerifyDB({{"key1", value}});
1134
-
1135
- ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
1136
-
1137
- // Adding another 100 bytes blob would take the total size to 264 bytes
1138
- // (2*132). max_db_size will be exceeded
1139
- // than max_db_size and trigger FIFO eviction.
1140
- ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), "key2", value, 60));
1141
- ASSERT_EQ(1, evict_count);
1142
- // key1 will exist until corresponding file be deleted.
1143
- VerifyDB({{"key1", value}, {"key2", value}});
1144
-
1145
- // Adding another 100 bytes blob without TTL.
1146
- ASSERT_OK(blob_db_->Put(WriteOptions(), "key3", value));
1147
- ASSERT_EQ(2, evict_count);
1148
- // key1 and key2 will exist until corresponding file be deleted.
1149
- VerifyDB({{"key1", value}, {"key2", value}, {"key3", value}});
1150
-
1151
- // The fourth blob file, without TTL.
1152
- ASSERT_OK(blob_db_->Put(WriteOptions(), "key4", value));
1153
- ASSERT_EQ(3, evict_count);
1154
- VerifyDB(
1155
- {{"key1", value}, {"key2", value}, {"key3", value}, {"key4", value}});
1156
-
1157
- auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
1158
- ASSERT_EQ(4, blob_files.size());
1159
- ASSERT_TRUE(blob_files[0]->Obsolete());
1160
- ASSERT_TRUE(blob_files[1]->Obsolete());
1161
- ASSERT_TRUE(blob_files[2]->Obsolete());
1162
- ASSERT_FALSE(blob_files[3]->Obsolete());
1163
- auto obsolete_files = blob_db_impl()->TEST_GetObsoleteFiles();
1164
- ASSERT_EQ(3, obsolete_files.size());
1165
- ASSERT_EQ(blob_files[0], obsolete_files[0]);
1166
- ASSERT_EQ(blob_files[1], obsolete_files[1]);
1167
- ASSERT_EQ(blob_files[2], obsolete_files[2]);
1168
-
1169
- blob_db_impl()->TEST_DeleteObsoleteFiles();
1170
- obsolete_files = blob_db_impl()->TEST_GetObsoleteFiles();
1171
- ASSERT_TRUE(obsolete_files.empty());
1172
- VerifyDB({{"key4", value}});
1173
- }
1174
-
1175
- TEST_F(BlobDBTest, FIFOEviction_NoOldestFileToEvict) {
1176
- Options options;
1177
- BlobDBOptions bdb_options;
1178
- bdb_options.max_db_size = 1000;
1179
- bdb_options.blob_file_size = 5000;
1180
- bdb_options.is_fifo = true;
1181
- bdb_options.disable_background_tasks = true;
1182
- Open(bdb_options);
1183
-
1184
- std::atomic<int> evict_count{0};
1185
- SyncPoint::GetInstance()->SetCallBack(
1186
- "BlobDBImpl::EvictOldestBlobFile:Evicted",
1187
- [&](void *) { evict_count++; });
1188
- SyncPoint::GetInstance()->EnableProcessing();
1189
-
1190
- std::string value(2000, 'v');
1191
- ASSERT_TRUE(Put("foo", std::string(2000, 'v')).IsNoSpace());
1192
- ASSERT_EQ(0, evict_count);
1193
- }
1194
-
1195
- TEST_F(BlobDBTest, FIFOEviction_NoEnoughBlobFilesToEvict) {
1196
- BlobDBOptions bdb_options;
1197
- bdb_options.is_fifo = true;
1198
- bdb_options.min_blob_size = 100;
1199
- bdb_options.disable_background_tasks = true;
1200
- Options options;
1201
- // Use mock env to stop wall clock.
1202
- options.env = mock_env_.get();
1203
- options.disable_auto_compactions = true;
1204
- auto statistics = CreateDBStatistics();
1205
- options.statistics = statistics;
1206
- Open(bdb_options, options);
1207
-
1208
- SyncPoint::GetInstance()->LoadDependency(
1209
- {{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
1210
- "BlobDBTest.FIFOEviction_NoEnoughBlobFilesToEvict:AfterFlush"}});
1211
-
1212
- SyncPoint::GetInstance()->EnableProcessing();
1213
-
1214
- ASSERT_EQ(0, blob_db_impl()->TEST_live_sst_size());
1215
- std::string small_value(50, 'v');
1216
- std::map<std::string, std::string> data;
1217
- // Insert some data into LSM tree to make sure FIFO eviction take SST
1218
- // file size into account.
1219
- for (int i = 0; i < 1000; i++) {
1220
- ASSERT_OK(Put("key" + std::to_string(i), small_value, &data));
1221
- }
1222
- ASSERT_OK(blob_db_->Flush(FlushOptions()));
1223
-
1224
- uint64_t live_sst_size = 0;
1225
- ASSERT_TRUE(blob_db_->GetIntProperty(DB::Properties::kTotalSstFilesSize,
1226
- &live_sst_size));
1227
- ASSERT_TRUE(live_sst_size > 0);
1228
-
1229
- TEST_SYNC_POINT(
1230
- "BlobDBTest.FIFOEviction_NoEnoughBlobFilesToEvict:AfterFlush");
1231
-
1232
- ASSERT_EQ(live_sst_size, blob_db_impl()->TEST_live_sst_size());
1233
-
1234
- bdb_options.max_db_size = live_sst_size + 2000;
1235
- Reopen(bdb_options, options);
1236
- ASSERT_EQ(live_sst_size, blob_db_impl()->TEST_live_sst_size());
1237
-
1238
- std::string value_1k(1000, 'v');
1239
- ASSERT_OK(PutWithTTL("large_key1", value_1k, 60, &data));
1240
- ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1241
- VerifyDB(data);
1242
- // large_key2 evicts large_key1
1243
- ASSERT_OK(PutWithTTL("large_key2", value_1k, 60, &data));
1244
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1245
- blob_db_impl()->TEST_DeleteObsoleteFiles();
1246
- data.erase("large_key1");
1247
- VerifyDB(data);
1248
- // large_key3 get no enough space even after evicting large_key2, so it
1249
- // instead return no space error.
1250
- std::string value_2k(2000, 'v');
1251
- ASSERT_TRUE(PutWithTTL("large_key3", value_2k, 60).IsNoSpace());
1252
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1253
- // Verify large_key2 still exists.
1254
- VerifyDB(data);
1255
-
1256
- SyncPoint::GetInstance()->DisableProcessing();
1257
- }
1258
-
1259
- // Test flush or compaction will trigger FIFO eviction since they update
1260
- // total SST file size.
1261
- TEST_F(BlobDBTest, FIFOEviction_TriggerOnSSTSizeChange) {
1262
- BlobDBOptions bdb_options;
1263
- bdb_options.max_db_size = 1000;
1264
- bdb_options.is_fifo = true;
1265
- bdb_options.min_blob_size = 100;
1266
- bdb_options.disable_background_tasks = true;
1267
- Options options;
1268
- // Use mock env to stop wall clock.
1269
- options.env = mock_env_.get();
1270
- auto statistics = CreateDBStatistics();
1271
- options.statistics = statistics;
1272
- options.compression = kNoCompression;
1273
- Open(bdb_options, options);
1274
-
1275
- SyncPoint::GetInstance()->LoadDependency(
1276
- {{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
1277
- "BlobDBTest.FIFOEviction_TriggerOnSSTSizeChange:AfterFlush"}});
1278
-
1279
- SyncPoint::GetInstance()->EnableProcessing();
1280
-
1281
- std::string value(800, 'v');
1282
- ASSERT_OK(PutWithTTL("large_key", value, 60));
1283
- ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
1284
- ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1285
- VerifyDB({{"large_key", value}});
1286
-
1287
- // Insert some small keys and flush to bring DB out of space.
1288
- std::map<std::string, std::string> data;
1289
- for (int i = 0; i < 10; i++) {
1290
- ASSERT_OK(Put("key" + std::to_string(i), "v", &data));
1291
- }
1292
- ASSERT_OK(blob_db_->Flush(FlushOptions()));
1293
-
1294
- TEST_SYNC_POINT("BlobDBTest.FIFOEviction_TriggerOnSSTSizeChange:AfterFlush");
1295
-
1296
- // Verify large_key is deleted by FIFO eviction.
1297
- blob_db_impl()->TEST_DeleteObsoleteFiles();
1298
- ASSERT_EQ(0, blob_db_impl()->TEST_GetBlobFiles().size());
1299
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1300
- VerifyDB(data);
1301
-
1302
- SyncPoint::GetInstance()->DisableProcessing();
1303
- }
1304
-
1305
- TEST_F(BlobDBTest, InlineSmallValues) {
1306
- constexpr uint64_t kMaxExpiration = 1000;
1307
- Random rnd(301);
1308
- BlobDBOptions bdb_options;
1309
- bdb_options.ttl_range_secs = kMaxExpiration;
1310
- bdb_options.min_blob_size = 100;
1311
- bdb_options.blob_file_size = 256 * 1000 * 1000;
1312
- bdb_options.disable_background_tasks = true;
1313
- Options options;
1314
- options.env = mock_env_.get();
1315
- mock_clock_->SetCurrentTime(0);
1316
- Open(bdb_options, options);
1317
- std::map<std::string, std::string> data;
1318
- std::map<std::string, KeyVersion> versions;
1319
- for (size_t i = 0; i < 1000; i++) {
1320
- bool is_small_value = rnd.Next() % 2;
1321
- bool has_ttl = rnd.Next() % 2;
1322
- uint64_t expiration = rnd.Next() % kMaxExpiration;
1323
- int len = is_small_value ? 50 : 200;
1324
- std::string key = "key" + std::to_string(i);
1325
- std::string value = rnd.HumanReadableString(len);
1326
- std::string blob_index;
1327
- data[key] = value;
1328
- SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
1329
- if (!has_ttl) {
1330
- ASSERT_OK(blob_db_->Put(WriteOptions(), key, value));
1331
- } else {
1332
- ASSERT_OK(blob_db_->PutUntil(WriteOptions(), key, value, expiration));
1333
- }
1334
- ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
1335
- versions[key] =
1336
- KeyVersion(key, value, sequence,
1337
- (is_small_value && !has_ttl) ? kTypeValue : kTypeBlobIndex);
1338
- }
1339
- VerifyDB(data);
1340
- VerifyBaseDB(versions);
1341
- auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
1342
- auto blob_files = bdb_impl->TEST_GetBlobFiles();
1343
- ASSERT_EQ(2, blob_files.size());
1344
- std::shared_ptr<BlobFile> non_ttl_file;
1345
- std::shared_ptr<BlobFile> ttl_file;
1346
- if (blob_files[0]->HasTTL()) {
1347
- ttl_file = blob_files[0];
1348
- non_ttl_file = blob_files[1];
1349
- } else {
1350
- non_ttl_file = blob_files[0];
1351
- ttl_file = blob_files[1];
1352
- }
1353
- ASSERT_FALSE(non_ttl_file->HasTTL());
1354
- ASSERT_TRUE(ttl_file->HasTTL());
1355
- }
1356
-
1357
895
  TEST_F(BlobDBTest, UserCompactionFilter) {
1358
896
  class CustomerFilter : public CompactionFilter {
1359
897
  public:
1360
- bool Filter(int /*level*/, const Slice & /*key*/, const Slice &value,
1361
- std::string *new_value, bool *value_changed) const override {
898
+ bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
899
+ std::string* new_value, bool* value_changed) const override {
1362
900
  *value_changed = false;
1363
- // changing value size to test value transitions between inlined data
1364
- // and stored-in-blob data
901
+ // Test compaction filter modifying blob values
1365
902
  if (value.size() % 4 == 1) {
1366
903
  *new_value = value.ToString();
1367
904
  // double size by duplicating value
@@ -1380,31 +917,23 @@ TEST_F(BlobDBTest, UserCompactionFilter) {
1380
917
  return false;
1381
918
  }
1382
919
  bool IgnoreSnapshots() const override { return true; }
1383
- const char *Name() const override { return "CustomerFilter"; }
920
+ const char* Name() const override { return "CustomerFilter"; }
1384
921
  };
1385
922
  class CustomerFilterFactory : public CompactionFilterFactory {
1386
- const char *Name() const override { return "CustomerFilterFactory"; }
923
+ const char* Name() const override { return "CustomerFilterFactory"; }
1387
924
  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
1388
- const CompactionFilter::Context & /*context*/) override {
925
+ const CompactionFilter::Context& /*context*/) override {
1389
926
  return std::unique_ptr<CompactionFilter>(new CustomerFilter());
1390
927
  }
1391
928
  };
1392
929
 
1393
930
  constexpr size_t kNumPuts = 1 << 10;
1394
- // Generate both inlined and blob value
1395
931
  constexpr uint64_t kMinValueSize = 1 << 6;
1396
932
  constexpr uint64_t kMaxValueSize = 1 << 8;
1397
- constexpr uint64_t kMinBlobSize = 1 << 7;
1398
- static_assert(kMinValueSize < kMinBlobSize);
1399
- static_assert(kMaxValueSize > kMinBlobSize);
1400
933
 
1401
934
  BlobDBOptions bdb_options;
1402
- bdb_options.min_blob_size = kMinBlobSize;
1403
935
  bdb_options.blob_file_size = kMaxValueSize * 10;
1404
936
  bdb_options.disable_background_tasks = true;
1405
- if (Snappy_Supported()) {
1406
- bdb_options.compression = CompressionType::kSnappyCompression;
1407
- }
1408
937
  // case_num == 0: Test user defined compaction filter
1409
938
  // case_num == 1: Test user defined compaction filter factory
1410
939
  for (int case_num = 0; case_num < 2; case_num++) {
@@ -1467,24 +996,22 @@ TEST_F(BlobDBTest, UserCompactionFilter) {
1467
996
  TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
1468
997
  class CustomerFilter : public CompactionFilter {
1469
998
  public:
1470
- bool Filter(int /*level*/, const Slice & /*key*/, const Slice &value,
1471
- std::string *new_value, bool *value_changed) const override {
999
+ bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
1000
+ std::string* new_value, bool* value_changed) const override {
1472
1001
  *new_value = value.ToString() + "_new";
1473
1002
  *value_changed = true;
1474
1003
  return false;
1475
1004
  }
1476
1005
  bool IgnoreSnapshots() const override { return true; }
1477
- const char *Name() const override { return "CustomerFilter"; }
1006
+ const char* Name() const override { return "CustomerFilter"; }
1478
1007
  };
1479
1008
 
1480
1009
  constexpr size_t kNumPuts = 100;
1481
1010
  constexpr int kValueSize = 100;
1482
1011
 
1483
1012
  BlobDBOptions bdb_options;
1484
- bdb_options.min_blob_size = 0;
1485
1013
  bdb_options.blob_file_size = kValueSize * 10;
1486
1014
  bdb_options.disable_background_tasks = true;
1487
- bdb_options.compression = CompressionType::kNoCompression;
1488
1015
 
1489
1016
  std::vector<std::string> io_failure_cases = {
1490
1017
  "BlobDBImpl::CreateBlobFileAndWriter",
@@ -1518,7 +1045,7 @@ TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
1518
1045
  VerifyDB(data);
1519
1046
 
1520
1047
  SyncPoint::GetInstance()->SetCallBack(
1521
- io_failure_cases[case_num], [&](void * /*arg*/) {
1048
+ io_failure_cases[case_num], [&](void* /*arg*/) {
1522
1049
  fault_injection_env_->SetFilesystemActive(false, Status::IOError());
1523
1050
  });
1524
1051
  SyncPoint::GetInstance()->EnableProcessing();
@@ -1542,13 +1069,11 @@ TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
1542
1069
  TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
1543
1070
  constexpr size_t kNumKeys = 100;
1544
1071
  constexpr size_t kNumPuts = 1000;
1545
- constexpr uint64_t kMaxExpiration = 1000;
1072
+ constexpr uint64_t kMaxTTL = 1000;
1546
1073
  constexpr uint64_t kCompactTime = 500;
1547
- constexpr uint64_t kMinBlobSize = 100;
1548
1074
  Random rnd(301);
1549
1075
  mock_clock_->SetCurrentTime(0);
1550
1076
  BlobDBOptions bdb_options;
1551
- bdb_options.min_blob_size = kMinBlobSize;
1552
1077
  bdb_options.disable_background_tasks = true;
1553
1078
  Options options;
1554
1079
  options.env = mock_env_.get();
@@ -1557,25 +1082,18 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
1557
1082
  std::map<std::string, std::string> data;
1558
1083
  std::map<std::string, std::string> data_after_compact;
1559
1084
  for (size_t i = 0; i < kNumPuts; i++) {
1560
- bool is_small_value = rnd.Next() % 2;
1561
1085
  bool has_ttl = rnd.Next() % 2;
1562
- uint64_t expiration = rnd.Next() % kMaxExpiration;
1563
- int len = is_small_value ? 10 : 200;
1086
+ // At time 0, stored expiration equals TTL
1087
+ uint64_t ttl = rnd.Next() % kMaxTTL;
1088
+ int len = rnd.Next() % 200 + 10;
1564
1089
  std::string key = "key" + std::to_string(rnd.Next() % kNumKeys);
1565
1090
  std::string value = rnd.HumanReadableString(len);
1566
1091
  if (!has_ttl) {
1567
- if (is_small_value) {
1568
- std::string blob_entry;
1569
- BlobIndex::EncodeInlinedTTL(&blob_entry, expiration, value);
1570
- // Fake blob index with TTL. See what it will do.
1571
- ASSERT_GT(kMinBlobSize, blob_entry.size());
1572
- value = blob_entry;
1573
- }
1574
1092
  ASSERT_OK(Put(key, value));
1575
1093
  data_after_compact[key] = value;
1576
1094
  } else {
1577
- ASSERT_OK(PutUntil(key, value, expiration));
1578
- if (expiration <= kCompactTime) {
1095
+ ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), key, value, ttl));
1096
+ if (ttl <= kCompactTime) {
1579
1097
  data_after_compact.erase(key);
1580
1098
  } else {
1581
1099
  data_after_compact[key] = value;
@@ -1588,7 +1106,7 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
1588
1106
  mock_clock_->SetCurrentTime(kCompactTime);
1589
1107
  // Take a snapshot before compaction. Make sure expired blob indexes is
1590
1108
  // filtered regardless of snapshot.
1591
- const Snapshot *snapshot = blob_db_->GetSnapshot();
1109
+ const Snapshot* snapshot = blob_db_->GetSnapshot();
1592
1110
  // Issue manual compaction to trigger compaction filter.
1593
1111
  ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1594
1112
  blob_db_->ReleaseSnapshot(snapshot);
@@ -1597,7 +1115,7 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
1597
1115
  const size_t kMaxKeys = 10000;
1598
1116
  ASSERT_OK(GetAllKeyVersions(blob_db_, {}, {}, kMaxKeys, &versions));
1599
1117
  ASSERT_EQ(data_after_compact.size(), versions.size());
1600
- for (auto &version : versions) {
1118
+ for (auto& version : versions) {
1601
1119
  ASSERT_TRUE(data_after_compact.count(version.user_key) > 0);
1602
1120
  }
1603
1121
  VerifyDB(data_after_compact);
@@ -1607,7 +1125,6 @@ TEST_F(BlobDBTest, FilterExpiredBlobIndex) {
1607
1125
  // blob file has been removed.
1608
1126
  TEST_F(BlobDBTest, FilterFileNotAvailable) {
1609
1127
  BlobDBOptions bdb_options;
1610
- bdb_options.min_blob_size = 0;
1611
1128
  bdb_options.disable_background_tasks = true;
1612
1129
  Options options;
1613
1130
  options.disable_auto_compactions = true;
@@ -1627,7 +1144,7 @@ TEST_F(BlobDBTest, FilterFileNotAvailable) {
1627
1144
 
1628
1145
  const size_t kMaxKeys = 10000;
1629
1146
 
1630
- DB *base_db = blob_db_->GetRootDB();
1147
+ DB* base_db = blob_db_->GetRootDB();
1631
1148
  std::vector<KeyVersion> versions;
1632
1149
  ASSERT_OK(GetAllKeyVersions(base_db, {}, {}, kMaxKeys, &versions));
1633
1150
  ASSERT_EQ(2, versions.size());
@@ -1660,127 +1177,25 @@ TEST_F(BlobDBTest, FilterFileNotAvailable) {
1660
1177
  VerifyDB({});
1661
1178
  }
1662
1179
 
1663
- // Test compaction filter should filter any inlined TTL keys that would have
1664
- // been dropped by last FIFO eviction if they are store out-of-line.
1665
- TEST_F(BlobDBTest, FilterForFIFOEviction) {
1666
- Random rnd(215);
1667
- BlobDBOptions bdb_options;
1668
- bdb_options.min_blob_size = 100;
1669
- bdb_options.ttl_range_secs = 60;
1670
- bdb_options.max_db_size = 0;
1671
- bdb_options.disable_background_tasks = true;
1672
- Options options;
1673
- // Use mock env to stop wall clock.
1674
- mock_clock_->SetCurrentTime(0);
1675
- options.env = mock_env_.get();
1676
- auto statistics = CreateDBStatistics();
1677
- options.statistics = statistics;
1678
- options.disable_auto_compactions = true;
1679
- Open(bdb_options, options);
1680
-
1681
- SyncPoint::GetInstance()->LoadDependency(
1682
- {{"DBImpl::NotifyOnFlushCompleted::PostAllOnFlushCompleted",
1683
- "BlobDBTest.FilterForFIFOEviction:AfterFlush"}});
1684
-
1685
- SyncPoint::GetInstance()->EnableProcessing();
1686
-
1687
- std::map<std::string, std::string> data;
1688
- std::map<std::string, std::string> data_after_compact;
1689
- // Insert some small values that will be inlined.
1690
- for (int i = 0; i < 1000; i++) {
1691
- std::string key = "key" + std::to_string(i);
1692
- std::string value = rnd.HumanReadableString(50);
1693
- uint64_t ttl = rnd.Next() % 120 + 1;
1694
- ASSERT_OK(PutWithTTL(key, value, ttl, &data));
1695
- if (ttl >= 60) {
1696
- data_after_compact[key] = value;
1697
- }
1698
- }
1699
- uint64_t num_keys_to_evict = data.size() - data_after_compact.size();
1700
- ASSERT_OK(blob_db_->Flush(FlushOptions()));
1701
-
1702
- TEST_SYNC_POINT("BlobDBTest.FilterForFIFOEviction:AfterFlush");
1703
-
1704
- uint64_t live_sst_size = blob_db_impl()->TEST_live_sst_size();
1705
- ASSERT_GT(live_sst_size, 0);
1706
- VerifyDB(data);
1707
-
1708
- bdb_options.max_db_size = live_sst_size + 30000;
1709
- bdb_options.is_fifo = true;
1710
- Reopen(bdb_options, options);
1711
- VerifyDB(data);
1712
-
1713
- // Put two large values, each on a different blob file.
1714
- std::string large_value(10000, 'v');
1715
- ASSERT_OK(PutWithTTL("large_key1", large_value, 90));
1716
- ASSERT_OK(PutWithTTL("large_key2", large_value, 150));
1717
- ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
1718
- ASSERT_EQ(0, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1719
- data["large_key1"] = large_value;
1720
- data["large_key2"] = large_value;
1721
- VerifyDB(data);
1722
-
1723
- // Put a third large value which will bring the DB out of space.
1724
- // FIFO eviction will evict the file of large_key1.
1725
- ASSERT_OK(PutWithTTL("large_key3", large_value, 150));
1726
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1727
- ASSERT_EQ(2, blob_db_impl()->TEST_GetBlobFiles().size());
1728
- blob_db_impl()->TEST_DeleteObsoleteFiles();
1729
- ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
1730
- data.erase("large_key1");
1731
- data["large_key3"] = large_value;
1732
- VerifyDB(data);
1733
-
1734
- // Putting some more small values. These values shouldn't be evicted by
1735
- // compaction filter since they are inserted after FIFO eviction.
1736
- ASSERT_OK(PutWithTTL("foo", "v", 30, &data_after_compact));
1737
- ASSERT_OK(PutWithTTL("bar", "v", 30, &data_after_compact));
1738
-
1739
- // FIFO eviction doesn't trigger again since there enough room for the flush.
1740
- ASSERT_OK(blob_db_->Flush(FlushOptions()));
1741
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1742
-
1743
- // Manual compact and check if compaction filter evict those keys with
1744
- // expiration < 60.
1745
- ASSERT_OK(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1746
- // All keys with expiration < 60, plus large_key1 is filtered by
1747
- // compaction filter.
1748
- ASSERT_EQ(num_keys_to_evict + 1,
1749
- statistics->getTickerCount(BLOB_DB_BLOB_INDEX_EVICTED_COUNT));
1750
- ASSERT_EQ(1, statistics->getTickerCount(BLOB_DB_FIFO_NUM_FILES_EVICTED));
1751
- ASSERT_EQ(1, blob_db_impl()->TEST_GetBlobFiles().size());
1752
- data_after_compact["large_key2"] = large_value;
1753
- data_after_compact["large_key3"] = large_value;
1754
- VerifyDB(data_after_compact);
1755
-
1756
- SyncPoint::GetInstance()->DisableProcessing();
1757
- }
1758
-
1759
1180
  TEST_F(BlobDBTest, GarbageCollection) {
1760
1181
  constexpr size_t kNumPuts = 1 << 10;
1761
1182
 
1762
- constexpr uint64_t kExpiration = 1000;
1183
+ // At time 0, stored expiration equals TTL
1184
+ constexpr uint64_t kTTL = 1000;
1763
1185
  constexpr uint64_t kCompactTime = 500;
1764
1186
 
1765
1187
  constexpr uint64_t kKeySize = 7; // "key" + 4 digits
1766
-
1767
- constexpr uint64_t kSmallValueSize = 1 << 6;
1768
- constexpr uint64_t kLargeValueSize = 1 << 8;
1769
- constexpr uint64_t kMinBlobSize = 1 << 7;
1770
- static_assert(kSmallValueSize < kMinBlobSize);
1771
- static_assert(kLargeValueSize > kMinBlobSize);
1188
+ constexpr uint64_t kValueSize = 1 << 8;
1772
1189
 
1773
1190
  constexpr size_t kBlobsPerFile = 8;
1774
1191
  constexpr size_t kNumBlobFiles = kNumPuts / kBlobsPerFile;
1775
1192
  constexpr uint64_t kBlobFileSize =
1776
1193
  BlobLogHeader::kSize +
1777
- (BlobLogRecord::kHeaderSize + kKeySize + kLargeValueSize) * kBlobsPerFile;
1194
+ (BlobLogRecord::kHeaderSize + kKeySize + kValueSize) * kBlobsPerFile;
1778
1195
 
1779
1196
  BlobDBOptions bdb_options;
1780
- bdb_options.min_blob_size = kMinBlobSize;
1781
1197
  bdb_options.blob_file_size = kBlobFileSize;
1782
1198
  bdb_options.enable_garbage_collection = true;
1783
- bdb_options.garbage_collection_cutoff = 0.25;
1784
1199
  bdb_options.disable_background_tasks = true;
1785
1200
 
1786
1201
  Options options;
@@ -1795,14 +1210,14 @@ TEST_F(BlobDBTest, GarbageCollection) {
1795
1210
 
1796
1211
  Random rnd(301);
1797
1212
 
1798
- // Add a bunch of large non-TTL values. These will be written to non-TTL
1213
+ // Add a bunch of non-TTL values. These will be written to non-TTL
1799
1214
  // blob files and will be subject to GC.
1800
1215
  for (size_t i = 0; i < kNumPuts; ++i) {
1801
1216
  std::ostringstream oss;
1802
1217
  oss << "key" << std::setw(4) << std::setfill('0') << i;
1803
1218
 
1804
1219
  const std::string key(oss.str());
1805
- const std::string value = rnd.HumanReadableString(kLargeValueSize);
1220
+ const std::string value = rnd.HumanReadableString(kValueSize);
1806
1221
  const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
1807
1222
 
1808
1223
  ASSERT_OK(Put(key, value));
@@ -1815,54 +1230,23 @@ TEST_F(BlobDBTest, GarbageCollection) {
1815
1230
  sequence, kTypeBlobIndex);
1816
1231
  }
1817
1232
 
1818
- // Add some small and/or TTL values that will be ignored during GC.
1819
- // First, add a large TTL value will be written to its own TTL blob file.
1233
+ // Add a TTL value that will be written to its own TTL blob file (ignored
1234
+ // during GC).
1820
1235
  {
1821
1236
  const std::string key("key2000");
1822
- const std::string value = rnd.HumanReadableString(kLargeValueSize);
1237
+ const std::string value = rnd.HumanReadableString(kValueSize);
1823
1238
  const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
1824
1239
 
1825
- ASSERT_OK(PutUntil(key, value, kExpiration));
1240
+ ASSERT_OK(blob_db_->PutWithTTL(WriteOptions(), key, value, kTTL));
1826
1241
  ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
1827
1242
 
1828
1243
  data[key] = value;
1829
1244
  blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeBlobIndex);
1830
1245
  blob_index_versions[key] =
1831
- BlobIndexVersion(key, /* file_number */ kNumBlobFiles + 1, kExpiration,
1246
+ BlobIndexVersion(key, /* file_number */ kNumBlobFiles + 1, kTTL,
1832
1247
  sequence, kTypeBlobIndex);
1833
1248
  }
1834
1249
 
1835
- // Now add a small TTL value (which will be inlined).
1836
- {
1837
- const std::string key("key3000");
1838
- const std::string value = rnd.HumanReadableString(kSmallValueSize);
1839
- const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
1840
-
1841
- ASSERT_OK(PutUntil(key, value, kExpiration));
1842
- ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
1843
-
1844
- data[key] = value;
1845
- blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeBlobIndex);
1846
- blob_index_versions[key] = BlobIndexVersion(
1847
- key, kInvalidBlobFileNumber, kExpiration, sequence, kTypeBlobIndex);
1848
- }
1849
-
1850
- // Finally, add a small non-TTL value (which will be stored as a regular
1851
- // value).
1852
- {
1853
- const std::string key("key4000");
1854
- const std::string value = rnd.HumanReadableString(kSmallValueSize);
1855
- const SequenceNumber sequence = blob_db_->GetLatestSequenceNumber() + 1;
1856
-
1857
- ASSERT_OK(Put(key, value));
1858
- ASSERT_EQ(blob_db_->GetLatestSequenceNumber(), sequence);
1859
-
1860
- data[key] = value;
1861
- blob_value_versions[key] = KeyVersion(key, value, sequence, kTypeValue);
1862
- blob_index_versions[key] = BlobIndexVersion(
1863
- key, kInvalidBlobFileNumber, kNoExpiration, sequence, kTypeValue);
1864
- }
1865
-
1866
1250
  VerifyDB(data);
1867
1251
  VerifyBaseDB(blob_value_versions);
1868
1252
  VerifyBaseDBBlobIndex(blob_index_versions);
@@ -1888,17 +1272,18 @@ TEST_F(BlobDBTest, GarbageCollection) {
1888
1272
  // compaction.
1889
1273
  VerifyDB(data);
1890
1274
 
1891
- for (auto &pair : blob_value_versions) {
1892
- KeyVersion &version = pair.second;
1275
+ for (auto& pair : blob_value_versions) {
1276
+ KeyVersion& version = pair.second;
1893
1277
  version.sequence = 0;
1894
1278
  }
1895
1279
 
1896
1280
  VerifyBaseDB(blob_value_versions);
1897
1281
 
1898
- const uint64_t cutoff = static_cast<uint64_t>(
1899
- bdb_options.garbage_collection_cutoff * kNumBlobFiles);
1900
- for (auto &pair : blob_index_versions) {
1901
- BlobIndexVersion &version = pair.second;
1282
+ // GC cutoff is fixed at 0.25
1283
+ constexpr double kGCCutoff = 0.25;
1284
+ const uint64_t cutoff = static_cast<uint64_t>(kGCCutoff * kNumBlobFiles);
1285
+ for (auto& pair : blob_index_versions) {
1286
+ BlobIndexVersion& version = pair.second;
1902
1287
 
1903
1288
  version.sequence = 0;
1904
1289
 
@@ -1915,7 +1300,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
1915
1300
 
1916
1301
  VerifyBaseDBBlobIndex(blob_index_versions);
1917
1302
 
1918
- const Statistics *const statistics = options.statistics.get();
1303
+ const Statistics* const statistics = options.statistics.get();
1919
1304
  assert(statistics);
1920
1305
 
1921
1306
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_FILES), cutoff);
@@ -1924,7 +1309,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
1924
1309
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_KEYS_RELOCATED),
1925
1310
  cutoff * kBlobsPerFile);
1926
1311
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_BYTES_RELOCATED),
1927
- cutoff * kBlobsPerFile * kLargeValueSize);
1312
+ cutoff * kBlobsPerFile * kValueSize);
1928
1313
 
1929
1314
  // At this point, we should have 128 immutable non-TTL files with file numbers
1930
1315
  // 33..128 and 130..161. (129 was taken by the TTL blob file.)
@@ -1946,9 +1331,7 @@ TEST_F(BlobDBTest, GarbageCollection) {
1946
1331
 
1947
1332
  TEST_F(BlobDBTest, GarbageCollectionFailure) {
1948
1333
  BlobDBOptions bdb_options;
1949
- bdb_options.min_blob_size = 0;
1950
1334
  bdb_options.enable_garbage_collection = true;
1951
- bdb_options.garbage_collection_cutoff = 1.0;
1952
1335
  bdb_options.disable_background_tasks = true;
1953
1336
 
1954
1337
  Options db_options;
@@ -1956,14 +1339,31 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
1956
1339
 
1957
1340
  Open(bdb_options, db_options);
1958
1341
 
1959
- // Write a couple of valid blobs.
1342
+ // Create 4 blob files. With fixed GC cutoff of 0.25, the oldest file
1343
+ // (floor(0.25 * 4) = 1) will be in the GC zone.
1344
+ // The first file contains valid blobs for "foo" and "dead".
1960
1345
  ASSERT_OK(Put("foo", "bar"));
1961
1346
  ASSERT_OK(Put("dead", "beef"));
1962
1347
 
1963
- // Write a fake blob reference into the base DB that points to a non-existing
1964
- // blob file.
1348
+ auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
1349
+ ASSERT_EQ(blob_files.size(), 1);
1350
+ auto first_file = blob_files[0];
1351
+ uint64_t first_file_number = first_file->BlobFileNumber();
1352
+ ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(first_file));
1353
+
1354
+ // Create 3 more blob files (files 2-4, outside GC zone).
1355
+ for (int i = 1; i < 4; i++) {
1356
+ ASSERT_OK(Put("key" + std::to_string(i), "value"));
1357
+ blob_files = blob_db_impl()->TEST_GetBlobFiles();
1358
+ ASSERT_EQ(static_cast<size_t>(i + 1), blob_files.size());
1359
+ ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_files[i]));
1360
+ }
1361
+
1362
+ // Write a fake blob index that points to the first file (in GC zone)
1363
+ // but with an invalid offset beyond the file size. This will cause
1364
+ // GC to fail when it tries to read this blob.
1965
1365
  std::string blob_index;
1966
- BlobIndex::EncodeBlob(&blob_index, /* file_number */ 1000, /* offset */ 1234,
1366
+ BlobIndex::EncodeBlob(&blob_index, first_file_number, /* offset */ 999999,
1967
1367
  /* size */ 5678, kNoCompression);
1968
1368
 
1969
1369
  WriteBatch batch;
@@ -1971,17 +1371,17 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
1971
1371
  &batch, blob_db_->DefaultColumnFamily()->GetID(), "key", blob_index));
1972
1372
  ASSERT_OK(blob_db_->GetRootDB()->Write(WriteOptions(), &batch));
1973
1373
 
1974
- auto blob_files = blob_db_impl()->TEST_GetBlobFiles();
1975
- ASSERT_EQ(blob_files.size(), 1);
1976
- auto blob_file = blob_files[0];
1977
- ASSERT_OK(blob_db_impl()->TEST_CloseBlobFile(blob_file));
1978
-
1374
+ // Verify compaction fails with IO error due to invalid blob offset.
1979
1375
  ASSERT_TRUE(blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)
1980
1376
  .IsIOError());
1981
1377
 
1982
- const Statistics *const statistics = db_options.statistics.get();
1378
+ const Statistics* const statistics = db_options.statistics.get();
1983
1379
  assert(statistics);
1984
1380
 
1381
+ // Verify GC statistics:
1382
+ // - Relocated 2 keys ("foo" and "dead") with 7 bytes ("bar" + "beef")
1383
+ // - Failed on "key" which has invalid blob offset
1384
+ // - Created 1 new GC output file before failing
1985
1385
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_FILES), 0);
1986
1386
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_NUM_NEW_FILES), 1);
1987
1387
  ASSERT_EQ(statistics->getTickerCount(BLOB_DB_GC_FAILURES), 1);
@@ -1993,7 +1393,6 @@ TEST_F(BlobDBTest, GarbageCollectionFailure) {
1993
1393
  TEST_F(BlobDBTest, EvictExpiredFile) {
1994
1394
  BlobDBOptions bdb_options;
1995
1395
  bdb_options.ttl_range_secs = 100;
1996
- bdb_options.min_blob_size = 0;
1997
1396
  bdb_options.disable_background_tasks = true;
1998
1397
  Options options;
1999
1398
  options.env = mock_env_.get();
@@ -2116,7 +1515,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2116
1515
  const std::vector<bool> expected_obsolete{false, false, false, false,
2117
1516
  false};
2118
1517
  for (size_t i = 0; i < 5; ++i) {
2119
- const auto &blob_file = blob_files[i];
1518
+ const auto& blob_file = blob_files[i];
2120
1519
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2121
1520
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2122
1521
  }
@@ -2144,7 +1543,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2144
1543
  const std::vector<bool> expected_obsolete{false, false, false, false,
2145
1544
  false};
2146
1545
  for (size_t i = 0; i < 5; ++i) {
2147
- const auto &blob_file = blob_files[i];
1546
+ const auto& blob_file = blob_files[i];
2148
1547
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2149
1548
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2150
1549
  }
@@ -2173,7 +1572,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2173
1572
  const std::vector<bool> expected_obsolete{false, false, false, false,
2174
1573
  false};
2175
1574
  for (size_t i = 0; i < 5; ++i) {
2176
- const auto &blob_file = blob_files[i];
1575
+ const auto& blob_file = blob_files[i];
2177
1576
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2178
1577
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2179
1578
  }
@@ -2211,7 +1610,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2211
1610
  {}, {7}, {3, 8, 23}, {4, 9}, {5, 10, 22}};
2212
1611
  const std::vector<bool> expected_obsolete{true, false, false, false, false};
2213
1612
  for (size_t i = 0; i < 5; ++i) {
2214
- const auto &blob_file = blob_files[i];
1613
+ const auto& blob_file = blob_files[i];
2215
1614
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2216
1615
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2217
1616
  }
@@ -2241,7 +1640,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2241
1640
  {}, {7}, {3, 8, 23}, {4, 9}, {5, 10, 22}};
2242
1641
  const std::vector<bool> expected_obsolete{true, false, false, false, false};
2243
1642
  for (size_t i = 0; i < 5; ++i) {
2244
- const auto &blob_file = blob_files[i];
1643
+ const auto& blob_file = blob_files[i];
2245
1644
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2246
1645
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2247
1646
  }
@@ -2272,7 +1671,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2272
1671
  {}, {}, {3, 8, 23, 25}, {4, 9}, {5, 10}};
2273
1672
  const std::vector<bool> expected_obsolete{true, false, false, false, false};
2274
1673
  for (size_t i = 0; i < 5; ++i) {
2275
- const auto &blob_file = blob_files[i];
1674
+ const auto& blob_file = blob_files[i];
2276
1675
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2277
1676
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2278
1677
  }
@@ -2302,7 +1701,7 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2302
1701
  {}, {}, {3, 8, 23, 25}, {4, 9}, {5, 10}};
2303
1702
  const std::vector<bool> expected_obsolete{true, true, false, false, false};
2304
1703
  for (size_t i = 0; i < 5; ++i) {
2305
- const auto &blob_file = blob_files[i];
1704
+ const auto& blob_file = blob_files[i];
2306
1705
  ASSERT_EQ(blob_file->GetLinkedSstFiles(), expected_sst_files[i]);
2307
1706
  ASSERT_EQ(blob_file->Obsolete(), expected_obsolete[i]);
2308
1707
  }
@@ -2323,7 +1722,6 @@ TEST_F(BlobDBTest, MaintainBlobFileToSstMapping) {
2323
1722
  TEST_F(BlobDBTest, ShutdownWait) {
2324
1723
  BlobDBOptions bdb_options;
2325
1724
  bdb_options.ttl_range_secs = 100;
2326
- bdb_options.min_blob_size = 0;
2327
1725
  bdb_options.disable_background_tasks = false;
2328
1726
  Options options;
2329
1727
  options.env = mock_env_.get();
@@ -2336,15 +1734,15 @@ TEST_F(BlobDBTest, ShutdownWait) {
2336
1734
  });
2337
1735
  // Force all tasks to be scheduled immediately.
2338
1736
  SyncPoint::GetInstance()->SetCallBack(
2339
- "TimeQueue::Add:item.end", [&](void *arg) {
2340
- std::chrono::steady_clock::time_point *tp =
2341
- static_cast<std::chrono::steady_clock::time_point *>(arg);
1737
+ "TimeQueue::Add:item.end", [&](void* arg) {
1738
+ std::chrono::steady_clock::time_point* tp =
1739
+ static_cast<std::chrono::steady_clock::time_point*>(arg);
2342
1740
  *tp =
2343
1741
  std::chrono::steady_clock::now() - std::chrono::milliseconds(10000);
2344
1742
  });
2345
1743
 
2346
1744
  SyncPoint::GetInstance()->SetCallBack(
2347
- "BlobDBImpl::EvictExpiredFiles:cb", [&](void * /*arg*/) {
1745
+ "BlobDBImpl::EvictExpiredFiles:cb", [&](void* /*arg*/) {
2348
1746
  // Sleep 3 ms to increase the chance of data race.
2349
1747
  // We've synced up the code so that EvictExpiredFiles()
2350
1748
  // is called concurrently with ~BlobDBImpl().
@@ -2387,8 +1785,6 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeClose) {
2387
1785
  options.statistics = CreateDBStatistics();
2388
1786
 
2389
1787
  BlobDBOptions blob_options;
2390
- blob_options.min_blob_size = 0;
2391
- blob_options.bytes_per_sync = 1 << 20;
2392
1788
  blob_options.disable_background_tasks = true;
2393
1789
 
2394
1790
  Open(blob_options, options);
@@ -2407,8 +1803,6 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
2407
1803
  options.env = fault_injection_env_.get();
2408
1804
 
2409
1805
  BlobDBOptions blob_options;
2410
- blob_options.min_blob_size = 0;
2411
- blob_options.bytes_per_sync = 1 << 20;
2412
1806
  blob_options.disable_background_tasks = true;
2413
1807
 
2414
1808
  Open(blob_options, options);
@@ -2419,7 +1813,7 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
2419
1813
  ASSERT_EQ(blob_files.size(), 1);
2420
1814
 
2421
1815
  SyncPoint::GetInstance()->SetCallBack(
2422
- "BlobLogWriter::Sync", [this](void * /* arg */) {
1816
+ "BlobLogWriter::Sync", [this](void* /* arg */) {
2423
1817
  fault_injection_env_->SetFilesystemActive(false, Status::IOError());
2424
1818
  });
2425
1819
  SyncPoint::GetInstance()->EnableProcessing();
@@ -2436,7 +1830,7 @@ TEST_F(BlobDBTest, SyncBlobFileBeforeCloseIOError) {
2436
1830
  } // namespace ROCKSDB_NAMESPACE::blob_db
2437
1831
 
2438
1832
  // A black-box test for the ttl wrapper around rocksdb
2439
- int main(int argc, char **argv) {
1833
+ int main(int argc, char** argv) {
2440
1834
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
2441
1835
  ::testing::InitGoogleTest(&argc, argv);
2442
1836
  return RUN_ALL_TESTS();