@nxtedition/rocksdb 15.4.0 → 15.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (402) hide show
  1. package/binding.cc +24 -19
  2. package/cache.js +1 -1
  3. package/chained-batch.js +12 -3
  4. package/deps/rocksdb/rocksdb/.clang-tidy +86 -0
  5. package/deps/rocksdb/rocksdb/BUCK +42 -0
  6. package/deps/rocksdb/rocksdb/CMakeLists.txt +11 -0
  7. package/deps/rocksdb/rocksdb/Makefile +59 -32
  8. package/deps/rocksdb/rocksdb/cache/cache.cc +0 -5
  9. package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +9 -9
  10. package/deps/rocksdb/rocksdb/cache/cache_key.cc +3 -3
  11. package/deps/rocksdb/rocksdb/cache/cache_key.h +5 -5
  12. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +16 -16
  13. package/deps/rocksdb/rocksdb/cache/cache_test.cc +1 -1
  14. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +258 -294
  15. package/deps/rocksdb/rocksdb/cache/clock_cache.h +98 -49
  16. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +1 -5
  17. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +2 -3
  18. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +18 -18
  19. package/deps/rocksdb/rocksdb/crash_test.mk +5 -1
  20. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +23 -22
  21. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -1
  22. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +14 -16
  23. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +38 -26
  24. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +5 -1
  25. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +101 -18
  26. package/deps/rocksdb/rocksdb/db/blob/blob_index.h +12 -0
  27. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +6 -9
  28. package/deps/rocksdb/rocksdb/db/builder.cc +23 -0
  29. package/deps/rocksdb/rocksdb/db/builder.h +7 -0
  30. package/deps/rocksdb/rocksdb/db/c.cc +373 -57
  31. package/deps/rocksdb/rocksdb/db/c_test.c +101 -1
  32. package/deps/rocksdb/rocksdb/db/column_family.cc +31 -3
  33. package/deps/rocksdb/rocksdb/db/column_family_test.cc +10 -13
  34. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +35 -48
  35. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +13 -5
  36. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +201 -39
  37. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +15 -10
  38. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +7 -7
  39. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +2 -455
  40. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +4 -2
  41. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +19 -0
  42. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +72 -9
  43. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -10
  44. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +405 -83
  45. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +25 -1
  46. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +23 -10
  47. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -0
  48. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +1410 -106
  49. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -5
  50. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +2 -1
  51. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +19 -10
  52. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +505 -45
  53. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
  54. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +9 -1
  55. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +4 -4
  56. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +7 -9
  57. package/deps/rocksdb/rocksdb/db/convenience.cc +4 -4
  58. package/deps/rocksdb/rocksdb/db/convenience_impl.h +2 -1
  59. package/deps/rocksdb/rocksdb/db/corruption_test.cc +60 -88
  60. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +10 -12
  61. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +471 -40
  62. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +116 -2
  63. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +5 -15
  64. package/deps/rocksdb/rocksdb/db/db_compaction_abort_test.cc +993 -0
  65. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +329 -29
  66. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +155 -13
  67. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +54 -31
  68. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -0
  69. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +232 -70
  70. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +57 -9
  71. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +224 -31
  72. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
  73. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +4 -2
  74. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +1 -1
  75. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -0
  76. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +164 -8
  77. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +6 -0
  78. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +5 -0
  79. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +47 -35
  80. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +22 -9
  81. package/deps/rocksdb/rocksdb/db/db_iter.cc +9 -0
  82. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +371 -6
  83. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -5
  84. package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +22 -23
  85. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +0 -2
  86. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +4 -4
  87. package/deps/rocksdb/rocksdb/db/db_options_test.cc +40 -0
  88. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +32 -13
  89. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +1 -1
  90. package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +4 -4
  91. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +68 -15
  92. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +1 -1
  93. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +2 -3
  94. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +6 -21
  95. package/deps/rocksdb/rocksdb/db/db_test.cc +644 -128
  96. package/deps/rocksdb/rocksdb/db/db_test2.cc +198 -81
  97. package/deps/rocksdb/rocksdb/db/db_test_util.cc +35 -10
  98. package/deps/rocksdb/rocksdb/db/db_test_util.h +8 -2
  99. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +36 -32
  100. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +11 -7
  101. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +499 -0
  102. package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +284 -20
  103. package/deps/rocksdb/rocksdb/db/db_write_test.cc +3 -3
  104. package/deps/rocksdb/rocksdb/db/dbformat.h +0 -5
  105. package/deps/rocksdb/rocksdb/db/error_handler.cc +24 -0
  106. package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +12 -14
  107. package/deps/rocksdb/rocksdb/db/experimental.cc +13 -10
  108. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +1 -1
  109. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +22 -3
  110. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +21 -15
  111. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +4 -6
  112. package/deps/rocksdb/rocksdb/db/flush_job.cc +11 -3
  113. package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +5 -6
  114. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +4 -2
  115. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +17 -17
  116. package/deps/rocksdb/rocksdb/db/internal_stats.cc +13 -0
  117. package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -0
  118. package/deps/rocksdb/rocksdb/db/listener_test.cc +154 -27
  119. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +6 -6
  120. package/deps/rocksdb/rocksdb/db/memtable.cc +197 -51
  121. package/deps/rocksdb/rocksdb/db/memtable.h +6 -0
  122. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +3 -4
  123. package/deps/rocksdb/rocksdb/db/merge_test.cc +37 -35
  124. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +2 -1
  125. package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
  126. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +9 -11
  127. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +10 -1
  128. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +292 -15
  129. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +10 -17
  130. package/deps/rocksdb/rocksdb/db/prefix_test.cc +6 -8
  131. package/deps/rocksdb/rocksdb/db/repair.cc +10 -10
  132. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +5 -5
  133. package/deps/rocksdb/rocksdb/db/table_cache.cc +142 -135
  134. package/deps/rocksdb/rocksdb/db/table_cache.h +30 -6
  135. package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +7 -7
  136. package/deps/rocksdb/rocksdb/db/version_builder.cc +11 -50
  137. package/deps/rocksdb/rocksdb/db/version_builder.h +2 -1
  138. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +2 -1
  139. package/deps/rocksdb/rocksdb/db/version_edit.cc +51 -2
  140. package/deps/rocksdb/rocksdb/db/version_edit.h +91 -29
  141. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -7
  142. package/deps/rocksdb/rocksdb/db/version_set.cc +211 -50
  143. package/deps/rocksdb/rocksdb/db/version_set.h +40 -3
  144. package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +5 -0
  145. package/deps/rocksdb/rocksdb/db/version_set_test.cc +294 -21
  146. package/deps/rocksdb/rocksdb/db/version_util.cc +96 -0
  147. package/deps/rocksdb/rocksdb/db/version_util.h +24 -0
  148. package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +5 -5
  149. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +647 -31
  150. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +219 -1
  151. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +549 -12
  152. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -3
  153. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +1 -1
  154. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -0
  155. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +21 -4
  156. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +32 -0
  157. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +74 -22
  158. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +9 -0
  159. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -61
  160. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +15 -2
  161. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +76 -2
  162. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +92 -72
  163. package/deps/rocksdb/rocksdb/env/env.cc +1 -0
  164. package/deps/rocksdb/rocksdb/env/env_test.cc +365 -2
  165. package/deps/rocksdb/rocksdb/env/fs_posix.cc +31 -30
  166. package/deps/rocksdb/rocksdb/env/io_posix.cc +8 -11
  167. package/deps/rocksdb/rocksdb/env/io_posix.h +30 -1
  168. package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
  169. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +1 -1
  170. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +108 -0
  171. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +32 -4
  172. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +4 -4
  173. package/deps/rocksdb/rocksdb/file/file_util.cc +8 -2
  174. package/deps/rocksdb/rocksdb/file/file_util.h +2 -1
  175. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +331 -12
  176. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +52 -35
  177. package/deps/rocksdb/rocksdb/folly.mk +22 -5
  178. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +1 -1
  179. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +100 -54
  180. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +67 -2
  181. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +149 -13
  182. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -12
  183. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +78 -97
  184. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +3 -3
  185. package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +2 -2
  186. package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +5 -0
  187. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +17 -2
  188. package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +1 -1
  189. package/deps/rocksdb/rocksdb/include/rocksdb/io_dispatcher.h +358 -0
  190. package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +13 -0
  191. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +43 -0
  192. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +20 -0
  193. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +63 -21
  194. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +10 -1
  195. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +1 -1
  196. package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +2 -7
  197. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +13 -0
  198. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +3 -14
  199. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +49 -9
  200. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +8 -0
  201. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +77 -6
  202. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +15 -0
  203. package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +16 -10
  204. package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +5 -5
  205. package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +2 -4
  206. package/deps/rocksdb/rocksdb/include/rocksdb/user_defined_index.h +106 -46
  207. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/db_ttl.h +1 -1
  208. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +14 -1
  209. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +5 -1
  210. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +2 -1
  211. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +7 -9
  212. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  213. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +1 -2
  214. package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +2 -2
  215. package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +226 -8
  216. package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +490 -0
  217. package/deps/rocksdb/rocksdb/memtable/skiplist.h +3 -3
  218. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +11 -0
  219. package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +4 -12
  220. package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +5 -5
  221. package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +21 -4
  222. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +9 -3
  223. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +21 -2
  224. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +2 -2
  225. package/deps/rocksdb/rocksdb/options/cf_options.cc +21 -1
  226. package/deps/rocksdb/rocksdb/options/cf_options.h +2 -0
  227. package/deps/rocksdb/rocksdb/options/customizable_test.cc +0 -2
  228. package/deps/rocksdb/rocksdb/options/db_options.cc +26 -5
  229. package/deps/rocksdb/rocksdb/options/db_options.h +3 -1
  230. package/deps/rocksdb/rocksdb/options/options.cc +5 -1
  231. package/deps/rocksdb/rocksdb/options/options_helper.cc +7 -2
  232. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +109 -103
  233. package/deps/rocksdb/rocksdb/options/options_test.cc +14 -0
  234. package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +15 -17
  235. package/deps/rocksdb/rocksdb/port/lang.h +4 -0
  236. package/deps/rocksdb/rocksdb/port/port_example.h +0 -23
  237. package/deps/rocksdb/rocksdb/port/stack_trace.cc +36 -0
  238. package/deps/rocksdb/rocksdb/port/stack_trace.h +9 -0
  239. package/deps/rocksdb/rocksdb/src.mk +12 -0
  240. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -2
  241. package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc +2 -1
  242. package/deps/rocksdb/rocksdb/table/block_based/block.cc +571 -292
  243. package/deps/rocksdb/rocksdb/table/block_based/block.h +143 -53
  244. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +154 -90
  245. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +5 -1
  246. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +51 -14
  247. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +0 -2
  248. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +147 -734
  249. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +30 -233
  250. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +178 -108
  251. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +13 -0
  252. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +17 -4
  253. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -2
  254. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +70 -0
  255. package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +168 -24
  256. package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +25 -9
  257. package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +7 -4
  258. package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +9 -2
  259. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +548 -169
  260. package/deps/rocksdb/rocksdb/table/block_based/block_type.h +30 -0
  261. package/deps/rocksdb/rocksdb/table/block_based/block_util.h +156 -0
  262. package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.cc +73 -30
  263. package/deps/rocksdb/rocksdb/table/block_based/data_block_footer.h +74 -7
  264. package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index.h +1 -1
  265. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +20 -14
  266. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +22 -12
  267. package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +1 -1
  268. package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.cc +332 -0
  269. package/deps/rocksdb/rocksdb/table/block_based/multi_scan_index_iterator.h +133 -0
  270. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -2
  271. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +1 -1
  272. package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +3 -2
  273. package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -1
  274. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +0 -1
  275. package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +126 -46
  276. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +31 -3
  277. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +1 -2
  278. package/deps/rocksdb/rocksdb/table/cleanable_test.cc +3 -1
  279. package/deps/rocksdb/rocksdb/table/external_table.cc +25 -4
  280. package/deps/rocksdb/rocksdb/table/format.cc +27 -15
  281. package/deps/rocksdb/rocksdb/table/format.h +41 -15
  282. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +1 -0
  283. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +22 -12
  284. package/deps/rocksdb/rocksdb/table/meta_blocks.h +0 -1
  285. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +7 -21
  286. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
  287. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +88 -13
  288. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +53 -42
  289. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +3 -12
  290. package/deps/rocksdb/rocksdb/table/table_builder.h +0 -4
  291. package/deps/rocksdb/rocksdb/table/table_properties.cc +18 -0
  292. package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +2 -3
  293. package/deps/rocksdb/rocksdb/table/table_test.cc +848 -172
  294. package/deps/rocksdb/rocksdb/table/unique_id.cc +24 -20
  295. package/deps/rocksdb/rocksdb/table/unique_id_impl.h +8 -8
  296. package/deps/rocksdb/rocksdb/test_util/sync_point.h +5 -4
  297. package/deps/rocksdb/rocksdb/test_util/testutil.cc +2 -1
  298. package/deps/rocksdb/rocksdb/test_util/testutil.h +2 -2
  299. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +2 -1
  300. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +238 -120
  301. package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +2 -2
  302. package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +2 -4
  303. package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +4 -8
  304. package/deps/rocksdb/rocksdb/tools/dump/rocksdb_undump.cc +1 -1
  305. package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +2 -3
  306. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +82 -20
  307. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +41 -47
  308. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +9 -0
  309. package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +5 -6
  310. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +1 -1
  311. package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +6 -5
  312. package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +4 -4
  313. package/deps/rocksdb/rocksdb/tools/write_stress.cc +1 -3
  314. package/deps/rocksdb/rocksdb/util/atomic.h +30 -23
  315. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +6 -7
  316. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +3 -3
  317. package/deps/rocksdb/rocksdb/util/bit_fields.h +68 -46
  318. package/deps/rocksdb/rocksdb/util/bloom_impl.h +16 -16
  319. package/deps/rocksdb/rocksdb/util/coding.h +14 -27
  320. package/deps/rocksdb/rocksdb/util/compression.cc +365 -207
  321. package/deps/rocksdb/rocksdb/util/compression.h +16 -1298
  322. package/deps/rocksdb/rocksdb/util/compression_test.cc +347 -61
  323. package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +8 -9
  324. package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +1 -1
  325. package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -1
  326. package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -3
  327. package/deps/rocksdb/rocksdb/util/filter_bench.cc +18 -18
  328. package/deps/rocksdb/rocksdb/util/gflags_compat.h +3 -3
  329. package/deps/rocksdb/rocksdb/util/hash_test.cc +19 -7
  330. package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.cc +1099 -0
  331. package/deps/rocksdb/rocksdb/util/io_dispatcher_imp.h +36 -0
  332. package/deps/rocksdb/rocksdb/util/io_dispatcher_test.cc +1919 -0
  333. package/deps/rocksdb/rocksdb/util/math.h +3 -1
  334. package/deps/rocksdb/rocksdb/util/mutexlock.h +19 -19
  335. package/deps/rocksdb/rocksdb/util/ribbon_alg.h +25 -25
  336. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +5 -7
  337. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -5
  338. package/deps/rocksdb/rocksdb/util/slice.cc +0 -10
  339. package/deps/rocksdb/rocksdb/util/slice_test.cc +35 -1
  340. package/deps/rocksdb/rocksdb/util/slice_transform_test.cc +5 -7
  341. package/deps/rocksdb/rocksdb/util/status.cc +3 -1
  342. package/deps/rocksdb/rocksdb/util/stop_watch.h +2 -0
  343. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +4 -1
  344. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -78
  345. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +12 -93
  346. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +1 -4
  347. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +0 -21
  348. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +6 -48
  349. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +94 -307
  350. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +12 -58
  351. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +2 -8
  352. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +2 -3
  353. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +205 -811
  354. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +18 -9
  355. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +2 -7
  356. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +1 -9
  357. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +17 -11
  358. package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +1 -1
  359. package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.h +1 -1
  360. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
  361. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +68 -61
  362. package/deps/rocksdb/rocksdb/utilities/debug.cc +2 -1
  363. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +105 -59
  364. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +274 -7
  365. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs_test.cc +94 -0
  366. package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +13 -17
  367. package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +16 -3
  368. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +25 -25
  369. package/deps/rocksdb/rocksdb/utilities/object_registry.cc +40 -40
  370. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +2 -5
  371. package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +17 -19
  372. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +2 -2
  373. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +2 -2
  374. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +1 -1
  375. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -2
  376. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +4 -13
  377. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +3 -3
  378. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
  379. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_seqno_test.cc +431 -0
  380. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +1 -2
  381. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +91 -0
  382. package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.cc +562 -0
  383. package/deps/rocksdb/rocksdb/utilities/trie_index/bitvector.h +615 -0
  384. package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.cc +2575 -0
  385. package/deps/rocksdb/rocksdb/utilities/trie_index/louds_trie.h +685 -0
  386. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_db_test.cc +2843 -0
  387. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.cc +567 -0
  388. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_factory.h +275 -0
  389. package/deps/rocksdb/rocksdb/utilities/trie_index/trie_index_test.cc +5183 -0
  390. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +4 -3
  391. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +1 -1
  392. package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +2 -2
  393. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +3 -3
  394. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +93 -88
  395. package/deps/rocksdb/rocksdb.gyp +7 -0
  396. package/index.js +11 -2
  397. package/iterator.js +15 -7
  398. package/package.json +1 -1
  399. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  400. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
  401. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_custom_library.h +0 -43
  402. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/lua/rocks_lua_util.h +0 -55
@@ -576,9 +576,8 @@ Status GetDecompressor(const std::string& compression_name,
576
576
  std::shared_ptr<Decompressor>* out_decompressor) {
577
577
  if (compression_name.empty()) {
578
578
  // Very old file (before RocksDB 4.9.0) that might contain compressed
579
- // blocks. Get a general decompressor for the format version.
580
- auto mgr_to_use = GetBuiltinCompressionManager(
581
- GetCompressFormatForVersion(table_format_version));
579
+ // blocks. Get a general decompressor (for all supported format_versions)
580
+ auto mgr_to_use = GetBuiltinV2CompressionManager();
582
581
  *out_decompressor = mgr_to_use->GetDecompressor();
583
582
  return Status::OK();
584
583
  }
@@ -664,8 +663,7 @@ Status GetDecompressor(const std::string& compression_name,
664
663
  compression_name);
665
664
  } else if (saved_comp_type != kNoCompression) {
666
665
  // Use built-in compression manager
667
- auto mgr_to_use = GetBuiltinCompressionManager(
668
- GetCompressFormatForVersion(table_format_version));
666
+ auto mgr_to_use = GetBuiltinV2CompressionManager();
669
667
  *out_decompressor =
670
668
  mgr_to_use->GetDecompressorOptimizeFor(saved_comp_type);
671
669
  } else {
@@ -810,7 +808,8 @@ Status BlockBasedTable::Open(
810
808
  }
811
809
  return s;
812
810
  }
813
- if (!IsSupportedFormatVersion(footer.format_version()) &&
811
+ if (!IsSupportedFormatVersionForRead(kBlockBasedTableMagicNumber,
812
+ footer.format_version()) &&
814
813
  !TEST_AllowUnsupportedFormatVersion()) {
815
814
  return Status::Corruption(
816
815
  "Unknown Footer version. Maybe this file was created with newer "
@@ -824,13 +823,6 @@ Status BlockBasedTable::Open(
824
823
  rep->file = std::move(file);
825
824
  rep->footer = footer;
826
825
 
827
- // Some ancient versions (~2.5 - 2.7, format_version=1) could compress the
828
- // metaindex block, so we need to allow for that
829
- if (footer.format_version() < 2) {
830
- auto mgr = GetBuiltinCompressionManager(/*compression_format_version=*/1);
831
- rep->decompressor = mgr->GetDecompressor();
832
- }
833
-
834
826
  // For fully portable/stable cache keys, we need to read the properties
835
827
  // block before setting up cache keys. TODO: consider setting up a bootstrap
836
828
  // cache key for PersistentCache to use for metaindex and properties blocks.
@@ -875,7 +867,8 @@ Status BlockBasedTable::Open(
875
867
  &rep->table_options, &rep->ioptions, rep->ioptions.stats,
876
868
  rep->decompressor.get(), block_protection_bytes_per_key,
877
869
  rep->internal_comparator.user_comparator(), rep->index_value_is_full,
878
- rep->index_has_first_key);
870
+ rep->index_has_first_key, rep->data_block_restart_interval,
871
+ rep->index_block_restart_interval);
879
872
 
880
873
  // Check expected unique id if provided
881
874
  if (expected_unique_id != kNullUniqueId64x2) {
@@ -935,20 +928,18 @@ Status BlockBasedTable::Open(
935
928
  rep->table_prefix_extractor = prefix_extractor;
936
929
  } else {
937
930
  // Current prefix_extractor doesn't match table
938
- if (rep->table_properties) {
939
- //**TODO: If/When the DBOptions has a registry in it, the ConfigOptions
940
- // will need to use it
941
- ConfigOptions config_options;
942
- Status st = SliceTransform::CreateFromString(
943
- config_options, rep->table_properties->prefix_extractor_name,
944
- &(rep->table_prefix_extractor));
945
- if (!st.ok()) {
946
- //**TODO: Should this be error be returned or swallowed?
947
- ROCKS_LOG_ERROR(rep->ioptions.logger,
948
- "Failed to create prefix extractor[%s]: %s",
949
- rep->table_properties->prefix_extractor_name.c_str(),
950
- st.ToString().c_str());
951
- }
931
+ //**TODO: If/When the DBOptions has a registry in it, the ConfigOptions
932
+ // will need to use it
933
+ ConfigOptions config_options;
934
+ Status st = SliceTransform::CreateFromString(
935
+ config_options, rep->table_properties->prefix_extractor_name,
936
+ &(rep->table_prefix_extractor));
937
+ if (!st.ok()) {
938
+ //**TODO: Should this be error be returned or swallowed?
939
+ ROCKS_LOG_ERROR(rep->ioptions.logger,
940
+ "Failed to create prefix extractor[%s]: %s",
941
+ rep->table_properties->prefix_extractor_name.c_str(),
942
+ st.ToString().c_str());
952
943
  }
953
944
  }
954
945
 
@@ -1094,86 +1085,82 @@ Status BlockBasedTable::ReadPropertiesBlock(
1094
1085
  BlockHandle handle;
1095
1086
  s = FindOptionalMetaBlock(meta_iter, kPropertiesBlockName, &handle);
1096
1087
 
1088
+ if (!s.ok()) {
1089
+ return s;
1090
+ } else if (handle.IsNull()) {
1091
+ return Status::Corruption("Cannot find Properties block from file.");
1092
+ }
1093
+
1094
+ s = meta_iter->status();
1095
+ std::unique_ptr<TableProperties> table_properties;
1096
+ if (s.ok()) {
1097
+ s = ReadTablePropertiesHelper(
1098
+ ro, handle, rep_->file.get(), prefetch_buffer, rep_->footer,
1099
+ rep_->ioptions, &table_properties, nullptr /* memory_allocator */);
1100
+ }
1101
+
1102
+ if (!s.ok()) {
1103
+ return s;
1104
+ }
1105
+
1106
+ assert(table_properties != nullptr);
1107
+ rep_->table_properties = std::move(table_properties);
1108
+
1109
+ rep_->data_block_restart_interval = static_cast<uint32_t>(
1110
+ rep_->table_properties->data_block_restart_interval);
1111
+ rep_->index_block_restart_interval = static_cast<uint32_t>(
1112
+ rep_->table_properties->index_block_restart_interval);
1113
+ rep_->separate_key_value_in_data_block =
1114
+ rep_->table_properties->separate_key_value_in_data_block > 0;
1115
+
1116
+ rep_->index_has_first_key =
1117
+ rep_->index_type == BlockBasedTableOptions::kBinarySearchWithFirstKey;
1118
+
1119
+ s = rep_->seqno_to_time_mapping.DecodeFrom(
1120
+ rep_->table_properties->seqno_to_time_mapping);
1097
1121
  if (!s.ok()) {
1098
1122
  ROCKS_LOG_WARN(rep_->ioptions.logger,
1099
- "Error when seeking to properties block from file: %s",
1123
+ "Problem reading or processing seqno-to-time mapping: %s",
1100
1124
  s.ToString().c_str());
1101
- } else if (!handle.IsNull()) {
1102
- s = meta_iter->status();
1103
- std::unique_ptr<TableProperties> table_properties;
1104
- if (s.ok()) {
1105
- s = ReadTablePropertiesHelper(
1106
- ro, handle, rep_->file.get(), prefetch_buffer, rep_->footer,
1107
- rep_->ioptions, &table_properties, nullptr /* memory_allocator */);
1108
- }
1109
- IGNORE_STATUS_IF_ERROR(s);
1125
+ }
1110
1126
 
1111
- if (!s.ok()) {
1112
- ROCKS_LOG_WARN(rep_->ioptions.logger,
1113
- "Encountered error while reading data from properties "
1114
- "block %s",
1115
- s.ToString().c_str());
1116
- } else {
1117
- assert(table_properties != nullptr);
1118
- rep_->table_properties = std::move(table_properties);
1127
+ // Read the table properties
1128
+ rep_->whole_key_filtering &= IsFeatureSupported(
1129
+ *(rep_->table_properties),
1130
+ BlockBasedTablePropertyNames::kWholeKeyFiltering, rep_->ioptions.logger);
1131
+ rep_->prefix_filtering &= IsFeatureSupported(
1132
+ *(rep_->table_properties), BlockBasedTablePropertyNames::kPrefixFiltering,
1133
+ rep_->ioptions.logger);
1119
1134
 
1120
- if (s.ok()) {
1121
- s = rep_->seqno_to_time_mapping.DecodeFrom(
1122
- rep_->table_properties->seqno_to_time_mapping);
1123
- }
1124
- if (!s.ok()) {
1125
- ROCKS_LOG_WARN(
1126
- rep_->ioptions.logger,
1127
- "Problem reading or processing seqno-to-time mapping: %s",
1128
- s.ToString().c_str());
1129
- }
1130
- }
1131
- } else {
1132
- ROCKS_LOG_ERROR(rep_->ioptions.logger,
1133
- "Cannot find Properties block from file.");
1135
+ rep_->index_key_includes_seq =
1136
+ rep_->table_properties->index_key_is_user_key == 0;
1137
+ rep_->index_value_is_full =
1138
+ rep_->table_properties->index_value_is_delta_encoded == 0;
1139
+
1140
+ // Read index_type from properties (required for format_version >= 2)
1141
+ auto& props = rep_->table_properties->user_collected_properties;
1142
+ auto index_type_pos = props.find(BlockBasedTablePropertyNames::kIndexType);
1143
+ if (index_type_pos == props.end()) {
1144
+ return Status::Corruption("Missing index type property");
1145
+ }
1146
+ rep_->index_type = static_cast<BlockBasedTableOptions::IndexType>(
1147
+ DecodeFixed32(index_type_pos->second.c_str()));
1148
+ auto min_ts_pos = props.find("rocksdb.timestamp_min");
1149
+ if (min_ts_pos != props.end()) {
1150
+ rep_->min_timestamp = Slice(min_ts_pos->second);
1151
+ }
1152
+ auto max_ts_pos = props.find("rocksdb.timestamp_max");
1153
+ if (max_ts_pos != props.end()) {
1154
+ rep_->max_timestamp = Slice(max_ts_pos->second);
1134
1155
  }
1135
1156
 
1136
- // Read the table properties, if provided.
1137
- if (rep_->table_properties) {
1138
- rep_->whole_key_filtering &=
1139
- IsFeatureSupported(*(rep_->table_properties),
1140
- BlockBasedTablePropertyNames::kWholeKeyFiltering,
1141
- rep_->ioptions.logger);
1142
- rep_->prefix_filtering &= IsFeatureSupported(
1143
- *(rep_->table_properties),
1144
- BlockBasedTablePropertyNames::kPrefixFiltering, rep_->ioptions.logger);
1145
-
1146
- rep_->index_key_includes_seq =
1147
- rep_->table_properties->index_key_is_user_key == 0;
1148
- rep_->index_value_is_full =
1149
- rep_->table_properties->index_value_is_delta_encoded == 0;
1150
-
1151
- // Update index_type with the true type.
1152
- // If table properties don't contain index type, we assume that the table
1153
- // is in very old format and has kBinarySearch index type.
1154
- auto& props = rep_->table_properties->user_collected_properties;
1155
- auto index_type_pos = props.find(BlockBasedTablePropertyNames::kIndexType);
1156
- if (index_type_pos != props.end()) {
1157
- rep_->index_type = static_cast<BlockBasedTableOptions::IndexType>(
1158
- DecodeFixed32(index_type_pos->second.c_str()));
1159
- }
1160
- auto min_ts_pos = props.find("rocksdb.timestamp_min");
1161
- if (min_ts_pos != props.end()) {
1162
- rep_->min_timestamp = Slice(min_ts_pos->second);
1163
- }
1164
- auto max_ts_pos = props.find("rocksdb.timestamp_max");
1165
- if (max_ts_pos != props.end()) {
1166
- rep_->max_timestamp = Slice(max_ts_pos->second);
1167
- }
1168
-
1169
- rep_->index_has_first_key =
1170
- rep_->index_type == BlockBasedTableOptions::kBinarySearchWithFirstKey;
1171
-
1172
- s = GetGlobalSequenceNumber(*(rep_->table_properties), largest_seqno,
1173
- &(rep_->global_seqno));
1174
- if (!s.ok()) {
1175
- ROCKS_LOG_ERROR(rep_->ioptions.logger, "%s", s.ToString().c_str());
1176
- }
1157
+ rep_->index_has_first_key =
1158
+ rep_->index_type == BlockBasedTableOptions::kBinarySearchWithFirstKey;
1159
+
1160
+ s = GetGlobalSequenceNumber(*(rep_->table_properties), largest_seqno,
1161
+ &(rep_->global_seqno));
1162
+ if (!s.ok()) {
1163
+ ROCKS_LOG_ERROR(rep_->ioptions.logger, "%s", s.ToString().c_str());
1177
1164
  }
1178
1165
  return s;
1179
1166
  }
@@ -1706,7 +1693,8 @@ IndexBlockIter* BlockBasedTable::InitBlockIterator<IndexBlockIter>(
1706
1693
  rep->get_global_seqno(block_type), input_iter, rep->ioptions.stats,
1707
1694
  /* total_order_seek */ true, rep->index_has_first_key,
1708
1695
  rep->index_key_includes_seq, rep->index_value_is_full,
1709
- block_contents_pinned, rep->user_defined_timestamps_persisted);
1696
+ block_contents_pinned, rep->user_defined_timestamps_persisted,
1697
+ nullptr /* prefix_index */, rep->table_options.index_block_search_type);
1710
1698
  }
1711
1699
 
1712
1700
  // Right now only called for Data blocks.
@@ -3060,12 +3048,7 @@ uint64_t BlockBasedTable::ApproximateDataOffsetOf(
3060
3048
  }
3061
3049
 
3062
3050
  uint64_t BlockBasedTable::GetApproximateDataSize() {
3063
- // Should be in table properties unless super old version
3064
- if (rep_->table_properties) {
3065
- return rep_->table_properties->data_size;
3066
- }
3067
- // Fall back to rough estimate from footer
3068
- return rep_->footer.metaindex_handle().offset();
3051
+ return rep_->table_properties->data_size;
3069
3052
  }
3070
3053
 
3071
3054
  uint64_t BlockBasedTable::ApproximateOffsetOf(const ReadOptions& read_options,
@@ -3241,6 +3224,17 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file,
3241
3224
  "--------------------------------------\n";
3242
3225
  out_stream << " " << rep_->footer.ToString() << "\n";
3243
3226
 
3227
+ // Output Checksum Type Legend
3228
+ out_stream << "Block Checksum Type Legend:\n"
3229
+ "--------------------------------------\n";
3230
+ out_stream << " 0 = kNoChecksum\n";
3231
+ out_stream << " 1 = kCRC32c\n";
3232
+ out_stream << " 2 = kxxHash\n";
3233
+ out_stream << " 3 = kxxHash64\n";
3234
+ out_stream << " 4 = kXXH3\n";
3235
+ out_stream << " (This file uses checksum type: "
3236
+ << static_cast<int>(rep_->footer.checksum_type()) << ")\n\n";
3237
+
3244
3238
  // Output MetaIndex
3245
3239
  out_stream << "Metaindex Details:\n"
3246
3240
  "--------------------------------------\n";
@@ -3251,25 +3245,47 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file,
3251
3245
  Status s = ReadMetaIndexBlock(ro, nullptr /* prefetch_buffer */, &metaindex,
3252
3246
  &metaindex_iter);
3253
3247
  if (s.ok()) {
3248
+ // Print metaindex block checksum
3249
+ DumpBlockChecksumInfo(rep_->footer.metaindex_handle(), ro,
3250
+ "Metaindex block", out_stream);
3251
+
3254
3252
  for (metaindex_iter->SeekToFirst(); metaindex_iter->Valid();
3255
3253
  metaindex_iter->Next()) {
3256
3254
  s = metaindex_iter->status();
3257
3255
  if (!s.ok()) {
3258
3256
  return s;
3259
3257
  }
3258
+ // Parse block handle from metaindex value
3259
+ BlockHandle block_handle;
3260
+ Slice input = metaindex_iter->value();
3261
+ Status handle_status = block_handle.DecodeFrom(&input);
3262
+
3263
+ if (!handle_status.ok()) {
3264
+ out_stream << " Skip the block with type "
3265
+ << metaindex_iter->key().ToString()
3266
+ << " due to error: " << handle_status.ToString() << "\n\n";
3267
+ continue;
3268
+ }
3269
+
3260
3270
  if (metaindex_iter->key() == kPropertiesBlockName) {
3261
3271
  out_stream << " Properties block handle: "
3262
3272
  << metaindex_iter->value().ToString(true) << "\n";
3273
+ DumpBlockChecksumInfo(block_handle, ro, "Properties block", out_stream);
3263
3274
  } else if (metaindex_iter->key() == kCompressionDictBlockName) {
3264
3275
  out_stream << " Compression dictionary block handle: "
3265
3276
  << metaindex_iter->value().ToString(true) << "\n";
3277
+ DumpBlockChecksumInfo(block_handle, ro, "Compression dictionary block",
3278
+ out_stream);
3266
3279
  } else if (strstr(metaindex_iter->key().ToString().c_str(),
3267
3280
  "filter.rocksdb.") != nullptr) {
3268
3281
  out_stream << " Filter block handle: "
3269
3282
  << metaindex_iter->value().ToString(true) << "\n";
3283
+ DumpBlockChecksumInfo(block_handle, ro, "Filter block", out_stream);
3270
3284
  } else if (metaindex_iter->key() == kRangeDelBlockName) {
3271
3285
  out_stream << " Range deletion block handle: "
3272
3286
  << metaindex_iter->value().ToString(true) << "\n";
3287
+ DumpBlockChecksumInfo(block_handle, ro, "Range deletion block",
3288
+ out_stream);
3273
3289
  }
3274
3290
  }
3275
3291
  out_stream << "\n";
@@ -3346,11 +3362,61 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file,
3346
3362
  return Status::OK();
3347
3363
  }
3348
3364
 
3365
+ void BlockBasedTable::DumpBlockChecksumInfo(const BlockHandle& block_handle,
3366
+ const ReadOptions& read_options,
3367
+ const char* block_name,
3368
+ std::ostream& out_stream) const {
3369
+ if (rep_->footer.GetBlockTrailerSize() == 0) {
3370
+ return;
3371
+ }
3372
+
3373
+ size_t block_size = static_cast<size_t>(block_handle.size());
3374
+ size_t block_size_with_trailer = block_size + kBlockTrailerSize;
3375
+ std::unique_ptr<char[]> raw_block(new char[block_size_with_trailer]);
3376
+ Slice raw_block_slice;
3377
+ IOOptions opts;
3378
+ IODebugContext dbg;
3379
+ IOStatus io_s = rep_->file->PrepareIOOptions(read_options, opts, &dbg);
3380
+ if (io_s.ok()) {
3381
+ io_s = rep_->file->Read(opts, block_handle.offset(),
3382
+ block_size_with_trailer, &raw_block_slice,
3383
+ raw_block.get(), /*aligned_buf=*/nullptr, &dbg);
3384
+ }
3385
+ if (io_s.ok() && raw_block_slice.size() == block_size_with_trailer) {
3386
+ const char* data = raw_block_slice.data();
3387
+ uint8_t compression_type_byte = static_cast<uint8_t>(data[block_size]);
3388
+ uint32_t stored_checksum = DecodeFixed32(data + block_size + 1);
3389
+ uint32_t modifier = ChecksumModifierForContext(
3390
+ rep_->footer.base_context_checksum(), block_handle.offset());
3391
+ uint32_t actual_checksum = stored_checksum - modifier;
3392
+ out_stream << " " << block_name << " checksum type: "
3393
+ << static_cast<int>(rep_->footer.checksum_type())
3394
+ << " checksum value: 0x" << std::hex << actual_checksum
3395
+ << std::dec << " offset: " << block_handle.offset()
3396
+ << " size: " << block_size << " compression type: "
3397
+ << static_cast<int>(compression_type_byte) << "\n";
3398
+ } else {
3399
+ out_stream << " ERROR: Failed to read " << block_name << " checksum info";
3400
+ if (!io_s.ok()) {
3401
+ out_stream << " - " << io_s.ToString();
3402
+ } else if (raw_block_slice.size() != block_size_with_trailer) {
3403
+ out_stream << " - read " << raw_block_slice.size() << " bytes, expected "
3404
+ << block_size_with_trailer;
3405
+ }
3406
+ out_stream << "\n";
3407
+ }
3408
+ }
3409
+
3349
3410
  Status BlockBasedTable::DumpIndexBlock(std::ostream& out_stream) {
3350
3411
  out_stream << "Index Details:\n"
3351
3412
  "--------------------------------------\n";
3352
3413
  // TODO: plumb Env::IOActivity, Env::IOPriority
3353
3414
  const ReadOptions read_options;
3415
+
3416
+ // Print index block checksum information
3417
+ DumpBlockChecksumInfo(rep_->index_handle, read_options, "Index block",
3418
+ out_stream);
3419
+
3354
3420
  std::unique_ptr<InternalIteratorBase<IndexValue>> blockhandles_iter(
3355
3421
  NewIndexIterator(read_options, /*disable_prefix_seek=*/false,
3356
3422
  /*input_iter=*/nullptr, /*get_context=*/nullptr,
@@ -3433,6 +3499,10 @@ Status BlockBasedTable::DumpDataBlocks(std::ostream& out_stream,
3433
3499
 
3434
3500
  out_stream << "Data Block # " << block_id << " @ "
3435
3501
  << blockhandles_iter->value().handle.ToString(true) << "\n";
3502
+
3503
+ // Read block checksum information
3504
+ DumpBlockChecksumInfo(bh, read_options, "Data block", out_stream);
3505
+
3436
3506
  out_stream << "--------------------------------------\n";
3437
3507
 
3438
3508
  std::unique_ptr<InternalIterator> datablock_iter;
@@ -555,6 +555,10 @@ class BlockBasedTable : public TableReader {
555
555
  void DumpKeyValue(const Slice& key, const Slice& value,
556
556
  std::ostream& out_stream,
557
557
  bool show_sequence_number_type = false);
558
+ void DumpBlockChecksumInfo(const BlockHandle& block_handle,
559
+ const ReadOptions& read_options,
560
+ const char* block_name,
561
+ std::ostream& out_stream) const;
558
562
 
559
563
  // Returns false if prefix_extractor exists and is compatible with that used
560
564
  // in building the table file, otherwise true.
@@ -575,6 +579,8 @@ class BlockBasedTable : public TableReader {
575
579
  friend class PartitionedFilterBlockReader;
576
580
  friend class PartitionedFilterBlockTest;
577
581
  friend class DBBasicTest_MultiGetIOBufferOverrun_Test;
582
+ friend class ReadSet;
583
+ friend class IODispatcherTest;
578
584
  };
579
585
 
580
586
  // Maintaining state of a two-level iteration on a partitioned index structure.
@@ -688,6 +694,13 @@ struct BlockBasedTable::Rep {
688
694
  bool index_key_includes_seq = true;
689
695
  bool index_value_is_full = true;
690
696
 
697
+ // Restart intervals read from table properties (0 if not available)
698
+ uint32_t data_block_restart_interval = 0;
699
+ uint32_t index_block_restart_interval = 0;
700
+
701
+ // If true, then data blocks have keys and values separated.
702
+ bool separate_key_value_in_data_block = false;
703
+
691
704
  // Whether block checksums in metadata blocks were verified on open.
692
705
  // This is only to mostly maintain current dubious behavior of VerifyChecksum
693
706
  // with respect to index blocks, but only when the checksum was previously
@@ -83,10 +83,18 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(
83
83
  decomp = dict.GetValue()->decompressor_.get();
84
84
  }
85
85
  }
86
- s = RetrieveBlock(
87
- prefetch_buffer, ro, handle, decomp, &block.As<IterBlocklike>(),
88
- get_context, lookup_context, for_compaction,
89
- /* use_cache */ true, async_read, use_block_cache_for_lookup);
86
+
87
+ if (block_type == BlockType::kRangeDeletion) {
88
+ s = RetrieveBlock(prefetch_buffer, ro, handle, decomp,
89
+ &block.As<Block_kRangeDeletion>(), get_context,
90
+ lookup_context, for_compaction, /* use_cache */ true,
91
+ async_read, use_block_cache_for_lookup);
92
+ } else {
93
+ s = RetrieveBlock(
94
+ prefetch_buffer, ro, handle, decomp, &block.As<IterBlocklike>(),
95
+ get_context, lookup_context, for_compaction,
96
+ /* use_cache */ true, async_read, use_block_cache_for_lookup);
97
+ }
90
98
  }
91
99
 
92
100
  if (s.IsTryAgain() && async_read) {
@@ -100,6 +108,9 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(
100
108
  }
101
109
 
102
110
  assert(block.GetValue() != nullptr);
111
+ assert(block_type != BlockType::kData ||
112
+ block.GetValue()->HasSeparatedKV() ==
113
+ rep_->separate_key_value_in_data_block);
103
114
 
104
115
  // Block contents are pinned and it is still pinned after the iterator
105
116
  // is destroyed as long as cleanup functions are moved to another object,
@@ -161,6 +172,8 @@ TBlockIter* BlockBasedTable::NewDataBlockIterator(const ReadOptions& ro,
161
172
  }
162
173
 
163
174
  assert(block.GetValue() != nullptr);
175
+ assert(block.GetValue()->HasSeparatedKV() ==
176
+ rep_->separate_key_value_in_data_block);
164
177
  // Block contents are pinned and it is still pinned after the iterator
165
178
  // is destroyed as long as cleanup functions are moved to another object,
166
179
  // when:
@@ -117,6 +117,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
117
117
 
118
118
  PERF_COUNTER_ADD(block_read_count, 1);
119
119
  PERF_COUNTER_ADD(block_read_byte, BlockSizeWithTrailer(handle));
120
+ PERF_COUNTER_ADD(data_block_read_byte, BlockSizeWithTrailer(handle));
120
121
  }
121
122
  // Handle the last block and process the pending last request
122
123
  if (prev_len != 0) {
@@ -220,7 +221,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
220
221
  // in each read request. Checksum is stored in the block trailer,
221
222
  // beyond the payload size.
222
223
  s = VerifyBlockChecksum(footer, data, handle.size(),
223
- rep_->file->file_name(), handle.offset());
224
+ rep_->file->file_name(), handle.offset(),
225
+ BlockType::kData);
224
226
  RecordTick(ioptions.stats, BLOCK_CHECKSUM_COMPUTE_COUNT);
225
227
  if (!s.ok()) {
226
228
  RecordTick(ioptions.stats, BLOCK_CHECKSUM_MISMATCH_COUNT);
@@ -248,7 +250,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
248
250
  assert(result.data() == data);
249
251
  assert(result.size() == BlockSizeWithTrailer(handle));
250
252
  s = VerifyBlockChecksum(footer, data, handle.size(),
251
- rep_->file->file_name(), handle.offset());
253
+ rep_->file->file_name(), handle.offset(),
254
+ BlockType::kData);
252
255
  if (s.ok()) {
253
256
  RecordTick(ioptions.stats,
254
257
  FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT);
@@ -1624,6 +1624,76 @@ TEST_P(BlockBasedTableReaderMultiScanTest, MultiScanUnpinPreviousBlocks) {
1624
1624
  }
1625
1625
  }
1626
1626
 
1627
+ // Regression test for assertion failure when re-seeking to an already-exhausted
1628
+ // scan range. This can happen when MergingIterator re-seeks a child iterator
1629
+ // after all scan ranges have been consumed (e.g., due to range tombstone
1630
+ // adjustments in other levels). The bug was that SeekToBlockIdx() set
1631
+ // valid_=true without bounds checking, while cur_idx_ was past
1632
+ // block_handles_.size().
1633
+ TEST_P(BlockBasedTableReaderMultiScanTest, MultiScanReseekAfterExhaustion) {
1634
+ std::vector<std::pair<std::string, std::string>> kv =
1635
+ BlockBasedTableReaderBaseTest::GenerateKVMap(
1636
+ 10 /* num_block */, true /* mixed_with_human_readable_string_value */,
1637
+ comparator_->timestamp_size(), same_key_diff_ts_, comparator_);
1638
+ std::string table_name = "BlockBasedTableReaderTest_ReseekAfterExhaustion" +
1639
+ CompressionTypeToString(compression_type_);
1640
+ ImmutableOptions ioptions(options_);
1641
+ CreateTable(table_name, ioptions, compression_type_, kv,
1642
+ compression_parallel_threads_, compression_dict_bytes_);
1643
+
1644
+ std::unique_ptr<BlockBasedTable> table;
1645
+ FileOptions foptions;
1646
+ foptions.use_direct_reads = use_direct_reads_;
1647
+ InternalKeyComparator comparator(options_.comparator);
1648
+ NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table,
1649
+ true /* bool prefetch_index_and_filter_in_cache */,
1650
+ nullptr /* status */, persist_udt_);
1651
+
1652
+ ReadOptions read_opts;
1653
+ std::unique_ptr<InternalIterator> iter;
1654
+ iter.reset(table->NewIterator(
1655
+ read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
1656
+ /*skip_filters=*/false, TableReaderCaller::kUncategorized));
1657
+
1658
+ // Set up two scan ranges covering blocks 0-4 and 5-9
1659
+ MultiScanArgs scan_options(BytewiseComparator());
1660
+ scan_options.insert(ExtractUserKey(kv[0].first),
1661
+ ExtractUserKey(kv[5 * kEntriesPerBlock - 1].first));
1662
+ scan_options.insert(ExtractUserKey(kv[5 * kEntriesPerBlock].first),
1663
+ ExtractUserKey(kv[10 * kEntriesPerBlock - 1].first));
1664
+
1665
+ iter->Prepare(&scan_options);
1666
+
1667
+ // Seek to range 1 and iterate through all blocks
1668
+ iter->Seek(kv[0].first);
1669
+ ASSERT_TRUE(iter->Valid());
1670
+ ASSERT_OK(iter->status());
1671
+ while (iter->Valid()) {
1672
+ iter->Next();
1673
+ }
1674
+
1675
+ // Re-seek to the first range's start key after range 1 is exhausted.
1676
+ // The forward-only check in MultiScanIndexIterator rejects this seek,
1677
+ // so the iterator remains in the scan_range_exhausted state (valid but
1678
+ // out-of-bound, signaling the caller to seek to the next range).
1679
+ iter->Seek(kv[0].first);
1680
+
1681
+ // Seek to range 2 and iterate through all blocks
1682
+ iter->Seek(kv[5 * kEntriesPerBlock].first);
1683
+ while (iter->Valid()) {
1684
+ iter->Next();
1685
+ }
1686
+ // Now all scan ranges are fully exhausted.
1687
+
1688
+ // Re-seek to the last range's start key. This simulates what happens when
1689
+ // MergingIterator re-seeks a child due to range tombstone key adjustment.
1690
+ // Before the fix, this would trigger:
1691
+ // assert(cur_idx_ < block_handles_.size()) in value()
1692
+ // because SeekToBlockIdx set valid_=true with cur_idx_ past bounds.
1693
+ iter->Seek(kv[5 * kEntriesPerBlock].first);
1694
+ ASSERT_FALSE(iter->Valid());
1695
+ }
1696
+
1627
1697
  // Test that fs_prefetch_support flag is correctly initialized during table
1628
1698
  // construction based on filesystem capabilities
1629
1699
  TEST_P(BlockBasedTableReaderTest, FSPrefetchSupportInitializedCorrectly) {