@nxtedition/rocksdb 13.5.7 → 13.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (509) hide show
  1. package/binding.cc +248 -70
  2. package/binding.gyp +2 -2
  3. package/deps/rocksdb/rocksdb/BUCK +12 -0
  4. package/deps/rocksdb/rocksdb/CMakeLists.txt +7 -0
  5. package/deps/rocksdb/rocksdb/Makefile +28 -23
  6. package/deps/rocksdb/rocksdb/cache/cache.cc +0 -1
  7. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +1 -2
  8. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +43 -39
  9. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +2 -0
  10. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +0 -1
  11. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +2 -3
  12. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +2 -2
  13. package/deps/rocksdb/rocksdb/cache/secondary_cache.cc +1 -3
  14. package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +11 -1
  15. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache_test.cc +13 -5
  16. package/deps/rocksdb/rocksdb/crash_test.mk +61 -15
  17. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +136 -45
  18. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +34 -16
  19. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +10 -7
  20. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +1 -2
  21. package/deps/rocksdb/rocksdb/db/blob/blob_file_meta.h +1 -0
  22. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +12 -9
  23. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +3 -4
  24. package/deps/rocksdb/rocksdb/db/blob/blob_source.cc +2 -2
  25. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +3 -4
  26. package/deps/rocksdb/rocksdb/db/builder.cc +22 -8
  27. package/deps/rocksdb/rocksdb/db/builder.h +5 -4
  28. package/deps/rocksdb/rocksdb/db/c.cc +556 -15
  29. package/deps/rocksdb/rocksdb/db/c_test.c +133 -12
  30. package/deps/rocksdb/rocksdb/db/column_family.cc +114 -50
  31. package/deps/rocksdb/rocksdb/db/column_family.h +53 -36
  32. package/deps/rocksdb/rocksdb/db/column_family_test.cc +6 -6
  33. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +0 -1
  34. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +95 -70
  35. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +71 -51
  36. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -86
  37. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +26 -68
  38. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +0 -122
  39. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +453 -258
  40. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +117 -92
  41. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +0 -1
  42. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +38 -38
  43. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +24 -17
  44. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +34 -45
  45. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +32 -31
  46. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +12 -3
  47. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +1 -1
  48. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +2 -1
  49. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +10 -10
  50. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +2 -1
  51. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +82 -34
  52. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +267 -179
  53. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +4 -1
  54. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +273 -89
  55. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +300 -14
  56. package/deps/rocksdb/rocksdb/db/compaction/compaction_state.cc +4 -4
  57. package/deps/rocksdb/rocksdb/db/compaction/compaction_state.h +2 -2
  58. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +28 -23
  59. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +69 -51
  60. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +522 -245
  61. package/deps/rocksdb/rocksdb/db/convenience.cc +15 -4
  62. package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -3
  63. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +0 -2
  64. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +196 -17
  65. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +74 -62
  66. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +48 -0
  67. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +682 -250
  68. package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +0 -1
  69. package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +3 -4
  70. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +11 -16
  71. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +57 -0
  72. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +2 -2
  73. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +1 -1
  74. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +540 -490
  75. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +347 -188
  76. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +584 -217
  77. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +13 -9
  78. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +5 -7
  79. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +40 -36
  80. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +1 -3
  81. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +751 -372
  82. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +35 -32
  83. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +24 -2
  84. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +125 -63
  85. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +2 -2
  86. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +311 -196
  87. package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +15 -5
  88. package/deps/rocksdb/rocksdb/db/db_iter.cc +42 -29
  89. package/deps/rocksdb/rocksdb/db/db_iter.h +96 -31
  90. package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +3 -4
  91. package/deps/rocksdb/rocksdb/db/db_iter_test.cc +168 -228
  92. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +454 -0
  93. package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +8 -8
  94. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +0 -1
  95. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +90 -0
  96. package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +60 -2
  97. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +7 -3
  98. package/deps/rocksdb/rocksdb/db/db_options_test.cc +85 -27
  99. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +3 -1
  100. package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +0 -2
  101. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +114 -2
  102. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +0 -1
  103. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +0 -1
  104. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +51 -3
  105. package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +0 -1
  106. package/deps/rocksdb/rocksdb/db/db_test.cc +325 -18
  107. package/deps/rocksdb/rocksdb/db/db_test2.cc +644 -20
  108. package/deps/rocksdb/rocksdb/db/db_test_util.cc +14 -6
  109. package/deps/rocksdb/rocksdb/db/db_test_util.h +9 -0
  110. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +64 -45
  111. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +203 -14
  112. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +259 -30
  113. package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +0 -1
  114. package/deps/rocksdb/rocksdb/db/db_write_test.cc +75 -1
  115. package/deps/rocksdb/rocksdb/db/dbformat.h +70 -6
  116. package/deps/rocksdb/rocksdb/db/deletefile_test.cc +0 -190
  117. package/deps/rocksdb/rocksdb/db/error_handler.cc +22 -7
  118. package/deps/rocksdb/rocksdb/db/error_handler.h +16 -1
  119. package/deps/rocksdb/rocksdb/db/event_helpers.cc +41 -26
  120. package/deps/rocksdb/rocksdb/db/experimental.cc +4 -3
  121. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +464 -78
  122. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +166 -69
  123. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +54 -25
  124. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +1 -3
  125. package/deps/rocksdb/rocksdb/db/flush_job.cc +98 -81
  126. package/deps/rocksdb/rocksdb/db/flush_job.h +4 -9
  127. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +80 -84
  128. package/deps/rocksdb/rocksdb/db/forward_iterator.cc +1 -1
  129. package/deps/rocksdb/rocksdb/db/forward_iterator.h +2 -2
  130. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +12 -19
  131. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +0 -2
  132. package/deps/rocksdb/rocksdb/db/internal_stats.cc +41 -15
  133. package/deps/rocksdb/rocksdb/db/internal_stats.h +63 -52
  134. package/deps/rocksdb/rocksdb/db/job_context.h +59 -24
  135. package/deps/rocksdb/rocksdb/db/listener_test.cc +69 -10
  136. package/deps/rocksdb/rocksdb/db/log_format.h +11 -2
  137. package/deps/rocksdb/rocksdb/db/log_reader.cc +147 -34
  138. package/deps/rocksdb/rocksdb/db/log_reader.h +40 -11
  139. package/deps/rocksdb/rocksdb/db/log_test.cc +16 -3
  140. package/deps/rocksdb/rocksdb/db/log_writer.cc +102 -55
  141. package/deps/rocksdb/rocksdb/db/log_writer.h +21 -2
  142. package/deps/rocksdb/rocksdb/db/malloc_stats.h +0 -2
  143. package/deps/rocksdb/rocksdb/db/memtable.cc +16 -47
  144. package/deps/rocksdb/rocksdb/db/memtable.h +76 -12
  145. package/deps/rocksdb/rocksdb/db/memtable_list.cc +23 -20
  146. package/deps/rocksdb/rocksdb/db/memtable_list.h +9 -11
  147. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +18 -37
  148. package/deps/rocksdb/rocksdb/db/merge_context.h +2 -1
  149. package/deps/rocksdb/rocksdb/db/merge_test.cc +8 -0
  150. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +3 -5
  151. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +15 -7
  152. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.h +6 -3
  153. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +22 -4
  154. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +41 -1
  155. package/deps/rocksdb/rocksdb/db/prefix_test.cc +0 -1
  156. package/deps/rocksdb/rocksdb/db/repair.cc +29 -34
  157. package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -1
  158. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +14 -15
  159. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +1 -3
  160. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +47 -1
  161. package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -3
  162. package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +1 -3
  163. package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +2 -1
  164. package/deps/rocksdb/rocksdb/db/version_builder.cc +2 -2
  165. package/deps/rocksdb/rocksdb/db/version_edit.cc +8 -37
  166. package/deps/rocksdb/rocksdb/db/version_edit.h +32 -1
  167. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +26 -18
  168. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +7 -5
  169. package/deps/rocksdb/rocksdb/db/version_set.cc +282 -197
  170. package/deps/rocksdb/rocksdb/db/version_set.h +54 -57
  171. package/deps/rocksdb/rocksdb/db/version_set_test.cc +28 -35
  172. package/deps/rocksdb/rocksdb/db/version_util.h +2 -3
  173. package/deps/rocksdb/rocksdb/db/wal_manager.cc +3 -2
  174. package/deps/rocksdb/rocksdb/db/wal_manager.h +0 -1
  175. package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +0 -1
  176. package/deps/rocksdb/rocksdb/db/wide/wide_columns.cc +1 -0
  177. package/deps/rocksdb/rocksdb/db/write_batch.cc +22 -8
  178. package/deps/rocksdb/rocksdb/db/write_batch_internal.h +5 -4
  179. package/deps/rocksdb/rocksdb/db/write_batch_test.cc +7 -6
  180. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +3 -4
  181. package/deps/rocksdb/rocksdb/db/write_thread.h +3 -3
  182. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +13 -5
  183. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +9 -2
  184. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_service.h +39 -0
  185. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compression_manager.h +65 -0
  186. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +45 -22
  187. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +7 -4
  188. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +22 -5
  189. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_table_properties_collector.h +28 -3
  190. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +143 -38
  191. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +4 -3
  192. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +80 -32
  193. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +51 -2
  194. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +23 -1
  195. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +305 -15
  196. package/deps/rocksdb/rocksdb/env/env.cc +32 -2
  197. package/deps/rocksdb/rocksdb/env/env_encryption.cc +0 -2
  198. package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +2 -4
  199. package/deps/rocksdb/rocksdb/env/env_posix.cc +4 -2
  200. package/deps/rocksdb/rocksdb/env/env_test.cc +0 -1
  201. package/deps/rocksdb/rocksdb/env/fs_posix.cc +20 -11
  202. package/deps/rocksdb/rocksdb/env/fs_readonly.h +0 -2
  203. package/deps/rocksdb/rocksdb/env/fs_remap.cc +0 -2
  204. package/deps/rocksdb/rocksdb/env/fs_remap.h +0 -2
  205. package/deps/rocksdb/rocksdb/env/io_posix.cc +6 -4
  206. package/deps/rocksdb/rocksdb/env/io_posix.h +3 -2
  207. package/deps/rocksdb/rocksdb/env/mock_env.cc +0 -1
  208. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +2 -2
  209. package/deps/rocksdb/rocksdb/file/delete_scheduler.h +0 -2
  210. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +0 -2
  211. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +30 -21
  212. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +16 -0
  213. package/deps/rocksdb/rocksdb/file/file_util.cc +32 -14
  214. package/deps/rocksdb/rocksdb/file/file_util.h +22 -5
  215. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +229 -76
  216. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +21 -12
  217. package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +10 -7
  218. package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc +12 -8
  219. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +1 -2
  220. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +0 -2
  221. package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +3 -3
  222. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +598 -0
  223. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_iterator.h +36 -0
  224. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +70 -11
  225. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +232 -11
  226. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +1 -1
  227. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +3 -1
  228. package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h +149 -15
  229. package/deps/rocksdb/rocksdb/include/rocksdb/convenience.h +17 -2
  230. package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +132 -34
  231. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +158 -79
  232. package/deps/rocksdb/rocksdb/include/rocksdb/db_bench_tool.h +2 -1
  233. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +4 -5
  234. package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +1 -3
  235. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +5 -0
  236. package/deps/rocksdb/rocksdb/include/rocksdb/external_table.h +275 -0
  237. package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +2 -1
  238. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +50 -5
  239. package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +10 -0
  240. package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +13 -0
  241. package/deps/rocksdb/rocksdb/include/rocksdb/ldb_tool.h +0 -1
  242. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +5 -2
  243. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +13 -0
  244. package/deps/rocksdb/rocksdb/include/rocksdb/multi_scan.h +237 -0
  245. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +230 -39
  246. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +15 -0
  247. package/deps/rocksdb/rocksdb/include/rocksdb/perf_level.h +31 -11
  248. package/deps/rocksdb/rocksdb/include/rocksdb/slice.h +41 -0
  249. package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +1 -1
  250. package/deps/rocksdb/rocksdb/include/rocksdb/sst_dump_tool.h +0 -1
  251. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +5 -1
  252. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +0 -1
  253. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +18 -3
  254. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +2 -0
  255. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +20 -8
  256. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +19 -2
  257. package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +1 -1
  258. package/deps/rocksdb/rocksdb/include/rocksdb/tool_hooks.h +124 -0
  259. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +1 -0
  260. package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +26 -1
  261. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +55 -6
  262. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/debug.h +3 -5
  263. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_mirror.h +0 -2
  264. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -2
  265. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/memory_util.h +0 -1
  266. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/object_registry.h +1 -2
  267. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_util.h +0 -1
  268. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index.h +96 -8
  269. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/secondary_index_faiss.h +117 -0
  270. package/deps/rocksdb/rocksdb/{utilities/secondary_index/faiss_ivf_index.h → include/rocksdb/utilities/secondary_index_simple.h} +11 -14
  271. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +26 -11
  272. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +16 -3
  273. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +0 -2
  274. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +63 -7
  275. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h +0 -1
  276. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +28 -12
  277. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +3 -3
  278. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +0 -2
  279. package/deps/rocksdb/rocksdb/logging/event_logger_test.cc +1 -2
  280. package/deps/rocksdb/rocksdb/memory/memory_allocator_impl.h +1 -1
  281. package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +0 -1
  282. package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +0 -1
  283. package/deps/rocksdb/rocksdb/memtable/memtablerep_bench.cc +3 -1
  284. package/deps/rocksdb/rocksdb/memtable/skiplist.h +2 -2
  285. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +2 -4
  286. package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +69 -8
  287. package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.cc +32 -9
  288. package/deps/rocksdb/rocksdb/memtable/wbwi_memtable.h +58 -45
  289. package/deps/rocksdb/rocksdb/monitoring/histogram.h +1 -1
  290. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +5 -3
  291. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +5 -0
  292. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +1 -1
  293. package/deps/rocksdb/rocksdb/monitoring/thread_status_util_debug.cc +3 -2
  294. package/deps/rocksdb/rocksdb/options/cf_options.cc +44 -13
  295. package/deps/rocksdb/rocksdb/options/cf_options.h +21 -7
  296. package/deps/rocksdb/rocksdb/options/configurable.cc +5 -5
  297. package/deps/rocksdb/rocksdb/options/configurable_test.h +1 -2
  298. package/deps/rocksdb/rocksdb/options/customizable.cc +0 -1
  299. package/deps/rocksdb/rocksdb/options/customizable_test.cc +4 -11
  300. package/deps/rocksdb/rocksdb/options/db_options.cc +18 -15
  301. package/deps/rocksdb/rocksdb/options/db_options.h +2 -2
  302. package/deps/rocksdb/rocksdb/options/options.cc +296 -305
  303. package/deps/rocksdb/rocksdb/options/options_helper.cc +188 -62
  304. package/deps/rocksdb/rocksdb/options/options_helper.h +3 -3
  305. package/deps/rocksdb/rocksdb/options/options_parser.cc +2 -4
  306. package/deps/rocksdb/rocksdb/options/options_parser.h +0 -1
  307. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +17 -4
  308. package/deps/rocksdb/rocksdb/options/options_test.cc +101 -76
  309. package/deps/rocksdb/rocksdb/port/lang.h +2 -1
  310. package/deps/rocksdb/rocksdb/port/port_posix.cc +2 -1
  311. package/deps/rocksdb/rocksdb/port/stack_trace.cc +5 -4
  312. package/deps/rocksdb/rocksdb/port/win/env_win.cc +3 -2
  313. package/deps/rocksdb/rocksdb/port/win/xpress_win.cc +99 -1
  314. package/deps/rocksdb/rocksdb/port/win/xpress_win.h +6 -0
  315. package/deps/rocksdb/rocksdb/src.mk +17 -11
  316. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.h +0 -1
  317. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1094 -929
  318. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +6 -19
  319. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +76 -22
  320. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +2 -0
  321. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +221 -131
  322. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +12 -9
  323. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +23 -24
  324. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +38 -38
  325. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +7 -4
  326. package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +5 -5
  327. package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +10 -12
  328. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +6 -4
  329. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +35 -43
  330. package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index_test.cc +2 -1
  331. package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +1 -1
  332. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +1 -2
  333. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +0 -4
  334. package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +0 -1
  335. package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +3 -3
  336. package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +3 -3
  337. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +4 -4
  338. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +1 -1
  339. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +4 -5
  340. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +4 -4
  341. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +37 -35
  342. package/deps/rocksdb/rocksdb/table/block_fetcher.h +11 -7
  343. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +4 -3
  344. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +31 -5
  345. package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +2 -1
  346. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.h +0 -1
  347. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc +0 -1
  348. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +0 -1
  349. package/deps/rocksdb/rocksdb/table/external_table.cc +483 -0
  350. package/deps/rocksdb/rocksdb/table/format.cc +62 -44
  351. package/deps/rocksdb/rocksdb/table/format.h +35 -12
  352. package/deps/rocksdb/rocksdb/table/internal_iterator.h +3 -13
  353. package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +8 -0
  354. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +6 -0
  355. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +150 -141
  356. package/deps/rocksdb/rocksdb/table/meta_blocks.h +5 -0
  357. package/deps/rocksdb/rocksdb/table/multiget_context.h +3 -2
  358. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +8 -0
  359. package/deps/rocksdb/rocksdb/table/plain/plain_table_index.cc +0 -1
  360. package/deps/rocksdb/rocksdb/table/plain/plain_table_index.h +0 -2
  361. package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.h +0 -2
  362. package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +0 -1
  363. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +6 -6
  364. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +0 -1
  365. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +86 -7
  366. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +88 -2
  367. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +0 -1
  368. package/deps/rocksdb/rocksdb/table/table_builder.h +10 -1
  369. package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +3 -2
  370. package/deps/rocksdb/rocksdb/table/table_test.cc +899 -22
  371. package/deps/rocksdb/rocksdb/test_util/testutil.cc +3 -4
  372. package/deps/rocksdb/rocksdb/test_util/testutil.h +132 -1
  373. package/deps/rocksdb/rocksdb/test_util/transaction_test_util.cc +0 -1
  374. package/deps/rocksdb/rocksdb/test_util/transaction_test_util.h +0 -2
  375. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +163 -77
  376. package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +0 -2
  377. package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +0 -1
  378. package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +0 -1
  379. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +120 -52
  380. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +1 -0
  381. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +1 -1
  382. package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +0 -2
  383. package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.cc +2 -2
  384. package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.h +0 -2
  385. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +2 -1
  386. package/deps/rocksdb/rocksdb/tools/tool_hooks.cc +94 -0
  387. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +0 -1
  388. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.h +0 -1
  389. package/deps/rocksdb/rocksdb/trace_replay/io_tracer.cc +1 -1
  390. package/deps/rocksdb/rocksdb/trace_replay/io_tracer_test.cc +2 -1
  391. package/deps/rocksdb/rocksdb/trace_replay/trace_replay.cc +3 -5
  392. package/deps/rocksdb/rocksdb/util/async_file_reader.cc +1 -1
  393. package/deps/rocksdb/rocksdb/util/async_file_reader.h +15 -8
  394. package/deps/rocksdb/rocksdb/util/auto_skip_compressor.cc +131 -0
  395. package/deps/rocksdb/rocksdb/util/auto_skip_compressor.h +90 -0
  396. package/deps/rocksdb/rocksdb/util/autovector.h +1 -1
  397. package/deps/rocksdb/rocksdb/util/autovector_test.cc +2 -2
  398. package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc +0 -2
  399. package/deps/rocksdb/rocksdb/util/compression.cc +936 -4
  400. package/deps/rocksdb/rocksdb/util/compression.h +348 -232
  401. package/deps/rocksdb/rocksdb/util/compression_test.cc +229 -0
  402. package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +10 -10
  403. package/deps/rocksdb/rocksdb/util/crc32c_ppc.c +1 -0
  404. package/deps/rocksdb/rocksdb/util/data_structure.cc +2 -0
  405. package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +1 -3
  406. package/deps/rocksdb/rocksdb/util/ppc-opcode.h +5 -5
  407. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +108 -0
  408. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +67 -0
  409. package/deps/rocksdb/rocksdb/util/slice_test.cc +83 -0
  410. package/deps/rocksdb/rocksdb/util/string_util.cc +0 -2
  411. package/deps/rocksdb/rocksdb/util/string_util.h +10 -0
  412. package/deps/rocksdb/rocksdb/util/thread_operation.h +2 -1
  413. package/deps/rocksdb/rocksdb/util/udt_util.cc +18 -5
  414. package/deps/rocksdb/rocksdb/util/udt_util.h +10 -7
  415. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +650 -154
  416. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +438 -144
  417. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +0 -1
  418. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +0 -1
  419. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_gc_stats.h +0 -1
  420. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +16 -17
  421. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +2 -1
  422. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +0 -1
  423. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +0 -1
  424. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +7 -8
  425. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +4 -3
  426. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.h +0 -1
  427. package/deps/rocksdb/rocksdb/utilities/cache_dump_load.cc +0 -1
  428. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +2 -2
  429. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +1 -1
  430. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +0 -48
  431. package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc +0 -1
  432. package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.h +0 -1
  433. package/deps/rocksdb/rocksdb/utilities/debug.cc +7 -14
  434. package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +0 -1
  435. package/deps/rocksdb/rocksdb/utilities/env_mirror_test.cc +0 -2
  436. package/deps/rocksdb/rocksdb/utilities/env_timed.cc +0 -1
  437. package/deps/rocksdb/rocksdb/utilities/env_timed_test.cc +0 -2
  438. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +5 -3
  439. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +10 -9
  440. package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +0 -1
  441. package/deps/rocksdb/rocksdb/utilities/memory/memory_util.cc +0 -1
  442. package/deps/rocksdb/rocksdb/utilities/memory_allocators.h +1 -0
  443. package/deps/rocksdb/rocksdb/utilities/object_registry_test.cc +0 -2
  444. package/deps/rocksdb/rocksdb/utilities/options/options_util.cc +0 -1
  445. package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +0 -1
  446. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.cc +0 -1
  447. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.h +0 -2
  448. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +0 -2
  449. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.cc +0 -1
  450. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.h +0 -2
  451. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table.h +0 -2
  452. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_evictable.h +0 -2
  453. package/deps/rocksdb/rocksdb/utilities/persistent_cache/lrulist.h +0 -2
  454. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.h +0 -2
  455. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.cc +0 -1
  456. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.h +0 -2
  457. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.cc +0 -1
  458. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.h +0 -2
  459. package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index.cc +183 -32
  460. package/deps/rocksdb/rocksdb/utilities/secondary_index/faiss_ivf_index_test.cc +258 -12
  461. package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_helper.h +33 -0
  462. package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_iterator.cc +99 -0
  463. package/deps/rocksdb/rocksdb/utilities/secondary_index/secondary_index_mixin.h +280 -120
  464. package/deps/rocksdb/rocksdb/utilities/secondary_index/simple_secondary_index.cc +79 -0
  465. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +52 -16
  466. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.h +10 -6
  467. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +55 -0
  468. package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.cc +0 -1
  469. package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.cc +0 -2
  470. package/deps/rocksdb/rocksdb/utilities/transactions/lock/lock_manager.h +0 -1
  471. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +37 -12
  472. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.h +2 -0
  473. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.cc +0 -2
  474. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_tracker.cc +0 -2
  475. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +1 -1
  476. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/db.h +1 -1
  477. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +1 -1
  478. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/dbt.cc +2 -1
  479. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +2 -2
  480. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.cc +0 -1
  481. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.h +0 -2
  482. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +1 -3
  483. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +36 -10
  484. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +5 -7
  485. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +4 -5
  486. package/deps/rocksdb/rocksdb/utilities/transactions/snapshot_checker.cc +1 -4
  487. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +1 -2
  488. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.cc +0 -2
  489. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.h +0 -1
  490. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +1118 -37
  491. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +4 -7
  492. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +0 -2
  493. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +0 -2
  494. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +3 -3
  495. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +0 -1
  496. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +0 -2
  497. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +1 -2
  498. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.h +1 -2
  499. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +0 -1
  500. package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +0 -3
  501. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +125 -127
  502. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +45 -23
  503. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +54 -22
  504. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +477 -58
  505. package/deps/rocksdb/rocksdb.gyp +9 -4
  506. package/index.js +50 -9
  507. package/package.json +8 -1
  508. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  509. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
@@ -24,7 +24,10 @@ Reader::Reporter::~Reporter() = default;
24
24
 
25
25
  Reader::Reader(std::shared_ptr<Logger> info_log,
26
26
  std::unique_ptr<SequentialFileReader>&& _file,
27
- Reporter* reporter, bool checksum, uint64_t log_num)
27
+ Reporter* reporter, bool checksum, uint64_t log_num,
28
+ bool track_and_verify_wals, bool stop_replay_for_corruption,
29
+ uint64_t min_wal_number_to_keep,
30
+ const PredecessorWALInfo& observed_predecessor_wal_info)
28
31
  : info_log_(info_log),
29
32
  file_(std::move(_file)),
30
33
  reporter_(reporter),
@@ -37,13 +40,17 @@ Reader::Reader(std::shared_ptr<Logger> info_log,
37
40
  last_record_offset_(0),
38
41
  end_of_buffer_offset_(0),
39
42
  log_number_(log_num),
43
+ track_and_verify_wals_(track_and_verify_wals),
44
+ stop_replay_for_corruption_(stop_replay_for_corruption),
45
+ min_wal_number_to_keep_(min_wal_number_to_keep),
46
+ observed_predecessor_wal_info_(observed_predecessor_wal_info),
40
47
  recycled_(false),
41
48
  first_record_read_(false),
42
49
  compression_type_(kNoCompression),
43
50
  compression_type_record_read_(false),
44
51
  uncompress_(nullptr),
45
52
  hash_state_(nullptr),
46
- uncompress_hash_state_(nullptr){}
53
+ uncompress_hash_state_(nullptr) {}
47
54
 
48
55
  Reader::~Reader() {
49
56
  delete[] backing_store_;
@@ -65,6 +72,9 @@ Reader::~Reader() {
65
72
  //
66
73
  // TODO krad: Evaluate if we need to move to a more strict mode where we
67
74
  // restrict the inconsistency to only the last log
75
+ // TODO (hx235): move `wal_recovery_mode` to be a member data like other
76
+ // information (e.g, `stop_replay_for_corruption`) to decide whether to
77
+ // check for and surface corruption in `ReadRecord()`
68
78
  bool Reader::ReadRecord(Slice* record, std::string* scratch,
69
79
  WALRecoveryMode wal_recovery_mode,
70
80
  uint64_t* record_checksum) {
@@ -88,7 +98,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
88
98
  while (true) {
89
99
  uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
90
100
  size_t drop_size = 0;
91
- const unsigned int record_type =
101
+ const uint8_t record_type =
92
102
  ReadPhysicalRecord(&fragment, &drop_size, record_checksum);
93
103
  switch (record_type) {
94
104
  case kFullType:
@@ -185,6 +195,23 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
185
195
  }
186
196
  break;
187
197
  }
198
+ case kPredecessorWALInfoType:
199
+ case kRecyclePredecessorWALInfoType: {
200
+ prospective_record_offset = physical_record_offset;
201
+ scratch->clear();
202
+ last_record_offset_ = prospective_record_offset;
203
+
204
+ PredecessorWALInfo recorded_predecessor_wal_info;
205
+ Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
206
+ if (!s.ok()) {
207
+ ReportCorruption(fragment.size(),
208
+ "could not decode PredecessorWALInfoType record");
209
+ } else {
210
+ MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
211
+ recorded_predecessor_wal_info);
212
+ }
213
+ break;
214
+ }
188
215
  case kUserDefinedTimestampSizeType:
189
216
  case kRecyclableUserDefinedTimestampSizeType: {
190
217
  if (in_fragmented_record && !scratch->empty()) {
@@ -313,11 +340,13 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
313
340
  break;
314
341
 
315
342
  default: {
316
- std::string reason =
317
- "unknown record type " + std::to_string(record_type);
318
- ReportCorruption(
319
- (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
320
- reason.c_str());
343
+ if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
344
+ std::string reason =
345
+ "unknown record type " + std::to_string(record_type);
346
+ ReportCorruption(
347
+ (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
348
+ reason.c_str());
349
+ }
321
350
  in_fragmented_record = false;
322
351
  scratch->clear();
323
352
  break;
@@ -327,6 +356,61 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
327
356
  return false;
328
357
  }
329
358
 
359
+ void Reader::MaybeVerifyPredecessorWALInfo(
360
+ WALRecoveryMode wal_recovery_mode, Slice fragment,
361
+ const PredecessorWALInfo& recorded_predecessor_wal_info) {
362
+ if (!track_and_verify_wals_ ||
363
+ wal_recovery_mode == WALRecoveryMode::kSkipAnyCorruptedRecords ||
364
+ stop_replay_for_corruption_) {
365
+ return;
366
+ }
367
+ assert(recorded_predecessor_wal_info.IsInitialized());
368
+ uint64_t recorded_predecessor_log_number =
369
+ recorded_predecessor_wal_info.GetLogNumber();
370
+
371
+ // This is the first WAL recovered thus with no predecessor WAL info has been
372
+ // initialized
373
+ if (!observed_predecessor_wal_info_.IsInitialized()) {
374
+ if (recorded_predecessor_log_number >= min_wal_number_to_keep_) {
375
+ std::string reason = "Missing WAL of log number " +
376
+ std::to_string(recorded_predecessor_log_number);
377
+ ReportCorruption(fragment.size(), reason.c_str(),
378
+ recorded_predecessor_log_number);
379
+ }
380
+ } else {
381
+ if (observed_predecessor_wal_info_.GetLogNumber() !=
382
+ recorded_predecessor_log_number) {
383
+ std::string reason = "Missing WAL of log number " +
384
+ std::to_string(recorded_predecessor_log_number);
385
+ ReportCorruption(fragment.size(), reason.c_str(),
386
+ recorded_predecessor_log_number);
387
+ } else if (observed_predecessor_wal_info_.GetLastSeqnoRecorded() !=
388
+ recorded_predecessor_wal_info.GetLastSeqnoRecorded()) {
389
+ std::string reason =
390
+ "Mismatched last sequence number recorded in the WAL of log number " +
391
+ std::to_string(recorded_predecessor_log_number) + ". Recorded " +
392
+ std::to_string(recorded_predecessor_wal_info.GetLastSeqnoRecorded()) +
393
+ ". Observed " +
394
+ std::to_string(
395
+ observed_predecessor_wal_info_.GetLastSeqnoRecorded()) +
396
+ ". (Last sequence number equal to 0 indicates no WAL records)";
397
+ ReportCorruption(fragment.size(), reason.c_str(),
398
+ recorded_predecessor_log_number);
399
+ } else if (observed_predecessor_wal_info_.GetSizeBytes() !=
400
+ recorded_predecessor_wal_info.GetSizeBytes()) {
401
+ std::string reason =
402
+ "Mismatched size of the WAL of log number " +
403
+ std::to_string(recorded_predecessor_log_number) + ". Recorded " +
404
+ std::to_string(recorded_predecessor_wal_info.GetSizeBytes()) +
405
+ " bytes. Observed " +
406
+ std::to_string(observed_predecessor_wal_info_.GetSizeBytes()) +
407
+ " bytes.";
408
+ ReportCorruption(fragment.size(), reason.c_str(),
409
+ recorded_predecessor_log_number);
410
+ }
411
+ }
412
+ }
413
+
330
414
  uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
331
415
 
332
416
  uint64_t Reader::LastRecordEnd() {
@@ -403,13 +487,15 @@ void Reader::UnmarkEOFInternal() {
403
487
  }
404
488
  }
405
489
 
406
- void Reader::ReportCorruption(size_t bytes, const char* reason) {
407
- ReportDrop(bytes, Status::Corruption(reason));
490
+ void Reader::ReportCorruption(size_t bytes, const char* reason,
491
+ uint64_t log_number) {
492
+ ReportDrop(bytes, Status::Corruption(reason), log_number);
408
493
  }
409
494
 
410
- void Reader::ReportDrop(size_t bytes, const Status& reason) {
495
+ void Reader::ReportDrop(size_t bytes, const Status& reason,
496
+ uint64_t log_number) {
411
497
  if (reporter_ != nullptr) {
412
- reporter_->Corruption(bytes, reason);
498
+ reporter_->Corruption(bytes, reason, log_number);
413
499
  }
414
500
  }
415
501
 
@@ -419,7 +505,7 @@ void Reader::ReportOldLogRecord(size_t bytes) {
419
505
  }
420
506
  }
421
507
 
422
- bool Reader::ReadMore(size_t* drop_size, int* error) {
508
+ bool Reader::ReadMore(size_t* drop_size, uint8_t* error) {
423
509
  if (!eof_ && !read_error_) {
424
510
  // Last read was a full read, so this is a trailer to skip
425
511
  buffer_.clear();
@@ -460,15 +546,15 @@ bool Reader::ReadMore(size_t* drop_size, int* error) {
460
546
  }
461
547
  }
462
548
 
463
- unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
464
- uint64_t* fragment_checksum) {
549
+ uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
550
+ uint64_t* fragment_checksum) {
465
551
  while (true) {
466
552
  // We need at least the minimum header size
467
553
  if (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
468
554
  // the default value of r is meaningless because ReadMore will overwrite
469
555
  // it if it returns false; in case it returns true, the return value will
470
556
  // not be used anyway
471
- int r = kEof;
557
+ uint8_t r = kEof;
472
558
  if (!ReadMore(drop_size, &r)) {
473
559
  return r;
474
560
  }
@@ -479,12 +565,13 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
479
565
  const char* header = buffer_.data();
480
566
  const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
481
567
  const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
482
- const unsigned int type = header[6];
568
+ const uint8_t type = static_cast<uint8_t>(header[6]);
483
569
  const uint32_t length = a | (b << 8);
484
570
  int header_size = kHeaderSize;
485
571
  const bool is_recyclable_type =
486
572
  ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
487
- type == kRecyclableUserDefinedTimestampSizeType);
573
+ type == kRecyclableUserDefinedTimestampSizeType ||
574
+ type == kRecyclePredecessorWALInfoType);
488
575
  if (is_recyclable_type) {
489
576
  header_size = kRecyclableHeaderSize;
490
577
  if (first_record_read_ && !recycled_) {
@@ -494,7 +581,7 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
494
581
  recycled_ = true;
495
582
  // We need enough for the larger header
496
583
  if (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
497
- int r = kEof;
584
+ uint8_t r = kEof;
498
585
  if (!ReadMore(drop_size, &r)) {
499
586
  return r;
500
587
  }
@@ -549,6 +636,8 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
549
636
  buffer_.remove_prefix(header_size + length);
550
637
 
551
638
  if (!uncompress_ || type == kSetCompressionType ||
639
+ type == kPredecessorWALInfoType ||
640
+ type == kRecyclePredecessorWALInfoType ||
552
641
  type == kUserDefinedTimestampSizeType ||
553
642
  type == kRecyclableUserDefinedTimestampSizeType) {
554
643
  *result = Slice(header + header_size, length);
@@ -638,7 +727,9 @@ Status Reader::UpdateRecordedTimestampSize(
638
727
  }
639
728
 
640
729
  bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
641
- WALRecoveryMode /*unused*/,
730
+ WALRecoveryMode wal_recovery_mode
731
+
732
+ ,
642
733
  uint64_t* /* checksum */) {
643
734
  assert(record != nullptr);
644
735
  assert(scratch != nullptr);
@@ -651,7 +742,7 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
651
742
  uint64_t prospective_record_offset = 0;
652
743
  uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
653
744
  size_t drop_size = 0;
654
- unsigned int fragment_type_or_err = 0; // Initialize to make compiler happy
745
+ uint8_t fragment_type_or_err = 0; // Initialize to make compiler happy
655
746
  Slice fragment;
656
747
  while (TryReadFragment(&fragment, &drop_size, &fragment_type_or_err)) {
657
748
  switch (fragment_type_or_err) {
@@ -728,7 +819,24 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
728
819
  }
729
820
  break;
730
821
  }
822
+ case kPredecessorWALInfoType:
823
+ case kRecyclePredecessorWALInfoType: {
824
+ fragments_.clear();
825
+ prospective_record_offset = physical_record_offset;
826
+ last_record_offset_ = prospective_record_offset;
827
+ in_fragmented_record_ = false;
731
828
 
829
+ PredecessorWALInfo recorded_predecessor_wal_info;
830
+ Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
831
+ if (!s.ok()) {
832
+ ReportCorruption(fragment.size(),
833
+ "could not decode PredecessorWALInfoType record");
834
+ } else {
835
+ MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
836
+ recorded_predecessor_wal_info);
837
+ }
838
+ break;
839
+ }
732
840
  case kUserDefinedTimestampSizeType:
733
841
  case kRecyclableUserDefinedTimestampSizeType: {
734
842
  if (in_fragmented_record_ && !scratch->empty()) {
@@ -781,11 +889,13 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
781
889
  break;
782
890
 
783
891
  default: {
784
- std::string reason =
785
- "unknown record type " + std::to_string(fragment_type_or_err);
786
- ReportCorruption(
787
- fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
788
- reason.c_str());
892
+ if ((fragment_type_or_err & kRecordTypeSafeIgnoreMask) == 0) {
893
+ std::string reason =
894
+ "unknown record type " + std::to_string(fragment_type_or_err);
895
+ ReportCorruption(
896
+ fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
897
+ reason.c_str());
898
+ }
789
899
  in_fragmented_record_ = false;
790
900
  fragments_.clear();
791
901
  break;
@@ -803,7 +913,7 @@ void FragmentBufferedReader::UnmarkEOF() {
803
913
  UnmarkEOFInternal();
804
914
  }
805
915
 
806
- bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
916
+ bool FragmentBufferedReader::TryReadMore(size_t* drop_size, uint8_t* error) {
807
917
  if (!eof_ && !read_error_) {
808
918
  // Last read was a full read, so this is a trailer to skip
809
919
  buffer_.clear();
@@ -844,15 +954,15 @@ bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
844
954
  }
845
955
 
846
956
  // return true if the caller should process the fragment_type_or_err.
847
- bool FragmentBufferedReader::TryReadFragment(
848
- Slice* fragment, size_t* drop_size, unsigned int* fragment_type_or_err) {
957
+ bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
958
+ uint8_t* fragment_type_or_err) {
849
959
  assert(fragment != nullptr);
850
960
  assert(drop_size != nullptr);
851
961
  assert(fragment_type_or_err != nullptr);
852
962
 
853
963
  while (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
854
964
  size_t old_size = buffer_.size();
855
- int error = kEof;
965
+ uint8_t error = kEof;
856
966
  if (!TryReadMore(drop_size, &error)) {
857
967
  *fragment_type_or_err = error;
858
968
  return false;
@@ -863,11 +973,12 @@ bool FragmentBufferedReader::TryReadFragment(
863
973
  const char* header = buffer_.data();
864
974
  const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
865
975
  const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
866
- const unsigned int type = header[6];
976
+ const uint8_t type = static_cast<uint8_t>(header[6]);
867
977
  const uint32_t length = a | (b << 8);
868
978
  int header_size = kHeaderSize;
869
979
  if ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
870
- type == kRecyclableUserDefinedTimestampSizeType) {
980
+ type == kRecyclableUserDefinedTimestampSizeType ||
981
+ type == kRecyclePredecessorWALInfoType) {
871
982
  if (first_record_read_ && !recycled_) {
872
983
  // A recycled log should have started with a recycled record
873
984
  *fragment_type_or_err = kBadRecord;
@@ -877,7 +988,7 @@ bool FragmentBufferedReader::TryReadFragment(
877
988
  header_size = kRecyclableHeaderSize;
878
989
  while (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
879
990
  size_t old_size = buffer_.size();
880
- int error = kEof;
991
+ uint8_t error = kEof;
881
992
  if (!TryReadMore(drop_size, &error)) {
882
993
  *fragment_type_or_err = error;
883
994
  return false;
@@ -894,7 +1005,7 @@ bool FragmentBufferedReader::TryReadFragment(
894
1005
 
895
1006
  while (header_size + length > buffer_.size()) {
896
1007
  size_t old_size = buffer_.size();
897
- int error = kEof;
1008
+ uint8_t error = kEof;
898
1009
  if (!TryReadMore(drop_size, &error)) {
899
1010
  *fragment_type_or_err = error;
900
1011
  return false;
@@ -923,6 +1034,8 @@ bool FragmentBufferedReader::TryReadFragment(
923
1034
  buffer_.remove_prefix(header_size + length);
924
1035
 
925
1036
  if (!uncompress_ || type == kSetCompressionType ||
1037
+ type == kPredecessorWALInfoType ||
1038
+ type == kRecyclePredecessorWALInfoType ||
926
1039
  type == kUserDefinedTimestampSizeType ||
927
1040
  type == kRecyclableUserDefinedTimestampSizeType) {
928
1041
  *fragment = Slice(header + header_size, length);
@@ -10,6 +10,7 @@
10
10
  #pragma once
11
11
  #include <stdint.h>
12
12
 
13
+ #include <cstdint>
13
14
  #include <memory>
14
15
  #include <unordered_map>
15
16
  #include <vector>
@@ -44,7 +45,8 @@ class Reader {
44
45
 
45
46
  // Some corruption was detected. "size" is the approximate number
46
47
  // of bytes dropped due to the corruption.
47
- virtual void Corruption(size_t bytes, const Status& status) = 0;
48
+ virtual void Corruption(size_t bytes, const Status& status,
49
+ uint64_t log_number = kMaxSequenceNumber) = 0;
48
50
 
49
51
  virtual void OldLogRecord(size_t /*bytes*/) {}
50
52
  };
@@ -57,9 +59,15 @@ class Reader {
57
59
  // live while this Reader is in use.
58
60
  //
59
61
  // If "checksum" is true, verify checksums if available.
62
+ // TODO(hx235): seperate WAL related parameters from general `Reader`
63
+ // parameters
60
64
  Reader(std::shared_ptr<Logger> info_log,
61
65
  std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
62
- bool checksum, uint64_t log_num);
66
+ bool checksum, uint64_t log_num, bool track_and_verify_wals = false,
67
+ bool stop_replay_for_corruption = false,
68
+ uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
69
+ const PredecessorWALInfo& observed_predecessor_wal_info =
70
+ PredecessorWALInfo());
63
71
  // No copying allowed
64
72
  Reader(const Reader&) = delete;
65
73
  void operator=(const Reader&) = delete;
@@ -147,6 +155,17 @@ class Reader {
147
155
  // which log number this is
148
156
  uint64_t const log_number_;
149
157
 
158
+ // See `Optinos::track_and_verify_wals`
159
+ bool track_and_verify_wals_;
160
+ // Below variables are used for WAL verification
161
+ // TODO(hx235): To revise `stop_replay_for_corruption_` inside `LogReader`
162
+ // since we have `observed_predecessor_wal_info_` to verify against the
163
+ // `recorded_predecessor_wal_info_` recorded in current WAL. If there is no
164
+ // WAL hole, we can revise `stop_replay_for_corruption_` to be false.
165
+ bool stop_replay_for_corruption_;
166
+ uint64_t min_wal_number_to_keep_;
167
+ PredecessorWALInfo observed_predecessor_wal_info_;
168
+
150
169
  // Whether this is a recycled log file
151
170
  bool recycled_;
152
171
 
@@ -171,7 +190,7 @@ class Reader {
171
190
  UnorderedMap<uint32_t, size_t> recorded_cf_to_ts_sz_;
172
191
 
173
192
  // Extend record types with the following special values
174
- enum {
193
+ enum : uint8_t {
175
194
  kEof = kMaxRecordType + 1,
176
195
  // Returned whenever we find an invalid physical record.
177
196
  // Currently there are three situations in which this happens:
@@ -192,24 +211,30 @@ class Reader {
192
211
  // If WAL compressioned is enabled, fragment_checksum is the checksum of the
193
212
  // fragment computed from the orginal buffer containinng uncompressed
194
213
  // fragment.
195
- unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size,
196
- uint64_t* fragment_checksum = nullptr);
214
+ uint8_t ReadPhysicalRecord(Slice* result, size_t* drop_size,
215
+ uint64_t* fragment_checksum = nullptr);
197
216
 
198
217
  // Read some more
199
- bool ReadMore(size_t* drop_size, int* error);
218
+ bool ReadMore(size_t* drop_size, uint8_t* error);
200
219
 
201
220
  void UnmarkEOFInternal();
202
221
 
203
222
  // Reports dropped bytes to the reporter.
204
223
  // buffer_ must be updated to remove the dropped bytes prior to invocation.
205
- void ReportCorruption(size_t bytes, const char* reason);
206
- void ReportDrop(size_t bytes, const Status& reason);
224
+ void ReportCorruption(size_t bytes, const char* reason,
225
+ uint64_t log_number = kMaxSequenceNumber);
226
+ void ReportDrop(size_t bytes, const Status& reason,
227
+ uint64_t log_number = kMaxSequenceNumber);
207
228
  void ReportOldLogRecord(size_t bytes);
208
229
 
209
230
  void InitCompression(const CompressionTypeRecord& compression_record);
210
231
 
211
232
  Status UpdateRecordedTimestampSize(
212
233
  const std::vector<std::pair<uint32_t, size_t>>& cf_to_ts_sz);
234
+
235
+ void MaybeVerifyPredecessorWALInfo(
236
+ WALRecoveryMode wal_recovery_mode, Slice fragment,
237
+ const PredecessorWALInfo& recorded_predecessor_wal_info);
213
238
  };
214
239
 
215
240
  class FragmentBufferedReader : public Reader {
@@ -217,7 +242,11 @@ class FragmentBufferedReader : public Reader {
217
242
  FragmentBufferedReader(std::shared_ptr<Logger> info_log,
218
243
  std::unique_ptr<SequentialFileReader>&& _file,
219
244
  Reporter* reporter, bool checksum, uint64_t log_num)
220
- : Reader(info_log, std::move(_file), reporter, checksum, log_num),
245
+ : Reader(info_log, std::move(_file), reporter, checksum, log_num,
246
+ false /*verify_and_track_wals*/,
247
+ false /*stop_replay_for_corruption*/,
248
+ std::numeric_limits<uint64_t>::max() /*min_wal_number_to_keep*/,
249
+ PredecessorWALInfo() /*observed_predecessor_wal_info*/),
221
250
  fragments_(),
222
251
  in_fragmented_record_(false) {}
223
252
  ~FragmentBufferedReader() override {}
@@ -232,9 +261,9 @@ class FragmentBufferedReader : public Reader {
232
261
  bool in_fragmented_record_;
233
262
 
234
263
  bool TryReadFragment(Slice* result, size_t* drop_size,
235
- unsigned int* fragment_type_or_err);
264
+ uint8_t* fragment_type_or_err);
236
265
 
237
- bool TryReadMore(size_t* drop_size, int* error);
266
+ bool TryReadMore(size_t* drop_size, uint8_t* error);
238
267
 
239
268
  // No copy allowed
240
269
  FragmentBufferedReader(const FragmentBufferedReader&);
@@ -129,7 +129,8 @@ class LogTest
129
129
  std::string message_;
130
130
 
131
131
  ReportCollector() : dropped_bytes_(0) {}
132
- void Corruption(size_t bytes, const Status& status) override {
132
+ void Corruption(size_t bytes, const Status& status,
133
+ uint64_t /*log_number*/ = kMaxSequenceNumber) override {
133
134
  dropped_bytes_ += bytes;
134
135
  message_.append(status.ToString());
135
136
  }
@@ -436,6 +437,17 @@ TEST_P(LogTest, BadRecordType) {
436
437
  ASSERT_EQ("OK", MatchError("unknown record type"));
437
438
  }
438
439
 
440
+ TEST_P(LogTest, IgnorableRecordType) {
441
+ Write("foo");
442
+ // Type is stored in header[6]
443
+ SetByte(6, static_cast<char>(kRecordTypeSafeIgnoreMask + 100));
444
+ FixChecksum(0, 3, false);
445
+ ASSERT_EQ("EOF", Read());
446
+ // The new type has value 129 and masked to be ignorable if unknown
447
+ ASSERT_EQ(0U, DroppedBytes());
448
+ ASSERT_EQ("", ReportMessage());
449
+ }
450
+
439
451
  TEST_P(LogTest, TruncatedTrailingRecordIsIgnored) {
440
452
  Write("foo");
441
453
  ShrinkSize(4); // Drop all payload as well as a header byte
@@ -454,7 +466,7 @@ TEST_P(LogTest, TruncatedTrailingRecordIsNotIgnored) {
454
466
  Write("foo");
455
467
  ShrinkSize(4); // Drop all payload as well as a header byte
456
468
  ASSERT_EQ("EOF", Read(WALRecoveryMode::kAbsoluteConsistency));
457
- // Truncated last record is ignored, not treated as an error
469
+ // Truncated last record is not ignored, treated as an error
458
470
  ASSERT_GT(DroppedBytes(), 0U);
459
471
  ASSERT_EQ("OK", MatchError("Corruption: truncated header"));
460
472
  }
@@ -814,7 +826,8 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
814
826
  std::string message_;
815
827
 
816
828
  ReportCollector() : dropped_bytes_(0) {}
817
- void Corruption(size_t bytes, const Status& status) override {
829
+ void Corruption(size_t bytes, const Status& status,
830
+ uint64_t /*log_number*/ = kMaxSequenceNumber) override {
818
831
  dropped_bytes_ += bytes;
819
832
  message_.append(status.ToString());
820
833
  }