@nxtedition/rocksdb 10.0.13 → 10.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (700) hide show
  1. package/.tap/coverage/213879bb-47ae-470e-bb54-f4157d9f60e2.json +162898 -0
  2. package/.tap/processinfo/{4c642b53-6895-4ef0-9391-7c6b8d484262.json → 213879bb-47ae-470e-bb54-f4157d9f60e2.json} +117 -121
  3. package/.tap/test-results/node_modules/abstract-level/test/get-test.js.tap +0 -0
  4. package/.tap/test-results/test/abstract-level-test.js.tap +1705 -0
  5. package/.tap/test-results/test/batch-test.js.tap +12 -0
  6. package/.tap/test-results/test/chained-batch-gc-test.js.tap +11 -0
  7. package/.tap/test-results/test/cleanup-hanging-iterators-test.js.tap +58 -0
  8. package/.tap/test-results/test/clear-gc-test.js.tap +13 -0
  9. package/.tap/test-results/test/column-test.js.tap +55 -0
  10. package/.tap/test-results/test/common.js.tap +0 -0
  11. package/.tap/test-results/test/compression-test.js.tap +30 -0
  12. package/.tap/test-results/test/db-identity.js.tap +12 -0
  13. package/.tap/test-results/test/electron.js.tap +0 -0
  14. package/.tap/test-results/test/env-cleanup-hook-test.js.tap +2 -0
  15. package/.tap/test-results/test/env-cleanup-hook.js.tap +0 -0
  16. package/.tap/test-results/test/gc.js.tap +0 -0
  17. package/.tap/test-results/test/getproperty-test.js.tap +29 -0
  18. package/.tap/test-results/test/iterator-gc-test.js.tap +4 -0
  19. package/.tap/test-results/test/iterator-hwm-test.js.tap +24 -0
  20. package/.tap/test-results/test/iterator-recursion-test.js.tap +12 -0
  21. package/.tap/test-results/test/iterator-starvation-test.js.tap +2 -0
  22. package/.tap/test-results/test/iterator-test.js.tap +0 -0
  23. package/.tap/test-results/test/leak-tester-batch.js.tap +0 -0
  24. package/.tap/test-results/test/leak-tester-iterator.js.tap +0 -0
  25. package/.tap/test-results/test/leak-tester.js.tap +0 -0
  26. package/.tap/test-results/test/lock-test.js.tap +0 -0
  27. package/.tap/test-results/test/lock.js.tap +0 -0
  28. package/.tap/test-results/test/make.js.tap +0 -0
  29. package/.tap/test-results/test/max-rev-merge.js.tap +0 -12
  30. package/.tap/test-results/test/merge-operator-test.js.tap +0 -0
  31. package/.tap/test-results/test/mkdir-test.js.tap +0 -0
  32. package/.tap/test-results/test/segfault-test.js.tap +0 -0
  33. package/.tap/test-results/test/stack-blower.js.tap +0 -0
  34. package/binding.cc +42 -34
  35. package/deps/rocksdb/rocksdb/CMakeLists.txt +44 -14
  36. package/deps/rocksdb/rocksdb/Makefile +34 -17
  37. package/deps/rocksdb/rocksdb/TARGETS +27 -0
  38. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +95 -42
  39. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager_test.cc +0 -1
  40. package/deps/rocksdb/rocksdb/cache/cache_test.cc +5 -3
  41. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +381 -393
  42. package/deps/rocksdb/rocksdb/cache/clock_cache.h +88 -51
  43. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +14 -7
  44. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +1 -1
  45. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +21 -19
  46. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +8 -7
  47. package/deps/rocksdb/rocksdb/cache/lru_cache.h +1 -1
  48. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +89 -13
  49. package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +16 -8
  50. package/deps/rocksdb/rocksdb/cache/sharded_cache.h +6 -6
  51. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache.cc +15 -9
  52. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache.h +16 -13
  53. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache_test.cc +306 -23
  54. package/deps/rocksdb/rocksdb/cache/typed_cache.h +3 -4
  55. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +30 -27
  56. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +8 -10
  57. package/deps/rocksdb/rocksdb/db/attribute_group_iterator_impl.cc +20 -0
  58. package/deps/rocksdb/rocksdb/db/attribute_group_iterator_impl.h +83 -0
  59. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +19 -16
  60. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +6 -5
  61. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +18 -21
  62. package/deps/rocksdb/rocksdb/db/blob/blob_file_cache_test.cc +5 -4
  63. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +7 -6
  64. package/deps/rocksdb/rocksdb/db/blob/blob_log_writer.cc +56 -27
  65. package/deps/rocksdb/rocksdb/db/blob/blob_log_writer.h +11 -7
  66. package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc +20 -19
  67. package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +106 -31
  68. package/deps/rocksdb/rocksdb/db/blob/db_blob_index_test.cc +5 -8
  69. package/deps/rocksdb/rocksdb/db/blob/prefetch_buffer_collection.cc +4 -2
  70. package/deps/rocksdb/rocksdb/db/builder.cc +89 -44
  71. package/deps/rocksdb/rocksdb/db/builder.h +4 -6
  72. package/deps/rocksdb/rocksdb/db/c.cc +205 -57
  73. package/deps/rocksdb/rocksdb/db/c_test.c +141 -20
  74. package/deps/rocksdb/rocksdb/db/coalescing_iterator.cc +47 -0
  75. package/deps/rocksdb/rocksdb/db/coalescing_iterator.h +79 -0
  76. package/deps/rocksdb/rocksdb/db/column_family.cc +100 -36
  77. package/deps/rocksdb/rocksdb/db/column_family.h +46 -28
  78. package/deps/rocksdb/rocksdb/db/column_family_test.cc +168 -46
  79. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +51 -6
  80. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +72 -24
  81. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +11 -14
  82. package/deps/rocksdb/rocksdb/db/compaction/compaction_iteration_stats.h +7 -0
  83. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +87 -8
  84. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +17 -3
  85. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +310 -70
  86. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +78 -62
  87. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +0 -1
  88. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +2 -3
  89. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +33 -96
  90. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +18 -10
  91. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +7 -8
  92. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +42 -22
  93. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +20 -16
  94. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +7 -3
  95. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +18 -17
  96. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +95 -10
  97. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +6 -7
  98. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +213 -34
  99. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +4 -4
  100. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +7 -8
  101. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +16 -16
  102. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +75 -142
  103. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +577 -34
  104. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +4 -4
  105. package/deps/rocksdb/rocksdb/db/convenience.cc +1 -1
  106. package/deps/rocksdb/rocksdb/db/corruption_test.cc +7 -38
  107. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +5 -5
  108. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +366 -136
  109. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +9 -10
  110. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +282 -6
  111. package/deps/rocksdb/rocksdb/db/db_compaction_filter_test.cc +6 -6
  112. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +551 -150
  113. package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +1 -1
  114. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +41 -13
  115. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +54 -17
  116. package/deps/rocksdb/rocksdb/db/db_follower_test.cc +527 -0
  117. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +40 -58
  118. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +34 -45
  119. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +699 -580
  120. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +308 -230
  121. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +180 -107
  122. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +6 -2
  123. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +10 -6
  124. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +36 -61
  125. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +347 -0
  126. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.h +54 -0
  127. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +206 -155
  128. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +5 -8
  129. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +41 -39
  130. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +26 -22
  131. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +14 -9
  132. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +126 -46
  133. package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +21 -9
  134. package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +294 -0
  135. package/deps/rocksdb/rocksdb/db/db_iter.cc +116 -78
  136. package/deps/rocksdb/rocksdb/db/db_iter.h +31 -7
  137. package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +34 -14
  138. package/deps/rocksdb/rocksdb/db/db_iter_test.cc +69 -8
  139. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +153 -18
  140. package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +9 -11
  141. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +9 -1
  142. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +6 -2
  143. package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +105 -0
  144. package/deps/rocksdb/rocksdb/db/db_options_test.cc +97 -111
  145. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +12 -3
  146. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +82 -30
  147. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +5 -7
  148. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +64 -22
  149. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +74 -0
  150. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +175 -47
  151. package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +4 -3
  152. package/deps/rocksdb/rocksdb/db/db_test.cc +133 -109
  153. package/deps/rocksdb/rocksdb/db/db_test2.cc +227 -139
  154. package/deps/rocksdb/rocksdb/db/db_test_util.cc +42 -19
  155. package/deps/rocksdb/rocksdb/db/db_test_util.h +67 -24
  156. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +140 -2
  157. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +17 -19
  158. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +332 -87
  159. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +4 -4
  160. package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +4 -4
  161. package/deps/rocksdb/rocksdb/db/db_write_test.cc +250 -30
  162. package/deps/rocksdb/rocksdb/db/dbformat.cc +41 -8
  163. package/deps/rocksdb/rocksdb/db/dbformat.h +100 -17
  164. package/deps/rocksdb/rocksdb/db/deletefile_test.cc +48 -8
  165. package/deps/rocksdb/rocksdb/db/error_handler.cc +102 -116
  166. package/deps/rocksdb/rocksdb/db/error_handler.h +28 -6
  167. package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +263 -99
  168. package/deps/rocksdb/rocksdb/db/event_helpers.cc +3 -1
  169. package/deps/rocksdb/rocksdb/db/experimental.cc +1073 -7
  170. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +261 -111
  171. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +263 -183
  172. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +31 -12
  173. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +698 -7
  174. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +5 -3
  175. package/deps/rocksdb/rocksdb/db/flush_job.cc +41 -41
  176. package/deps/rocksdb/rocksdb/db/flush_job.h +11 -8
  177. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +90 -13
  178. package/deps/rocksdb/rocksdb/db/forward_iterator.cc +19 -6
  179. package/deps/rocksdb/rocksdb/db/forward_iterator.h +12 -12
  180. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +83 -52
  181. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +169 -2
  182. package/deps/rocksdb/rocksdb/db/internal_stats.cc +12 -6
  183. package/deps/rocksdb/rocksdb/db/internal_stats.h +2 -2
  184. package/deps/rocksdb/rocksdb/db/job_context.h +17 -0
  185. package/deps/rocksdb/rocksdb/db/listener_test.cc +4 -4
  186. package/deps/rocksdb/rocksdb/db/log_reader.cc +44 -20
  187. package/deps/rocksdb/rocksdb/db/log_reader.h +3 -0
  188. package/deps/rocksdb/rocksdb/db/log_test.cc +81 -17
  189. package/deps/rocksdb/rocksdb/db/log_writer.cc +135 -92
  190. package/deps/rocksdb/rocksdb/db/log_writer.h +11 -10
  191. package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -3
  192. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +1 -1
  193. package/deps/rocksdb/rocksdb/db/memtable.cc +82 -62
  194. package/deps/rocksdb/rocksdb/db/memtable.h +17 -9
  195. package/deps/rocksdb/rocksdb/db/memtable_list.cc +19 -13
  196. package/deps/rocksdb/rocksdb/db/memtable_list.h +3 -1
  197. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +38 -5
  198. package/deps/rocksdb/rocksdb/db/merge_context.h +3 -0
  199. package/deps/rocksdb/rocksdb/db/merge_helper.cc +34 -156
  200. package/deps/rocksdb/rocksdb/db/merge_helper.h +52 -46
  201. package/deps/rocksdb/rocksdb/db/merge_test.cc +16 -11
  202. package/deps/rocksdb/rocksdb/db/multi_cf_iterator_impl.h +289 -0
  203. package/deps/rocksdb/rocksdb/db/multi_cf_iterator_test.cc +918 -0
  204. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +4 -6
  205. package/deps/rocksdb/rocksdb/db/options_file_test.cc +4 -4
  206. package/deps/rocksdb/rocksdb/db/output_validator.cc +8 -12
  207. package/deps/rocksdb/rocksdb/db/output_validator.h +1 -4
  208. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +23 -4
  209. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.cc +2 -3
  210. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +1 -1
  211. package/deps/rocksdb/rocksdb/db/pinned_iterators_manager.h +2 -2
  212. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +3 -6
  213. package/deps/rocksdb/rocksdb/db/prefix_test.cc +21 -8
  214. package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +0 -2
  215. package/deps/rocksdb/rocksdb/db/range_del_aggregator.h +0 -1
  216. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.cc +17 -4
  217. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +6 -3
  218. package/deps/rocksdb/rocksdb/db/repair.cc +35 -29
  219. package/deps/rocksdb/rocksdb/db/repair_test.cc +1 -2
  220. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +428 -156
  221. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +393 -227
  222. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +154 -82
  223. package/deps/rocksdb/rocksdb/db/snapshot_checker.h +2 -2
  224. package/deps/rocksdb/rocksdb/db/table_cache.cc +9 -11
  225. package/deps/rocksdb/rocksdb/db/table_cache.h +2 -1
  226. package/deps/rocksdb/rocksdb/db/table_cache_sync_and_async.h +8 -3
  227. package/deps/rocksdb/rocksdb/db/table_properties_collector.h +27 -26
  228. package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +23 -23
  229. package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +5 -5
  230. package/deps/rocksdb/rocksdb/db/version_builder.cc +6 -0
  231. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +62 -29
  232. package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -2
  233. package/deps/rocksdb/rocksdb/db/version_edit.h +30 -11
  234. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +291 -116
  235. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +54 -11
  236. package/deps/rocksdb/rocksdb/db/version_edit_test.cc +3 -1
  237. package/deps/rocksdb/rocksdb/db/version_set.cc +301 -141
  238. package/deps/rocksdb/rocksdb/db/version_set.h +90 -33
  239. package/deps/rocksdb/rocksdb/db/version_set_test.cc +817 -193
  240. package/deps/rocksdb/rocksdb/db/version_util.h +10 -5
  241. package/deps/rocksdb/rocksdb/db/wal_manager.cc +15 -11
  242. package/deps/rocksdb/rocksdb/db/wal_manager.h +2 -1
  243. package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +10 -9
  244. package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +403 -28
  245. package/deps/rocksdb/rocksdb/db/wide/wide_columns.cc +2 -2
  246. package/deps/rocksdb/rocksdb/db/write_batch.cc +204 -18
  247. package/deps/rocksdb/rocksdb/db/write_batch_internal.h +4 -0
  248. package/deps/rocksdb/rocksdb/db/write_batch_test.cc +149 -5
  249. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +7 -7
  250. package/deps/rocksdb/rocksdb/db/write_stall_stats.h +7 -7
  251. package/deps/rocksdb/rocksdb/db/write_thread.cc +86 -47
  252. package/deps/rocksdb/rocksdb/db/write_thread.h +2 -0
  253. package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +176 -68
  254. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +337 -125
  255. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +38 -9
  256. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +126 -34
  257. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +1 -1
  258. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.h +2 -2
  259. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +161 -1
  260. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +267 -9
  261. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.cc +13 -7
  262. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_table_properties_collector.h +9 -9
  263. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +535 -86
  264. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +27 -5
  265. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +8 -15
  266. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +46 -21
  267. package/deps/rocksdb/rocksdb/db_stress_tool/expected_value.h +74 -4
  268. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +28 -11
  269. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +426 -151
  270. package/deps/rocksdb/rocksdb/env/composite_env.cc +1 -1
  271. package/deps/rocksdb/rocksdb/env/emulated_clock.h +6 -6
  272. package/deps/rocksdb/rocksdb/env/env.cc +21 -11
  273. package/deps/rocksdb/rocksdb/env/env_basic_test.cc +9 -9
  274. package/deps/rocksdb/rocksdb/env/env_chroot.cc +3 -2
  275. package/deps/rocksdb/rocksdb/env/env_encryption.cc +3 -3
  276. package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +2 -2
  277. package/deps/rocksdb/rocksdb/env/env_posix.cc +12 -10
  278. package/deps/rocksdb/rocksdb/env/env_test.cc +24 -18
  279. package/deps/rocksdb/rocksdb/env/file_system.cc +7 -6
  280. package/deps/rocksdb/rocksdb/env/file_system_tracer.cc +3 -3
  281. package/deps/rocksdb/rocksdb/env/file_system_tracer.h +3 -3
  282. package/deps/rocksdb/rocksdb/env/fs_on_demand.cc +331 -0
  283. package/deps/rocksdb/rocksdb/env/fs_on_demand.h +139 -0
  284. package/deps/rocksdb/rocksdb/env/fs_posix.cc +14 -15
  285. package/deps/rocksdb/rocksdb/env/io_posix.cc +15 -10
  286. package/deps/rocksdb/rocksdb/env/io_posix.h +86 -92
  287. package/deps/rocksdb/rocksdb/env/mock_env.cc +1 -1
  288. package/deps/rocksdb/rocksdb/env/mock_env_test.cc +6 -6
  289. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +8 -6
  290. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +1 -1
  291. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +599 -592
  292. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +312 -200
  293. package/deps/rocksdb/rocksdb/file/file_util.cc +19 -14
  294. package/deps/rocksdb/rocksdb/file/file_util.h +34 -24
  295. package/deps/rocksdb/rocksdb/file/filename.cc +32 -17
  296. package/deps/rocksdb/rocksdb/file/filename.h +51 -53
  297. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +82 -441
  298. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +7 -6
  299. package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +5 -5
  300. package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc +9 -3
  301. package/deps/rocksdb/rocksdb/file/read_write_util.h +3 -3
  302. package/deps/rocksdb/rocksdb/file/sequence_file_reader.cc +6 -2
  303. package/deps/rocksdb/rocksdb/file/sequence_file_reader.h +9 -4
  304. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +8 -4
  305. package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +110 -89
  306. package/deps/rocksdb/rocksdb/file/writable_file_writer.h +34 -17
  307. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +16 -0
  308. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +39 -189
  309. package/deps/rocksdb/rocksdb/include/rocksdb/attribute_groups.h +114 -0
  310. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +91 -6
  311. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +52 -7
  312. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -1
  313. package/deps/rocksdb/rocksdb/include/rocksdb/comparator.h +27 -0
  314. package/deps/rocksdb/rocksdb/include/rocksdb/compression_type.h +146 -0
  315. package/deps/rocksdb/rocksdb/include/rocksdb/concurrent_task_limiter.h +2 -2
  316. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +254 -151
  317. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +39 -36
  318. package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +1 -1
  319. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +431 -0
  320. package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +1 -1
  321. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +57 -23
  322. package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +27 -57
  323. package/deps/rocksdb/rocksdb/include/rocksdb/iterator_base.h +74 -0
  324. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +2 -1
  325. package/deps/rocksdb/rocksdb/include/rocksdb/memory_allocator.h +1 -1
  326. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +11 -11
  327. package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +1 -1
  328. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +146 -71
  329. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +6 -1
  330. package/deps/rocksdb/rocksdb/include/rocksdb/perf_level.h +7 -4
  331. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +17 -13
  332. package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +17 -19
  333. package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +5 -5
  334. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_manager.h +10 -8
  335. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_reader.h +18 -2
  336. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +12 -2
  337. package/deps/rocksdb/rocksdb/include/rocksdb/sst_partitioner.h +3 -3
  338. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +118 -116
  339. package/deps/rocksdb/rocksdb/include/rocksdb/system_clock.h +4 -4
  340. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +26 -21
  341. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +18 -6
  342. package/deps/rocksdb/rocksdb/include/rocksdb/threadpool.h +1 -1
  343. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +5 -5
  344. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record_result.h +8 -8
  345. package/deps/rocksdb/rocksdb/include/rocksdb/types.h +12 -0
  346. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/cache_dump_load.h +4 -0
  347. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_mirror.h +3 -3
  348. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +8 -0
  349. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/optimistic_transaction_db.h +1 -1
  350. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/option_change_migration.h +4 -4
  351. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_type.h +3 -3
  352. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/sim_cache.h +5 -6
  353. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +159 -167
  354. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +1 -2
  355. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +52 -17
  356. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +3 -4
  357. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/types_util.h +36 -0
  358. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +118 -20
  359. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  360. package/deps/rocksdb/rocksdb/include/rocksdb/wal_filter.h +1 -1
  361. package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h +87 -45
  362. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +26 -3
  363. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch_base.h +22 -1
  364. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.cc +7 -9
  365. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.h +5 -6
  366. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +2 -2
  367. package/deps/rocksdb/rocksdb/logging/env_logger.h +3 -3
  368. package/deps/rocksdb/rocksdb/logging/env_logger_test.cc +1 -1
  369. package/deps/rocksdb/rocksdb/logging/log_buffer.h +3 -3
  370. package/deps/rocksdb/rocksdb/memory/arena.h +11 -0
  371. package/deps/rocksdb/rocksdb/memory/arena_test.cc +17 -2
  372. package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.cc +1 -1
  373. package/deps/rocksdb/rocksdb/memory/memory_allocator.cc +1 -0
  374. package/deps/rocksdb/rocksdb/memory/memory_allocator_impl.h +1 -1
  375. package/deps/rocksdb/rocksdb/memtable/alloc_tracker.cc +1 -1
  376. package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +11 -10
  377. package/deps/rocksdb/rocksdb/memtable/hash_skiplist_rep.cc +8 -7
  378. package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +2 -2
  379. package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +2 -2
  380. package/deps/rocksdb/rocksdb/memtable/skiplist_test.cc +1 -1
  381. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +3 -3
  382. package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +2 -2
  383. package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +3 -1
  384. package/deps/rocksdb/rocksdb/monitoring/file_read_sample.h +2 -2
  385. package/deps/rocksdb/rocksdb/monitoring/histogram.cc +20 -10
  386. package/deps/rocksdb/rocksdb/monitoring/histogram.h +15 -15
  387. package/deps/rocksdb/rocksdb/monitoring/histogram_windowing.cc +7 -3
  388. package/deps/rocksdb/rocksdb/monitoring/histogram_windowing.h +14 -14
  389. package/deps/rocksdb/rocksdb/monitoring/in_memory_stats_history.cc +1 -1
  390. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +4 -0
  391. package/deps/rocksdb/rocksdb/monitoring/perf_context_imp.h +8 -1
  392. package/deps/rocksdb/rocksdb/monitoring/perf_level.cc +1 -1
  393. package/deps/rocksdb/rocksdb/monitoring/persistent_stats_history.cc +5 -2
  394. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +64 -58
  395. package/deps/rocksdb/rocksdb/monitoring/statistics_impl.h +11 -12
  396. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +1 -1
  397. package/deps/rocksdb/rocksdb/monitoring/thread_status_impl.cc +1 -1
  398. package/deps/rocksdb/rocksdb/monitoring/thread_status_util.cc +7 -1
  399. package/deps/rocksdb/rocksdb/options/cf_options.cc +20 -28
  400. package/deps/rocksdb/rocksdb/options/cf_options.h +7 -14
  401. package/deps/rocksdb/rocksdb/options/configurable.cc +20 -12
  402. package/deps/rocksdb/rocksdb/options/configurable_test.cc +4 -6
  403. package/deps/rocksdb/rocksdb/options/configurable_test.h +2 -2
  404. package/deps/rocksdb/rocksdb/options/customizable.cc +1 -1
  405. package/deps/rocksdb/rocksdb/options/customizable_test.cc +3 -2
  406. package/deps/rocksdb/rocksdb/options/db_options.cc +22 -48
  407. package/deps/rocksdb/rocksdb/options/db_options.h +3 -3
  408. package/deps/rocksdb/rocksdb/options/offpeak_time_info.cc +59 -0
  409. package/deps/rocksdb/rocksdb/options/offpeak_time_info.h +37 -0
  410. package/deps/rocksdb/rocksdb/options/options.cc +14 -6
  411. package/deps/rocksdb/rocksdb/options/options_helper.cc +8 -13
  412. package/deps/rocksdb/rocksdb/options/options_helper.h +2 -3
  413. package/deps/rocksdb/rocksdb/options/options_parser.cc +35 -26
  414. package/deps/rocksdb/rocksdb/options/options_parser.h +4 -2
  415. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +3 -3
  416. package/deps/rocksdb/rocksdb/options/options_test.cc +30 -23
  417. package/deps/rocksdb/rocksdb/port/lang.h +1 -1
  418. package/deps/rocksdb/rocksdb/port/port_example.h +7 -7
  419. package/deps/rocksdb/rocksdb/port/port_posix.cc +7 -7
  420. package/deps/rocksdb/rocksdb/port/port_posix.h +7 -7
  421. package/deps/rocksdb/rocksdb/port/stack_trace.cc +85 -24
  422. package/deps/rocksdb/rocksdb/port/win/env_win.cc +1 -2
  423. package/deps/rocksdb/rocksdb/port/win/io_win.cc +4 -4
  424. package/deps/rocksdb/rocksdb/port/win/io_win.h +13 -17
  425. package/deps/rocksdb/rocksdb/port/win/port_win.cc +3 -2
  426. package/deps/rocksdb/rocksdb/port/win/port_win.h +4 -4
  427. package/deps/rocksdb/rocksdb/src.mk +16 -1
  428. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +1 -7
  429. package/deps/rocksdb/rocksdb/table/block_based/block.cc +39 -15
  430. package/deps/rocksdb/rocksdb/table/block_based/block.h +7 -7
  431. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +80 -40
  432. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +2 -3
  433. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +20 -5
  434. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +228 -122
  435. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +50 -9
  436. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +82 -25
  437. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +17 -20
  438. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +47 -13
  439. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +163 -50
  440. package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +1 -2
  441. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +23 -23
  442. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +2 -12
  443. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +1 -2
  444. package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index_test.cc +6 -4
  445. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +32 -27
  446. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +1 -1
  447. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +7 -10
  448. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +2 -2
  449. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +1 -2
  450. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +25 -7
  451. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +3 -5
  452. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +4 -6
  453. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +6 -6
  454. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +1 -1
  455. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +3 -5
  456. package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +2 -2
  457. package/deps/rocksdb/rocksdb/table/block_based/reader_common.h +4 -5
  458. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +138 -86
  459. package/deps/rocksdb/rocksdb/table/block_fetcher.h +22 -0
  460. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +26 -13
  461. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.cc +11 -11
  462. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc +11 -11
  463. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader.cc +10 -5
  464. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +6 -7
  465. package/deps/rocksdb/rocksdb/table/format.cc +15 -21
  466. package/deps/rocksdb/rocksdb/table/format.h +8 -0
  467. package/deps/rocksdb/rocksdb/table/get_context.cc +121 -78
  468. package/deps/rocksdb/rocksdb/table/get_context.h +11 -6
  469. package/deps/rocksdb/rocksdb/table/internal_iterator.h +12 -5
  470. package/deps/rocksdb/rocksdb/table/iterator.cc +4 -0
  471. package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +8 -1
  472. package/deps/rocksdb/rocksdb/table/merger_test.cc +1 -1
  473. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +5 -0
  474. package/deps/rocksdb/rocksdb/table/merging_iterator.h +4 -3
  475. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +15 -11
  476. package/deps/rocksdb/rocksdb/table/meta_blocks.h +8 -4
  477. package/deps/rocksdb/rocksdb/table/mock_table.cc +12 -8
  478. package/deps/rocksdb/rocksdb/table/mock_table.h +1 -3
  479. package/deps/rocksdb/rocksdb/table/persistent_cache_helper.cc +0 -1
  480. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +29 -21
  481. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.h +4 -14
  482. package/deps/rocksdb/rocksdb/table/plain/plain_table_factory.cc +7 -8
  483. package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.cc +7 -5
  484. package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +15 -8
  485. package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.h +1 -1
  486. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +75 -30
  487. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +7 -2
  488. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +76 -11
  489. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +358 -2
  490. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +93 -16
  491. package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +12 -12
  492. package/deps/rocksdb/rocksdb/table/table_builder.h +12 -6
  493. package/deps/rocksdb/rocksdb/table/table_iterator.h +69 -0
  494. package/deps/rocksdb/rocksdb/table/table_properties.cc +1 -1
  495. package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +8 -6
  496. package/deps/rocksdb/rocksdb/table/table_test.cc +732 -121
  497. package/deps/rocksdb/rocksdb/table/two_level_iterator.cc +3 -1
  498. package/deps/rocksdb/rocksdb/table/two_level_iterator.h +1 -1
  499. package/deps/rocksdb/rocksdb/table/unique_id.cc +2 -2
  500. package/deps/rocksdb/rocksdb/test_util/mock_time_env.cc +2 -2
  501. package/deps/rocksdb/rocksdb/test_util/mock_time_env.h +5 -5
  502. package/deps/rocksdb/rocksdb/test_util/secondary_cache_test_util.cc +2 -6
  503. package/deps/rocksdb/rocksdb/test_util/testharness.cc +2 -4
  504. package/deps/rocksdb/rocksdb/test_util/testutil.cc +56 -32
  505. package/deps/rocksdb/rocksdb/test_util/testutil.h +57 -34
  506. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.cc +18 -1
  507. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.h +1 -0
  508. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +3 -3
  509. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +210 -112
  510. package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +3 -3
  511. package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +1 -1
  512. package/deps/rocksdb/rocksdb/tools/db_sanity_test.cc +22 -27
  513. package/deps/rocksdb/rocksdb/tools/dump/db_dump_tool.cc +10 -4
  514. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +328 -59
  515. package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +34 -2
  516. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +64 -17
  517. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +8 -1
  518. package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.cc +3 -2
  519. package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc +136 -10
  520. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +4 -2
  521. package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +50 -8
  522. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +51 -45
  523. package/deps/rocksdb/rocksdb/trace_replay/trace_record.cc +1 -1
  524. package/deps/rocksdb/rocksdb/trace_replay/trace_record_handler.h +10 -10
  525. package/deps/rocksdb/rocksdb/trace_replay/trace_replay.cc +14 -4
  526. package/deps/rocksdb/rocksdb/trace_replay/trace_replay.h +2 -1
  527. package/deps/rocksdb/rocksdb/unreleased_history/README.txt +2 -2
  528. package/deps/rocksdb/rocksdb/util/async_file_reader.cc +2 -4
  529. package/deps/rocksdb/rocksdb/util/atomic.h +111 -0
  530. package/deps/rocksdb/rocksdb/util/autovector.h +5 -1
  531. package/deps/rocksdb/rocksdb/util/bloom_test.cc +12 -2
  532. package/deps/rocksdb/rocksdb/util/cast_util.h +22 -0
  533. package/deps/rocksdb/rocksdb/util/coding.h +34 -38
  534. package/deps/rocksdb/rocksdb/util/coding_test.cc +1 -1
  535. package/deps/rocksdb/rocksdb/util/comparator.cc +18 -5
  536. package/deps/rocksdb/rocksdb/util/compression_context_cache.cc +1 -1
  537. package/deps/rocksdb/rocksdb/util/concurrent_task_limiter_impl.h +4 -4
  538. package/deps/rocksdb/rocksdb/util/crc32c.cc +3 -6
  539. package/deps/rocksdb/rocksdb/util/crc32c.h +3 -3
  540. package/deps/rocksdb/rocksdb/util/crc32c_arm64.h +3 -4
  541. package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +1 -2
  542. package/deps/rocksdb/rocksdb/util/crc32c_test.cc +2 -4
  543. package/deps/rocksdb/rocksdb/util/data_structure.cc +2 -4
  544. package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +3 -2
  545. package/deps/rocksdb/rocksdb/util/file_checksum_helper.cc +3 -3
  546. package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +83 -59
  547. package/deps/rocksdb/rocksdb/util/filelock_test.cc +3 -3
  548. package/deps/rocksdb/rocksdb/util/hash.h +4 -4
  549. package/deps/rocksdb/rocksdb/util/log_write_bench.cc +3 -3
  550. package/deps/rocksdb/rocksdb/util/murmurhash.cc +1 -1
  551. package/deps/rocksdb/rocksdb/util/random.cc +3 -4
  552. package/deps/rocksdb/rocksdb/util/rate_limiter.cc +23 -43
  553. package/deps/rocksdb/rocksdb/util/rate_limiter_impl.h +21 -16
  554. package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +44 -24
  555. package/deps/rocksdb/rocksdb/util/repeatable_thread_test.cc +2 -3
  556. package/deps/rocksdb/rocksdb/util/ribbon_config.cc +2 -10
  557. package/deps/rocksdb/rocksdb/util/ribbon_impl.h +2 -2
  558. package/deps/rocksdb/rocksdb/util/slice.cc +4 -5
  559. package/deps/rocksdb/rocksdb/util/slice_test.cc +70 -4
  560. package/deps/rocksdb/rocksdb/util/status.cc +1 -1
  561. package/deps/rocksdb/rocksdb/util/stderr_logger.cc +39 -7
  562. package/deps/rocksdb/rocksdb/util/stderr_logger.h +12 -2
  563. package/deps/rocksdb/rocksdb/util/string_util.cc +31 -18
  564. package/deps/rocksdb/rocksdb/util/string_util.h +9 -9
  565. package/deps/rocksdb/rocksdb/util/string_util_test.cc +36 -0
  566. package/deps/rocksdb/rocksdb/util/thread_list_test.cc +5 -5
  567. package/deps/rocksdb/rocksdb/util/thread_local.cc +1 -1
  568. package/deps/rocksdb/rocksdb/util/threadpool_imp.cc +3 -4
  569. package/deps/rocksdb/rocksdb/util/threadpool_imp.h +1 -1
  570. package/deps/rocksdb/rocksdb/util/udt_util_test.cc +4 -4
  571. package/deps/rocksdb/rocksdb/util/vector_iterator.h +12 -16
  572. package/deps/rocksdb/rocksdb/util/write_batch_util.h +5 -0
  573. package/deps/rocksdb/rocksdb/util/xxhash.h +116 -116
  574. package/deps/rocksdb/rocksdb/utilities/agg_merge/agg_merge.cc +4 -5
  575. package/deps/rocksdb/rocksdb/utilities/agg_merge/agg_merge_impl.h +2 -2
  576. package/deps/rocksdb/rocksdb/utilities/agg_merge/test_agg_merge.cc +1 -2
  577. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +33 -31
  578. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +123 -22
  579. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +9 -7
  580. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +3 -6
  581. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +22 -55
  582. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +113 -74
  583. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +36 -24
  584. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +27 -11
  585. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +5 -3
  586. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +68 -54
  587. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +3 -6
  588. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +16 -16
  589. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +3 -2
  590. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +29 -9
  591. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.h +27 -14
  592. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_compaction_filter.cc +2 -4
  593. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_compaction_filter.h +3 -3
  594. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_format_test.cc +2 -4
  595. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +2 -4
  596. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_row_merge_test.cc +2 -4
  597. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_serialize_test.cc +2 -4
  598. package/deps/rocksdb/rocksdb/utilities/cassandra/format.cc +2 -4
  599. package/deps/rocksdb/rocksdb/utilities/cassandra/format.h +8 -8
  600. package/deps/rocksdb/rocksdb/utilities/cassandra/merge_operator.cc +3 -7
  601. package/deps/rocksdb/rocksdb/utilities/cassandra/merge_operator.h +7 -8
  602. package/deps/rocksdb/rocksdb/utilities/cassandra/test_utils.cc +2 -4
  603. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +9 -7
  604. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +7 -10
  605. package/deps/rocksdb/rocksdb/utilities/debug.cc +6 -2
  606. package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +18 -11
  607. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.cc +2 -2
  608. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.h +18 -25
  609. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +32 -23
  610. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +51 -48
  611. package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.cc +5 -3
  612. package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.h +2 -2
  613. package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +1 -1
  614. package/deps/rocksdb/rocksdb/utilities/merge_operators/bytesxor.h +2 -3
  615. package/deps/rocksdb/rocksdb/utilities/merge_operators/sortlist.cc +3 -1
  616. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend.cc +7 -7
  617. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend.h +10 -10
  618. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend2.cc +5 -6
  619. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend2.h +5 -6
  620. package/deps/rocksdb/rocksdb/utilities/object_registry.cc +1 -1
  621. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +1 -8
  622. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc +29 -21
  623. package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +14 -15
  624. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.cc +2 -2
  625. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +1 -1
  626. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_bench.cc +3 -3
  627. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_test.cc +3 -5
  628. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.cc +1 -1
  629. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.cc +2 -2
  630. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.h +8 -8
  631. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache.cc +4 -4
  632. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc +1 -1
  633. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.h +6 -9
  634. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector_test.cc +2 -2
  635. package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.cc +1 -1
  636. package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.h +6 -6
  637. package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.cc +1 -2
  638. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +2 -3
  639. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.h +1 -1
  640. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_tracker.h +1 -1
  641. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +1 -1
  642. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/db.h +5 -0
  643. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc +1 -1
  644. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc +3 -3
  645. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc +1 -1
  646. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_assert_subst.h +4 -0
  647. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h +4 -0
  648. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +1 -1
  649. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/txn_subst.h +4 -0
  650. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/standalone_port.cc +12 -3
  651. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/dbt.cc +17 -3
  652. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/growable_array.h +3 -3
  653. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/partitioned_counter.h +1 -1
  654. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +3 -3
  655. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc +4 -2
  656. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.h +1 -1
  657. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction.cc +1 -2
  658. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.cc +2 -3
  659. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +3 -4
  660. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +442 -10
  661. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +27 -9
  662. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +28 -7
  663. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +70 -12
  664. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +50 -22
  665. package/deps/rocksdb/rocksdb/utilities/transactions/snapshot_checker.cc +1 -1
  666. package/deps/rocksdb/rocksdb/utilities/transactions/timestamped_snapshot_test.cc +1 -1
  667. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.cc +64 -3
  668. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +76 -17
  669. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_db_mutex_impl.cc +6 -6
  670. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +573 -77
  671. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -2
  672. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +65 -3
  673. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +13 -11
  674. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +1 -0
  675. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.h +14 -16
  676. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +37 -24
  677. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.h +23 -29
  678. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +4 -2
  679. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.h +31 -40
  680. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +8 -6
  681. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.h +3 -4
  682. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +34 -18
  683. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +21 -25
  684. package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +26 -5
  685. package/deps/rocksdb/rocksdb/utilities/types_util.cc +88 -0
  686. package/deps/rocksdb/rocksdb/utilities/types_util_test.cc +98 -0
  687. package/deps/rocksdb/rocksdb/utilities/util_merge_operators_test.cc +1 -1
  688. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +463 -90
  689. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +259 -76
  690. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +70 -14
  691. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +609 -10
  692. package/deps/rocksdb/rocksdb.gyp +8 -4
  693. package/index.js +18 -4
  694. package/max_rev_operator.h +2 -0
  695. package/package.json +2 -2
  696. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  697. package/prebuilds/linux-x64/@nxtedition+rocksdb.node +0 -0
  698. package/util.h +29 -5
  699. package/.tap/coverage/4c642b53-6895-4ef0-9391-7c6b8d484262.json +0 -14334
  700. package/deps/rocksdb/rocksdb/table/scoped_arena_iterator.h +0 -57
@@ -0,0 +1,331 @@
1
+ // Copyright (c) 2024-present, Facebook, Inc. All rights reserved.
2
+ // This source code is licensed under both the GPLv2 (found in the
3
+ // COPYING file in the root directory) and Apache 2.0 License
4
+ // (found in the LICENSE.Apache file in the root directory).
5
+
6
+ #include "env/fs_on_demand.h"
7
+
8
+ #include <algorithm>
9
+ #include <set>
10
+
11
+ #include "file/filename.h"
12
+ #include "port/port.h"
13
+ #include "rocksdb/types.h"
14
+
15
+ namespace ROCKSDB_NAMESPACE {
16
+ // Check if the input path is under orig (typically the local directory), and if
17
+ // so, change it to the equivalent path under replace (typically the remote
18
+ // directory). For example, if orig is "/data/follower", replace is
19
+ // "/data/leader", and the given path is "/data/follower/000010.sst", on return
20
+ // the path would be changed to
21
+ // "/data/leader/000010.sst".
22
+ // Return value is true if the path was modified, false otherwise
23
+ bool OnDemandFileSystem::CheckPathAndAdjust(const std::string& orig,
24
+ const std::string& replace,
25
+ std::string& path) {
26
+ size_t pos = path.find(orig);
27
+ if (pos > 0) {
28
+ return false;
29
+ }
30
+ path.replace(pos, orig.length(), replace);
31
+ return true;
32
+ }
33
+
34
+ bool OnDemandFileSystem::LookupFileType(const std::string& name,
35
+ FileType* type) {
36
+ std::size_t found = name.find_last_of('/');
37
+ std::string file_name = name.substr(found);
38
+ uint64_t number = 0;
39
+ return ParseFileName(file_name, &number, type);
40
+ }
41
+
42
+ // RocksDB opens non-SST files for reading in sequential file mode. This
43
+ // includes CURRENT, OPTIONS, MANIFEST etc. For these files, we open them
44
+ // in place in the source directory. For files that are appendable or
45
+ // can be renamed, which is MANIFEST and CURRENT files, we wrap the
46
+ // underlying FSSequentialFile with another class that checks when EOF
47
+ // has been reached and re-opens the file to see the latest data. On some
48
+ // distributed file systems, this is necessary.
49
+ IOStatus OnDemandFileSystem::NewSequentialFile(
50
+ const std::string& fname, const FileOptions& file_opts,
51
+ std::unique_ptr<FSSequentialFile>* result, IODebugContext* dbg) {
52
+ FileType type;
53
+ static std::unordered_set<FileType> valid_types(
54
+ {kWalFile, kDescriptorFile, kCurrentFile, kIdentityFile, kOptionsFile});
55
+ if (!LookupFileType(fname, &type) ||
56
+ (valid_types.find(type) == valid_types.end())) {
57
+ return IOStatus::NotSupported();
58
+ }
59
+
60
+ IOStatus s;
61
+ std::string rname = fname;
62
+ if (CheckPathAndAdjust(local_path_, remote_path_, rname)) {
63
+ // First clear any local directory cache as it may be out of date
64
+ target()->DiscardCacheForDirectory(rname);
65
+
66
+ std::unique_ptr<FSSequentialFile> inner_file;
67
+ s = target()->NewSequentialFile(rname, file_opts, &inner_file, dbg);
68
+ if (s.ok() && type == kDescriptorFile) {
69
+ result->reset(new OnDemandSequentialFile(std::move(inner_file), this,
70
+ file_opts, rname));
71
+ } else {
72
+ *result = std::move(inner_file);
73
+ }
74
+ } else {
75
+ s = target()->NewSequentialFile(fname, file_opts, result, dbg);
76
+ }
77
+ return s;
78
+ }
79
+
80
+ // This is only supported for SST files. If the file is present locally,
81
+ // i.e in the destination dir, we just open it and return. If its in the
82
+ // remote, i.e source dir, we link it locally and open the link.
83
+ // TODO: Add support for blob files belonging to the new BlobDB
84
+ IOStatus OnDemandFileSystem::NewRandomAccessFile(
85
+ const std::string& fname, const FileOptions& file_opts,
86
+ std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* dbg) {
87
+ FileType type;
88
+ if (!LookupFileType(fname, &type) || type != kTableFile) {
89
+ return IOStatus::NotSupported();
90
+ }
91
+
92
+ IOStatus s = target()->FileExists(fname, file_opts.io_options, nullptr);
93
+ if (s.IsNotFound() || s.IsPathNotFound()) {
94
+ std::string rname = fname;
95
+ if (CheckPathAndAdjust(local_path_, remote_path_, rname)) {
96
+ // First clear any local directory cache as it may be out of date
97
+ target()->DiscardCacheForDirectory(rname);
98
+
99
+ s = target()->LinkFile(rname, fname, IOOptions(), nullptr);
100
+ if (!s.ok()) {
101
+ return s;
102
+ }
103
+ }
104
+ }
105
+
106
+ return s.ok() ? target()->NewRandomAccessFile(fname, file_opts, result, dbg)
107
+ : s;
108
+ }
109
+
110
+ // We don't expect to create any writable file other than info LOG files.
111
+ IOStatus OnDemandFileSystem::NewWritableFile(
112
+ const std::string& fname, const FileOptions& file_opts,
113
+ std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
114
+ FileType type;
115
+ if (!LookupFileType(fname, &type) || type != kInfoLogFile) {
116
+ return IOStatus::NotSupported();
117
+ }
118
+
119
+ std::string rname = fname;
120
+ if (CheckPathAndAdjust(local_path_, remote_path_, rname)) {
121
+ // First clear any local directory cache as it may be out of date
122
+ target()->DiscardCacheForDirectory(rname);
123
+
124
+ IOStatus s = target()->FileExists(rname, file_opts.io_options, dbg);
125
+ if (s.ok()) {
126
+ return IOStatus::InvalidArgument(
127
+ "Writing to a file present in the remote directory not supoprted");
128
+ }
129
+ }
130
+
131
+ return target()->NewWritableFile(fname, file_opts, result, dbg);
132
+ }
133
+
134
+ // Currently not supported, as there's no need for RocksDB to create a
135
+ // directory object for a DB in read-only mode.
136
+ IOStatus OnDemandFileSystem::NewDirectory(
137
+ const std::string& /*name*/, const IOOptions& /*io_opts*/,
138
+ std::unique_ptr<FSDirectory>* /*result*/, IODebugContext* /*dbg*/) {
139
+ return IOStatus::NotSupported();
140
+ }
141
+
142
+ // Check if the given file exists, either locally or remote. If the file is an
143
+ // SST file, then link it locally. We assume if the file existence is being
144
+ // checked, its for verification purposes, for example while replaying the
145
+ // MANIFEST. The file will be opened for reading some time in the future.
146
+ IOStatus OnDemandFileSystem::FileExists(const std::string& fname,
147
+ const IOOptions& options,
148
+ IODebugContext* dbg) {
149
+ IOStatus s = target()->FileExists(fname, options, dbg);
150
+ if (!s.IsNotFound() && !s.IsPathNotFound()) {
151
+ return s;
152
+ }
153
+
154
+ std::string rname = fname;
155
+ if (CheckPathAndAdjust(local_path_, remote_path_, rname)) {
156
+ // First clear any local directory cache as it may be out of date
157
+ target()->DiscardCacheForDirectory(rname);
158
+
159
+ FileType type;
160
+ if (LookupFileType(fname, &type) && type == kTableFile) {
161
+ s = target()->LinkFile(rname, fname, options, dbg);
162
+ } else {
163
+ s = target()->FileExists(rname, options, dbg);
164
+ }
165
+ }
166
+ return s;
167
+ }
168
+
169
+ // Doa listing of both the local and remote directories and merge the two.
170
+ IOStatus OnDemandFileSystem::GetChildren(const std::string& dir,
171
+ const IOOptions& options,
172
+ std::vector<std::string>* result,
173
+ IODebugContext* dbg) {
174
+ std::string rdir = dir;
175
+ IOStatus s = target()->GetChildren(dir, options, result, dbg);
176
+ if (!s.ok() || !CheckPathAndAdjust(local_path_, remote_path_, rdir)) {
177
+ return s;
178
+ }
179
+
180
+ std::vector<std::string> rchildren;
181
+ // First clear any local directory cache as it may be out of date
182
+ target()->DiscardCacheForDirectory(rdir);
183
+ s = target()->GetChildren(rdir, options, &rchildren, dbg);
184
+ if (s.ok()) {
185
+ std::for_each(rchildren.begin(), rchildren.end(), [&](std::string& name) {
186
+ // Adjust name
187
+ (void)CheckPathAndAdjust(remote_path_, local_path_, name);
188
+ });
189
+ std::sort(result->begin(), result->end());
190
+ std::sort(rchildren.begin(), rchildren.end());
191
+
192
+ std::vector<std::string> output;
193
+ output.reserve(result->size() + rchildren.size());
194
+ std::set_union(result->begin(), result->end(), rchildren.begin(),
195
+ rchildren.end(), std::back_inserter(output));
196
+ *result = std::move(output);
197
+ }
198
+ return s;
199
+ }
200
+
201
+ // Doa listing of both the local and remote directories and merge the two.
202
+ IOStatus OnDemandFileSystem::GetChildrenFileAttributes(
203
+ const std::string& dir, const IOOptions& options,
204
+ std::vector<FileAttributes>* result, IODebugContext* dbg) {
205
+ std::string rdir = dir;
206
+ IOStatus s = target()->GetChildrenFileAttributes(dir, options, result, dbg);
207
+ if (!s.ok() || !CheckPathAndAdjust(local_path_, remote_path_, rdir)) {
208
+ return s;
209
+ }
210
+
211
+ std::vector<FileAttributes> rchildren;
212
+ // First clear any local directory cache as it may be out of date
213
+ target()->DiscardCacheForDirectory(rdir);
214
+ s = target()->GetChildrenFileAttributes(rdir, options, &rchildren, dbg);
215
+ if (s.ok()) {
216
+ struct FileAttributeSorter {
217
+ bool operator()(const FileAttributes& lhs, const FileAttributes& rhs) {
218
+ return lhs.name < rhs.name;
219
+ }
220
+ } file_attr_sorter;
221
+
222
+ std::for_each(
223
+ rchildren.begin(), rchildren.end(), [&](FileAttributes& file) {
224
+ // Adjust name
225
+ (void)CheckPathAndAdjust(remote_path_, local_path_, file.name);
226
+ });
227
+ std::sort(result->begin(), result->end(), file_attr_sorter);
228
+ std::sort(rchildren.begin(), rchildren.end(), file_attr_sorter);
229
+
230
+ std::vector<FileAttributes> output;
231
+ output.reserve(result->size() + rchildren.size());
232
+ std::set_union(rchildren.begin(), rchildren.end(), result->begin(),
233
+ result->end(), std::back_inserter(output), file_attr_sorter);
234
+ *result = std::move(output);
235
+ }
236
+ return s;
237
+ }
238
+
239
+ IOStatus OnDemandFileSystem::GetFileSize(const std::string& fname,
240
+ const IOOptions& options,
241
+ uint64_t* file_size,
242
+ IODebugContext* dbg) {
243
+ uint64_t local_size = 0;
244
+ IOStatus s = target()->GetFileSize(fname, options, &local_size, dbg);
245
+ if (!s.ok() && !s.IsNotFound() && !s.IsPathNotFound()) {
246
+ return s;
247
+ }
248
+
249
+ if (s.IsNotFound() || s.IsPathNotFound()) {
250
+ std::string rname = fname;
251
+ if (CheckPathAndAdjust(local_path_, remote_path_, rname)) {
252
+ // First clear any local directory cache as it may be out of date
253
+ target()->DiscardCacheForDirectory(rname);
254
+
255
+ FileType type;
256
+ if (LookupFileType(fname, &type) && type == kTableFile) {
257
+ s = target()->LinkFile(rname, fname, options, dbg);
258
+ if (s.ok()) {
259
+ s = target()->GetFileSize(fname, options, &local_size, dbg);
260
+ }
261
+ } else {
262
+ s = target()->GetFileSize(rname, options, &local_size, dbg);
263
+ }
264
+ }
265
+ }
266
+ *file_size = local_size;
267
+ return s;
268
+ }
269
+
270
+ // An implementation of Read that tracks whether we've reached EOF. If so,
271
+ // re-open the file to try to read past the previous EOF offset. After
272
+ // re-opening, positing it back to the last read offset.
273
+ IOStatus OnDemandSequentialFile::Read(size_t n, const IOOptions& options,
274
+ Slice* result, char* scratch,
275
+ IODebugContext* dbg) {
276
+ IOStatus s;
277
+ if (eof_) {
278
+ // Reopen the file. With some distributed file systems, this is required
279
+ // in order to get the new size
280
+ file_.reset();
281
+ s = fs_->NewSequentialFile(path_, file_opts_, &file_, dbg);
282
+ if (!s.ok()) {
283
+ return IOStatus::IOError("While opening file after relinking, got error ",
284
+ s.ToString());
285
+ }
286
+ s = file_->Skip(offset_);
287
+ if (!s.ok()) {
288
+ return IOStatus::IOError(
289
+ "While seeking to offset" + std::to_string(offset_) + "got error",
290
+ s.ToString());
291
+ }
292
+ eof_ = false;
293
+ }
294
+
295
+ s = file_->Read(n, options, result, scratch, dbg);
296
+ if (s.ok()) {
297
+ offset_ += result->size();
298
+ if (result->size() < n) {
299
+ // We reached EOF. Mark it so we know to relink next time
300
+ eof_ = true;
301
+ }
302
+ }
303
+ return s;
304
+ }
305
+
306
+ IOStatus OnDemandSequentialFile::Skip(uint64_t n) {
307
+ IOStatus s = file_->Skip(n);
308
+ if (s.ok()) {
309
+ offset_ += n;
310
+ }
311
+ return s;
312
+ }
313
+
314
+ bool OnDemandSequentialFile::use_direct_io() const {
315
+ return file_->use_direct_io();
316
+ }
317
+
318
+ size_t OnDemandSequentialFile::GetRequiredBufferAlignment() const {
319
+ return file_->GetRequiredBufferAlignment();
320
+ }
321
+
322
+ Temperature OnDemandSequentialFile::GetTemperature() const {
323
+ return file_->GetTemperature();
324
+ }
325
+
326
+ std::shared_ptr<FileSystem> NewOnDemandFileSystem(
327
+ const std::shared_ptr<FileSystem>& fs, std::string src_path,
328
+ std::string dest_path) {
329
+ return std::make_shared<OnDemandFileSystem>(fs, src_path, dest_path);
330
+ }
331
+ } // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,139 @@
1
+ // Copyright (c) 2024-present, Facebook, Inc. All rights reserved.
2
+ // This source code is licensed under both the GPLv2 (found in the
3
+ // COPYING file in the root directory) and Apache 2.0 License
4
+ // (found in the LICENSE.Apache file in the root directory).
5
+
6
+ #pragma once
7
+ #include <string>
8
+
9
+ #include "rocksdb/file_system.h"
10
+
11
+ namespace ROCKSDB_NAMESPACE {
12
+ // A FileSystem that links files to a local (destination) directory from a
13
+ // corresponding remote (source) directory on demand. The decision to link
14
+ // depends on the file type, with appendable or rename-able files, such as,
15
+ // descriptors, logs, CURRENT, being read in place in the remote directory,
16
+ // and SST files being linked. In the future, files read in place may be
17
+ // mirrored to the local directory, so the local dir has a complete database
18
+ // for troubleshooting purposes.
19
+
20
+ class OnDemandFileSystem : public FileSystemWrapper {
21
+ public:
22
+ OnDemandFileSystem(const std::shared_ptr<FileSystem>& target,
23
+ const std::string& remote_path,
24
+ const std::string& local_path)
25
+ : FileSystemWrapper(target),
26
+ remote_path_(remote_path),
27
+ local_path_(local_path) {}
28
+
29
+ const char* Name() const override { return "OnDemandFileSystem"; }
30
+
31
+ IOStatus NewSequentialFile(const std::string& fname,
32
+ const FileOptions& file_opts,
33
+ std::unique_ptr<FSSequentialFile>* result,
34
+ IODebugContext* dbg) override;
35
+
36
+ IOStatus NewRandomAccessFile(const std::string& fname,
37
+ const FileOptions& file_opts,
38
+ std::unique_ptr<FSRandomAccessFile>* result,
39
+ IODebugContext* dbg) override;
40
+
41
+ IOStatus NewWritableFile(const std::string& fname,
42
+ const FileOptions& file_opts,
43
+ std::unique_ptr<FSWritableFile>* result,
44
+ IODebugContext* dbg) override;
45
+
46
+ IOStatus ReuseWritableFile(const std::string& /*fname*/,
47
+ const std::string& /*old_fname*/,
48
+ const FileOptions& /*fopts*/,
49
+ std::unique_ptr<FSWritableFile>* /*result*/,
50
+ IODebugContext* /*dbg*/) override {
51
+ return IOStatus::NotSupported("ReuseWritableFile");
52
+ }
53
+
54
+ IOStatus NewDirectory(const std::string& name, const IOOptions& io_opts,
55
+ std::unique_ptr<FSDirectory>* result,
56
+ IODebugContext* dbg) override;
57
+
58
+ IOStatus FileExists(const std::string& fname, const IOOptions& options,
59
+ IODebugContext* dbg) override;
60
+
61
+ IOStatus GetChildren(const std::string& dir, const IOOptions& options,
62
+ std::vector<std::string>* result,
63
+ IODebugContext* dbg) override;
64
+
65
+ IOStatus GetChildrenFileAttributes(const std::string& dir,
66
+ const IOOptions& options,
67
+ std::vector<FileAttributes>* result,
68
+ IODebugContext* dbg) override;
69
+
70
+ IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
71
+ uint64_t* file_size, IODebugContext* dbg) override;
72
+
73
+ private:
74
+ bool CheckPathAndAdjust(const std::string& orig, const std::string& replace,
75
+ std::string& path);
76
+ bool LookupFileType(const std::string& name, FileType* type);
77
+
78
+ const std::string remote_path_;
79
+ const std::string local_path_;
80
+ };
81
+
82
+ // A wrapper class around an FSSequentialFile object. Its mainly
83
+ // intended to be used for appendable files like MANIFEST and logs.
84
+ // Beneath the covers, it tracks when EOF is reached, and reopens
85
+ // the file in order to read the latest appended data. This is
86
+ // necessary on some distributed file systems as they may have
87
+ // stale metadata about the file.
88
+ // TODO: Mirror the data read to a local file for troubleshooting
89
+ // purposes, as well as recovery in case the source dir is
90
+ // deleted.
91
+ class OnDemandSequentialFile : public FSSequentialFile {
92
+ public:
93
+ OnDemandSequentialFile(std::unique_ptr<FSSequentialFile>&& file,
94
+ OnDemandFileSystem* fs, const FileOptions& file_opts,
95
+ const std::string& path)
96
+ : file_(std::move(file)),
97
+ fs_(fs),
98
+ file_opts_(file_opts),
99
+ path_(path),
100
+ eof_(false),
101
+ offset_(0) {}
102
+
103
+ virtual ~OnDemandSequentialFile() {}
104
+
105
+ IOStatus Read(size_t n, const IOOptions& options, Slice* result,
106
+ char* scratch, IODebugContext* dbg) override;
107
+
108
+ IOStatus Skip(uint64_t n) override;
109
+
110
+ bool use_direct_io() const override;
111
+
112
+ size_t GetRequiredBufferAlignment() const override;
113
+
114
+ IOStatus InvalidateCache(size_t /*offset*/, size_t /*length*/) override {
115
+ return IOStatus::NotSupported("InvalidateCache not supported.");
116
+ }
117
+
118
+ IOStatus PositionedRead(uint64_t /*offset*/, size_t /*n*/,
119
+ const IOOptions& /*options*/, Slice* /*result*/,
120
+ char* /*scratch*/, IODebugContext* /*dbg*/) override {
121
+ return IOStatus::NotSupported("PositionedRead");
122
+ }
123
+
124
+ Temperature GetTemperature() const override;
125
+
126
+ private:
127
+ std::unique_ptr<FSSequentialFile> file_;
128
+ OnDemandFileSystem* fs_;
129
+ const FileOptions file_opts_;
130
+ const std::string path_;
131
+ bool eof_;
132
+ uint64_t offset_;
133
+ };
134
+
135
+ std::shared_ptr<FileSystem> NewOnDemandFileSystem(
136
+ const std::shared_ptr<FileSystem>& fs, std::string remote_path,
137
+ std::string local_path);
138
+
139
+ } // namespace ROCKSDB_NAMESPACE
@@ -13,16 +13,17 @@
13
13
  #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
14
14
  #include <dlfcn.h>
15
15
  #endif
16
- #include <errno.h>
17
16
  #include <fcntl.h>
18
17
  #include <pthread.h>
19
- #include <signal.h>
20
- #include <stdio.h>
21
- #include <stdlib.h>
22
- #include <string.h>
23
18
  #include <sys/ioctl.h>
24
19
  #include <sys/mman.h>
25
20
  #include <sys/stat.h>
21
+
22
+ #include <cerrno>
23
+ #include <csignal>
24
+ #include <cstdio>
25
+ #include <cstdlib>
26
+ #include <cstring>
26
27
  #if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID)
27
28
  #include <sys/statfs.h>
28
29
  #include <sys/sysmacros.h>
@@ -30,9 +31,9 @@
30
31
  #include <sys/statvfs.h>
31
32
  #include <sys/time.h>
32
33
  #include <sys/types.h>
33
- #include <time.h>
34
34
 
35
35
  #include <algorithm>
36
+ #include <ctime>
36
37
  // Get nano time includes
37
38
  #if defined(OS_LINUX) || defined(OS_FREEBSD)
38
39
  #elif defined(__MACH__)
@@ -115,7 +116,7 @@ class PosixFileLock : public FileLock {
115
116
  filename.clear();
116
117
  }
117
118
 
118
- virtual ~PosixFileLock() override {
119
+ ~PosixFileLock() override {
119
120
  // Check for destruction without UnlockFile
120
121
  assert(fd_ == -1);
121
122
  }
@@ -143,7 +144,7 @@ class PosixFileSystem : public FileSystem {
143
144
  const char* Name() const override { return kClassName(); }
144
145
  const char* NickName() const override { return kDefaultName(); }
145
146
 
146
- ~PosixFileSystem() override {}
147
+ ~PosixFileSystem() override = default;
147
148
  bool IsInstanceOf(const std::string& name) const override {
148
149
  if (name == "posix") {
149
150
  return true;
@@ -805,7 +806,7 @@ class PosixFileSystem : public FileSystem {
805
806
 
806
807
  IOStatus UnlockFile(FileLock* lock, const IOOptions& /*opts*/,
807
808
  IODebugContext* /*dbg*/) override {
808
- PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
809
+ PosixFileLock* my_lock = static_cast<PosixFileLock*>(lock);
809
810
  IOStatus result;
810
811
  mutex_locked_files.Lock();
811
812
  // If we are unlocking, then verify that we had locked it earlier,
@@ -997,16 +998,14 @@ class PosixFileSystem : public FileSystem {
997
998
  }
998
999
  #endif // ROCKSDB_IOURING_PRESENT
999
1000
 
1000
- // EXPERIMENTAL
1001
- //
1002
- // TODO akankshamahajan:
1001
+ // TODO:
1003
1002
  // 1. Update Poll API to take into account min_completions
1004
1003
  // and returns if number of handles in io_handles (any order) completed is
1005
1004
  // equal to atleast min_completions.
1006
1005
  // 2. Currently in case of direct_io, Read API is called because of which call
1007
1006
  // to Poll API fails as it expects IOHandle to be populated.
1008
- virtual IOStatus Poll(std::vector<void*>& io_handles,
1009
- size_t /*min_completions*/) override {
1007
+ IOStatus Poll(std::vector<void*>& io_handles,
1008
+ size_t /*min_completions*/) override {
1010
1009
  #if defined(ROCKSDB_IOURING_PRESENT)
1011
1010
  // io_uring_queue_init.
1012
1011
  struct io_uring* iu = nullptr;
@@ -1078,7 +1077,7 @@ class PosixFileSystem : public FileSystem {
1078
1077
  #endif
1079
1078
  }
1080
1079
 
1081
- virtual IOStatus AbortIO(std::vector<void*>& io_handles) override {
1080
+ IOStatus AbortIO(std::vector<void*>& io_handles) override {
1082
1081
  #if defined(ROCKSDB_IOURING_PRESENT)
1083
1082
  // io_uring_queue_init.
1084
1083
  struct io_uring* iu = nullptr;
@@ -10,23 +10,24 @@
10
10
  #ifdef ROCKSDB_LIB_IO_POSIX
11
11
  #include "env/io_posix.h"
12
12
 
13
- #include <errno.h>
14
13
  #include <fcntl.h>
15
14
 
16
15
  #include <algorithm>
16
+ #include <cerrno>
17
17
  #if defined(OS_LINUX)
18
18
  #include <linux/fs.h>
19
19
  #ifndef FALLOC_FL_KEEP_SIZE
20
20
  #include <linux/falloc.h>
21
21
  #endif
22
22
  #endif
23
- #include <stdio.h>
24
- #include <stdlib.h>
25
- #include <string.h>
26
23
  #include <sys/ioctl.h>
27
24
  #include <sys/mman.h>
28
25
  #include <sys/stat.h>
29
26
  #include <sys/types.h>
27
+
28
+ #include <cstdio>
29
+ #include <cstdlib>
30
+ #include <cstring>
30
31
  #ifdef OS_LINUX
31
32
  #include <sys/statfs.h>
32
33
  #include <sys/sysmacros.h>
@@ -437,7 +438,7 @@ void LogicalBlockSizeCache::UnrefAndTryRemoveCachedLogicalBlockSize(
437
438
 
438
439
  size_t LogicalBlockSizeCache::GetLogicalBlockSize(const std::string& fname,
439
440
  int fd) {
440
- std::string dir = fname.substr(0, fname.find_last_of("/"));
441
+ std::string dir = fname.substr(0, fname.find_last_of('/'));
441
442
  if (dir.empty()) {
442
443
  dir = "/";
443
444
  }
@@ -654,7 +655,9 @@ IOStatus PosixRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs,
654
655
  size_t this_reqs = (num_reqs - reqs_off) + incomplete_rq_list.size();
655
656
 
656
657
  // If requests exceed depth, split it into batches
657
- if (this_reqs > kIoUringDepth) this_reqs = kIoUringDepth;
658
+ if (this_reqs > kIoUringDepth) {
659
+ this_reqs = kIoUringDepth;
660
+ }
658
661
 
659
662
  assert(incomplete_rq_list.size() <= this_reqs);
660
663
  for (size_t i = 0; i < this_reqs; i++) {
@@ -854,7 +857,7 @@ IOStatus PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
854
857
 
855
858
  IOStatus PosixRandomAccessFile::ReadAsync(
856
859
  FSReadRequest& req, const IOOptions& /*opts*/,
857
- std::function<void(const FSReadRequest&, void*)> cb, void* cb_arg,
860
+ std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
858
861
  void** io_handle, IOHandleDeleter* del_fn, IODebugContext* /*dbg*/) {
859
862
  if (use_direct_io()) {
860
863
  assert(IsSectorAligned(req.offset, GetRequiredBufferAlignment()));
@@ -965,7 +968,7 @@ IOStatus PosixMmapReadableFile::Read(uint64_t offset, size_t n,
965
968
  } else if (offset + n > length_) {
966
969
  n = static_cast<size_t>(length_ - offset);
967
970
  }
968
- *result = Slice(reinterpret_cast<char*>(mmapped_region_) + offset, n);
971
+ *result = Slice(static_cast<char*>(mmapped_region_) + offset, n);
969
972
  return s;
970
973
  }
971
974
 
@@ -1064,7 +1067,7 @@ IOStatus PosixMmapFile::MapNewRegion() {
1064
1067
  }
1065
1068
  TEST_KILL_RANDOM("PosixMmapFile::Append:2");
1066
1069
 
1067
- base_ = reinterpret_cast<char*>(ptr);
1070
+ base_ = static_cast<char*>(ptr);
1068
1071
  limit_ = base_ + map_size_;
1069
1072
  dst_ = base_;
1070
1073
  last_sync_ = base_;
@@ -1438,10 +1441,12 @@ void PosixWritableFile::SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) {
1438
1441
  #ifdef OS_LINUX
1439
1442
  // Suppress Valgrind "Unimplemented functionality" error.
1440
1443
  #ifndef ROCKSDB_VALGRIND_RUN
1444
+ uint64_t fcntl_hint = hint;
1445
+
1441
1446
  if (hint == write_hint_) {
1442
1447
  return;
1443
1448
  }
1444
- if (fcntl(fd_, F_SET_RW_HINT, &hint) == 0) {
1449
+ if (fcntl(fd_, F_SET_RW_HINT, &fcntl_hint) == 0) {
1445
1450
  write_hint_ = hint;
1446
1451
  }
1447
1452
  #else