@nxtedition/rocksdb 9.0.0 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (307) hide show
  1. package/binding.cc +244 -177
  2. package/deps/rocksdb/rocksdb/CMakeLists.txt +13 -9
  3. package/deps/rocksdb/rocksdb/Makefile +15 -6
  4. package/deps/rocksdb/rocksdb/README.md +29 -0
  5. package/deps/rocksdb/rocksdb/TARGETS +17 -2
  6. package/deps/rocksdb/rocksdb/cache/cache.cc +35 -0
  7. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +74 -15
  8. package/deps/rocksdb/rocksdb/cache/cache_helpers.cc +2 -1
  9. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +4 -3
  10. package/deps/rocksdb/rocksdb/cache/cache_test.cc +16 -4
  11. package/deps/rocksdb/rocksdb/cache/charged_cache.cc +4 -2
  12. package/deps/rocksdb/rocksdb/cache/charged_cache.h +5 -3
  13. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +2024 -14
  14. package/deps/rocksdb/rocksdb/cache/clock_cache.h +349 -23
  15. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +126 -51
  16. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +9 -0
  17. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +202 -7
  18. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +31 -14
  19. package/deps/rocksdb/rocksdb/cache/secondary_cache.cc +0 -33
  20. package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.cc +314 -25
  21. package/deps/rocksdb/rocksdb/cache/secondary_cache_adapter.h +29 -4
  22. package/deps/rocksdb/rocksdb/cache/sharded_cache.cc +10 -0
  23. package/deps/rocksdb/rocksdb/cache/sharded_cache.h +8 -3
  24. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache.cc +119 -0
  25. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache.h +155 -0
  26. package/deps/rocksdb/rocksdb/cache/tiered_secondary_cache_test.cc +711 -0
  27. package/deps/rocksdb/rocksdb/cache/typed_cache.h +17 -11
  28. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +25 -11
  29. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +1 -0
  30. package/deps/rocksdb/rocksdb/db/blob/blob_contents.h +2 -1
  31. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +2 -1
  32. package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +8 -0
  33. package/deps/rocksdb/rocksdb/db/blob/db_blob_index_test.cc +7 -3
  34. package/deps/rocksdb/rocksdb/db/builder.cc +3 -3
  35. package/deps/rocksdb/rocksdb/db/c.cc +64 -0
  36. package/deps/rocksdb/rocksdb/db/c_test.c +36 -0
  37. package/deps/rocksdb/rocksdb/db/column_family.cc +23 -15
  38. package/deps/rocksdb/rocksdb/db/column_family.h +9 -0
  39. package/deps/rocksdb/rocksdb/db/column_family_test.cc +101 -5
  40. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +36 -23
  41. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +24 -10
  42. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +3 -5
  43. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +42 -18
  44. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +7 -3
  45. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +4 -2
  46. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +8 -6
  47. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +1 -1
  48. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +3 -0
  49. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +61 -0
  50. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +146 -64
  51. package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +13 -39
  52. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +1 -0
  53. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +29 -7
  54. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +8 -3
  55. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +59 -0
  56. package/deps/rocksdb/rocksdb/db/db_compaction_filter_test.cc +27 -3
  57. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +186 -2
  58. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +1 -0
  59. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +17 -5
  60. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +519 -240
  61. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +104 -43
  62. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +169 -66
  63. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +2 -1
  64. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +12 -4
  65. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +50 -14
  66. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +85 -53
  67. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.h +3 -7
  68. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +99 -82
  69. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +4 -14
  70. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +24 -21
  71. package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +6 -0
  72. package/deps/rocksdb/rocksdb/db/db_iter.cc +83 -55
  73. package/deps/rocksdb/rocksdb/db/db_iter.h +10 -2
  74. package/deps/rocksdb/rocksdb/db/db_iter_test.cc +29 -0
  75. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +276 -21
  76. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +35 -0
  77. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +187 -1
  78. package/deps/rocksdb/rocksdb/db/db_options_test.cc +258 -0
  79. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +258 -0
  80. package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +1 -0
  81. package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +52 -0
  82. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +74 -1
  83. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +22 -4
  84. package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +3 -1
  85. package/deps/rocksdb/rocksdb/db/db_test.cc +134 -30
  86. package/deps/rocksdb/rocksdb/db/db_test2.cc +3 -0
  87. package/deps/rocksdb/rocksdb/db/db_test_util.cc +11 -6
  88. package/deps/rocksdb/rocksdb/db/db_test_util.h +5 -2
  89. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +1 -0
  90. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +12 -0
  91. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +337 -1
  92. package/deps/rocksdb/rocksdb/db/deletefile_test.cc +2 -0
  93. package/deps/rocksdb/rocksdb/db/error_handler.cc +51 -34
  94. package/deps/rocksdb/rocksdb/db/error_handler.h +7 -6
  95. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +58 -0
  96. package/deps/rocksdb/rocksdb/db/flush_job.cc +17 -19
  97. package/deps/rocksdb/rocksdb/db/flush_job.h +3 -3
  98. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +2 -1
  99. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +2 -0
  100. package/deps/rocksdb/rocksdb/db/memtable.cc +18 -70
  101. package/deps/rocksdb/rocksdb/db/memtable_list.cc +1 -1
  102. package/deps/rocksdb/rocksdb/db/memtable_list.h +11 -1
  103. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +1 -1
  104. package/deps/rocksdb/rocksdb/db/merge_helper.cc +330 -115
  105. package/deps/rocksdb/rocksdb/db/merge_helper.h +100 -12
  106. package/deps/rocksdb/rocksdb/db/merge_operator.cc +82 -0
  107. package/deps/rocksdb/rocksdb/db/merge_test.cc +267 -0
  108. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +3 -0
  109. package/deps/rocksdb/rocksdb/db/periodic_task_scheduler.h +4 -4
  110. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +2 -0
  111. package/deps/rocksdb/rocksdb/db/prefix_test.cc +1 -0
  112. package/deps/rocksdb/rocksdb/db/range_del_aggregator.h +4 -0
  113. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +4 -0
  114. package/deps/rocksdb/rocksdb/db/repair.cc +4 -3
  115. package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +454 -70
  116. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc +105 -69
  117. package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h +83 -46
  118. package/deps/rocksdb/rocksdb/db/table_cache.cc +32 -19
  119. package/deps/rocksdb/rocksdb/db/table_cache.h +12 -6
  120. package/deps/rocksdb/rocksdb/db/version_edit.h +10 -4
  121. package/deps/rocksdb/rocksdb/db/version_set.cc +75 -73
  122. package/deps/rocksdb/rocksdb/db/version_set.h +8 -8
  123. package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +2 -5
  124. package/deps/rocksdb/rocksdb/db/version_set_test.cc +22 -11
  125. package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +525 -0
  126. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +6 -22
  127. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +0 -20
  128. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +0 -29
  129. package/deps/rocksdb/rocksdb/db/wide/wide_columns_helper.cc +46 -0
  130. package/deps/rocksdb/rocksdb/db/wide/wide_columns_helper.h +40 -0
  131. package/deps/rocksdb/rocksdb/db/wide/wide_columns_helper_test.cc +39 -0
  132. package/deps/rocksdb/rocksdb/db/write_batch.cc +44 -20
  133. package/deps/rocksdb/rocksdb/db_stress_tool/CMakeLists.txt +1 -0
  134. package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +4 -4
  135. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +4 -7
  136. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +88 -10
  137. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +15 -10
  138. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +108 -58
  139. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +36 -14
  140. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +34 -0
  141. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +1 -1
  142. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +195 -130
  143. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +4 -2
  144. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +12 -12
  145. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_wide_merge_operator.cc +51 -0
  146. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_wide_merge_operator.h +27 -0
  147. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +3 -6
  148. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +14 -11
  149. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +44 -38
  150. package/deps/rocksdb/rocksdb/env/env.cc +5 -0
  151. package/deps/rocksdb/rocksdb/env/unique_id_gen.cc +1 -0
  152. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +50 -29
  153. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +32 -2
  154. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +513 -30
  155. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +8 -0
  156. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h +38 -13
  157. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +14 -7
  158. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +42 -0
  159. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +65 -12
  160. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +11 -0
  161. package/deps/rocksdb/rocksdb/include/rocksdb/comparator.h +26 -0
  162. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +37 -4
  163. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +2 -0
  164. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +1 -0
  165. package/deps/rocksdb/rocksdb/include/rocksdb/filter_policy.h +8 -3
  166. package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +10 -4
  167. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +4 -0
  168. package/deps/rocksdb/rocksdb/include/rocksdb/memory_allocator.h +1 -1
  169. package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +55 -4
  170. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +45 -5
  171. package/deps/rocksdb/rocksdb/include/rocksdb/port_defs.h +4 -0
  172. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +9 -0
  173. package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +79 -8
  174. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +16 -0
  175. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +35 -0
  176. package/deps/rocksdb/rocksdb/include/rocksdb/system_clock.h +15 -0
  177. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +14 -3
  178. package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +2 -0
  179. package/deps/rocksdb/rocksdb/include/rocksdb/types.h +7 -0
  180. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +6 -1
  181. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_type.h +2 -1
  182. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +9 -0
  183. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  184. package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h +53 -2
  185. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +0 -2
  186. package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.cc +2 -2
  187. package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.h +1 -1
  188. package/deps/rocksdb/rocksdb/microbench/README.md +60 -0
  189. package/deps/rocksdb/rocksdb/monitoring/instrumented_mutex.h +1 -1
  190. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +6 -0
  191. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +18 -7
  192. package/deps/rocksdb/rocksdb/monitoring/thread_status_util_debug.cc +4 -0
  193. package/deps/rocksdb/rocksdb/options/customizable_test.cc +4 -0
  194. package/deps/rocksdb/rocksdb/options/db_options.cc +47 -2
  195. package/deps/rocksdb/rocksdb/options/db_options.h +3 -0
  196. package/deps/rocksdb/rocksdb/options/options_helper.cc +12 -0
  197. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +3 -1
  198. package/deps/rocksdb/rocksdb/options/options_test.cc +6 -1
  199. package/deps/rocksdb/rocksdb/plugin/README.md +43 -0
  200. package/deps/rocksdb/rocksdb/port/README +10 -0
  201. package/deps/rocksdb/rocksdb/port/port_example.h +1 -1
  202. package/deps/rocksdb/rocksdb/port/port_posix.cc +1 -1
  203. package/deps/rocksdb/rocksdb/port/port_posix.h +7 -4
  204. package/deps/rocksdb/rocksdb/port/stack_trace.cc +5 -0
  205. package/deps/rocksdb/rocksdb/port/win/port_win.h +5 -2
  206. package/deps/rocksdb/rocksdb/src.mk +7 -1
  207. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +1 -1
  208. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +3 -1
  209. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +275 -61
  210. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +96 -4
  211. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +179 -62
  212. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +35 -22
  213. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +12 -8
  214. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +14 -9
  215. package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc +3 -1
  216. package/deps/rocksdb/rocksdb/table/block_based/block_cache.h +26 -7
  217. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +15 -12
  218. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +10 -5
  219. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +39 -18
  220. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +6 -6
  221. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +44 -26
  222. package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +2 -1
  223. package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +1 -1
  224. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +10 -8
  225. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +4 -2
  226. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +3 -2
  227. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +1 -1
  228. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +3 -2
  229. package/deps/rocksdb/rocksdb/table/block_fetcher.h +4 -0
  230. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +6 -2
  231. package/deps/rocksdb/rocksdb/table/get_context.cc +52 -89
  232. package/deps/rocksdb/rocksdb/table/get_context.h +12 -3
  233. package/deps/rocksdb/rocksdb/table/internal_iterator.h +11 -0
  234. package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +29 -1
  235. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +12 -0
  236. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +33 -6
  237. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +1 -0
  238. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +2 -4
  239. package/deps/rocksdb/rocksdb/table/table_reader.h +6 -0
  240. package/deps/rocksdb/rocksdb/test_util/mock_time_env.h +31 -0
  241. package/deps/rocksdb/rocksdb/test_util/secondary_cache_test_util.cc +2 -1
  242. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_pysim.py +3 -3
  243. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +26 -43
  244. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +213 -28
  245. package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +36 -0
  246. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +0 -1
  247. package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc +33 -10
  248. package/deps/rocksdb/rocksdb/util/bloom_test.cc +32 -11
  249. package/deps/rocksdb/rocksdb/util/cast_util.h +10 -0
  250. package/deps/rocksdb/rocksdb/util/comparator.cc +26 -1
  251. package/deps/rocksdb/rocksdb/util/compression.h +9 -3
  252. package/deps/rocksdb/rocksdb/util/crc32c.cc +7 -1
  253. package/deps/rocksdb/rocksdb/util/distributed_mutex.h +1 -1
  254. package/deps/rocksdb/rocksdb/util/overload.h +23 -0
  255. package/deps/rocksdb/rocksdb/util/rate_limiter.cc +53 -18
  256. package/deps/rocksdb/rocksdb/util/rate_limiter_impl.h +6 -1
  257. package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +90 -19
  258. package/deps/rocksdb/rocksdb/util/slice_test.cc +30 -0
  259. package/deps/rocksdb/rocksdb/util/status.cc +1 -0
  260. package/deps/rocksdb/rocksdb/util/string_util.cc +39 -0
  261. package/deps/rocksdb/rocksdb/util/string_util.h +10 -0
  262. package/deps/rocksdb/rocksdb/util/thread_operation.h +2 -0
  263. package/deps/rocksdb/rocksdb/util/udt_util.cc +42 -0
  264. package/deps/rocksdb/rocksdb/util/udt_util.h +19 -0
  265. package/deps/rocksdb/rocksdb/util/udt_util_test.cc +14 -0
  266. package/deps/rocksdb/rocksdb/util/xxhash.h +0 -3
  267. package/deps/rocksdb/rocksdb/util/xxph3.h +0 -4
  268. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +2 -1
  269. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.h +1 -0
  270. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +19 -15
  271. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +11 -7
  272. package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.h +5 -0
  273. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +3 -0
  274. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc +9 -0
  275. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache.cc +7 -4
  276. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/README +13 -0
  277. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +41 -0
  278. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +15 -9
  279. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +4 -0
  280. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +155 -0
  281. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +6 -0
  282. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +81 -1
  283. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +2 -6
  284. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +7 -5
  285. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +2 -1
  286. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +3 -2
  287. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +57 -27
  288. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +127 -120
  289. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +129 -59
  290. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +105 -8
  291. package/deps/rocksdb/rocksdb.gyp +4 -2
  292. package/index.js +38 -55
  293. package/package.json +4 -4
  294. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  295. package/util.h +7 -1
  296. package/deps/rocksdb/rocksdb/cmake/modules/CxxFlags.cmake +0 -7
  297. package/deps/rocksdb/rocksdb/cmake/modules/FindJeMalloc.cmake +0 -29
  298. package/deps/rocksdb/rocksdb/cmake/modules/FindNUMA.cmake +0 -29
  299. package/deps/rocksdb/rocksdb/cmake/modules/FindSnappy.cmake +0 -29
  300. package/deps/rocksdb/rocksdb/cmake/modules/FindTBB.cmake +0 -33
  301. package/deps/rocksdb/rocksdb/cmake/modules/Findgflags.cmake +0 -29
  302. package/deps/rocksdb/rocksdb/cmake/modules/Findlz4.cmake +0 -29
  303. package/deps/rocksdb/rocksdb/cmake/modules/Finduring.cmake +0 -26
  304. package/deps/rocksdb/rocksdb/cmake/modules/Findzstd.cmake +0 -29
  305. package/deps/rocksdb/rocksdb/cmake/modules/ReadVersion.cmake +0 -10
  306. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  307. package/prebuilds/linux-x64/node.napi.node +0 -0
@@ -12,13 +12,13 @@
12
12
  #include "rocksdb/utilities/debug.h"
13
13
  #include "test_util/mock_time_env.h"
14
14
 
15
-
16
15
  namespace ROCKSDB_NAMESPACE {
17
16
 
18
17
  class SeqnoTimeTest : public DBTestBase {
19
18
  public:
20
19
  SeqnoTimeTest() : DBTestBase("seqno_time_test", /*env_do_fsync=*/false) {
21
20
  mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
21
+ mock_clock_->SetCurrentTime(kMockStartTime);
22
22
  mock_env_ = std::make_unique<CompositeEnvWrapper>(env_, mock_clock_);
23
23
  }
24
24
 
@@ -26,6 +26,10 @@ class SeqnoTimeTest : public DBTestBase {
26
26
  std::unique_ptr<Env> mock_env_;
27
27
  std::shared_ptr<MockSystemClock> mock_clock_;
28
28
 
29
+ // Sufficient starting time that preserve time doesn't under-flow into
30
+ // pre-history
31
+ static constexpr uint32_t kMockStartTime = 10000000;
32
+
29
33
  void SetUp() override {
30
34
  mock_clock_->InstallTimedWaitFixCallback();
31
35
  SyncPoint::GetInstance()->SetCallBack(
@@ -34,6 +38,7 @@ class SeqnoTimeTest : public DBTestBase {
34
38
  reinterpret_cast<PeriodicTaskScheduler*>(arg);
35
39
  periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock_.get());
36
40
  });
41
+ mock_clock_->SetCurrentTime(kMockStartTime);
37
42
  }
38
43
 
39
44
  // make sure the file is not in cache, otherwise it won't have IO info
@@ -77,11 +82,6 @@ TEST_F(SeqnoTimeTest, TemperatureBasicUniversal) {
77
82
  options.num_levels = kNumLevels;
78
83
  DestroyAndReopen(options);
79
84
 
80
- // pass some time first, otherwise the first a few keys write time are going
81
- // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
82
- dbfull()->TEST_WaitForPeriodicTaskRun(
83
- [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
84
-
85
85
  int sst_num = 0;
86
86
  // Write files that are overlap and enough to trigger compaction
87
87
  for (; sst_num < kNumTrigger; sst_num++) {
@@ -189,11 +189,6 @@ TEST_F(SeqnoTimeTest, TemperatureBasicLevel) {
189
189
  options.disable_auto_compactions = true;
190
190
  DestroyAndReopen(options);
191
191
 
192
- // pass some time first, otherwise the first a few keys write time are going
193
- // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
194
- dbfull()->TEST_WaitForPeriodicTaskRun(
195
- [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
196
-
197
192
  int sst_num = 0;
198
193
  // Write files that are overlap
199
194
  for (; sst_num < 4; sst_num++) {
@@ -320,7 +315,9 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
320
315
  DestroyAndReopen(options);
321
316
 
322
317
  std::set<uint64_t> checked_file_nums;
323
- SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber();
318
+ SequenceNumber start_seq = dbfull()->GetLatestSequenceNumber() + 1;
319
+ uint64_t start_time = mock_clock_->NowSeconds();
320
+
324
321
  // Write a key every 10 seconds
325
322
  for (int i = 0; i < 200; i++) {
326
323
  ASSERT_OK(Put(Key(i), "value"));
@@ -338,21 +335,20 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
338
335
  ASSERT_FALSE(tp_mapping.Empty());
339
336
  auto seqs = tp_mapping.TEST_GetInternalMapping();
340
337
  // about ~20 seqs->time entries, because the sample rate is 10000/100, and it
341
- // passes 2k time.
342
- ASSERT_GE(seqs.size(), 19);
343
- ASSERT_LE(seqs.size(), 21);
344
- SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber();
345
- for (auto i = start_seq; i < start_seq + 10; i++) {
346
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 1) * 10);
347
- }
348
- start_seq += 10;
338
+ // passes 2k time. Add (roughly) one for starting entry.
339
+ ASSERT_GE(seqs.size(), 20);
340
+ ASSERT_LE(seqs.size(), 22);
341
+ SequenceNumber seq_end = dbfull()->GetLatestSequenceNumber() + 1;
349
342
  for (auto i = start_seq; i < seq_end; i++) {
350
343
  // The result is within the range
351
- ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - 10) * 10);
352
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i + 10) * 10);
344
+ ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i),
345
+ start_time + (i - start_seq) * 10 - 100);
346
+ ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i),
347
+ start_time + (i - start_seq) * 10);
353
348
  }
354
349
  checked_file_nums.insert(it->second->orig_file_number);
355
350
  start_seq = seq_end;
351
+ start_time = mock_clock_->NowSeconds();
356
352
 
357
353
  // Write a key every 1 seconds
358
354
  for (int i = 0; i < 200; i++) {
@@ -360,7 +356,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
360
356
  dbfull()->TEST_WaitForPeriodicTaskRun(
361
357
  [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1)); });
362
358
  }
363
- seq_end = dbfull()->GetLatestSequenceNumber();
359
+ seq_end = dbfull()->GetLatestSequenceNumber() + 1;
364
360
  ASSERT_OK(Flush());
365
361
  tables_props.clear();
366
362
  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props));
@@ -382,13 +378,14 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
382
378
  ASSERT_GE(seqs.size(), 1);
383
379
  ASSERT_LE(seqs.size(), 3);
384
380
  for (auto i = start_seq; i < seq_end; i++) {
385
- // The result is not very accurate, as there is more data write within small
386
- // range of time
387
- ASSERT_GE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 1000);
388
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000);
381
+ ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i),
382
+ start_time + (i - start_seq) - 100);
383
+ ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i),
384
+ start_time + (i - start_seq));
389
385
  }
390
386
  checked_file_nums.insert(it->second->orig_file_number);
391
387
  start_seq = seq_end;
388
+ start_time = mock_clock_->NowSeconds();
392
389
 
393
390
  // Write a key every 200 seconds
394
391
  for (int i = 0; i < 200; i++) {
@@ -396,7 +393,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
396
393
  dbfull()->TEST_WaitForPeriodicTaskRun(
397
394
  [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(200)); });
398
395
  }
399
- seq_end = dbfull()->GetLatestSequenceNumber();
396
+ seq_end = dbfull()->GetLatestSequenceNumber() + 1;
400
397
  ASSERT_OK(Flush());
401
398
  tables_props.clear();
402
399
  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props));
@@ -417,20 +414,18 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
417
414
  // The sequence number -> time entries should be maxed
418
415
  ASSERT_GE(seqs.size(), 99);
419
416
  ASSERT_LE(seqs.size(), 101);
420
- for (auto i = start_seq; i < seq_end - 99; i++) {
421
- // likely the first 100 entries reports 0
422
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i), (i - start_seq) + 3000);
423
- }
424
- start_seq += 101;
425
-
426
417
  for (auto i = start_seq; i < seq_end; i++) {
427
- ASSERT_GE(tp_mapping.GetOldestApproximateTime(i),
428
- (i - start_seq) * 200 + 22200);
429
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i),
430
- (i - start_seq) * 200 + 22600);
418
+ // aged out entries allowed to report time=0
419
+ if ((seq_end - i) * 200 <= 10000) {
420
+ ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i),
421
+ start_time + (i - start_seq) * 200 - 100);
422
+ }
423
+ ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i),
424
+ start_time + (i - start_seq) * 200);
431
425
  }
432
426
  checked_file_nums.insert(it->second->orig_file_number);
433
427
  start_seq = seq_end;
428
+ start_time = mock_clock_->NowSeconds();
434
429
 
435
430
  // Write a key every 100 seconds
436
431
  for (int i = 0; i < 200; i++) {
@@ -438,7 +433,7 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
438
433
  dbfull()->TEST_WaitForPeriodicTaskRun(
439
434
  [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
440
435
  }
441
- seq_end = dbfull()->GetLatestSequenceNumber();
436
+ seq_end = dbfull()->GetLatestSequenceNumber() + 1;
442
437
  ASSERT_OK(Flush());
443
438
  tables_props.clear();
444
439
  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(&tables_props));
@@ -484,18 +479,15 @@ TEST_P(SeqnoTimeTablePropTest, BasicSeqnoToTimeMapping) {
484
479
  seqs = tp_mapping.TEST_GetInternalMapping();
485
480
  ASSERT_GE(seqs.size(), 99);
486
481
  ASSERT_LE(seqs.size(), 101);
487
- for (auto i = start_seq; i < seq_end - 99; i++) {
488
- // likely the first 100 entries reports 0
489
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i),
490
- (i - start_seq) * 100 + 50000);
491
- }
492
- start_seq += 101;
493
-
494
482
  for (auto i = start_seq; i < seq_end; i++) {
495
- ASSERT_GE(tp_mapping.GetOldestApproximateTime(i),
496
- (i - start_seq) * 100 + 52200);
497
- ASSERT_LE(tp_mapping.GetOldestApproximateTime(i),
498
- (i - start_seq) * 100 + 52400);
483
+ // aged out entries allowed to report time=0
484
+ // FIXME: should be <=
485
+ if ((seq_end - i) * 100 < 10000) {
486
+ ASSERT_GE(tp_mapping.GetProximalTimeBeforeSeqno(i),
487
+ start_time + (i - start_seq) * 100 - 100);
488
+ }
489
+ ASSERT_LE(tp_mapping.GetProximalTimeBeforeSeqno(i),
490
+ start_time + (i - start_seq) * 100);
499
491
  }
500
492
  ASSERT_OK(db_->Close());
501
493
  }
@@ -620,14 +612,12 @@ TEST_P(SeqnoTimeTablePropTest, MultiCFs) {
620
612
  ASSERT_GE(seqs.size(), 99);
621
613
  ASSERT_LE(seqs.size(), 101);
622
614
 
623
- for (int j = 0; j < 2; j++) {
624
615
  for (int i = 0; i < 200; i++) {
625
616
  ASSERT_OK(Put(0, Key(i), "value"));
626
617
  dbfull()->TEST_WaitForPeriodicTaskRun(
627
618
  [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(100)); });
628
619
  }
629
620
  ASSERT_OK(Flush(0));
630
- }
631
621
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
632
622
  tables_props.clear();
633
623
  ASSERT_OK(dbfull()->GetPropertiesOfAllTables(handles_[0], &tables_props));
@@ -735,8 +725,9 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
735
725
  ASSERT_OK(tp_mapping.Sort());
736
726
  ASSERT_FALSE(tp_mapping.Empty());
737
727
  auto seqs = tp_mapping.TEST_GetInternalMapping();
738
- ASSERT_GE(seqs.size(), 10 - 1);
739
- ASSERT_LE(seqs.size(), 10 + 1);
728
+ // Add (roughly) one for starting entry.
729
+ ASSERT_GE(seqs.size(), 10);
730
+ ASSERT_LE(seqs.size(), 10 + 2);
740
731
  }
741
732
 
742
733
  // Trigger a compaction
@@ -826,6 +817,203 @@ TEST_P(SeqnoTimeTablePropTest, SeqnoToTimeMappingUniversal) {
826
817
  Close();
827
818
  }
828
819
 
820
+ TEST_P(SeqnoTimeTablePropTest, PrePopulateInDB) {
821
+ Options base_options = CurrentOptions();
822
+ base_options.env = mock_env_.get();
823
+ base_options.disable_auto_compactions = true;
824
+ base_options.create_missing_column_families = true;
825
+ Options track_options = base_options;
826
+ constexpr uint32_t kPreserveSecs = 1234567;
827
+ SetTrackTimeDurationOptions(kPreserveSecs, track_options);
828
+ SeqnoToTimeMapping sttm;
829
+ SequenceNumber latest_seqno;
830
+ uint64_t start_time, end_time;
831
+
832
+ // #### DB#1, #2: No pre-population without preserve/preclude ####
833
+ // #### But a single entry is added when preserve/preclude enabled ####
834
+ for (bool with_write : {false, true}) {
835
+ SCOPED_TRACE("with_write=" + std::to_string(with_write));
836
+ DestroyAndReopen(base_options);
837
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
838
+ ASSERT_TRUE(sttm.Empty());
839
+ ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
840
+
841
+ if (with_write) {
842
+ // Ensure that writes before new CF with preserve/preclude option don't
843
+ // interfere with the seqno-to-time mapping getting a starting entry.
844
+ ASSERT_OK(Put("foo", "bar"));
845
+ ASSERT_OK(Flush());
846
+ } else {
847
+ // FIXME: currently, starting entry after CreateColumnFamily requires
848
+ // non-zero seqno
849
+ ASSERT_OK(Delete("blah"));
850
+ }
851
+
852
+ // Unfortunately, if we add a CF with preserve/preclude option after
853
+ // open, that does not reserve seqnos with pre-populated time mappings.
854
+ CreateColumnFamilies({"one"}, track_options);
855
+
856
+ // No pre-population (unfortunately), just a single starting entry
857
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
858
+ latest_seqno = db_->GetLatestSequenceNumber();
859
+ start_time = mock_clock_->NowSeconds();
860
+ ASSERT_EQ(sttm.Size(), 1);
861
+ ASSERT_EQ(latest_seqno, 1U);
862
+ // Current time maps to starting entry / seqno
863
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), 1U);
864
+ // Any older times are unknown.
865
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - 1),
866
+ kUnknownSeqnoBeforeAll);
867
+
868
+ // Now check that writes can proceed normally (passing about 20% of preserve
869
+ // time)
870
+ for (int i = 0; i < 20; i++) {
871
+ ASSERT_OK(Put(Key(i), "value"));
872
+ dbfull()->TEST_WaitForPeriodicTaskRun([&] {
873
+ mock_clock_->MockSleepForSeconds(static_cast<int>(kPreserveSecs / 99));
874
+ });
875
+ }
876
+ ASSERT_OK(Flush());
877
+
878
+ // Check that mappings are getting populated
879
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
880
+ latest_seqno = db_->GetLatestSequenceNumber();
881
+ end_time = mock_clock_->NowSeconds();
882
+ ASSERT_EQ(sttm.Size(), 21);
883
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(end_time), latest_seqno);
884
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), 1U);
885
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - 1),
886
+ kUnknownSeqnoBeforeAll);
887
+ }
888
+
889
+ // ### DB#3, #4: Read-only DB with preserve/preclude after not ####
890
+ // Make sure we don't hit issues with read-only DBs, which don't need
891
+ // the mapping in the DB state (though it wouldn't hurt anything)
892
+ for (bool with_write : {false, true}) {
893
+ SCOPED_TRACE("with_write=" + std::to_string(with_write));
894
+ DestroyAndReopen(base_options);
895
+ if (with_write) {
896
+ ASSERT_OK(Put("foo", "bar"));
897
+ ASSERT_OK(Flush());
898
+ }
899
+
900
+ ASSERT_OK(ReadOnlyReopen(base_options));
901
+ if (with_write) {
902
+ ASSERT_EQ(Get("foo"), "bar");
903
+ }
904
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
905
+ ASSERT_EQ(sttm.Size(), 0);
906
+ if (!with_write) {
907
+ ASSERT_EQ(db_->GetLatestSequenceNumber(), 0);
908
+ }
909
+
910
+ ASSERT_OK(ReadOnlyReopen(track_options));
911
+ if (with_write) {
912
+ ASSERT_EQ(Get("foo"), "bar");
913
+ }
914
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
915
+ ASSERT_EQ(sttm.Size(), 0);
916
+ if (!with_write) {
917
+ ASSERT_EQ(db_->GetLatestSequenceNumber(), 0);
918
+
919
+ // And even if we re-open read-write, we do not get pre-population,
920
+ // because that's only for new DBs.
921
+ Reopen(track_options);
922
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
923
+ ASSERT_EQ(sttm.Size(), 0);
924
+ ASSERT_EQ(db_->GetLatestSequenceNumber(), 0);
925
+ }
926
+ }
927
+
928
+ // #### DB#5: Destroy and open with preserve/preclude option ####
929
+ DestroyAndReopen(track_options);
930
+
931
+ // Ensure pre-population
932
+ constexpr auto kPrePopPairs = SeqnoToTimeMapping::kMaxSeqnoTimePairsPerSST;
933
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
934
+ latest_seqno = db_->GetLatestSequenceNumber();
935
+ start_time = mock_clock_->NowSeconds();
936
+ ASSERT_EQ(sttm.Size(), kPrePopPairs);
937
+ // One nono-zero sequence number per pre-populated pair (this could be
938
+ // revised if we want to use interpolation for better approximate time
939
+ // mappings with no guarantee of erring in just one direction).
940
+ ASSERT_EQ(latest_seqno, kPrePopPairs);
941
+ // Current time maps to last pre-allocated seqno
942
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), latest_seqno);
943
+ // Oldest tracking time maps to first pre-allocated seqno
944
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs), 1);
945
+
946
+ // In more detail, check that estimated seqnos (pre-allocated) are uniformly
947
+ // spread over the tracked time.
948
+ for (auto ratio : {0.0, 0.433, 0.678, 0.987, 1.0}) {
949
+ // Round up query time
950
+ uint64_t t = start_time - kPreserveSecs +
951
+ static_cast<uint64_t>(ratio * kPreserveSecs + 0.9999999);
952
+ // Round down estimated seqno
953
+ SequenceNumber s =
954
+ static_cast<SequenceNumber>(ratio * (latest_seqno - 1)) + 1;
955
+ // Match
956
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(t), s);
957
+ }
958
+
959
+ // Now check that writes can proceed normally (passing about 20% of preserve
960
+ // time)
961
+ for (int i = 0; i < 20; i++) {
962
+ ASSERT_OK(Put(Key(i), "value"));
963
+ dbfull()->TEST_WaitForPeriodicTaskRun([&] {
964
+ mock_clock_->MockSleepForSeconds(static_cast<int>(kPreserveSecs / 99));
965
+ });
966
+ }
967
+ ASSERT_OK(Flush());
968
+
969
+ // Can still see some pre-populated mappings, though some displaced
970
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
971
+ latest_seqno = db_->GetLatestSequenceNumber();
972
+ end_time = mock_clock_->NowSeconds();
973
+ ASSERT_EQ(sttm.Size(), kPrePopPairs);
974
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(end_time), latest_seqno);
975
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs / 2),
976
+ kPrePopPairs / 2);
977
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs),
978
+ kUnknownSeqnoBeforeAll);
979
+
980
+ // Make sure we don't hit issues with read-only DBs, which don't need
981
+ // the mapping in the DB state (though it wouldn't hurt anything)
982
+ ASSERT_OK(ReadOnlyReopen(track_options));
983
+ ASSERT_EQ(Get(Key(0)), "value");
984
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
985
+ ASSERT_EQ(sttm.Size(), 0);
986
+
987
+ // #### DB#6: Destroy and open+create an extra CF with preserve/preclude ####
988
+ // (default CF does not have the option)
989
+ Destroy(track_options);
990
+ ReopenWithColumnFamilies({"default", "one"},
991
+ List({base_options, track_options}));
992
+
993
+ // Ensure pre-population (not as exhaustive checking here)
994
+ sttm = dbfull()->TEST_GetSeqnoToTimeMapping();
995
+ latest_seqno = db_->GetLatestSequenceNumber();
996
+ start_time = mock_clock_->NowSeconds();
997
+ ASSERT_EQ(sttm.Size(), kPrePopPairs);
998
+ // One nono-zero sequence number per pre-populated pair (this could be
999
+ // revised if we want to use interpolation for better approximate time
1000
+ // mappings with no guarantee of erring in just one direction).
1001
+ ASSERT_EQ(latest_seqno, kPrePopPairs);
1002
+ // Current time maps to last pre-allocated seqno
1003
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time), latest_seqno);
1004
+ // Oldest tracking time maps to first pre-allocated seqno
1005
+ ASSERT_EQ(sttm.GetProximalSeqnoBeforeTime(start_time - kPreserveSecs), 1);
1006
+
1007
+ // Even after no writes and DB re-open without tracking options, sequence
1008
+ // numbers should not go backward into those that were pre-allocated.
1009
+ // (Future work: persist the mapping)
1010
+ ReopenWithColumnFamilies({"default", "one"},
1011
+ List({base_options, base_options}));
1012
+ ASSERT_EQ(latest_seqno, db_->GetLatestSequenceNumber());
1013
+
1014
+ Close();
1015
+ }
1016
+
829
1017
  TEST_F(SeqnoTimeTest, MappingAppend) {
830
1018
  SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10);
831
1019
 
@@ -843,8 +1031,9 @@ TEST_F(SeqnoTimeTest, MappingAppend) {
843
1031
  ASSERT_FALSE(test.Append(8, 12));
844
1032
  ASSERT_EQ(size, test.Size());
845
1033
 
846
- // Append with the same seqno, newer time will be accepted
847
- ASSERT_TRUE(test.Append(10, 12));
1034
+ // Append with the same seqno, newer time is rejected because that makes
1035
+ // GetProximalSeqnoBeforeTime queries worse (see later test)
1036
+ ASSERT_FALSE(test.Append(10, 12));
848
1037
  ASSERT_EQ(size, test.Size());
849
1038
  // older time will be ignored
850
1039
  ASSERT_FALSE(test.Append(10, 9));
@@ -853,25 +1042,220 @@ TEST_F(SeqnoTimeTest, MappingAppend) {
853
1042
  // new seqno with old time will be ignored
854
1043
  ASSERT_FALSE(test.Append(12, 8));
855
1044
  ASSERT_EQ(size, test.Size());
1045
+
1046
+ // new seqno with same time is accepted by replacing last entry
1047
+ // (improves GetProximalSeqnoBeforeTime queries without blowing up size)
1048
+ ASSERT_TRUE(test.Append(12, 11));
1049
+ ASSERT_EQ(size, test.Size());
856
1050
  }
857
1051
 
858
- TEST_F(SeqnoTimeTest, GetOldestApproximateTime) {
1052
+ TEST_F(SeqnoTimeTest, ProximalFunctions) {
859
1053
  SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10);
860
1054
 
861
- ASSERT_EQ(test.GetOldestApproximateTime(10), kUnknownSeqnoTime);
1055
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1), kUnknownTimeBeforeAll);
1056
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U),
1057
+ kUnknownTimeBeforeAll);
1058
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1), kUnknownSeqnoBeforeAll);
1059
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U),
1060
+ kUnknownSeqnoBeforeAll);
1061
+
1062
+ // (Taken from example in SeqnoToTimeMapping class comment)
1063
+ // Time 500 is after seqno 10 and before seqno 11
1064
+ EXPECT_TRUE(test.Append(10, 500));
1065
+
1066
+ // Seqno too early
1067
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(9), kUnknownTimeBeforeAll);
1068
+ // We only know that 500 is after 10
1069
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll);
1070
+ // Found
1071
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U);
1072
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U), 500U);
1073
+
1074
+ // Time too early
1075
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(499), kUnknownSeqnoBeforeAll);
1076
+ // Found
1077
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U);
1078
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(501), 10U);
1079
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U), 10U);
1080
+
1081
+ // More samples
1082
+ EXPECT_TRUE(test.Append(20, 600));
1083
+ EXPECT_TRUE(test.Append(30, 700));
1084
+
1085
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll);
1086
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U);
1087
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(20), 500U);
1088
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(21), 600U);
1089
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U);
1090
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U);
1091
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(1000000000000U), 700U);
1092
+
1093
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(499), kUnknownSeqnoBeforeAll);
1094
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U);
1095
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(501), 10U);
1096
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), 10U);
1097
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U);
1098
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(601), 20U);
1099
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U);
1100
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U);
1101
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(701), 30U);
1102
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(1000000000000U), 30U);
1103
+
1104
+ // Redundant sample ignored
1105
+ EXPECT_EQ(test.Size(), 3U);
1106
+ EXPECT_FALSE(test.Append(30, 700));
1107
+ EXPECT_EQ(test.Size(), 3U);
1108
+
1109
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U);
1110
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U);
1111
+
1112
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U);
1113
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U);
1114
+
1115
+ // Later sample with same seqno is ignored, to provide best results
1116
+ // for GetProximalSeqnoBeforeTime function while saving entries
1117
+ // in SeqnoToTimeMapping.
1118
+ EXPECT_FALSE(test.Append(30, 800));
1119
+
1120
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U);
1121
+ // Could return 800, but saving space in SeqnoToTimeMapping instead.
1122
+ // Can reconsider if/when GetProximalTimeBeforeSeqno is used in
1123
+ // production.
1124
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(31), 700U);
1125
+
1126
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U);
1127
+ // If the existing {30, 700} entry were replaced with {30, 800}, this
1128
+ // would return seqno 20 instead of 30, which would preclude more than
1129
+ // necessary for "preclude_last_level_data_seconds" feature.
1130
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U);
1131
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(800), 30U);
1132
+
1133
+ // Still OK
1134
+ EXPECT_TRUE(test.Append(40, 900));
1135
+
1136
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(30), 600U);
1137
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(41), 900U);
1138
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 30U);
1139
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(900), 40U);
1140
+
1141
+ // Burst of writes during a short time creates an opportunity
1142
+ // for better results from GetProximalSeqnoBeforeTime(), at the
1143
+ // expense of GetProximalTimeBeforeSeqno().
1144
+ EXPECT_TRUE(test.Append(50, 900));
1145
+
1146
+ // These are subject to later revision depending on priorities
1147
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(49), 700U);
1148
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(51), 900U);
1149
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 30U);
1150
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(900), 50U);
1151
+ }
862
1152
 
863
- test.Append(3, 10);
1153
+ TEST_F(SeqnoTimeTest, PrePopulate) {
1154
+ SeqnoToTimeMapping test(/*max_time_duration=*/100, /*max_capacity=*/10);
864
1155
 
865
- ASSERT_EQ(test.GetOldestApproximateTime(2), kUnknownSeqnoTime);
866
- ASSERT_EQ(test.GetOldestApproximateTime(3), 10);
867
- ASSERT_EQ(test.GetOldestApproximateTime(10), 10);
1156
+ EXPECT_EQ(test.Size(), 0U);
868
1157
 
869
- test.Append(10, 100);
1158
+ // Smallest case is like two Appends
1159
+ test.PrePopulate(10, 11, 500, 600);
1160
+
1161
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(10), kUnknownTimeBeforeAll);
1162
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(11), 500U);
1163
+ EXPECT_EQ(test.GetProximalTimeBeforeSeqno(12), 600U);
1164
+
1165
+ test.Clear();
1166
+
1167
+ // Populate a small range
1168
+ uint64_t kTimeIncrement = 1234567;
1169
+ test.PrePopulate(1, 12, kTimeIncrement, kTimeIncrement * 2);
1170
+
1171
+ for (uint64_t i = 0; i <= 12; ++i) {
1172
+ // NOTE: with 1 and 12 as the pre-populated end points, the duration is
1173
+ // broken into 11 equal(-ish) spans
1174
+ uint64_t t = kTimeIncrement + (i * kTimeIncrement) / 11 - 1;
1175
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(t), i);
1176
+ }
1177
+
1178
+ test.Clear();
1179
+
1180
+ // Populate an excessively large range (in the future we might want to
1181
+ // interpolate estimated times for seqnos between entries)
1182
+ test.PrePopulate(1, 34567, kTimeIncrement, kTimeIncrement * 2);
1183
+
1184
+ for (auto ratio : {0.0, 0.433, 0.678, 0.987, 1.0}) {
1185
+ // Round up query time
1186
+ uint64_t t = kTimeIncrement +
1187
+ static_cast<uint64_t>(ratio * kTimeIncrement + 0.9999999);
1188
+ // Round down estimated seqno
1189
+ SequenceNumber s = static_cast<SequenceNumber>(ratio * (34567 - 1)) + 1;
1190
+ // Match
1191
+ // TODO: for now this is exact, but in the future might need approximation
1192
+ // bounds to account for limited samples.
1193
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(t), s);
1194
+ }
1195
+ }
1196
+
1197
+ TEST_F(SeqnoTimeTest, TruncateOldEntries) {
1198
+ constexpr uint64_t kMaxTimeDuration = 42;
1199
+ SeqnoToTimeMapping test(kMaxTimeDuration, /*max_capacity=*/10);
1200
+
1201
+ EXPECT_EQ(test.Size(), 0U);
1202
+
1203
+ // Safe on empty mapping
1204
+ test.TruncateOldEntries(500);
1205
+
1206
+ EXPECT_EQ(test.Size(), 0U);
1207
+
1208
+ // (Taken from example in SeqnoToTimeMapping class comment)
1209
+ // Time 500 is after seqno 10 and before seqno 11
1210
+ EXPECT_TRUE(test.Append(10, 500));
1211
+ EXPECT_TRUE(test.Append(20, 600));
1212
+ EXPECT_TRUE(test.Append(30, 700));
1213
+ EXPECT_TRUE(test.Append(40, 800));
1214
+ EXPECT_TRUE(test.Append(50, 900));
1215
+
1216
+ EXPECT_EQ(test.Size(), 5U);
1217
+
1218
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), 10U);
1219
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), 10U);
1220
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U);
1221
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U);
1222
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U);
1223
+ // etc.
1224
+
1225
+ // Must keep first entry
1226
+ test.TruncateOldEntries(500 + kMaxTimeDuration);
1227
+ EXPECT_EQ(test.Size(), 5U);
1228
+ test.TruncateOldEntries(599 + kMaxTimeDuration);
1229
+ EXPECT_EQ(test.Size(), 5U);
1230
+
1231
+ // Purges first entry
1232
+ test.TruncateOldEntries(600 + kMaxTimeDuration);
1233
+ EXPECT_EQ(test.Size(), 4U);
1234
+
1235
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(500), kUnknownSeqnoBeforeAll);
1236
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(599), kUnknownSeqnoBeforeAll);
1237
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(600), 20U);
1238
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(699), 20U);
1239
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(700), 30U);
1240
+
1241
+ // No effect
1242
+ test.TruncateOldEntries(600 + kMaxTimeDuration);
1243
+ EXPECT_EQ(test.Size(), 4U);
1244
+ test.TruncateOldEntries(699 + kMaxTimeDuration);
1245
+ EXPECT_EQ(test.Size(), 4U);
1246
+
1247
+ // Purges next two
1248
+ test.TruncateOldEntries(899 + kMaxTimeDuration);
1249
+ EXPECT_EQ(test.Size(), 2U);
1250
+
1251
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(799), kUnknownSeqnoBeforeAll);
1252
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(899), 40U);
1253
+
1254
+ // Always keep last entry, to have a non-trivial seqno bound
1255
+ test.TruncateOldEntries(10000000);
1256
+ EXPECT_EQ(test.Size(), 1U);
870
1257
 
871
- test.Append(100, 1000);
872
- ASSERT_EQ(test.GetOldestApproximateTime(10), 100);
873
- ASSERT_EQ(test.GetOldestApproximateTime(40), 100);
874
- ASSERT_EQ(test.GetOldestApproximateTime(111), 1000);
1258
+ EXPECT_EQ(test.GetProximalSeqnoBeforeTime(10000000), 50U);
875
1259
  }
876
1260
 
877
1261
  TEST_F(SeqnoTimeTest, Sort) {
@@ -930,10 +1314,10 @@ TEST_F(SeqnoTimeTest, EncodeDecodeBasic) {
930
1314
  for (SequenceNumber seq = 0; seq <= 1000; seq++) {
931
1315
  // test has the more accurate time mapping, encode only pick
932
1316
  // kMaxSeqnoTimePairsPerSST number of entries, which is less accurate
933
- uint64_t target_time = test.GetOldestApproximateTime(seq);
934
- ASSERT_GE(decoded.GetOldestApproximateTime(seq),
1317
+ uint64_t target_time = test.GetProximalTimeBeforeSeqno(seq);
1318
+ ASSERT_GE(decoded.GetProximalTimeBeforeSeqno(seq),
935
1319
  target_time < 200 ? 0 : target_time - 200);
936
- ASSERT_LE(decoded.GetOldestApproximateTime(seq), target_time);
1320
+ ASSERT_LE(decoded.GetProximalTimeBeforeSeqno(seq), target_time);
937
1321
  }
938
1322
  }
939
1323