@nxtedition/rocksdb 5.2.21 → 5.2.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (923) hide show
  1. package/binding.cc +510 -967
  2. package/binding.gyp +78 -72
  3. package/chained-batch.js +1 -2
  4. package/deps/rocksdb/build_version.cc +70 -4
  5. package/deps/rocksdb/rocksdb/CMakeLists.txt +281 -149
  6. package/deps/rocksdb/rocksdb/Makefile +459 -469
  7. package/deps/rocksdb/rocksdb/TARGETS +5244 -1500
  8. package/deps/rocksdb/rocksdb/cache/cache.cc +12 -3
  9. package/deps/rocksdb/rocksdb/cache/cache_bench.cc +7 -368
  10. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +924 -0
  11. package/deps/rocksdb/rocksdb/cache/cache_entry_roles.cc +128 -0
  12. package/deps/rocksdb/rocksdb/cache/cache_entry_roles.h +103 -0
  13. package/deps/rocksdb/rocksdb/cache/cache_entry_stats.h +183 -0
  14. package/deps/rocksdb/rocksdb/cache/cache_helpers.h +11 -0
  15. package/deps/rocksdb/rocksdb/cache/cache_key.cc +344 -0
  16. package/deps/rocksdb/rocksdb/cache/cache_key.h +132 -0
  17. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.cc +183 -0
  18. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager.h +288 -0
  19. package/deps/rocksdb/rocksdb/cache/cache_reservation_manager_test.cc +468 -0
  20. package/deps/rocksdb/rocksdb/cache/cache_test.cc +85 -8
  21. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +121 -51
  22. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc +171 -0
  23. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h +86 -0
  24. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +607 -0
  25. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +381 -154
  26. package/deps/rocksdb/rocksdb/cache/lru_cache.h +176 -33
  27. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +1659 -3
  28. package/deps/rocksdb/rocksdb/cache/sharded_cache.cc +94 -23
  29. package/deps/rocksdb/rocksdb/cache/sharded_cache.h +49 -28
  30. package/deps/rocksdb/rocksdb/cmake/modules/CxxFlags.cmake +7 -0
  31. package/deps/rocksdb/rocksdb/cmake/modules/FindJeMalloc.cmake +29 -0
  32. package/deps/rocksdb/rocksdb/cmake/modules/FindNUMA.cmake +29 -0
  33. package/deps/rocksdb/rocksdb/cmake/modules/FindSnappy.cmake +29 -0
  34. package/deps/rocksdb/rocksdb/cmake/modules/FindTBB.cmake +33 -0
  35. package/deps/rocksdb/rocksdb/cmake/modules/Findgflags.cmake +29 -0
  36. package/deps/rocksdb/rocksdb/cmake/modules/Findlz4.cmake +29 -0
  37. package/deps/rocksdb/rocksdb/cmake/modules/Finduring.cmake +26 -0
  38. package/deps/rocksdb/rocksdb/cmake/modules/Findzstd.cmake +29 -0
  39. package/deps/rocksdb/rocksdb/cmake/modules/ReadVersion.cmake +10 -0
  40. package/deps/rocksdb/rocksdb/crash_test.mk +93 -0
  41. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +54 -31
  42. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +10 -6
  43. package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +146 -0
  44. package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator_test.cc +326 -0
  45. package/deps/rocksdb/rocksdb/db/blob/blob_fetcher.cc +34 -0
  46. package/deps/rocksdb/rocksdb/db/blob/blob_fetcher.h +37 -0
  47. package/deps/rocksdb/rocksdb/db/blob/blob_file_addition.cc +4 -2
  48. package/deps/rocksdb/rocksdb/db/blob/blob_file_addition_test.cc +8 -4
  49. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.cc +99 -40
  50. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder.h +20 -8
  51. package/deps/rocksdb/rocksdb/db/blob/blob_file_builder_test.cc +95 -83
  52. package/deps/rocksdb/rocksdb/db/blob/blob_file_cache.cc +13 -10
  53. package/deps/rocksdb/rocksdb/db/blob/blob_file_cache.h +7 -4
  54. package/deps/rocksdb/rocksdb/db/blob/blob_file_cache_test.cc +37 -37
  55. package/deps/rocksdb/rocksdb/db/blob/blob_file_completion_callback.h +101 -0
  56. package/deps/rocksdb/rocksdb/db/blob/blob_file_meta.cc +8 -1
  57. package/deps/rocksdb/rocksdb/db/blob/blob_file_meta.h +6 -0
  58. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +209 -44
  59. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.h +37 -11
  60. package/deps/rocksdb/rocksdb/db/blob/blob_file_reader_test.cc +382 -179
  61. package/deps/rocksdb/rocksdb/db/blob/blob_garbage_meter.cc +100 -0
  62. package/deps/rocksdb/rocksdb/db/blob/blob_garbage_meter.h +102 -0
  63. package/deps/rocksdb/rocksdb/db/blob/blob_garbage_meter_test.cc +196 -0
  64. package/deps/rocksdb/rocksdb/db/blob/blob_index.h +3 -0
  65. package/deps/rocksdb/rocksdb/db/blob/blob_log_format.h +2 -1
  66. package/deps/rocksdb/rocksdb/db/blob/blob_log_sequential_reader.cc +7 -5
  67. package/deps/rocksdb/rocksdb/db/blob/blob_log_sequential_reader.h +10 -3
  68. package/deps/rocksdb/rocksdb/db/blob/blob_log_writer.cc +12 -8
  69. package/deps/rocksdb/rocksdb/db/blob/blob_log_writer.h +5 -5
  70. package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc +772 -9
  71. package/deps/rocksdb/rocksdb/db/blob/db_blob_compaction_test.cc +730 -0
  72. package/deps/rocksdb/rocksdb/db/blob/db_blob_corruption_test.cc +82 -0
  73. package/deps/rocksdb/rocksdb/db/blob/db_blob_index_test.cc +155 -17
  74. package/deps/rocksdb/rocksdb/db/blob/prefetch_buffer_collection.cc +21 -0
  75. package/deps/rocksdb/rocksdb/db/blob/prefetch_buffer_collection.h +38 -0
  76. package/deps/rocksdb/rocksdb/db/builder.cc +137 -89
  77. package/deps/rocksdb/rocksdb/db/builder.h +16 -37
  78. package/deps/rocksdb/rocksdb/db/c.cc +413 -208
  79. package/deps/rocksdb/rocksdb/db/c_test.c +227 -138
  80. package/deps/rocksdb/rocksdb/db/column_family.cc +118 -103
  81. package/deps/rocksdb/rocksdb/db/column_family.h +86 -44
  82. package/deps/rocksdb/rocksdb/db/column_family_test.cc +38 -24
  83. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +81 -0
  84. package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +275 -0
  85. package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator_test.cc +258 -0
  86. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +81 -28
  87. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +43 -12
  88. package/deps/rocksdb/rocksdb/db/compaction/compaction_iteration_stats.h +12 -0
  89. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +406 -215
  90. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +147 -50
  91. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +167 -61
  92. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +1321 -156
  93. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +197 -28
  94. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_stats_test.cc +2 -3
  95. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +246 -43
  96. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +65 -26
  97. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +7 -7
  98. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +122 -9
  99. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +8 -2
  100. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +18 -6
  101. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +1 -1
  102. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +536 -44
  103. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +311 -30
  104. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +1 -1
  105. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +849 -0
  106. package/deps/rocksdb/rocksdb/db/compaction/file_pri.h +92 -0
  107. package/deps/rocksdb/rocksdb/db/compaction/sst_partitioner.cc +46 -0
  108. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +1 -1
  109. package/deps/rocksdb/rocksdb/db/convenience.cc +6 -3
  110. package/deps/rocksdb/rocksdb/db/corruption_test.cc +383 -28
  111. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +7 -2
  112. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +154 -45
  113. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +1095 -33
  114. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +1249 -203
  115. package/deps/rocksdb/rocksdb/db/db_compaction_filter_test.cc +135 -9
  116. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +1348 -166
  117. package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +3 -5
  118. package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +1 -1
  119. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +312 -45
  120. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +1734 -48
  121. package/deps/rocksdb/rocksdb/db/{compacted_db_impl.cc → db_impl/compacted_db_impl.cc} +24 -7
  122. package/deps/rocksdb/rocksdb/db/{compacted_db_impl.h → db_impl/compacted_db_impl.h} +1 -1
  123. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +644 -333
  124. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +365 -92
  125. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +578 -210
  126. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +38 -16
  127. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +17 -10
  128. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +75 -74
  129. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +450 -183
  130. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +42 -9
  131. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +232 -15
  132. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +42 -4
  133. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +297 -100
  134. package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +16 -15
  135. package/deps/rocksdb/rocksdb/db/db_inplace_update_test.cc +31 -1
  136. package/deps/rocksdb/rocksdb/db/db_io_failure_test.cc +6 -5
  137. package/deps/rocksdb/rocksdb/db/db_iter.cc +218 -153
  138. package/deps/rocksdb/rocksdb/db/db_iter.h +14 -12
  139. package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +1 -1
  140. package/deps/rocksdb/rocksdb/db/db_iter_test.cc +84 -160
  141. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +47 -6
  142. package/deps/rocksdb/rocksdb/db/db_kv_checksum_test.cc +204 -0
  143. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +21 -13
  144. package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +17 -10
  145. package/deps/rocksdb/rocksdb/db/db_memtable_test.cc +38 -24
  146. package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +184 -19
  147. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +1 -1
  148. package/deps/rocksdb/rocksdb/db/db_options_test.cc +183 -3
  149. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +409 -9
  150. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +92 -23
  151. package/deps/rocksdb/rocksdb/db/db_rate_limiter_test.cc +446 -0
  152. package/deps/rocksdb/rocksdb/db/{db_impl/db_secondary_test.cc → db_secondary_test.cc} +363 -35
  153. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +520 -15
  154. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +50 -1
  155. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +139 -4
  156. package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +1 -1
  157. package/deps/rocksdb/rocksdb/db/db_test.cc +669 -359
  158. package/deps/rocksdb/rocksdb/db/db_test2.cc +2110 -304
  159. package/deps/rocksdb/rocksdb/db/db_test_util.cc +76 -43
  160. package/deps/rocksdb/rocksdb/db/db_test_util.h +231 -103
  161. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +19 -11
  162. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +490 -71
  163. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +980 -349
  164. package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +11 -12
  165. package/deps/rocksdb/rocksdb/db/db_write_buffer_manager_test.cc +793 -0
  166. package/deps/rocksdb/rocksdb/db/db_write_test.cc +2 -1
  167. package/deps/rocksdb/rocksdb/db/dbformat.cc +4 -12
  168. package/deps/rocksdb/rocksdb/db/dbformat.h +28 -18
  169. package/deps/rocksdb/rocksdb/db/dbformat_test.cc +3 -0
  170. package/deps/rocksdb/rocksdb/db/deletefile_test.cc +50 -15
  171. package/deps/rocksdb/rocksdb/db/error_handler.cc +127 -41
  172. package/deps/rocksdb/rocksdb/db/error_handler.h +12 -5
  173. package/deps/rocksdb/rocksdb/db/error_handler_fs_test.cc +524 -255
  174. package/deps/rocksdb/rocksdb/db/event_helpers.cc +136 -11
  175. package/deps/rocksdb/rocksdb/db/event_helpers.h +27 -2
  176. package/deps/rocksdb/rocksdb/db/experimental.cc +100 -0
  177. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +307 -4
  178. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +137 -60
  179. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +12 -8
  180. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +86 -55
  181. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +86 -5
  182. package/deps/rocksdb/rocksdb/db/filename_test.cc +63 -0
  183. package/deps/rocksdb/rocksdb/db/flush_job.cc +619 -64
  184. package/deps/rocksdb/rocksdb/db/flush_job.h +30 -7
  185. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +33 -16
  186. package/deps/rocksdb/rocksdb/db/flush_scheduler.h +2 -1
  187. package/deps/rocksdb/rocksdb/db/forward_iterator.cc +18 -17
  188. package/deps/rocksdb/rocksdb/db/forward_iterator.h +5 -4
  189. package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +0 -1
  190. package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +91 -0
  191. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +25 -14
  192. package/deps/rocksdb/rocksdb/db/import_column_family_job.h +6 -5
  193. package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +1 -1
  194. package/deps/rocksdb/rocksdb/db/internal_stats.cc +471 -50
  195. package/deps/rocksdb/rocksdb/db/internal_stats.h +129 -25
  196. package/deps/rocksdb/rocksdb/db/job_context.h +22 -9
  197. package/deps/rocksdb/rocksdb/db/kv_checksum.h +394 -0
  198. package/deps/rocksdb/rocksdb/db/listener_test.cc +518 -41
  199. package/deps/rocksdb/rocksdb/db/log_format.h +4 -1
  200. package/deps/rocksdb/rocksdb/db/log_reader.cc +129 -6
  201. package/deps/rocksdb/rocksdb/db/log_reader.h +17 -1
  202. package/deps/rocksdb/rocksdb/db/log_test.cc +161 -11
  203. package/deps/rocksdb/rocksdb/db/log_writer.cc +92 -13
  204. package/deps/rocksdb/rocksdb/db/log_writer.h +18 -5
  205. package/deps/rocksdb/rocksdb/db/logs_with_prep_tracker.h +1 -1
  206. package/deps/rocksdb/rocksdb/db/lookup_key.h +0 -1
  207. package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -2
  208. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +21 -8
  209. package/deps/rocksdb/rocksdb/db/memtable.cc +144 -54
  210. package/deps/rocksdb/rocksdb/db/memtable.h +72 -15
  211. package/deps/rocksdb/rocksdb/db/memtable_list.cc +95 -47
  212. package/deps/rocksdb/rocksdb/db/memtable_list.h +33 -13
  213. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +61 -31
  214. package/deps/rocksdb/rocksdb/db/merge_context.h +20 -8
  215. package/deps/rocksdb/rocksdb/db/merge_helper.cc +54 -11
  216. package/deps/rocksdb/rocksdb/db/merge_helper.h +17 -6
  217. package/deps/rocksdb/rocksdb/db/merge_helper_test.cc +13 -7
  218. package/deps/rocksdb/rocksdb/db/merge_test.cc +40 -19
  219. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +14 -25
  220. package/deps/rocksdb/rocksdb/db/output_validator.cc +3 -0
  221. package/deps/rocksdb/rocksdb/db/output_validator.h +5 -4
  222. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +32 -28
  223. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.cc +43 -29
  224. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler.h +9 -7
  225. package/deps/rocksdb/rocksdb/db/periodic_work_scheduler_test.cc +21 -16
  226. package/deps/rocksdb/rocksdb/db/pinned_iterators_manager.h +1 -1
  227. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +29 -36
  228. package/deps/rocksdb/rocksdb/db/pre_release_callback.h +1 -2
  229. package/deps/rocksdb/rocksdb/db/prefix_test.cc +4 -4
  230. package/deps/rocksdb/rocksdb/db/range_del_aggregator.h +2 -2
  231. package/deps/rocksdb/rocksdb/db/range_del_aggregator_bench.cc +11 -11
  232. package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +3 -2
  233. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.cc +14 -8
  234. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter.h +17 -0
  235. package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter_test.cc +4 -2
  236. package/deps/rocksdb/rocksdb/db/read_callback.h +1 -0
  237. package/deps/rocksdb/rocksdb/db/repair.cc +87 -58
  238. package/deps/rocksdb/rocksdb/db/repair_test.cc +35 -5
  239. package/deps/rocksdb/rocksdb/db/snapshot_impl.h +2 -1
  240. package/deps/rocksdb/rocksdb/db/table_cache.cc +95 -69
  241. package/deps/rocksdb/rocksdb/db/table_cache.h +63 -53
  242. package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +4 -4
  243. package/deps/rocksdb/rocksdb/db/table_properties_collector.h +78 -10
  244. package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +28 -33
  245. package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +30 -51
  246. package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +12 -8
  247. package/deps/rocksdb/rocksdb/db/version_builder.cc +564 -341
  248. package/deps/rocksdb/rocksdb/db/version_builder.h +8 -8
  249. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +327 -155
  250. package/deps/rocksdb/rocksdb/db/version_edit.cc +89 -27
  251. package/deps/rocksdb/rocksdb/db/version_edit.h +42 -17
  252. package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +324 -43
  253. package/deps/rocksdb/rocksdb/db/version_edit_handler.h +79 -22
  254. package/deps/rocksdb/rocksdb/db/version_edit_test.cc +165 -20
  255. package/deps/rocksdb/rocksdb/db/version_set.cc +935 -1034
  256. package/deps/rocksdb/rocksdb/db/version_set.h +183 -122
  257. package/deps/rocksdb/rocksdb/db/version_set_test.cc +556 -138
  258. package/deps/rocksdb/rocksdb/db/version_util.h +68 -0
  259. package/deps/rocksdb/rocksdb/db/wal_manager.cc +23 -21
  260. package/deps/rocksdb/rocksdb/db/wal_manager.h +5 -2
  261. package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +30 -27
  262. package/deps/rocksdb/rocksdb/db/write_batch.cc +704 -209
  263. package/deps/rocksdb/rocksdb/db/write_batch_internal.h +135 -2
  264. package/deps/rocksdb/rocksdb/db/write_batch_test.cc +209 -5
  265. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +2 -0
  266. package/deps/rocksdb/rocksdb/db/write_controller.cc +47 -54
  267. package/deps/rocksdb/rocksdb/db/write_controller.h +12 -9
  268. package/deps/rocksdb/rocksdb/db/write_controller_test.cc +215 -103
  269. package/deps/rocksdb/rocksdb/db/write_thread.cc +11 -0
  270. package/deps/rocksdb/rocksdb/db/write_thread.h +14 -8
  271. package/deps/rocksdb/rocksdb/db_stress_tool/CMakeLists.txt +7 -4
  272. package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +10 -3
  273. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +6 -0
  274. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress.cc +1 -1
  275. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +19 -2
  276. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +78 -25
  277. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_compaction_filter.h +13 -2
  278. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +29 -12
  279. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +5 -1
  280. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +199 -32
  281. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.cc +188 -0
  282. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_listener.h +59 -10
  283. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_shared_state.h +77 -109
  284. package/deps/rocksdb/rocksdb/{third-party/folly/folly/synchronization/WaitOptions.cpp → db_stress_tool/db_stress_stat.cc} +9 -4
  285. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_stat.h +7 -6
  286. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_table_properties_collector.h +1 -0
  287. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +699 -143
  288. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +20 -2
  289. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +49 -39
  290. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.cc +631 -0
  291. package/deps/rocksdb/rocksdb/db_stress_tool/expected_state.h +287 -0
  292. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +1565 -0
  293. package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.h +374 -0
  294. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +149 -18
  295. package/deps/rocksdb/rocksdb/env/composite_env.cc +464 -0
  296. package/deps/rocksdb/rocksdb/env/composite_env_wrapper.h +98 -646
  297. package/deps/rocksdb/rocksdb/env/emulated_clock.h +114 -0
  298. package/deps/rocksdb/rocksdb/env/env.cc +632 -42
  299. package/deps/rocksdb/rocksdb/env/env_basic_test.cc +84 -36
  300. package/deps/rocksdb/rocksdb/env/env_chroot.cc +88 -286
  301. package/deps/rocksdb/rocksdb/env/env_chroot.h +34 -1
  302. package/deps/rocksdb/rocksdb/env/env_encryption.cc +469 -277
  303. package/deps/rocksdb/rocksdb/env/env_encryption_ctr.h +9 -30
  304. package/deps/rocksdb/rocksdb/env/env_posix.cc +110 -119
  305. package/deps/rocksdb/rocksdb/env/env_test.cc +1128 -39
  306. package/deps/rocksdb/rocksdb/env/file_system.cc +147 -8
  307. package/deps/rocksdb/rocksdb/env/file_system_tracer.cc +207 -136
  308. package/deps/rocksdb/rocksdb/env/file_system_tracer.h +86 -54
  309. package/deps/rocksdb/rocksdb/env/fs_posix.cc +192 -64
  310. package/deps/rocksdb/rocksdb/env/fs_readonly.h +107 -0
  311. package/deps/rocksdb/rocksdb/env/fs_remap.cc +339 -0
  312. package/deps/rocksdb/rocksdb/env/fs_remap.h +139 -0
  313. package/deps/rocksdb/rocksdb/env/io_posix.cc +245 -41
  314. package/deps/rocksdb/rocksdb/env/io_posix.h +66 -1
  315. package/deps/rocksdb/rocksdb/env/mock_env.cc +147 -149
  316. package/deps/rocksdb/rocksdb/env/mock_env.h +113 -11
  317. package/deps/rocksdb/rocksdb/env/mock_env_test.cc +2 -4
  318. package/deps/rocksdb/rocksdb/env/unique_id_gen.cc +164 -0
  319. package/deps/rocksdb/rocksdb/env/unique_id_gen.h +71 -0
  320. package/deps/rocksdb/rocksdb/file/delete_scheduler.cc +9 -5
  321. package/deps/rocksdb/rocksdb/file/delete_scheduler.h +6 -4
  322. package/deps/rocksdb/rocksdb/file/delete_scheduler_test.cc +19 -12
  323. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +459 -70
  324. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +205 -28
  325. package/deps/rocksdb/rocksdb/file/file_util.cc +39 -28
  326. package/deps/rocksdb/rocksdb/file/file_util.h +18 -27
  327. package/deps/rocksdb/rocksdb/file/filename.cc +59 -22
  328. package/deps/rocksdb/rocksdb/file/filename.h +13 -8
  329. package/deps/rocksdb/rocksdb/file/line_file_reader.cc +68 -0
  330. package/deps/rocksdb/rocksdb/file/line_file_reader.h +59 -0
  331. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +1130 -6
  332. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +220 -36
  333. package/deps/rocksdb/rocksdb/file/random_access_file_reader.h +69 -17
  334. package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc +13 -12
  335. package/deps/rocksdb/rocksdb/file/read_write_util.cc +3 -38
  336. package/deps/rocksdb/rocksdb/file/read_write_util.h +0 -4
  337. package/deps/rocksdb/rocksdb/file/readahead_file_info.h +33 -0
  338. package/deps/rocksdb/rocksdb/file/sequence_file_reader.cc +57 -9
  339. package/deps/rocksdb/rocksdb/file/sequence_file_reader.h +58 -6
  340. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.cc +29 -54
  341. package/deps/rocksdb/rocksdb/file/sst_file_manager_impl.h +22 -29
  342. package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +424 -50
  343. package/deps/rocksdb/rocksdb/file/writable_file_writer.h +66 -19
  344. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +157 -66
  345. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +224 -121
  346. package/deps/rocksdb/rocksdb/include/rocksdb/cache.h +333 -30
  347. package/deps/rocksdb/rocksdb/include/rocksdb/cache_bench_tool.h +14 -0
  348. package/deps/rocksdb/rocksdb/include/rocksdb/cleanable.h +1 -1
  349. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +90 -50
  350. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_job_stats.h +13 -5
  351. package/deps/rocksdb/rocksdb/include/rocksdb/comparator.h +20 -4
  352. package/deps/rocksdb/rocksdb/include/rocksdb/concurrent_task_limiter.h +8 -3
  353. package/deps/rocksdb/rocksdb/include/rocksdb/configurable.h +53 -12
  354. package/deps/rocksdb/rocksdb/include/rocksdb/convenience.h +31 -6
  355. package/deps/rocksdb/rocksdb/include/rocksdb/customizable.h +102 -7
  356. package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +51 -0
  357. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +370 -262
  358. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +286 -87
  359. package/deps/rocksdb/rocksdb/include/rocksdb/env_encryption.h +124 -64
  360. package/deps/rocksdb/rocksdb/include/rocksdb/experimental.h +27 -0
  361. package/deps/rocksdb/rocksdb/include/rocksdb/file_checksum.h +21 -4
  362. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +384 -41
  363. package/deps/rocksdb/rocksdb/include/rocksdb/filter_policy.h +111 -143
  364. package/deps/rocksdb/rocksdb/include/rocksdb/flush_block_policy.h +20 -6
  365. package/deps/rocksdb/rocksdb/include/rocksdb/functor_wrapper.h +56 -0
  366. package/deps/rocksdb/rocksdb/include/rocksdb/io_status.h +15 -33
  367. package/deps/rocksdb/rocksdb/include/rocksdb/iostats_context.h +37 -1
  368. package/deps/rocksdb/rocksdb/include/rocksdb/iterator.h +1 -3
  369. package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +314 -26
  370. package/deps/rocksdb/rocksdb/include/rocksdb/memory_allocator.h +11 -7
  371. package/deps/rocksdb/rocksdb/include/rocksdb/memtablerep.h +50 -15
  372. package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +10 -3
  373. package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +186 -96
  374. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +373 -103
  375. package/deps/rocksdb/rocksdb/include/rocksdb/perf_context.h +13 -3
  376. package/deps/rocksdb/rocksdb/include/rocksdb/persistent_cache.h +2 -2
  377. package/deps/rocksdb/rocksdb/include/rocksdb/rate_limiter.h +37 -7
  378. package/deps/rocksdb/rocksdb/include/rocksdb/rocksdb_namespace.h +6 -0
  379. package/deps/rocksdb/rocksdb/include/rocksdb/secondary_cache.h +87 -0
  380. package/deps/rocksdb/rocksdb/include/rocksdb/slice.h +5 -12
  381. package/deps/rocksdb/rocksdb/include/rocksdb/slice_transform.h +59 -30
  382. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_manager.h +11 -11
  383. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +22 -0
  384. package/deps/rocksdb/rocksdb/include/rocksdb/sst_partitioner.h +17 -10
  385. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +121 -41
  386. package/deps/rocksdb/rocksdb/include/rocksdb/stats_history.h +1 -0
  387. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +114 -136
  388. package/deps/rocksdb/rocksdb/include/rocksdb/system_clock.h +116 -0
  389. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +160 -18
  390. package/deps/rocksdb/rocksdb/include/rocksdb/table_properties.h +57 -15
  391. package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +3 -1
  392. package/deps/rocksdb/rocksdb/include/rocksdb/trace_reader_writer.h +10 -6
  393. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +247 -0
  394. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record_result.h +187 -0
  395. package/deps/rocksdb/rocksdb/include/rocksdb/transaction_log.h +1 -1
  396. package/deps/rocksdb/rocksdb/include/rocksdb/types.h +14 -24
  397. package/deps/rocksdb/rocksdb/include/rocksdb/unique_id.h +46 -0
  398. package/deps/rocksdb/rocksdb/include/rocksdb/universal_compaction.h +14 -4
  399. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/agg_merge.h +138 -0
  400. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +631 -0
  401. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/cache_dump_load.h +142 -0
  402. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/checkpoint.h +12 -9
  403. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/customizable_util.h +368 -0
  404. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +24 -0
  405. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd_execute_result.h +4 -0
  406. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/object_registry.h +418 -63
  407. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_type.h +143 -73
  408. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/options_util.h +2 -2
  409. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/replayer.h +87 -0
  410. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/sim_cache.h +2 -2
  411. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +43 -5
  412. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +18 -23
  413. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +26 -0
  414. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +32 -6
  415. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db_mutex.h +1 -2
  416. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +20 -1
  417. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +30 -3
  418. package/deps/rocksdb/rocksdb/include/rocksdb/wal_filter.h +11 -2
  419. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +89 -11
  420. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch_base.h +11 -0
  421. package/deps/rocksdb/rocksdb/include/rocksdb/write_buffer_manager.h +108 -38
  422. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.cc +40 -23
  423. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.h +12 -5
  424. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +100 -49
  425. package/deps/rocksdb/rocksdb/logging/env_logger.h +7 -5
  426. package/deps/rocksdb/rocksdb/logging/env_logger_test.cc +0 -1
  427. package/deps/rocksdb/rocksdb/logging/posix_logger.h +3 -9
  428. package/deps/rocksdb/rocksdb/memory/arena.cc +3 -1
  429. package/deps/rocksdb/rocksdb/memory/arena.h +1 -1
  430. package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.cc +171 -106
  431. package/deps/rocksdb/rocksdb/memory/jemalloc_nodump_allocator.h +31 -15
  432. package/deps/rocksdb/rocksdb/memory/memkind_kmem_allocator.cc +15 -4
  433. package/deps/rocksdb/rocksdb/memory/memkind_kmem_allocator.h +24 -8
  434. package/deps/rocksdb/rocksdb/memory/memory_allocator.cc +91 -0
  435. package/deps/rocksdb/rocksdb/memory/memory_allocator_test.cc +239 -0
  436. package/deps/rocksdb/rocksdb/memory/memory_usage.h +14 -1
  437. package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +72 -9
  438. package/deps/rocksdb/rocksdb/memtable/hash_skiplist_rep.cc +52 -6
  439. package/deps/rocksdb/rocksdb/memtable/inlineskiplist.h +53 -0
  440. package/deps/rocksdb/rocksdb/memtable/inlineskiplist_test.cc +5 -5
  441. package/deps/rocksdb/rocksdb/memtable/memtablerep_bench.cc +17 -5
  442. package/deps/rocksdb/rocksdb/memtable/skiplist_test.cc +1 -1
  443. package/deps/rocksdb/rocksdb/memtable/skiplistrep.cc +87 -0
  444. package/deps/rocksdb/rocksdb/memtable/vectorrep.cc +20 -10
  445. package/deps/rocksdb/rocksdb/memtable/write_buffer_manager.cc +148 -94
  446. package/deps/rocksdb/rocksdb/memtable/write_buffer_manager_test.cc +160 -62
  447. package/deps/rocksdb/rocksdb/microbench/CMakeLists.txt +17 -0
  448. package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +1360 -0
  449. package/deps/rocksdb/rocksdb/microbench/ribbon_bench.cc +153 -0
  450. package/deps/rocksdb/rocksdb/monitoring/histogram.cc +8 -15
  451. package/deps/rocksdb/rocksdb/monitoring/histogram.h +0 -1
  452. package/deps/rocksdb/rocksdb/monitoring/histogram_test.cc +18 -16
  453. package/deps/rocksdb/rocksdb/monitoring/histogram_windowing.cc +9 -7
  454. package/deps/rocksdb/rocksdb/monitoring/histogram_windowing.h +5 -3
  455. package/deps/rocksdb/rocksdb/monitoring/instrumented_mutex.cc +7 -5
  456. package/deps/rocksdb/rocksdb/monitoring/instrumented_mutex.h +37 -12
  457. package/deps/rocksdb/rocksdb/monitoring/iostats_context.cc +26 -6
  458. package/deps/rocksdb/rocksdb/monitoring/iostats_context_imp.h +6 -10
  459. package/deps/rocksdb/rocksdb/monitoring/perf_context.cc +14 -13
  460. package/deps/rocksdb/rocksdb/monitoring/perf_context_imp.h +19 -20
  461. package/deps/rocksdb/rocksdb/monitoring/perf_step_timer.h +18 -18
  462. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +84 -2
  463. package/deps/rocksdb/rocksdb/monitoring/statistics.h +6 -0
  464. package/deps/rocksdb/rocksdb/monitoring/statistics_test.cc +47 -2
  465. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +67 -54
  466. package/deps/rocksdb/rocksdb/monitoring/thread_status_updater.cc +4 -1
  467. package/deps/rocksdb/rocksdb/monitoring/thread_status_util.cc +2 -1
  468. package/deps/rocksdb/rocksdb/monitoring/thread_status_util_debug.cc +2 -2
  469. package/deps/rocksdb/rocksdb/options/cf_options.cc +280 -212
  470. package/deps/rocksdb/rocksdb/options/cf_options.h +51 -57
  471. package/deps/rocksdb/rocksdb/options/configurable.cc +242 -138
  472. package/deps/rocksdb/rocksdb/options/configurable_helper.h +4 -68
  473. package/deps/rocksdb/rocksdb/options/configurable_test.cc +144 -21
  474. package/deps/rocksdb/rocksdb/options/configurable_test.h +2 -3
  475. package/deps/rocksdb/rocksdb/options/customizable.cc +67 -7
  476. package/deps/rocksdb/rocksdb/options/customizable_test.cc +1773 -151
  477. package/deps/rocksdb/rocksdb/options/db_options.cc +275 -47
  478. package/deps/rocksdb/rocksdb/options/db_options.h +36 -7
  479. package/deps/rocksdb/rocksdb/options/options.cc +49 -17
  480. package/deps/rocksdb/rocksdb/options/options_helper.cc +369 -352
  481. package/deps/rocksdb/rocksdb/options/options_helper.h +23 -23
  482. package/deps/rocksdb/rocksdb/options/options_parser.cc +18 -13
  483. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +67 -54
  484. package/deps/rocksdb/rocksdb/options/options_test.cc +1162 -187
  485. package/deps/rocksdb/rocksdb/port/jemalloc_helper.h +1 -1
  486. package/deps/rocksdb/rocksdb/port/lang.h +52 -0
  487. package/deps/rocksdb/rocksdb/port/port_example.h +1 -1
  488. package/deps/rocksdb/rocksdb/port/port_posix.cc +31 -2
  489. package/deps/rocksdb/rocksdb/port/port_posix.h +20 -2
  490. package/deps/rocksdb/rocksdb/port/stack_trace.cc +20 -4
  491. package/deps/rocksdb/rocksdb/port/sys_time.h +2 -2
  492. package/deps/rocksdb/rocksdb/port/win/env_default.cc +7 -7
  493. package/deps/rocksdb/rocksdb/port/win/env_win.cc +44 -74
  494. package/deps/rocksdb/rocksdb/port/win/env_win.h +25 -23
  495. package/deps/rocksdb/rocksdb/port/win/io_win.cc +32 -34
  496. package/deps/rocksdb/rocksdb/port/win/io_win.h +12 -6
  497. package/deps/rocksdb/rocksdb/port/win/port_win.cc +55 -35
  498. package/deps/rocksdb/rocksdb/port/win/port_win.h +22 -5
  499. package/deps/rocksdb/rocksdb/port/win/win_logger.cc +3 -3
  500. package/deps/rocksdb/rocksdb/port/win/win_logger.h +3 -5
  501. package/deps/rocksdb/rocksdb/port/win/win_thread.cc +7 -1
  502. package/deps/rocksdb/rocksdb/port/win/win_thread.h +12 -17
  503. package/deps/rocksdb/rocksdb/python.mk +9 -0
  504. package/deps/rocksdb/rocksdb/src.mk +82 -34
  505. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +3 -4
  506. package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.h +1 -1
  507. package/deps/rocksdb/rocksdb/table/block_based/block.cc +158 -80
  508. package/deps/rocksdb/rocksdb/table/block_based/block.h +64 -36
  509. package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.cc +23 -14
  510. package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.h +13 -5
  511. package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block_test.cc +3 -218
  512. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +603 -328
  513. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +28 -22
  514. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.cc +220 -82
  515. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_factory.h +8 -2
  516. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +3 -4
  517. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +28 -4
  518. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +598 -492
  519. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +151 -96
  520. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +31 -58
  521. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +330 -92
  522. package/deps/rocksdb/rocksdb/table/block_based/block_builder.cc +50 -19
  523. package/deps/rocksdb/rocksdb/table/block_based/block_builder.h +23 -0
  524. package/deps/rocksdb/rocksdb/table/block_based/block_like_traits.h +226 -0
  525. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.cc +56 -22
  526. package/deps/rocksdb/rocksdb/table/block_based/block_prefetcher.h +42 -4
  527. package/deps/rocksdb/rocksdb/table/block_based/block_test.cc +5 -2
  528. package/deps/rocksdb/rocksdb/table/block_based/block_type.h +2 -0
  529. package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h +34 -20
  530. package/deps/rocksdb/rocksdb/table/block_based/data_block_hash_index_test.cc +9 -10
  531. package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +26 -3
  532. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +2 -1
  533. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +844 -202
  534. package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +281 -81
  535. package/deps/rocksdb/rocksdb/table/block_based/flush_block_policy.cc +62 -2
  536. package/deps/rocksdb/rocksdb/table/block_based/flush_block_policy.h +2 -3
  537. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +28 -7
  538. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +22 -6
  539. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +28 -26
  540. package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +1 -1
  541. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +1 -2
  542. package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc +2 -1
  543. package/deps/rocksdb/rocksdb/table/block_based/mock_block_based_table.h +11 -4
  544. package/deps/rocksdb/rocksdb/table/block_based/parsed_full_filter_block.cc +2 -1
  545. package/deps/rocksdb/rocksdb/table/block_based/parsed_full_filter_block.h +2 -0
  546. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +68 -26
  547. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +44 -9
  548. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block_test.cc +12 -10
  549. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.cc +3 -4
  550. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_iterator.h +23 -4
  551. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc +44 -19
  552. package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.h +5 -1
  553. package/deps/rocksdb/rocksdb/table/block_based/reader_common.cc +16 -28
  554. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc +7 -4
  555. package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.h +2 -2
  556. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +77 -57
  557. package/deps/rocksdb/rocksdb/table/block_fetcher.h +23 -12
  558. package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +43 -56
  559. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.cc +8 -8
  560. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder.h +2 -1
  561. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_builder_test.cc +52 -70
  562. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_factory.cc +5 -8
  563. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_factory.h +1 -1
  564. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader.cc +17 -11
  565. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader.h +2 -3
  566. package/deps/rocksdb/rocksdb/table/cuckoo/cuckoo_table_reader_test.cc +42 -51
  567. package/deps/rocksdb/rocksdb/table/format.cc +258 -104
  568. package/deps/rocksdb/rocksdb/table/format.h +120 -109
  569. package/deps/rocksdb/rocksdb/table/get_context.cc +97 -65
  570. package/deps/rocksdb/rocksdb/table/get_context.h +19 -12
  571. package/deps/rocksdb/rocksdb/table/internal_iterator.h +14 -0
  572. package/deps/rocksdb/rocksdb/table/iterator_wrapper.h +8 -0
  573. package/deps/rocksdb/rocksdb/table/merger_test.cc +3 -2
  574. package/deps/rocksdb/rocksdb/table/merging_iterator.cc +11 -21
  575. package/deps/rocksdb/rocksdb/table/merging_iterator.h +3 -3
  576. package/deps/rocksdb/rocksdb/table/meta_blocks.cc +176 -171
  577. package/deps/rocksdb/rocksdb/table/meta_blocks.h +47 -33
  578. package/deps/rocksdb/rocksdb/table/mock_table.cc +7 -9
  579. package/deps/rocksdb/rocksdb/table/mock_table.h +3 -2
  580. package/deps/rocksdb/rocksdb/table/multiget_context.h +15 -8
  581. package/deps/rocksdb/rocksdb/table/persistent_cache_helper.cc +22 -29
  582. package/deps/rocksdb/rocksdb/table/persistent_cache_options.h +6 -3
  583. package/deps/rocksdb/rocksdb/table/plain/plain_table_bloom.h +5 -8
  584. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.cc +29 -26
  585. package/deps/rocksdb/rocksdb/table/plain/plain_table_builder.h +12 -16
  586. package/deps/rocksdb/rocksdb/table/plain/plain_table_factory.cc +145 -69
  587. package/deps/rocksdb/rocksdb/table/plain/plain_table_factory.h +1 -1
  588. package/deps/rocksdb/rocksdb/table/plain/plain_table_index.cc +7 -6
  589. package/deps/rocksdb/rocksdb/table/plain/plain_table_index.h +3 -4
  590. package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.cc +3 -1
  591. package/deps/rocksdb/rocksdb/table/plain/plain_table_key_coding.h +1 -1
  592. package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.cc +13 -18
  593. package/deps/rocksdb/rocksdb/table/plain/plain_table_reader.h +4 -9
  594. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +55 -37
  595. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +10 -5
  596. package/deps/rocksdb/rocksdb/table/sst_file_reader.cc +11 -8
  597. package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc +222 -16
  598. package/deps/rocksdb/rocksdb/table/sst_file_writer.cc +106 -58
  599. package/deps/rocksdb/rocksdb/table/sst_file_writer_collectors.h +6 -5
  600. package/deps/rocksdb/rocksdb/table/table_builder.h +68 -44
  601. package/deps/rocksdb/rocksdb/table/table_factory.cc +37 -10
  602. package/deps/rocksdb/rocksdb/table/table_properties.cc +109 -54
  603. package/deps/rocksdb/rocksdb/table/table_properties_internal.h +4 -20
  604. package/deps/rocksdb/rocksdb/table/table_reader_bench.cc +33 -32
  605. package/deps/rocksdb/rocksdb/table/table_reader_caller.h +2 -0
  606. package/deps/rocksdb/rocksdb/table/table_test.cc +989 -326
  607. package/deps/rocksdb/rocksdb/table/two_level_iterator.cc +4 -0
  608. package/deps/rocksdb/rocksdb/table/unique_id.cc +166 -0
  609. package/deps/rocksdb/rocksdb/table/unique_id_impl.h +59 -0
  610. package/deps/rocksdb/rocksdb/test_util/mock_time_env.cc +1 -1
  611. package/deps/rocksdb/rocksdb/test_util/mock_time_env.h +13 -10
  612. package/deps/rocksdb/rocksdb/test_util/sync_point.cc +1 -2
  613. package/deps/rocksdb/rocksdb/test_util/sync_point.h +35 -16
  614. package/deps/rocksdb/rocksdb/test_util/sync_point_impl.cc +32 -10
  615. package/deps/rocksdb/rocksdb/test_util/sync_point_impl.h +31 -4
  616. package/deps/rocksdb/rocksdb/test_util/testharness.cc +53 -1
  617. package/deps/rocksdb/rocksdb/test_util/testharness.h +67 -3
  618. package/deps/rocksdb/rocksdb/test_util/testutil.cc +236 -66
  619. package/deps/rocksdb/rocksdb/test_util/testutil.h +63 -100
  620. package/deps/rocksdb/rocksdb/test_util/transaction_test_util.cc +12 -1
  621. package/deps/rocksdb/rocksdb/tools/blob_dump.cc +2 -2
  622. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.cc +6 -3
  623. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer.h +1 -0
  624. package/deps/rocksdb/rocksdb/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +9 -3
  625. package/deps/rocksdb/rocksdb/tools/db_bench.cc +1 -1
  626. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +1420 -611
  627. package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +11 -8
  628. package/deps/rocksdb/rocksdb/tools/db_repl_stress.cc +11 -1
  629. package/deps/rocksdb/rocksdb/tools/io_tracer_parser_test.cc +4 -2
  630. package/deps/rocksdb/rocksdb/tools/io_tracer_parser_tool.cc +46 -22
  631. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +655 -179
  632. package/deps/rocksdb/rocksdb/tools/ldb_cmd_impl.h +58 -6
  633. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +472 -29
  634. package/deps/rocksdb/rocksdb/tools/ldb_tool.cc +23 -2
  635. package/deps/rocksdb/rocksdb/tools/reduce_levels_test.cc +2 -2
  636. package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.cc +246 -0
  637. package/deps/rocksdb/rocksdb/tools/simulated_hybrid_file_system.h +126 -0
  638. package/deps/rocksdb/rocksdb/tools/sst_dump_test.cc +83 -29
  639. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +38 -17
  640. package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +191 -55
  641. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +219 -296
  642. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.h +87 -53
  643. package/deps/rocksdb/rocksdb/tools/write_stress.cc +8 -7
  644. package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.cc +6 -5
  645. package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +5 -4
  646. package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer_test.cc +14 -9
  647. package/deps/rocksdb/rocksdb/trace_replay/io_tracer.cc +134 -60
  648. package/deps/rocksdb/rocksdb/trace_replay/io_tracer.h +49 -38
  649. package/deps/rocksdb/rocksdb/trace_replay/io_tracer_test.cc +152 -15
  650. package/deps/rocksdb/rocksdb/trace_replay/trace_record.cc +206 -0
  651. package/deps/rocksdb/rocksdb/trace_replay/trace_record_handler.cc +190 -0
  652. package/deps/rocksdb/rocksdb/trace_replay/trace_record_handler.h +46 -0
  653. package/deps/rocksdb/rocksdb/trace_replay/trace_record_result.cc +146 -0
  654. package/deps/rocksdb/rocksdb/trace_replay/trace_replay.cc +475 -344
  655. package/deps/rocksdb/rocksdb/trace_replay/trace_replay.h +83 -95
  656. package/deps/rocksdb/rocksdb/util/autovector.h +38 -18
  657. package/deps/rocksdb/rocksdb/util/autovector_test.cc +1 -1
  658. package/deps/rocksdb/rocksdb/util/bloom_impl.h +4 -0
  659. package/deps/rocksdb/rocksdb/util/bloom_test.cc +276 -94
  660. package/deps/rocksdb/rocksdb/util/build_version.cc.in +81 -4
  661. package/deps/rocksdb/rocksdb/util/cast_util.h +22 -0
  662. package/deps/rocksdb/rocksdb/util/channel.h +2 -0
  663. package/deps/rocksdb/rocksdb/util/coding.h +1 -33
  664. package/deps/rocksdb/rocksdb/util/compaction_job_stats_impl.cc +8 -0
  665. package/deps/rocksdb/rocksdb/util/comparator.cc +163 -3
  666. package/deps/rocksdb/rocksdb/util/compression.cc +122 -0
  667. package/deps/rocksdb/rocksdb/util/compression.h +212 -7
  668. package/deps/rocksdb/rocksdb/util/compression_context_cache.cc +1 -3
  669. package/deps/rocksdb/rocksdb/util/crc32c.cc +165 -2
  670. package/deps/rocksdb/rocksdb/util/crc32c.h +6 -0
  671. package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +14 -0
  672. package/deps/rocksdb/rocksdb/util/crc32c_ppc.h +3 -0
  673. package/deps/rocksdb/rocksdb/util/crc32c_test.cc +47 -0
  674. package/deps/rocksdb/rocksdb/util/defer.h +30 -1
  675. package/deps/rocksdb/rocksdb/util/defer_test.cc +11 -0
  676. package/deps/rocksdb/rocksdb/util/duplicate_detector.h +3 -1
  677. package/deps/rocksdb/rocksdb/util/dynamic_bloom.h +3 -3
  678. package/deps/rocksdb/rocksdb/util/dynamic_bloom_test.cc +5 -4
  679. package/deps/rocksdb/rocksdb/util/fastrange.h +2 -0
  680. package/deps/rocksdb/rocksdb/util/file_checksum_helper.cc +36 -0
  681. package/deps/rocksdb/rocksdb/util/file_checksum_helper.h +3 -1
  682. package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +512 -52
  683. package/deps/rocksdb/rocksdb/util/filter_bench.cc +65 -10
  684. package/deps/rocksdb/rocksdb/util/gflags_compat.h +6 -1
  685. package/deps/rocksdb/rocksdb/util/hash.cc +121 -3
  686. package/deps/rocksdb/rocksdb/util/hash.h +31 -1
  687. package/deps/rocksdb/rocksdb/util/hash128.h +26 -0
  688. package/deps/rocksdb/rocksdb/util/hash_containers.h +51 -0
  689. package/deps/rocksdb/rocksdb/util/hash_test.cc +194 -2
  690. package/deps/rocksdb/rocksdb/util/heap.h +6 -1
  691. package/deps/rocksdb/rocksdb/util/kv_map.h +1 -1
  692. package/deps/rocksdb/rocksdb/util/log_write_bench.cc +8 -6
  693. package/deps/rocksdb/rocksdb/util/math.h +74 -7
  694. package/deps/rocksdb/rocksdb/util/math128.h +13 -1
  695. package/deps/rocksdb/rocksdb/util/murmurhash.h +3 -3
  696. package/deps/rocksdb/rocksdb/util/random.cc +9 -0
  697. package/deps/rocksdb/rocksdb/util/random.h +6 -0
  698. package/deps/rocksdb/rocksdb/util/rate_limiter.cc +298 -144
  699. package/deps/rocksdb/rocksdb/util/rate_limiter.h +68 -19
  700. package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +335 -23
  701. package/deps/rocksdb/rocksdb/util/repeatable_thread.h +10 -12
  702. package/deps/rocksdb/rocksdb/util/repeatable_thread_test.cc +18 -15
  703. package/deps/rocksdb/rocksdb/util/ribbon_alg.h +98 -74
  704. package/deps/rocksdb/rocksdb/util/ribbon_config.cc +506 -0
  705. package/deps/rocksdb/rocksdb/util/ribbon_config.h +182 -0
  706. package/deps/rocksdb/rocksdb/util/ribbon_impl.h +154 -79
  707. package/deps/rocksdb/rocksdb/util/ribbon_test.cc +742 -365
  708. package/deps/rocksdb/rocksdb/util/set_comparator.h +2 -0
  709. package/deps/rocksdb/rocksdb/util/slice.cc +198 -35
  710. package/deps/rocksdb/rocksdb/util/slice_test.cc +30 -1
  711. package/deps/rocksdb/rocksdb/util/status.cc +32 -29
  712. package/deps/rocksdb/rocksdb/util/stop_watch.h +18 -18
  713. package/deps/rocksdb/rocksdb/util/string_util.cc +85 -6
  714. package/deps/rocksdb/rocksdb/util/string_util.h +47 -2
  715. package/deps/rocksdb/rocksdb/util/thread_guard.h +41 -0
  716. package/deps/rocksdb/rocksdb/util/thread_local.h +2 -2
  717. package/deps/rocksdb/rocksdb/util/thread_local_test.cc +22 -24
  718. package/deps/rocksdb/rocksdb/util/threadpool_imp.cc +7 -6
  719. package/deps/rocksdb/rocksdb/util/timer.h +55 -46
  720. package/deps/rocksdb/rocksdb/util/timer_test.cc +50 -48
  721. package/deps/rocksdb/rocksdb/util/user_comparator_wrapper.h +4 -0
  722. package/deps/rocksdb/rocksdb/util/vector_iterator.h +31 -15
  723. package/deps/rocksdb/rocksdb/util/work_queue.h +2 -0
  724. package/deps/rocksdb/rocksdb/util/xxhash.cc +35 -1144
  725. package/deps/rocksdb/rocksdb/util/xxhash.h +5117 -373
  726. package/deps/rocksdb/rocksdb/util/xxph3.h +1762 -0
  727. package/deps/rocksdb/rocksdb/utilities/agg_merge/agg_merge.cc +238 -0
  728. package/deps/rocksdb/rocksdb/utilities/agg_merge/agg_merge.h +49 -0
  729. package/deps/rocksdb/rocksdb/utilities/agg_merge/agg_merge_test.cc +134 -0
  730. package/deps/rocksdb/rocksdb/utilities/agg_merge/test_agg_merge.cc +104 -0
  731. package/deps/rocksdb/rocksdb/utilities/agg_merge/test_agg_merge.h +47 -0
  732. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +3164 -0
  733. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_impl.h +29 -0
  734. package/deps/rocksdb/rocksdb/utilities/{backupable/backupable_db_test.cc → backup/backup_engine_test.cc} +1679 -485
  735. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.cc +6 -4
  736. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_compaction_filter.h +14 -9
  737. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.cc +2 -0
  738. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db.h +1 -0
  739. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_gc_stats.h +4 -0
  740. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.cc +37 -27
  741. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl.h +8 -4
  742. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +1 -1
  743. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_iterator.h +13 -10
  744. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_listener.h +5 -0
  745. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_db_test.cc +44 -25
  746. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_dump_tool.cc +3 -4
  747. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.cc +27 -19
  748. package/deps/rocksdb/rocksdb/utilities/blob_db/blob_file.h +4 -2
  749. package/deps/rocksdb/rocksdb/utilities/cache_dump_load.cc +69 -0
  750. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +489 -0
  751. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.h +366 -0
  752. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_compaction_filter.cc +67 -4
  753. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_compaction_filter.h +21 -6
  754. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_functional_test.cc +107 -7
  755. package/deps/rocksdb/rocksdb/utilities/cassandra/cassandra_options.h +43 -0
  756. package/deps/rocksdb/rocksdb/utilities/cassandra/format.h +1 -1
  757. package/deps/rocksdb/rocksdb/utilities/cassandra/merge_operator.cc +24 -8
  758. package/deps/rocksdb/rocksdb/utilities/cassandra/merge_operator.h +7 -7
  759. package/deps/rocksdb/rocksdb/utilities/cassandra/serialize.h +5 -0
  760. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +99 -218
  761. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.h +8 -24
  762. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +114 -1
  763. package/deps/rocksdb/rocksdb/utilities/compaction_filters/layered_compaction_filter_base.h +6 -2
  764. package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc +0 -4
  765. package/deps/rocksdb/rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.h +7 -6
  766. package/deps/rocksdb/rocksdb/utilities/compaction_filters.cc +56 -0
  767. package/deps/rocksdb/rocksdb/utilities/convenience/info_log_finder.cc +2 -2
  768. package/deps/rocksdb/rocksdb/utilities/counted_fs.cc +355 -0
  769. package/deps/rocksdb/rocksdb/utilities/counted_fs.h +152 -0
  770. package/deps/rocksdb/rocksdb/utilities/env_mirror.cc +13 -0
  771. package/deps/rocksdb/rocksdb/utilities/env_timed.cc +164 -122
  772. package/deps/rocksdb/rocksdb/utilities/env_timed.h +97 -0
  773. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.cc +75 -17
  774. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.h +19 -3
  775. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +539 -126
  776. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +162 -17
  777. package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.cc +110 -0
  778. package/deps/rocksdb/rocksdb/utilities/fault_injection_secondary_cache.h +94 -0
  779. package/deps/rocksdb/rocksdb/utilities/memory/memory_test.cc +5 -2
  780. package/deps/rocksdb/rocksdb/utilities/memory_allocators.h +104 -0
  781. package/deps/rocksdb/rocksdb/utilities/merge_operators/bytesxor.h +5 -3
  782. package/deps/rocksdb/rocksdb/utilities/merge_operators/max.cc +4 -1
  783. package/deps/rocksdb/rocksdb/utilities/merge_operators/put.cc +11 -3
  784. package/deps/rocksdb/rocksdb/utilities/merge_operators/sortlist.cc +0 -2
  785. package/deps/rocksdb/rocksdb/utilities/merge_operators/sortlist.h +5 -1
  786. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend.cc +29 -10
  787. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend.h +6 -3
  788. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend2.cc +29 -14
  789. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend2.h +6 -3
  790. package/deps/rocksdb/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc +71 -18
  791. package/deps/rocksdb/rocksdb/utilities/merge_operators/uint64add.cc +15 -9
  792. package/deps/rocksdb/rocksdb/utilities/merge_operators.cc +120 -0
  793. package/deps/rocksdb/rocksdb/utilities/merge_operators.h +3 -23
  794. package/deps/rocksdb/rocksdb/utilities/object_registry.cc +267 -42
  795. package/deps/rocksdb/rocksdb/utilities/object_registry_test.cc +702 -76
  796. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +1 -1
  797. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc +26 -5
  798. package/deps/rocksdb/rocksdb/utilities/options/options_util.cc +1 -1
  799. package/deps/rocksdb/rocksdb/utilities/options/options_util_test.cc +124 -1
  800. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.cc +2 -3
  801. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier.h +8 -9
  802. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.cc +15 -13
  803. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_file.h +1 -1
  804. package/deps/rocksdb/rocksdb/utilities/persistent_cache/block_cache_tier_metadata.h +4 -4
  805. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_evictable.h +2 -2
  806. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_bench.cc +8 -9
  807. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_test.cc +1 -1
  808. package/deps/rocksdb/rocksdb/utilities/persistent_cache/persistent_cache_tier.h +6 -3
  809. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.h +2 -2
  810. package/deps/rocksdb/rocksdb/utilities/simulator_cache/cache_simulator.cc +3 -0
  811. package/deps/rocksdb/rocksdb/utilities/simulator_cache/cache_simulator_test.cc +2 -0
  812. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache.cc +43 -35
  813. package/deps/rocksdb/rocksdb/utilities/simulator_cache/sim_cache_test.cc +20 -18
  814. package/deps/rocksdb/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +107 -2
  815. package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.cc +23 -15
  816. package/deps/rocksdb/rocksdb/utilities/trace/file_trace_reader_writer.h +2 -2
  817. package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.cc +316 -0
  818. package/deps/rocksdb/rocksdb/utilities/trace/replayer_impl.h +86 -0
  819. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.cc +4 -5
  820. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager.h +4 -3
  821. package/deps/rocksdb/rocksdb/utilities/transactions/lock/point/point_lock_manager_test.h +1 -1
  822. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_locking_test.cc +119 -3
  823. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc +20 -3
  824. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/locktree/locktree.h +20 -0
  825. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_external_pthread.h +3 -2
  826. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +4 -0
  827. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc +38 -14
  828. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/range_tree_lock_manager.h +17 -10
  829. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_db_impl.h +1 -0
  830. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +1 -2
  831. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +423 -34
  832. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +82 -2
  833. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +72 -40
  834. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +32 -1
  835. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.cc +13 -5
  836. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +7 -3
  837. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +207 -43
  838. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +50 -7
  839. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.cc +28 -10
  840. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_util.h +11 -6
  841. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +516 -0
  842. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +506 -15
  843. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn.cc +27 -13
  844. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.cc +14 -14
  845. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_txn_db.h +3 -0
  846. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_transaction_test.cc +2 -2
  847. package/deps/rocksdb/rocksdb/utilities/transactions/write_unprepared_txn.cc +14 -5
  848. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.cc +305 -27
  849. package/deps/rocksdb/rocksdb/utilities/ttl/db_ttl_impl.h +55 -159
  850. package/deps/rocksdb/rocksdb/utilities/ttl/ttl_test.cc +209 -2
  851. package/deps/rocksdb/rocksdb/utilities/wal_filter.cc +23 -0
  852. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +157 -88
  853. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +501 -114
  854. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +91 -316
  855. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_test.cc +1212 -672
  856. package/deps/rocksdb/rocksdb.gyp +425 -446
  857. package/index.js +5 -87
  858. package/package-lock.json +23687 -0
  859. package/package.json +8 -9
  860. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  861. package/prebuilds/darwin-x64/node.napi.node +0 -0
  862. package/prebuilds/{darwin-x64+arm64 → linux-x64}/node.napi.node +0 -0
  863. package/deps/rocksdb/rocksdb/README.md +0 -32
  864. package/deps/rocksdb/rocksdb/env/env_hdfs.cc +0 -648
  865. package/deps/rocksdb/rocksdb/hdfs/README +0 -23
  866. package/deps/rocksdb/rocksdb/hdfs/env_hdfs.h +0 -386
  867. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backupable_db.h +0 -535
  868. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_librados.h +0 -175
  869. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/utility_db.h +0 -34
  870. package/deps/rocksdb/rocksdb/memory/memkind_kmem_allocator_test.cc +0 -102
  871. package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.h +0 -49
  872. package/deps/rocksdb/rocksdb/memtable/hash_skiplist_rep.h +0 -44
  873. package/deps/rocksdb/rocksdb/options/customizable_helper.h +0 -216
  874. package/deps/rocksdb/rocksdb/port/README +0 -10
  875. package/deps/rocksdb/rocksdb/third-party/folly/folly/CPortability.h +0 -27
  876. package/deps/rocksdb/rocksdb/third-party/folly/folly/ConstexprMath.h +0 -45
  877. package/deps/rocksdb/rocksdb/third-party/folly/folly/Indestructible.h +0 -166
  878. package/deps/rocksdb/rocksdb/third-party/folly/folly/Optional.h +0 -570
  879. package/deps/rocksdb/rocksdb/third-party/folly/folly/Portability.h +0 -92
  880. package/deps/rocksdb/rocksdb/third-party/folly/folly/ScopeGuard.h +0 -54
  881. package/deps/rocksdb/rocksdb/third-party/folly/folly/Traits.h +0 -152
  882. package/deps/rocksdb/rocksdb/third-party/folly/folly/Unit.h +0 -59
  883. package/deps/rocksdb/rocksdb/third-party/folly/folly/Utility.h +0 -141
  884. package/deps/rocksdb/rocksdb/third-party/folly/folly/chrono/Hardware.h +0 -33
  885. package/deps/rocksdb/rocksdb/third-party/folly/folly/container/Array.h +0 -74
  886. package/deps/rocksdb/rocksdb/third-party/folly/folly/detail/Futex-inl.h +0 -117
  887. package/deps/rocksdb/rocksdb/third-party/folly/folly/detail/Futex.cpp +0 -263
  888. package/deps/rocksdb/rocksdb/third-party/folly/folly/detail/Futex.h +0 -96
  889. package/deps/rocksdb/rocksdb/third-party/folly/folly/functional/Invoke.h +0 -40
  890. package/deps/rocksdb/rocksdb/third-party/folly/folly/hash/Hash.h +0 -29
  891. package/deps/rocksdb/rocksdb/third-party/folly/folly/lang/Align.h +0 -144
  892. package/deps/rocksdb/rocksdb/third-party/folly/folly/lang/Bits.h +0 -30
  893. package/deps/rocksdb/rocksdb/third-party/folly/folly/lang/Launder.h +0 -51
  894. package/deps/rocksdb/rocksdb/third-party/folly/folly/portability/Asm.h +0 -28
  895. package/deps/rocksdb/rocksdb/third-party/folly/folly/portability/SysSyscall.h +0 -10
  896. package/deps/rocksdb/rocksdb/third-party/folly/folly/portability/SysTypes.h +0 -26
  897. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/AtomicNotification-inl.h +0 -138
  898. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/AtomicNotification.cpp +0 -23
  899. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/AtomicNotification.h +0 -57
  900. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/AtomicUtil-inl.h +0 -260
  901. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/AtomicUtil.h +0 -52
  902. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/Baton.h +0 -328
  903. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/DistributedMutex-inl.h +0 -1703
  904. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/DistributedMutex.cpp +0 -16
  905. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/DistributedMutex.h +0 -304
  906. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/DistributedMutexSpecializations.h +0 -39
  907. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/ParkingLot.cpp +0 -26
  908. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/ParkingLot.h +0 -318
  909. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/WaitOptions.h +0 -57
  910. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/detail/InlineFunctionRef.h +0 -219
  911. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h +0 -207
  912. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable.h +0 -164
  913. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/detail/Sleeper.h +0 -57
  914. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/detail/Spin.h +0 -77
  915. package/deps/rocksdb/rocksdb/third-party/folly/folly/synchronization/test/DistributedMutexTest.cpp +0 -1145
  916. package/deps/rocksdb/rocksdb/util/build_version.h +0 -15
  917. package/deps/rocksdb/rocksdb/util/xxh3p.h +0 -1392
  918. package/deps/rocksdb/rocksdb/utilities/backupable/backupable_db.cc +0 -2354
  919. package/deps/rocksdb/rocksdb/utilities/env_librados.cc +0 -1497
  920. package/deps/rocksdb/rocksdb/utilities/env_librados_test.cc +0 -1146
  921. package/deps/rocksdb/rocksdb/utilities/transactions/lock/range/range_tree/lib/README +0 -13
  922. package/deps/snappy/snappy-1.1.7/README.md +0 -149
  923. package/prebuilds/linux-x64/node.napi.glibc.node +0 -0
@@ -1,1392 +0,0 @@
1
- // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2
- // This source code is licensed under both the GPLv2 (found in the
3
- // COPYING file in the root directory) and Apache 2.0 License
4
- // (found in the LICENSE.Apache file in the root directory).
5
- /*
6
- xxHash - Extremely Fast Hash algorithm
7
- Development source file for `xxh3`
8
- Copyright (C) 2019-present, Yann Collet.
9
-
10
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
11
-
12
- Redistribution and use in source and binary forms, with or without
13
- modification, are permitted provided that the following conditions are
14
- met:
15
-
16
- * Redistributions of source code must retain the above copyright
17
- notice, this list of conditions and the following disclaimer.
18
- * Redistributions in binary form must reproduce the above
19
- copyright notice, this list of conditions and the following disclaimer
20
- in the documentation and/or other materials provided with the
21
- distribution.
22
-
23
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34
-
35
- You can contact the author at :
36
- - xxHash source repository : https://github.com/Cyan4973/xxHash
37
- */
38
-
39
- /* RocksDB Note: This file contains a preview release (xxhash repository
40
- version 0.7.2) of XXH3 that is unlikely to be compatible with the final
41
- version of XXH3. We have therefore renamed this XXH3p ("preview"), for
42
- clarity so that we can continue to use this version even after
43
- integrating a newer incompatible version.
44
- */
45
-
46
- /* Note :
47
- This file is separated for development purposes.
48
- It will be integrated into `xxhash.c` when development phase is complete.
49
- */
50
-
51
- #ifndef XXH3p_H
52
- #define XXH3p_H
53
-
54
-
55
- /* === Dependencies === */
56
-
57
- #undef XXH_INLINE_ALL /* in case it's already defined */
58
- #define XXH_INLINE_ALL
59
- #include "xxhash.h"
60
-
61
-
62
- /* === Compiler specifics === */
63
-
64
- #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
65
- # define XXH_RESTRICT restrict
66
- #else
67
- /* note : it might be useful to define __restrict or __restrict__ for some C++ compilers */
68
- # define XXH_RESTRICT /* disable */
69
- #endif
70
-
71
- #if defined(__GNUC__)
72
- # if defined(__AVX2__)
73
- # include <immintrin.h>
74
- # elif defined(__SSE2__)
75
- # include <emmintrin.h>
76
- # elif defined(__ARM_NEON__) || defined(__ARM_NEON)
77
- # define inline __inline__ /* clang bug */
78
- # include <arm_neon.h>
79
- # undef inline
80
- # endif
81
- #elif defined(_MSC_VER)
82
- # include <intrin.h>
83
- #endif
84
-
85
- /*
86
- * Sanity check.
87
- *
88
- * XXH3 only requires these features to be efficient:
89
- *
90
- * - Usable unaligned access
91
- * - A 32-bit or 64-bit ALU
92
- * - If 32-bit, a decent ADC instruction
93
- * - A 32 or 64-bit multiply with a 64-bit result
94
- *
95
- * Almost all 32-bit and 64-bit targets meet this, except for Thumb-1, the
96
- * classic 16-bit only subset of ARM's instruction set.
97
- *
98
- * First of all, Thumb-1 lacks support for the UMULL instruction which
99
- * performs the important long multiply. This means numerous __aeabi_lmul
100
- * calls.
101
- *
102
- * Second of all, the 8 functional registers are just not enough.
103
- * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
104
- * Lo registers, and this shuffling results in thousands more MOVs than A32.
105
- *
106
- * A32 and T32 don't have this limitation. They can access all 14 registers,
107
- * do a 32->64 multiply with UMULL, and the flexible operand is helpful too.
108
- *
109
- * If compiling Thumb-1 for a target which supports ARM instructions, we
110
- * will give a warning.
111
- *
112
- * Usually, if this happens, it is because of an accident and you probably
113
- * need to specify -march, as you probably meant to compileh for a newer
114
- * architecture.
115
- */
116
- #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
117
- # warning "XXH3 is highly inefficient without ARM or Thumb-2."
118
- #endif
119
-
120
- /* ==========================================
121
- * Vectorization detection
122
- * ========================================== */
123
- #define XXH_SCALAR 0
124
- #define XXH_SSE2 1
125
- #define XXH_AVX2 2
126
- #define XXH_NEON 3
127
- #define XXH_VSX 4
128
-
129
- #ifndef XXH_VECTOR /* can be defined on command line */
130
- # if defined(__AVX2__)
131
- # define XXH_VECTOR XXH_AVX2
132
- # elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
133
- # define XXH_VECTOR XXH_SSE2
134
- # elif defined(__GNUC__) /* msvc support maybe later */ \
135
- && (defined(__ARM_NEON__) || defined(__ARM_NEON)) \
136
- && (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \
137
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
138
- # define XXH_VECTOR XXH_NEON
139
- # elif defined(__PPC64__) && defined(__POWER8_VECTOR__) && defined(__GNUC__)
140
- # define XXH_VECTOR XXH_VSX
141
- # else
142
- # define XXH_VECTOR XXH_SCALAR
143
- # endif
144
- #endif
145
-
146
- /* control alignment of accumulator,
147
- * for compatibility with fast vector loads */
148
- #ifndef XXH_ACC_ALIGN
149
- # if XXH_VECTOR == 0 /* scalar */
150
- # define XXH_ACC_ALIGN 8
151
- # elif XXH_VECTOR == 1 /* sse2 */
152
- # define XXH_ACC_ALIGN 16
153
- # elif XXH_VECTOR == 2 /* avx2 */
154
- # define XXH_ACC_ALIGN 32
155
- # elif XXH_VECTOR == 3 /* neon */
156
- # define XXH_ACC_ALIGN 16
157
- # elif XXH_VECTOR == 4 /* vsx */
158
- # define XXH_ACC_ALIGN 16
159
- # endif
160
- #endif
161
-
162
- /* xxh_u64 XXH_mult32to64(xxh_u32 a, xxh_u64 b) { return (xxh_u64)a * (xxh_u64)b; } */
163
- #if defined(_MSC_VER) && defined(_M_IX86)
164
- # include <intrin.h>
165
- # define XXH_mult32to64(x, y) __emulu(x, y)
166
- #else
167
- # define XXH_mult32to64(x, y) ((xxh_u64)((x) & 0xFFFFFFFF) * (xxh_u64)((y) & 0xFFFFFFFF))
168
- #endif
169
-
170
- /* VSX stuff. It's a lot because VSX support is mediocre across compilers and
171
- * there is a lot of mischief with endianness. */
172
- #if XXH_VECTOR == XXH_VSX
173
- # include <altivec.h>
174
- # undef vector
175
- typedef __vector unsigned long long U64x2;
176
- typedef __vector unsigned char U8x16;
177
- typedef __vector unsigned U32x4;
178
-
179
- #ifndef XXH_VSX_BE
180
- # if defined(__BIG_ENDIAN__) \
181
- || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
182
- # define XXH_VSX_BE 1
183
- # elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
184
- # warning "-maltivec=be is not recommended. Please use native endianness."
185
- # define XXH_VSX_BE 1
186
- # else
187
- # define XXH_VSX_BE 0
188
- # endif
189
- #endif
190
-
191
- /* We need some helpers for big endian mode. */
192
- #if XXH_VSX_BE
193
- /* A wrapper for POWER9's vec_revb. */
194
- # ifdef __POWER9_VECTOR__
195
- # define XXH_vec_revb vec_revb
196
- # else
197
- XXH_FORCE_INLINE U64x2 XXH_vec_revb(U64x2 val)
198
- {
199
- U8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
200
- 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
201
- return vec_perm(val, val, vByteSwap);
202
- }
203
- # endif
204
-
205
- /* Power8 Crypto gives us vpermxor which is very handy for
206
- * PPC64EB.
207
- *
208
- * U8x16 vpermxor(U8x16 a, U8x16 b, U8x16 mask)
209
- * {
210
- * U8x16 ret;
211
- * for (int i = 0; i < 16; i++) {
212
- * ret[i] = a[mask[i] & 0xF] ^ b[mask[i] >> 4];
213
- * }
214
- * return ret;
215
- * }
216
- *
217
- * Because both of the main loops load the key, swap, and xor it with input,
218
- * we can combine the key swap into this instruction.
219
- */
220
- # ifdef vec_permxor
221
- # define XXH_vec_permxor vec_permxor
222
- # else
223
- # define XXH_vec_permxor __builtin_crypto_vpermxor
224
- # endif
225
- #endif /* XXH_VSX_BE */
226
- /*
227
- * Because we reinterpret the multiply, there are endian memes: vec_mulo actually becomes
228
- * vec_mule.
229
- *
230
- * Additionally, the intrinsic wasn't added until GCC 8, despite existing for a while.
231
- * Clang has an easy way to control this, we can just use the builtin which doesn't swap.
232
- * GCC needs inline assembly. */
233
- #if __has_builtin(__builtin_altivec_vmuleuw)
234
- # define XXH_vec_mulo __builtin_altivec_vmulouw
235
- # define XXH_vec_mule __builtin_altivec_vmuleuw
236
- #else
237
- /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
238
- XXH_FORCE_INLINE U64x2 XXH_vec_mulo(U32x4 a, U32x4 b) {
239
- U64x2 result;
240
- __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
241
- return result;
242
- }
243
- XXH_FORCE_INLINE U64x2 XXH_vec_mule(U32x4 a, U32x4 b) {
244
- U64x2 result;
245
- __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
246
- return result;
247
- }
248
- #endif /* __has_builtin(__builtin_altivec_vmuleuw) */
249
- #endif /* XXH_VECTOR == XXH_VSX */
250
-
251
- /* prefetch
252
- * can be disabled, by declaring XXH_NO_PREFETCH build macro */
253
- #if defined(XXH_NO_PREFETCH)
254
- # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
255
- #else
256
- #if defined(_MSC_VER) && \
257
- (defined(_M_X64) || \
258
- defined(_M_IX86)) /* _mm_prefetch() is not defined outside of x86/x64 */
259
- # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
260
- # define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
261
- # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
262
- # define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
263
- # else
264
- # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
265
- # endif
266
- #endif /* XXH_NO_PREFETCH */
267
-
268
-
269
- /* ==========================================
270
- * XXH3 default settings
271
- * ========================================== */
272
-
273
- #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3p_SECRET_SIZE_MIN */
274
-
275
- #if (XXH_SECRET_DEFAULT_SIZE < XXH3p_SECRET_SIZE_MIN)
276
- # error "default keyset is not large enough"
277
- #endif
278
-
279
- XXH_ALIGN(64) static const xxh_u8 kSecret[XXH_SECRET_DEFAULT_SIZE] = {
280
- 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
281
- 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
282
- 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
283
- 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
284
- 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
285
- 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
286
- 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
287
- 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
288
-
289
- 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
290
- 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
291
- 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
292
- 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
293
- };
294
-
295
- /*
296
- * GCC for x86 has a tendency to use SSE in this loop. While it
297
- * successfully avoids swapping (as MUL overwrites EAX and EDX), it
298
- * slows it down because instead of free register swap shifts, it
299
- * must use pshufd and punpckl/hd.
300
- *
301
- * To prevent this, we use this attribute to shut off SSE.
302
- */
303
- #if defined(__GNUC__) && !defined(__clang__) && defined(__i386__)
304
- __attribute__((__target__("no-sse")))
305
- #endif
306
- static XXH128_hash_t
307
- XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
308
- {
309
- /*
310
- * GCC/Clang __uint128_t method.
311
- *
312
- * On most 64-bit targets, GCC and Clang define a __uint128_t type.
313
- * This is usually the best way as it usually uses a native long 64-bit
314
- * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
315
- *
316
- * Usually.
317
- *
318
- * Despite being a 32-bit platform, Clang (and emscripten) define this
319
- * type despite not having the arithmetic for it. This results in a
320
- * laggy compiler builtin call which calculates a full 128-bit multiply.
321
- * In that case it is best to use the portable one.
322
- * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
323
- */
324
- #if defined(__GNUC__) && !defined(__wasm__) \
325
- && defined(__SIZEOF_INT128__) \
326
- || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
327
-
328
- __uint128_t product = (__uint128_t)lhs * (__uint128_t)rhs;
329
- XXH128_hash_t const r128 = { (xxh_u64)(product), (xxh_u64)(product >> 64) };
330
- return r128;
331
-
332
- /*
333
- * MSVC for x64's _umul128 method.
334
- *
335
- * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
336
- *
337
- * This compiles to single operand MUL on x64.
338
- */
339
- #elif defined(_M_X64) || defined(_M_IA64)
340
-
341
- #ifndef _MSC_VER
342
- # pragma intrinsic(_umul128)
343
- #endif
344
- xxh_u64 product_high;
345
- xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
346
- XXH128_hash_t const r128 = { product_low, product_high };
347
- return r128;
348
-
349
- #else
350
- /*
351
- * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
352
- *
353
- * This is a fast and simple grade school multiply, which is shown
354
- * below with base 10 arithmetic instead of base 0x100000000.
355
- *
356
- * 9 3 // D2 lhs = 93
357
- * x 7 5 // D2 rhs = 75
358
- * ----------
359
- * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10)
360
- * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10)
361
- * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10)
362
- * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10)
363
- * ---------
364
- * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21
365
- * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63
366
- * ---------
367
- * 6 9 7 5
368
- *
369
- * The reasons for adding the products like this are:
370
- * 1. It avoids manual carry tracking. Just like how
371
- * (9 * 9) + 9 + 9 = 99, the same applies with this for
372
- * UINT64_MAX. This avoids a lot of complexity.
373
- *
374
- * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
375
- * instruction available in ARMv6+ A32/T32, which is shown below:
376
- *
377
- * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
378
- * {
379
- * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
380
- * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
381
- * *RdHi = (xxh_u32)(product >> 32);
382
- * }
383
- *
384
- * This instruction was designed for efficient long multiplication,
385
- * and allows this to be calculated in only 4 instructions which
386
- * is comparable to some 64-bit ALUs.
387
- *
388
- * 3. It isn't terrible on other platforms. Usually this will be
389
- * a couple of 32-bit ADD/ADCs.
390
- */
391
-
392
- /* First calculate all of the cross products. */
393
- xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
394
- xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
395
- xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
396
- xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
397
-
398
- /* Now add the products together. These will never overflow. */
399
- xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
400
- xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
401
- xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
402
-
403
- XXH128_hash_t r128 = { lower, upper };
404
- return r128;
405
- #endif
406
- }
407
-
408
- /*
409
- * We want to keep the attribute here because a target switch
410
- * disables inlining.
411
- *
412
- * Does a 64-bit to 128-bit multiply, then XOR folds it.
413
- * The reason for the separate function is to prevent passing
414
- * too many structs around by value. This will hopefully inline
415
- * the multiply, but we don't force it.
416
- */
417
- #if defined(__GNUC__) && !defined(__clang__) && defined(__i386__)
418
- __attribute__((__target__("no-sse")))
419
- #endif
420
- static xxh_u64
421
- XXH3p_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
422
- {
423
- XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
424
- return product.low64 ^ product.high64;
425
- }
426
-
427
-
428
- static XXH64_hash_t XXH3p_avalanche(xxh_u64 h64)
429
- {
430
- h64 ^= h64 >> 37;
431
- h64 *= PRIME64_3;
432
- h64 ^= h64 >> 32;
433
- return h64;
434
- }
435
-
436
-
437
- /* ==========================================
438
- * Short keys
439
- * ========================================== */
440
-
441
- XXH_FORCE_INLINE XXH64_hash_t
442
- XXH3p_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
443
- {
444
- XXH_ASSERT(input != NULL);
445
- XXH_ASSERT(1 <= len && len <= 3);
446
- XXH_ASSERT(secret != NULL);
447
- { xxh_u8 const c1 = input[0];
448
- xxh_u8 const c2 = input[len >> 1];
449
- xxh_u8 const c3 = input[len - 1];
450
- xxh_u32 const combined = ((xxh_u32)c1) | (((xxh_u32)c2) << 8) | (((xxh_u32)c3) << 16) | (((xxh_u32)len) << 24);
451
- xxh_u64 const keyed = (xxh_u64)combined ^ (XXH_readLE32(secret) + seed);
452
- xxh_u64 const mixed = keyed * PRIME64_1;
453
- return XXH3p_avalanche(mixed);
454
- }
455
- }
456
-
457
- XXH_FORCE_INLINE XXH64_hash_t
458
- XXH3p_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
459
- {
460
- XXH_ASSERT(input != NULL);
461
- XXH_ASSERT(secret != NULL);
462
- XXH_ASSERT(4 <= len && len <= 8);
463
- { xxh_u32 const input_lo = XXH_readLE32(input);
464
- xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
465
- xxh_u64 const input_64 = input_lo | ((xxh_u64)input_hi << 32);
466
- xxh_u64 const keyed = input_64 ^ (XXH_readLE64(secret) + seed);
467
- xxh_u64 const mix64 = len + ((keyed ^ (keyed >> 51)) * PRIME32_1);
468
- return XXH3p_avalanche((mix64 ^ (mix64 >> 47)) * PRIME64_2);
469
- }
470
- }
471
-
472
- XXH_FORCE_INLINE XXH64_hash_t
473
- XXH3p_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
474
- {
475
- XXH_ASSERT(input != NULL);
476
- XXH_ASSERT(secret != NULL);
477
- XXH_ASSERT(9 <= len && len <= 16);
478
- { xxh_u64 const input_lo = XXH_readLE64(input) ^ (XXH_readLE64(secret) + seed);
479
- xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ (XXH_readLE64(secret + 8) - seed);
480
- xxh_u64 const acc = len + (input_lo + input_hi) + XXH3p_mul128_fold64(input_lo, input_hi);
481
- return XXH3p_avalanche(acc);
482
- }
483
- }
484
-
485
- XXH_FORCE_INLINE XXH64_hash_t
486
- XXH3p_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
487
- {
488
- XXH_ASSERT(len <= 16);
489
- { if (len > 8) return XXH3p_len_9to16_64b(input, len, secret, seed);
490
- if (len >= 4) return XXH3p_len_4to8_64b(input, len, secret, seed);
491
- if (len) return XXH3p_len_1to3_64b(input, len, secret, seed);
492
- /*
493
- * RocksDB modification from XXH3 preview: zero result for empty
494
- * string can be problematic for multiplication-based algorithms.
495
- * Return a hash of the seed instead.
496
- */
497
- return XXH3p_mul128_fold64(seed + XXH_readLE64(secret), PRIME64_2);
498
- }
499
- }
500
-
501
-
502
- /* === Long Keys === */
503
-
504
- #define STRIPE_LEN 64
505
- #define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
506
- #define ACC_NB (STRIPE_LEN / sizeof(xxh_u64))
507
-
508
- typedef enum { XXH3p_acc_64bits, XXH3p_acc_128bits } XXH3p_accWidth_e;
509
-
510
- XXH_FORCE_INLINE void
511
- XXH3p_accumulate_512( void* XXH_RESTRICT acc,
512
- const void* XXH_RESTRICT input,
513
- const void* XXH_RESTRICT secret,
514
- XXH3p_accWidth_e accWidth)
515
- {
516
- #if (XXH_VECTOR == XXH_AVX2)
517
-
518
- XXH_ASSERT((((size_t)acc) & 31) == 0);
519
- { XXH_ALIGN(32) __m256i* const xacc = (__m256i *) acc;
520
- const __m256i* const xinput = (const __m256i *) input; /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this type */
521
- const __m256i* const xsecret = (const __m256i *) secret; /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this type */
522
-
523
- size_t i;
524
- for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) {
525
- __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
526
- __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
527
- __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); /* uint32 dk[8] = {d0+k0, d1+k1, d2+k2, d3+k3, ...} */
528
- __m256i const product = _mm256_mul_epu32 (data_key, _mm256_shuffle_epi32 (data_key, 0x31)); /* uint64 mul[4] = {dk0*dk1, dk2*dk3, ...} */
529
- if (accWidth == XXH3p_acc_128bits) {
530
- __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
531
- __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
532
- xacc[i] = _mm256_add_epi64(product, sum);
533
- } else { /* XXH3p_acc_64bits */
534
- __m256i const sum = _mm256_add_epi64(xacc[i], data_vec);
535
- xacc[i] = _mm256_add_epi64(product, sum);
536
- }
537
- } }
538
-
539
- #elif (XXH_VECTOR == XXH_SSE2)
540
-
541
- XXH_ASSERT((((size_t)acc) & 15) == 0);
542
- { XXH_ALIGN(16) __m128i* const xacc = (__m128i *) acc;
543
- const __m128i* const xinput = (const __m128i *) input; /* not really aligned, just for ptr arithmetic, and because _mm_loadu_si128() requires this type */
544
- const __m128i* const xsecret = (const __m128i *) secret; /* not really aligned, just for ptr arithmetic, and because _mm_loadu_si128() requires this type */
545
-
546
- size_t i;
547
- for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) {
548
- __m128i const data_vec = _mm_loadu_si128 (xinput+i);
549
- __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
550
- __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); /* uint32 dk[8] = {d0+k0, d1+k1, d2+k2, d3+k3, ...} */
551
- __m128i const product = _mm_mul_epu32 (data_key, _mm_shuffle_epi32 (data_key, 0x31)); /* uint64 mul[4] = {dk0*dk1, dk2*dk3, ...} */
552
- if (accWidth == XXH3p_acc_128bits) {
553
- __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
554
- __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
555
- xacc[i] = _mm_add_epi64(product, sum);
556
- } else { /* XXH3p_acc_64bits */
557
- __m128i const sum = _mm_add_epi64(xacc[i], data_vec);
558
- xacc[i] = _mm_add_epi64(product, sum);
559
- }
560
- } }
561
-
562
- #elif (XXH_VECTOR == XXH_NEON)
563
-
564
- XXH_ASSERT((((size_t)acc) & 15) == 0);
565
- {
566
- XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t *) acc;
567
- /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
568
- uint8_t const* const xinput = (const uint8_t *) input;
569
- uint8_t const* const xsecret = (const uint8_t *) secret;
570
-
571
- size_t i;
572
- for (i=0; i < STRIPE_LEN / sizeof(uint64x2_t); i++) {
573
- #if !defined(__aarch64__) && !defined(__arm64__) && defined(__GNUC__) /* ARM32-specific hack */
574
- /* vzip on ARMv7 Clang generates a lot of vmovs (technically vorrs) without this.
575
- * vzip on 32-bit ARM NEON will overwrite the original register, and I think that Clang
576
- * assumes I don't want to destroy it and tries to make a copy. This slows down the code
577
- * a lot.
578
- * aarch64 not only uses an entirely different syntax, but it requires three
579
- * instructions...
580
- * ext v1.16B, v0.16B, #8 // select high bits because aarch64 can't address them directly
581
- * zip1 v3.2s, v0.2s, v1.2s // first zip
582
- * zip2 v2.2s, v0.2s, v1.2s // second zip
583
- * ...to do what ARM does in one:
584
- * vzip.32 d0, d1 // Interleave high and low bits and overwrite. */
585
-
586
- /* data_vec = xsecret[i]; */
587
- uint8x16_t const data_vec = vld1q_u8(xinput + (i * 16));
588
- /* key_vec = xsecret[i]; */
589
- uint8x16_t const key_vec = vld1q_u8(xsecret + (i * 16));
590
- /* data_key = data_vec ^ key_vec; */
591
- uint32x4_t data_key;
592
-
593
- if (accWidth == XXH3p_acc_64bits) {
594
- /* Add first to prevent register swaps */
595
- /* xacc[i] += data_vec; */
596
- xacc[i] = vaddq_u64 (xacc[i], vreinterpretq_u64_u8(data_vec));
597
- } else { /* XXH3p_acc_128bits */
598
- /* xacc[i] += swap(data_vec); */
599
- /* can probably be optimized better */
600
- uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
601
- uint64x2_t const swapped= vextq_u64(data64, data64, 1);
602
- xacc[i] = vaddq_u64 (xacc[i], swapped);
603
- }
604
-
605
- data_key = vreinterpretq_u32_u8(veorq_u8(data_vec, key_vec));
606
-
607
- /* Here's the magic. We use the quirkiness of vzip to shuffle data_key in place.
608
- * shuffle: data_key[0, 1, 2, 3] = data_key[0, 2, 1, 3] */
609
- __asm__("vzip.32 %e0, %f0" : "+w" (data_key));
610
- /* xacc[i] += (uint64x2_t) data_key[0, 1] * (uint64x2_t) data_key[2, 3]; */
611
- xacc[i] = vmlal_u32(xacc[i], vget_low_u32(data_key), vget_high_u32(data_key));
612
-
613
- #else
614
- /* On aarch64, vshrn/vmovn seems to be equivalent to, if not faster than, the vzip method. */
615
-
616
- /* data_vec = xsecret[i]; */
617
- uint8x16_t const data_vec = vld1q_u8(xinput + (i * 16));
618
- /* key_vec = xsecret[i]; */
619
- uint8x16_t const key_vec = vld1q_u8(xsecret + (i * 16));
620
- /* data_key = data_vec ^ key_vec; */
621
- uint64x2_t const data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
622
- /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); */
623
- uint32x2_t const data_key_lo = vmovn_u64 (data_key);
624
- /* data_key_hi = (uint32x2_t) (data_key >> 32); */
625
- uint32x2_t const data_key_hi = vshrn_n_u64 (data_key, 32);
626
- if (accWidth == XXH3p_acc_64bits) {
627
- /* xacc[i] += data_vec; */
628
- xacc[i] = vaddq_u64 (xacc[i], vreinterpretq_u64_u8(data_vec));
629
- } else { /* XXH3p_acc_128bits */
630
- /* xacc[i] += swap(data_vec); */
631
- uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
632
- uint64x2_t const swapped= vextq_u64(data64, data64, 1);
633
- xacc[i] = vaddq_u64 (xacc[i], swapped);
634
- }
635
- /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
636
- xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
637
-
638
- #endif
639
- }
640
- }
641
-
642
- #elif (XXH_VECTOR == XXH_VSX) && /* work around a compiler bug */ (__GNUC__ > 5)
643
- U64x2* const xacc = (U64x2*) acc; /* presumed aligned */
644
- U64x2 const* const xinput = (U64x2 const*) input; /* no alignment restriction */
645
- U64x2 const* const xsecret = (U64x2 const*) secret; /* no alignment restriction */
646
- U64x2 const v32 = { 32, 32 };
647
- #if XXH_VSX_BE
648
- U8x16 const vXorSwap = { 0x07, 0x16, 0x25, 0x34, 0x43, 0x52, 0x61, 0x70,
649
- 0x8F, 0x9E, 0xAD, 0xBC, 0xCB, 0xDA, 0xE9, 0xF8 };
650
- #endif
651
- size_t i;
652
- for (i = 0; i < STRIPE_LEN / sizeof(U64x2); i++) {
653
- /* data_vec = xinput[i]; */
654
- /* key_vec = xsecret[i]; */
655
- #if XXH_VSX_BE
656
- /* byteswap */
657
- U64x2 const data_vec = XXH_vec_revb(vec_vsx_ld(0, xinput + i));
658
- U64x2 const key_raw = vec_vsx_ld(0, xsecret + i);
659
- /* See comment above. data_key = data_vec ^ swap(xsecret[i]); */
660
- U64x2 const data_key = (U64x2)XXH_vec_permxor((U8x16)data_vec, (U8x16)key_raw, vXorSwap);
661
- #else
662
- U64x2 const data_vec = vec_vsx_ld(0, xinput + i);
663
- U64x2 const key_vec = vec_vsx_ld(0, xsecret + i);
664
- U64x2 const data_key = data_vec ^ key_vec;
665
- #endif
666
- /* shuffled = (data_key << 32) | (data_key >> 32); */
667
- U32x4 const shuffled = (U32x4)vec_rl(data_key, v32);
668
- /* product = ((U64x2)data_key & 0xFFFFFFFF) * ((U64x2)shuffled & 0xFFFFFFFF); */
669
- U64x2 const product = XXH_vec_mulo((U32x4)data_key, shuffled);
670
- xacc[i] += product;
671
-
672
- if (accWidth == XXH3p_acc_64bits) {
673
- xacc[i] += data_vec;
674
- } else { /* XXH3p_acc_128bits */
675
- /* swap high and low halves */
676
- U64x2 const data_swapped = vec_xxpermdi(data_vec, data_vec, 2);
677
- xacc[i] += data_swapped;
678
- }
679
- }
680
-
681
- #else /* scalar variant of Accumulator - universal */
682
-
683
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned on 32-bytes boundaries, little hint for the auto-vectorizer */
684
- const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */
685
- const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
686
- size_t i;
687
- XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
688
- for (i=0; i < ACC_NB; i++) {
689
- xxh_u64 const data_val = XXH_readLE64(xinput + 8*i);
690
- xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8);
691
-
692
- if (accWidth == XXH3p_acc_64bits) {
693
- xacc[i] += data_val;
694
- } else {
695
- xacc[i ^ 1] += data_val; /* swap adjacent lanes */
696
- }
697
- xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
698
- }
699
- #endif
700
- }
701
-
702
- XXH_FORCE_INLINE void
703
- XXH3p_scrambleAcc(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
704
- {
705
- #if (XXH_VECTOR == XXH_AVX2)
706
-
707
- XXH_ASSERT((((size_t)acc) & 31) == 0);
708
- { XXH_ALIGN(32) __m256i* const xacc = (__m256i*) acc;
709
- const __m256i* const xsecret = (const __m256i *) secret; /* not really aligned, just for ptr arithmetic, and because _mm256_loadu_si256() requires this argument type */
710
- const __m256i prime32 = _mm256_set1_epi32((int)PRIME32_1);
711
-
712
- size_t i;
713
- for (i=0; i < STRIPE_LEN/sizeof(__m256i); i++) {
714
- /* xacc[i] ^= (xacc[i] >> 47) */
715
- __m256i const acc_vec = xacc[i];
716
- __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
717
- __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
718
- /* xacc[i] ^= xsecret; */
719
- __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
720
- __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
721
-
722
- /* xacc[i] *= PRIME32_1; */
723
- __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, 0x31);
724
- __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
725
- __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
726
- xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
727
- }
728
- }
729
-
730
- #elif (XXH_VECTOR == XXH_SSE2)
731
-
732
- XXH_ASSERT((((size_t)acc) & 15) == 0);
733
- { XXH_ALIGN(16) __m128i* const xacc = (__m128i*) acc;
734
- const __m128i* const xsecret = (const __m128i *) secret; /* not really aligned, just for ptr arithmetic, and because _mm_loadu_si128() requires this argument type */
735
- const __m128i prime32 = _mm_set1_epi32((int)PRIME32_1);
736
-
737
- size_t i;
738
- for (i=0; i < STRIPE_LEN/sizeof(__m128i); i++) {
739
- /* xacc[i] ^= (xacc[i] >> 47) */
740
- __m128i const acc_vec = xacc[i];
741
- __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
742
- __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
743
- /* xacc[i] ^= xsecret; */
744
- __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
745
- __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
746
-
747
- /* xacc[i] *= PRIME32_1; */
748
- __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, 0x31);
749
- __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
750
- __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
751
- xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
752
- }
753
- }
754
-
755
- #elif (XXH_VECTOR == XXH_NEON)
756
-
757
- XXH_ASSERT((((size_t)acc) & 15) == 0);
758
-
759
- { uint64x2_t* const xacc = (uint64x2_t*) acc;
760
- uint8_t const* const xsecret = (uint8_t const*) secret;
761
- uint32x2_t const prime = vdup_n_u32 (PRIME32_1);
762
-
763
- size_t i;
764
- for (i=0; i < STRIPE_LEN/sizeof(uint64x2_t); i++) {
765
- /* data_vec = xacc[i] ^ (xacc[i] >> 47); */
766
- uint64x2_t const acc_vec = xacc[i];
767
- uint64x2_t const shifted = vshrq_n_u64 (acc_vec, 47);
768
- uint64x2_t const data_vec = veorq_u64 (acc_vec, shifted);
769
-
770
- /* key_vec = xsecret[i]; */
771
- uint32x4_t const key_vec = vreinterpretq_u32_u8(vld1q_u8(xsecret + (i * 16)));
772
- /* data_key = data_vec ^ key_vec; */
773
- uint32x4_t const data_key = veorq_u32 (vreinterpretq_u32_u64(data_vec), key_vec);
774
- /* shuffled = { data_key[0, 2], data_key[1, 3] }; */
775
- uint32x2x2_t const shuffled = vzip_u32 (vget_low_u32(data_key), vget_high_u32(data_key));
776
-
777
- /* data_key *= PRIME32_1 */
778
-
779
- /* prod_hi = (data_key >> 32) * PRIME32_1; */
780
- uint64x2_t const prod_hi = vmull_u32 (shuffled.val[1], prime);
781
- /* xacc[i] = prod_hi << 32; */
782
- xacc[i] = vshlq_n_u64(prod_hi, 32);
783
- /* xacc[i] += (prod_hi & 0xFFFFFFFF) * PRIME32_1; */
784
- xacc[i] = vmlal_u32(xacc[i], shuffled.val[0], prime);
785
- } }
786
-
787
- #elif (XXH_VECTOR == XXH_VSX) && /* work around a compiler bug */ (__GNUC__ > 5)
788
-
789
- U64x2* const xacc = (U64x2*) acc;
790
- const U64x2* const xsecret = (const U64x2*) secret;
791
- /* constants */
792
- U64x2 const v32 = { 32, 32 };
793
- U64x2 const v47 = { 47, 47 };
794
- U32x4 const prime = { PRIME32_1, PRIME32_1, PRIME32_1, PRIME32_1 };
795
- size_t i;
796
- #if XXH_VSX_BE
797
- /* endian swap */
798
- U8x16 const vXorSwap = { 0x07, 0x16, 0x25, 0x34, 0x43, 0x52, 0x61, 0x70,
799
- 0x8F, 0x9E, 0xAD, 0xBC, 0xCB, 0xDA, 0xE9, 0xF8 };
800
- #endif
801
- for (i = 0; i < STRIPE_LEN / sizeof(U64x2); i++) {
802
- U64x2 const acc_vec = xacc[i];
803
- U64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
804
- /* key_vec = xsecret[i]; */
805
- #if XXH_VSX_BE
806
- /* swap bytes words */
807
- U64x2 const key_raw = vec_vsx_ld(0, xsecret + i);
808
- U64x2 const data_key = (U64x2)XXH_vec_permxor((U8x16)data_vec, (U8x16)key_raw, vXorSwap);
809
- #else
810
- U64x2 const key_vec = vec_vsx_ld(0, xsecret + i);
811
- U64x2 const data_key = data_vec ^ key_vec;
812
- #endif
813
-
814
- /* data_key *= PRIME32_1 */
815
-
816
- /* prod_lo = ((U64x2)data_key & 0xFFFFFFFF) * ((U64x2)prime & 0xFFFFFFFF); */
817
- U64x2 const prod_even = XXH_vec_mule((U32x4)data_key, prime);
818
- /* prod_hi = ((U64x2)data_key >> 32) * ((U64x2)prime >> 32); */
819
- U64x2 const prod_odd = XXH_vec_mulo((U32x4)data_key, prime);
820
- xacc[i] = prod_odd + (prod_even << v32);
821
- }
822
-
823
- #else /* scalar variant of Scrambler - universal */
824
-
825
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned on 32-bytes boundaries, little hint for the auto-vectorizer */
826
- const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
827
- size_t i;
828
- XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
829
- for (i=0; i < ACC_NB; i++) {
830
- xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i);
831
- xxh_u64 acc64 = xacc[i];
832
- acc64 ^= acc64 >> 47;
833
- acc64 ^= key64;
834
- acc64 *= PRIME32_1;
835
- xacc[i] = acc64;
836
- }
837
-
838
- #endif
839
- }
840
-
841
- #define XXH_PREFETCH_DIST 384
842
-
843
- /* assumption : nbStripes will not overflow secret size */
844
- XXH_FORCE_INLINE void
845
- XXH3p_accumulate( xxh_u64* XXH_RESTRICT acc,
846
- const xxh_u8* XXH_RESTRICT input,
847
- const xxh_u8* XXH_RESTRICT secret,
848
- size_t nbStripes,
849
- XXH3p_accWidth_e accWidth)
850
- {
851
- size_t n;
852
- for (n = 0; n < nbStripes; n++ ) {
853
- const xxh_u8* const in = input + n*STRIPE_LEN;
854
- XXH_PREFETCH(in + XXH_PREFETCH_DIST);
855
- XXH3p_accumulate_512(acc,
856
- in,
857
- secret + n*XXH_SECRET_CONSUME_RATE,
858
- accWidth);
859
- }
860
- }
861
-
862
- /* note : clang auto-vectorizes well in SS2 mode _if_ this function is `static`,
863
- * and doesn't auto-vectorize it at all if it is `FORCE_INLINE`.
864
- * However, it auto-vectorizes better AVX2 if it is `FORCE_INLINE`
865
- * Pretty much every other modes and compilers prefer `FORCE_INLINE`.
866
- */
867
-
868
- #if defined(__clang__) && (XXH_VECTOR==0) && !defined(__AVX2__) && !defined(__arm__) && !defined(__thumb__)
869
- static void
870
- #else
871
- XXH_FORCE_INLINE void
872
- #endif
873
- XXH3p_hashLong_internal_loop( xxh_u64* XXH_RESTRICT acc,
874
- const xxh_u8* XXH_RESTRICT input, size_t len,
875
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
876
- XXH3p_accWidth_e accWidth)
877
- {
878
- size_t const nb_rounds = (secretSize - STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
879
- size_t const block_len = STRIPE_LEN * nb_rounds;
880
- size_t const nb_blocks = len / block_len;
881
-
882
- size_t n;
883
-
884
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN);
885
-
886
- for (n = 0; n < nb_blocks; n++) {
887
- XXH3p_accumulate(acc, input + n*block_len, secret, nb_rounds, accWidth);
888
- XXH3p_scrambleAcc(acc, secret + secretSize - STRIPE_LEN);
889
- }
890
-
891
- /* last partial block */
892
- XXH_ASSERT(len > STRIPE_LEN);
893
- { size_t const nbStripes = (len - (block_len * nb_blocks)) / STRIPE_LEN;
894
- XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
895
- XXH3p_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, accWidth);
896
-
897
- /* last stripe */
898
- if (len & (STRIPE_LEN - 1)) {
899
- const xxh_u8* const p = input + len - STRIPE_LEN;
900
- #define XXH_SECRET_LASTACC_START 7 /* do not align on 8, so that secret is different from scrambler */
901
- XXH3p_accumulate_512(acc, p, secret + secretSize - STRIPE_LEN - XXH_SECRET_LASTACC_START, accWidth);
902
- } }
903
- }
904
-
905
- XXH_FORCE_INLINE xxh_u64
906
- XXH3p_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
907
- {
908
- return XXH3p_mul128_fold64(
909
- acc[0] ^ XXH_readLE64(secret),
910
- acc[1] ^ XXH_readLE64(secret+8) );
911
- }
912
-
913
- static XXH64_hash_t
914
- XXH3p_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
915
- {
916
- xxh_u64 result64 = start;
917
-
918
- result64 += XXH3p_mix2Accs(acc+0, secret + 0);
919
- result64 += XXH3p_mix2Accs(acc+2, secret + 16);
920
- result64 += XXH3p_mix2Accs(acc+4, secret + 32);
921
- result64 += XXH3p_mix2Accs(acc+6, secret + 48);
922
-
923
- return XXH3p_avalanche(result64);
924
- }
925
-
926
- #define XXH3p_INIT_ACC { PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3, \
927
- PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1 };
928
-
929
- XXH_FORCE_INLINE XXH64_hash_t
930
- XXH3p_hashLong_internal(const xxh_u8* XXH_RESTRICT input, size_t len,
931
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize)
932
- {
933
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[ACC_NB] = XXH3p_INIT_ACC;
934
-
935
- XXH3p_hashLong_internal_loop(acc, input, len, secret, secretSize, XXH3p_acc_64bits);
936
-
937
- /* converge into final hash */
938
- XXH_STATIC_ASSERT(sizeof(acc) == 64);
939
- #define XXH_SECRET_MERGEACCS_START 11 /* do not align on 8, so that secret is different from accumulator */
940
- XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
941
- return XXH3p_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * PRIME64_1);
942
- }
943
-
944
-
945
- XXH_NO_INLINE XXH64_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
946
- XXH3p_hashLong_64b_defaultSecret(const xxh_u8* XXH_RESTRICT input, size_t len)
947
- {
948
- return XXH3p_hashLong_internal(input, len, kSecret, sizeof(kSecret));
949
- }
950
-
951
- XXH_NO_INLINE XXH64_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
952
- XXH3p_hashLong_64b_withSecret(const xxh_u8* XXH_RESTRICT input, size_t len,
953
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize)
954
- {
955
- return XXH3p_hashLong_internal(input, len, secret, secretSize);
956
- }
957
-
958
-
959
- XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
960
- {
961
- if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
962
- memcpy(dst, &v64, sizeof(v64));
963
- }
964
-
965
- /* XXH3p_initCustomSecret() :
966
- * destination `customSecret` is presumed allocated and same size as `kSecret`.
967
- */
968
- XXH_FORCE_INLINE void XXH3p_initCustomSecret(xxh_u8* customSecret, xxh_u64 seed64)
969
- {
970
- int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
971
- int i;
972
-
973
- XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
974
-
975
- for (i=0; i < nbRounds; i++) {
976
- XXH_writeLE64(customSecret + 16*i, XXH_readLE64(kSecret + 16*i) + seed64);
977
- XXH_writeLE64(customSecret + 16*i + 8, XXH_readLE64(kSecret + 16*i + 8) - seed64);
978
- }
979
- }
980
-
981
-
982
- /* XXH3p_hashLong_64b_withSeed() :
983
- * Generate a custom key,
984
- * based on alteration of default kSecret with the seed,
985
- * and then use this key for long mode hashing.
986
- * This operation is decently fast but nonetheless costs a little bit of time.
987
- * Try to avoid it whenever possible (typically when seed==0).
988
- */
989
- XXH_NO_INLINE XXH64_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
990
- XXH3p_hashLong_64b_withSeed(const xxh_u8* input, size_t len, XXH64_hash_t seed)
991
- {
992
- XXH_ALIGN(8) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
993
- if (seed==0) return XXH3p_hashLong_64b_defaultSecret(input, len);
994
- XXH3p_initCustomSecret(secret, seed);
995
- return XXH3p_hashLong_internal(input, len, secret, sizeof(secret));
996
- }
997
-
998
-
999
- XXH_FORCE_INLINE xxh_u64 XXH3p_mix16B(const xxh_u8* XXH_RESTRICT input,
1000
- const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
1001
- {
1002
- xxh_u64 const input_lo = XXH_readLE64(input);
1003
- xxh_u64 const input_hi = XXH_readLE64(input+8);
1004
- return XXH3p_mul128_fold64(
1005
- input_lo ^ (XXH_readLE64(secret) + seed64),
1006
- input_hi ^ (XXH_readLE64(secret+8) - seed64) );
1007
- }
1008
-
1009
-
1010
- XXH_FORCE_INLINE XXH64_hash_t
1011
- XXH3p_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
1012
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
1013
- XXH64_hash_t seed)
1014
- {
1015
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN); (void)secretSize;
1016
- XXH_ASSERT(16 < len && len <= 128);
1017
-
1018
- { xxh_u64 acc = len * PRIME64_1;
1019
- if (len > 32) {
1020
- if (len > 64) {
1021
- if (len > 96) {
1022
- acc += XXH3p_mix16B(input+48, secret+96, seed);
1023
- acc += XXH3p_mix16B(input+len-64, secret+112, seed);
1024
- }
1025
- acc += XXH3p_mix16B(input+32, secret+64, seed);
1026
- acc += XXH3p_mix16B(input+len-48, secret+80, seed);
1027
- }
1028
- acc += XXH3p_mix16B(input+16, secret+32, seed);
1029
- acc += XXH3p_mix16B(input+len-32, secret+48, seed);
1030
- }
1031
- acc += XXH3p_mix16B(input+0, secret+0, seed);
1032
- acc += XXH3p_mix16B(input+len-16, secret+16, seed);
1033
-
1034
- return XXH3p_avalanche(acc);
1035
- }
1036
- }
1037
-
1038
- #define XXH3p_MIDSIZE_MAX 240
1039
-
1040
- XXH_NO_INLINE XXH64_hash_t
1041
- XXH3p_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
1042
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
1043
- XXH64_hash_t seed)
1044
- {
1045
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN); (void)secretSize;
1046
- XXH_ASSERT(128 < len && len <= XXH3p_MIDSIZE_MAX);
1047
-
1048
- #define XXH3p_MIDSIZE_STARTOFFSET 3
1049
- #define XXH3p_MIDSIZE_LASTOFFSET 17
1050
-
1051
- { xxh_u64 acc = len * PRIME64_1;
1052
- int const nbRounds = (int)len / 16;
1053
- int i;
1054
- for (i=0; i<8; i++) {
1055
- acc += XXH3p_mix16B(input+(16*i), secret+(16*i), seed);
1056
- }
1057
- acc = XXH3p_avalanche(acc);
1058
- XXH_ASSERT(nbRounds >= 8);
1059
- for (i=8 ; i < nbRounds; i++) {
1060
- acc += XXH3p_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3p_MIDSIZE_STARTOFFSET, seed);
1061
- }
1062
- /* last bytes */
1063
- acc += XXH3p_mix16B(input + len - 16, secret + XXH3p_SECRET_SIZE_MIN - XXH3p_MIDSIZE_LASTOFFSET, seed);
1064
- return XXH3p_avalanche(acc);
1065
- }
1066
- }
1067
-
1068
- /* === Public entry point === */
1069
-
1070
- XXH_PUBLIC_API XXH64_hash_t XXH3p_64bits(const void* input, size_t len)
1071
- {
1072
- if (len <= 16) return XXH3p_len_0to16_64b((const xxh_u8*)input, len, kSecret, 0);
1073
- if (len <= 128) return XXH3p_len_17to128_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0);
1074
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0);
1075
- return XXH3p_hashLong_64b_defaultSecret((const xxh_u8*)input, len);
1076
- }
1077
-
1078
- XXH_PUBLIC_API XXH64_hash_t
1079
- XXH3p_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
1080
- {
1081
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN);
1082
- /* if an action must be taken should `secret` conditions not be respected,
1083
- * it should be done here.
1084
- * For now, it's a contract pre-condition.
1085
- * Adding a check and a branch here would cost performance at every hash */
1086
- if (len <= 16) return XXH3p_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, 0);
1087
- if (len <= 128) return XXH3p_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0);
1088
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0);
1089
- return XXH3p_hashLong_64b_withSecret((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize);
1090
- }
1091
-
1092
- XXH_PUBLIC_API XXH64_hash_t
1093
- XXH3p_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
1094
- {
1095
- if (len <= 16) return XXH3p_len_0to16_64b((const xxh_u8*)input, len, kSecret, seed);
1096
- if (len <= 128) return XXH3p_len_17to128_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed);
1097
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_64b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed);
1098
- return XXH3p_hashLong_64b_withSeed((const xxh_u8*)input, len, seed);
1099
- }
1100
-
1101
- /* === XXH3 streaming === */
1102
-
1103
- /* RocksDB Note: unused & removed due to bug in preview version */
1104
-
1105
- /* ==========================================
1106
- * XXH3 128 bits (=> XXH128)
1107
- * ========================================== */
1108
-
1109
- XXH_FORCE_INLINE XXH128_hash_t
1110
- XXH3p_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
1111
- {
1112
- XXH_ASSERT(input != NULL);
1113
- XXH_ASSERT(1 <= len && len <= 3);
1114
- XXH_ASSERT(secret != NULL);
1115
- { xxh_u8 const c1 = input[0];
1116
- xxh_u8 const c2 = input[len >> 1];
1117
- xxh_u8 const c3 = input[len - 1];
1118
- xxh_u32 const combinedl = ((xxh_u32)c1) + (((xxh_u32)c2) << 8) + (((xxh_u32)c3) << 16) + (((xxh_u32)len) << 24);
1119
- xxh_u32 const combinedh = XXH_swap32(combinedl);
1120
- xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ (XXH_readLE32(secret) + seed);
1121
- xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ (XXH_readLE32(secret+4) - seed);
1122
- xxh_u64 const mixedl = keyed_lo * PRIME64_1;
1123
- xxh_u64 const mixedh = keyed_hi * PRIME64_5;
1124
- XXH128_hash_t const h128 = { XXH3p_avalanche(mixedl) /*low64*/, XXH3p_avalanche(mixedh) /*high64*/ };
1125
- return h128;
1126
- }
1127
- }
1128
-
1129
-
1130
- XXH_FORCE_INLINE XXH128_hash_t
1131
- XXH3p_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
1132
- {
1133
- XXH_ASSERT(input != NULL);
1134
- XXH_ASSERT(secret != NULL);
1135
- XXH_ASSERT(4 <= len && len <= 8);
1136
- { xxh_u32 const input_lo = XXH_readLE32(input);
1137
- xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
1138
- xxh_u64 const input_64_lo = input_lo + ((xxh_u64)input_hi << 32);
1139
- xxh_u64 const input_64_hi = XXH_swap64(input_64_lo);
1140
- xxh_u64 const keyed_lo = input_64_lo ^ (XXH_readLE64(secret) + seed);
1141
- xxh_u64 const keyed_hi = input_64_hi ^ (XXH_readLE64(secret + 8) - seed);
1142
- xxh_u64 const mix64l1 = len + ((keyed_lo ^ (keyed_lo >> 51)) * PRIME32_1);
1143
- xxh_u64 const mix64l2 = (mix64l1 ^ (mix64l1 >> 47)) * PRIME64_2;
1144
- xxh_u64 const mix64h1 = ((keyed_hi ^ (keyed_hi >> 47)) * PRIME64_1) - len;
1145
- xxh_u64 const mix64h2 = (mix64h1 ^ (mix64h1 >> 43)) * PRIME64_4;
1146
- { XXH128_hash_t const h128 = { XXH3p_avalanche(mix64l2) /*low64*/, XXH3p_avalanche(mix64h2) /*high64*/ };
1147
- return h128;
1148
- } }
1149
- }
1150
-
1151
- XXH_FORCE_INLINE XXH128_hash_t
1152
- XXH3p_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
1153
- {
1154
- XXH_ASSERT(input != NULL);
1155
- XXH_ASSERT(secret != NULL);
1156
- XXH_ASSERT(9 <= len && len <= 16);
1157
- { xxh_u64 const input_lo = XXH_readLE64(input) ^ (XXH_readLE64(secret) + seed);
1158
- xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ (XXH_readLE64(secret+8) - seed);
1159
- XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi, PRIME64_1);
1160
- xxh_u64 const lenContrib = XXH_mult32to64(len, PRIME32_5);
1161
- m128.low64 += lenContrib;
1162
- m128.high64 += input_hi * PRIME64_1;
1163
- m128.low64 ^= (m128.high64 >> 32);
1164
- { XXH128_hash_t h128 = XXH_mult64to128(m128.low64, PRIME64_2);
1165
- h128.high64 += m128.high64 * PRIME64_2;
1166
- h128.low64 = XXH3p_avalanche(h128.low64);
1167
- h128.high64 = XXH3p_avalanche(h128.high64);
1168
- return h128;
1169
- } }
1170
- }
1171
-
1172
- /* Assumption : `secret` size is >= 16
1173
- * Note : it should be >= XXH3p_SECRET_SIZE_MIN anyway */
1174
- XXH_FORCE_INLINE XXH128_hash_t
1175
- XXH3p_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
1176
- {
1177
- XXH_ASSERT(len <= 16);
1178
- { if (len > 8) return XXH3p_len_9to16_128b(input, len, secret, seed);
1179
- if (len >= 4) return XXH3p_len_4to8_128b(input, len, secret, seed);
1180
- if (len) return XXH3p_len_1to3_128b(input, len, secret, seed);
1181
- { XXH128_hash_t const h128 = { 0, 0 };
1182
- return h128;
1183
- } }
1184
- }
1185
-
1186
- XXH_FORCE_INLINE XXH128_hash_t
1187
- XXH3p_hashLong_128b_internal(const xxh_u8* XXH_RESTRICT input, size_t len,
1188
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize)
1189
- {
1190
- XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[ACC_NB] = XXH3p_INIT_ACC;
1191
-
1192
- XXH3p_hashLong_internal_loop(acc, input, len, secret, secretSize, XXH3p_acc_128bits);
1193
-
1194
- /* converge into final hash */
1195
- XXH_STATIC_ASSERT(sizeof(acc) == 64);
1196
- XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
1197
- { xxh_u64 const low64 = XXH3p_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * PRIME64_1);
1198
- xxh_u64 const high64 = XXH3p_mergeAccs(acc, secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((xxh_u64)len * PRIME64_2));
1199
- XXH128_hash_t const h128 = { low64, high64 };
1200
- return h128;
1201
- }
1202
- }
1203
-
1204
- XXH_NO_INLINE XXH128_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
1205
- XXH3p_hashLong_128b_defaultSecret(const xxh_u8* input, size_t len)
1206
- {
1207
- return XXH3p_hashLong_128b_internal(input, len, kSecret, sizeof(kSecret));
1208
- }
1209
-
1210
- XXH_NO_INLINE XXH128_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
1211
- XXH3p_hashLong_128b_withSecret(const xxh_u8* input, size_t len,
1212
- const xxh_u8* secret, size_t secretSize)
1213
- {
1214
- return XXH3p_hashLong_128b_internal(input, len, secret, secretSize);
1215
- }
1216
-
1217
- XXH_NO_INLINE XXH128_hash_t /* It's important for performance that XXH3p_hashLong is not inlined. Not sure why (uop cache maybe ?), but difference is large and easily measurable */
1218
- XXH3p_hashLong_128b_withSeed(const xxh_u8* input, size_t len, XXH64_hash_t seed)
1219
- {
1220
- XXH_ALIGN(8) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
1221
- if (seed == 0) return XXH3p_hashLong_128b_defaultSecret(input, len);
1222
- XXH3p_initCustomSecret(secret, seed);
1223
- return XXH3p_hashLong_128b_internal(input, len, secret, sizeof(secret));
1224
- }
1225
-
1226
-
1227
- XXH_FORCE_INLINE XXH128_hash_t
1228
- XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, const xxh_u8* secret, XXH64_hash_t seed)
1229
- {
1230
- acc.low64 += XXH3p_mix16B (input_1, secret+0, seed);
1231
- acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
1232
- acc.high64 += XXH3p_mix16B (input_2, secret+16, seed);
1233
- acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
1234
- return acc;
1235
- }
1236
-
1237
- XXH_NO_INLINE XXH128_hash_t
1238
- XXH3p_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
1239
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
1240
- XXH64_hash_t seed)
1241
- {
1242
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN); (void)secretSize;
1243
- XXH_ASSERT(128 < len && len <= XXH3p_MIDSIZE_MAX);
1244
-
1245
- { XXH128_hash_t acc;
1246
- int const nbRounds = (int)len / 32;
1247
- int i;
1248
- acc.low64 = len * PRIME64_1;
1249
- acc.high64 = 0;
1250
- for (i=0; i<4; i++) {
1251
- acc = XXH128_mix32B(acc, input+(32*i), input+(32*i)+16, secret+(32*i), seed);
1252
- }
1253
- acc.low64 = XXH3p_avalanche(acc.low64);
1254
- acc.high64 = XXH3p_avalanche(acc.high64);
1255
- XXH_ASSERT(nbRounds >= 4);
1256
- for (i=4 ; i < nbRounds; i++) {
1257
- acc = XXH128_mix32B(acc, input+(32*i), input+(32*i)+16, secret+XXH3p_MIDSIZE_STARTOFFSET+(32*(i-4)), seed);
1258
- }
1259
- /* last bytes */
1260
- acc = XXH128_mix32B(acc, input + len - 16, input + len - 32, secret + XXH3p_SECRET_SIZE_MIN - XXH3p_MIDSIZE_LASTOFFSET - 16, 0ULL - seed);
1261
-
1262
- { xxh_u64 const low64 = acc.low64 + acc.high64;
1263
- xxh_u64 const high64 = (acc.low64 * PRIME64_1) + (acc.high64 * PRIME64_4) + ((len - seed) * PRIME64_2);
1264
- XXH128_hash_t const h128 = { XXH3p_avalanche(low64), (XXH64_hash_t)0 - XXH3p_avalanche(high64) };
1265
- return h128;
1266
- }
1267
- }
1268
- }
1269
-
1270
-
1271
- XXH_FORCE_INLINE XXH128_hash_t
1272
- XXH3p_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
1273
- const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
1274
- XXH64_hash_t seed)
1275
- {
1276
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN); (void)secretSize;
1277
- XXH_ASSERT(16 < len && len <= 128);
1278
-
1279
- { XXH128_hash_t acc;
1280
- acc.low64 = len * PRIME64_1;
1281
- acc.high64 = 0;
1282
- if (len > 32) {
1283
- if (len > 64) {
1284
- if (len > 96) {
1285
- acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
1286
- }
1287
- acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
1288
- }
1289
- acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
1290
- }
1291
- acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
1292
- { xxh_u64 const low64 = acc.low64 + acc.high64;
1293
- xxh_u64 const high64 = (acc.low64 * PRIME64_1) + (acc.high64 * PRIME64_4) + ((len - seed) * PRIME64_2);
1294
- XXH128_hash_t const h128 = { XXH3p_avalanche(low64), (XXH64_hash_t)0 - XXH3p_avalanche(high64) };
1295
- return h128;
1296
- }
1297
- }
1298
- }
1299
-
1300
- XXH_PUBLIC_API XXH128_hash_t XXH3p_128bits(const void* input, size_t len)
1301
- {
1302
- if (len <= 16) return XXH3p_len_0to16_128b((const xxh_u8*)input, len, kSecret, 0);
1303
- if (len <= 128) return XXH3p_len_17to128_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0);
1304
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), 0);
1305
- return XXH3p_hashLong_128b_defaultSecret((const xxh_u8*)input, len);
1306
- }
1307
-
1308
- XXH_PUBLIC_API XXH128_hash_t
1309
- XXH3p_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
1310
- {
1311
- XXH_ASSERT(secretSize >= XXH3p_SECRET_SIZE_MIN);
1312
- /* if an action must be taken should `secret` conditions not be respected,
1313
- * it should be done here.
1314
- * For now, it's a contract pre-condition.
1315
- * Adding a check and a branch here would cost performance at every hash */
1316
- if (len <= 16) return XXH3p_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, 0);
1317
- if (len <= 128) return XXH3p_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0);
1318
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, 0);
1319
- return XXH3p_hashLong_128b_withSecret((const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize);
1320
- }
1321
-
1322
- XXH_PUBLIC_API XXH128_hash_t
1323
- XXH3p_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
1324
- {
1325
- if (len <= 16) return XXH3p_len_0to16_128b((const xxh_u8*)input, len, kSecret, seed);
1326
- if (len <= 128) return XXH3p_len_17to128_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed);
1327
- if (len <= XXH3p_MIDSIZE_MAX) return XXH3p_len_129to240_128b((const xxh_u8*)input, len, kSecret, sizeof(kSecret), seed);
1328
- return XXH3p_hashLong_128b_withSeed((const xxh_u8*)input, len, seed);
1329
- }
1330
-
1331
- XXH_PUBLIC_API XXH128_hash_t
1332
- XXH128(const void* input, size_t len, XXH64_hash_t seed)
1333
- {
1334
- return XXH3p_128bits_withSeed(input, len, seed);
1335
- }
1336
-
1337
-
1338
- /* === XXH3 128-bit streaming === */
1339
-
1340
- /* RocksDB Note: unused & removed due to bug in preview version */
1341
-
1342
- /* 128-bit utility functions */
1343
-
1344
- #include <string.h> /* memcmp */
1345
-
1346
- /* return : 1 is equal, 0 if different */
1347
- XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
1348
- {
1349
- /* note : XXH128_hash_t is compact, it has no padding byte */
1350
- return !(memcmp(&h1, &h2, sizeof(h1)));
1351
- }
1352
-
1353
- /* This prototype is compatible with stdlib's qsort().
1354
- * return : >0 if *h128_1 > *h128_2
1355
- * <0 if *h128_1 < *h128_2
1356
- * =0 if *h128_1 == *h128_2 */
1357
- XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
1358
- {
1359
- XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
1360
- XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
1361
- int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
1362
- /* note : bets that, in most cases, hash values are different */
1363
- if (hcmp) return hcmp;
1364
- return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
1365
- }
1366
-
1367
-
1368
- /*====== Canonical representation ======*/
1369
- XXH_PUBLIC_API void
1370
- XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
1371
- {
1372
- XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
1373
- if (XXH_CPU_LITTLE_ENDIAN) {
1374
- hash.high64 = XXH_swap64(hash.high64);
1375
- hash.low64 = XXH_swap64(hash.low64);
1376
- }
1377
- memcpy(dst, &hash.high64, sizeof(hash.high64));
1378
- memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
1379
- }
1380
-
1381
- XXH_PUBLIC_API XXH128_hash_t
1382
- XXH128_hashFromCanonical(const XXH128_canonical_t* src)
1383
- {
1384
- XXH128_hash_t h;
1385
- h.high64 = XXH_readBE64(src);
1386
- h.low64 = XXH_readBE64(src->digest + 8);
1387
- return h;
1388
- }
1389
-
1390
-
1391
-
1392
- #endif /* XXH3p_H */