@nxtedition/rocksdb 7.0.0-alpha.6 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/binding.cc +37 -36
  2. package/deps/rocksdb/rocksdb/CMakeLists.txt +10 -3
  3. package/deps/rocksdb/rocksdb/Makefile +8 -1
  4. package/deps/rocksdb/rocksdb/TARGETS +14 -0
  5. package/deps/rocksdb/rocksdb/cache/cache.cc +50 -2
  6. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +9 -3
  7. package/deps/rocksdb/rocksdb/cache/cache_test.cc +111 -33
  8. package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache_test.cc +71 -31
  9. package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc +31 -30
  10. package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h +21 -8
  11. package/deps/rocksdb/rocksdb/cache/lru_cache.cc +35 -38
  12. package/deps/rocksdb/rocksdb/cache/lru_cache.h +22 -9
  13. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +48 -0
  14. package/deps/rocksdb/rocksdb/db/blob/db_blob_compaction_test.cc +78 -0
  15. package/deps/rocksdb/rocksdb/db/builder.cc +7 -5
  16. package/deps/rocksdb/rocksdb/db/c.cc +777 -108
  17. package/deps/rocksdb/rocksdb/db/c_test.c +290 -30
  18. package/deps/rocksdb/rocksdb/db/column_family.cc +13 -0
  19. package/deps/rocksdb/rocksdb/db/column_family_test.cc +24 -36
  20. package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +18 -4
  21. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +24 -6
  22. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +6 -9
  23. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +38 -40
  24. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +4 -4
  25. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +14 -17
  26. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +3 -5
  27. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +253 -24
  28. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +9 -3
  29. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +3 -2
  30. package/deps/rocksdb/rocksdb/db/corruption_test.cc +67 -10
  31. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +83 -7
  32. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +5 -2
  33. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +68 -0
  34. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +40 -1
  35. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.cc +94 -23
  36. package/deps/rocksdb/rocksdb/db/db_impl/compacted_db_impl.h +17 -4
  37. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +263 -58
  38. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +186 -23
  39. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +43 -14
  40. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +24 -28
  41. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +116 -83
  42. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_readonly.cc +13 -5
  43. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +71 -34
  44. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.h +8 -3
  45. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +72 -33
  46. package/deps/rocksdb/rocksdb/db/db_readonly_with_timestamp_test.cc +629 -0
  47. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +438 -10
  48. package/deps/rocksdb/rocksdb/db/db_sst_test.cc +43 -2
  49. package/deps/rocksdb/rocksdb/db/db_test.cc +41 -1
  50. package/deps/rocksdb/rocksdb/db/db_test2.cc +41 -12
  51. package/deps/rocksdb/rocksdb/db/db_test_util.h +1 -0
  52. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +90 -0
  53. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +109 -16
  54. package/deps/rocksdb/rocksdb/db/dbformat.h +1 -1
  55. package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +54 -0
  56. package/deps/rocksdb/rocksdb/db/flush_job.cc +3 -3
  57. package/deps/rocksdb/rocksdb/db/log_reader.cc +22 -4
  58. package/deps/rocksdb/rocksdb/db/log_reader.h +4 -0
  59. package/deps/rocksdb/rocksdb/db/memtable.cc +4 -0
  60. package/deps/rocksdb/rocksdb/db/post_memtable_callback.h +25 -0
  61. package/deps/rocksdb/rocksdb/db/repair.cc +1 -1
  62. package/deps/rocksdb/rocksdb/db/repair_test.cc +3 -2
  63. package/deps/rocksdb/rocksdb/db/snapshot_impl.h +65 -2
  64. package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +3 -2
  65. package/deps/rocksdb/rocksdb/db/version_set.cc +52 -0
  66. package/deps/rocksdb/rocksdb/db/version_set.h +57 -43
  67. package/deps/rocksdb/rocksdb/db/wal_manager.cc +14 -4
  68. package/deps/rocksdb/rocksdb/db/wal_manager.h +16 -0
  69. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.cc +141 -0
  70. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization.h +55 -0
  71. package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc +292 -0
  72. package/deps/rocksdb/rocksdb/db/write_thread.h +6 -1
  73. package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc +2 -0
  74. package/deps/rocksdb/rocksdb/db_stress_tool/cf_consistency_stress.cc +42 -19
  75. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +28 -0
  76. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +6 -2
  77. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +11 -5
  78. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +18 -12
  79. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +74 -167
  80. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +4 -9
  81. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +16 -9
  82. package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +117 -10
  83. package/deps/rocksdb/rocksdb/env/composite_env.cc +7 -0
  84. package/deps/rocksdb/rocksdb/env/env.cc +4 -0
  85. package/deps/rocksdb/rocksdb/env/env_posix.cc +3 -3
  86. package/deps/rocksdb/rocksdb/env/env_test.cc +5 -5
  87. package/deps/rocksdb/rocksdb/env/file_system_tracer.cc +45 -0
  88. package/deps/rocksdb/rocksdb/env/file_system_tracer.h +14 -0
  89. package/deps/rocksdb/rocksdb/env/fs_posix.cc +1 -1
  90. package/deps/rocksdb/rocksdb/env/io_posix.cc +50 -24
  91. package/deps/rocksdb/rocksdb/env/io_posix.h +9 -7
  92. package/deps/rocksdb/rocksdb/env/mock_env.cc +9 -3
  93. package/deps/rocksdb/rocksdb/file/file_util.cc +4 -1
  94. package/deps/rocksdb/rocksdb/file/filename.cc +14 -0
  95. package/deps/rocksdb/rocksdb/file/line_file_reader.cc +9 -4
  96. package/deps/rocksdb/rocksdb/file/line_file_reader.h +3 -2
  97. package/deps/rocksdb/rocksdb/file/prefetch_test.cc +157 -0
  98. package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc +8 -1
  99. package/deps/rocksdb/rocksdb/file/sequence_file_reader.cc +68 -32
  100. package/deps/rocksdb/rocksdb/file/sequence_file_reader.h +20 -6
  101. package/deps/rocksdb/rocksdb/file/writable_file_writer.cc +10 -6
  102. package/deps/rocksdb/rocksdb/file/writable_file_writer.h +4 -2
  103. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +16 -0
  104. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +231 -2
  105. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +4 -2
  106. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +3 -0
  107. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +13 -1
  108. package/deps/rocksdb/rocksdb/include/rocksdb/io_status.h +4 -20
  109. package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +1 -1
  110. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +31 -0
  111. package/deps/rocksdb/rocksdb/include/rocksdb/snapshot.h +2 -0
  112. package/deps/rocksdb/rocksdb/include/rocksdb/status.h +4 -20
  113. package/deps/rocksdb/rocksdb/include/rocksdb/table.h +2 -2
  114. package/deps/rocksdb/rocksdb/include/rocksdb/trace_record.h +1 -0
  115. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd.h +1 -0
  116. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction.h +34 -0
  117. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/transaction_db.h +36 -0
  118. package/deps/rocksdb/rocksdb/include/rocksdb/wide_columns.h +74 -0
  119. package/deps/rocksdb/rocksdb/logging/auto_roll_logger.cc +36 -3
  120. package/deps/rocksdb/rocksdb/logging/auto_roll_logger_test.cc +16 -3
  121. package/deps/rocksdb/rocksdb/logging/env_logger.h +3 -3
  122. package/deps/rocksdb/rocksdb/logging/log_buffer.cc +2 -2
  123. package/deps/rocksdb/rocksdb/logging/log_buffer.h +1 -1
  124. package/deps/rocksdb/rocksdb/logging/posix_logger.h +3 -3
  125. package/deps/rocksdb/rocksdb/memory/arena.cc +0 -1
  126. package/deps/rocksdb/rocksdb/microbench/db_basic_bench.cc +61 -73
  127. package/deps/rocksdb/rocksdb/monitoring/histogram.cc +6 -5
  128. package/deps/rocksdb/rocksdb/monitoring/histogram_test.cc +6 -0
  129. package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +7 -3
  130. package/deps/rocksdb/rocksdb/options/cf_options.cc +6 -0
  131. package/deps/rocksdb/rocksdb/options/cf_options.h +3 -0
  132. package/deps/rocksdb/rocksdb/options/options.cc +4 -1
  133. package/deps/rocksdb/rocksdb/options/options_helper.cc +1 -0
  134. package/deps/rocksdb/rocksdb/options/options_parser.cc +1 -1
  135. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +1 -0
  136. package/deps/rocksdb/rocksdb/options/options_test.cc +4 -0
  137. package/deps/rocksdb/rocksdb/port/port_posix.h +0 -2
  138. package/deps/rocksdb/rocksdb/port/sys_time.h +27 -11
  139. package/deps/rocksdb/rocksdb/port/win/env_win.cc +1 -1
  140. package/deps/rocksdb/rocksdb/port/win/io_win.cc +16 -0
  141. package/deps/rocksdb/rocksdb/port/win/io_win.h +11 -2
  142. package/deps/rocksdb/rocksdb/port/win/port_win.cc +1 -1
  143. package/deps/rocksdb/rocksdb/port/win/port_win.h +2 -16
  144. package/deps/rocksdb/rocksdb/port/win/win_jemalloc.cc +2 -2
  145. package/deps/rocksdb/rocksdb/port/win/win_logger.cc +2 -2
  146. package/deps/rocksdb/rocksdb/rocksdb.pc.in +4 -5
  147. package/deps/rocksdb/rocksdb/src.mk +3 -0
  148. package/deps/rocksdb/rocksdb/table/block_based/block_based_filter_block.cc +7 -5
  149. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +39 -43
  150. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +2 -4
  151. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +42 -34
  152. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +1 -7
  153. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +2 -2
  154. package/deps/rocksdb/rocksdb/table/block_based/block_like_traits.h +2 -2
  155. package/deps/rocksdb/rocksdb/table/block_based/block_prefix_index.cc +7 -13
  156. package/deps/rocksdb/rocksdb/table/block_based/block_prefix_index.h +9 -5
  157. package/deps/rocksdb/rocksdb/table/block_based/block_type.h +5 -2
  158. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc +4 -4
  159. package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.h +6 -2
  160. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +8 -5
  161. package/deps/rocksdb/rocksdb/table/block_based/hash_index_reader.cc +2 -2
  162. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +19 -14
  163. package/deps/rocksdb/rocksdb/table/block_fetcher.cc +2 -0
  164. package/deps/rocksdb/rocksdb/table/format.h +1 -3
  165. package/deps/rocksdb/rocksdb/table/get_context.cc +5 -0
  166. package/deps/rocksdb/rocksdb/table/multiget_context.h +3 -0
  167. package/deps/rocksdb/rocksdb/table/scoped_arena_iterator.h +3 -4
  168. package/deps/rocksdb/rocksdb/table/table_test.cc +1 -1
  169. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +102 -6
  170. package/deps/rocksdb/rocksdb/tools/db_bench_tool_test.cc +1 -0
  171. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +19 -2
  172. package/deps/rocksdb/rocksdb/tools/trace_analyzer_test.cc +2 -1
  173. package/deps/rocksdb/rocksdb/tools/trace_analyzer_tool.cc +2 -1
  174. package/deps/rocksdb/rocksdb/util/aligned_buffer.h +2 -4
  175. package/deps/rocksdb/rocksdb/util/autovector.h +11 -1
  176. package/deps/rocksdb/rocksdb/util/cleanable.cc +1 -0
  177. package/deps/rocksdb/rocksdb/util/compression.h +5 -7
  178. package/deps/rocksdb/rocksdb/util/file_reader_writer_test.cc +14 -8
  179. package/deps/rocksdb/rocksdb/util/string_util.cc +1 -1
  180. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +33 -63
  181. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +1 -1
  182. package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc +3 -2
  183. package/deps/rocksdb/rocksdb/utilities/counted_fs.cc +14 -0
  184. package/deps/rocksdb/rocksdb/utilities/counted_fs.h +7 -1
  185. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.cc +7 -0
  186. package/deps/rocksdb/rocksdb/utilities/fault_injection_env.h +1 -0
  187. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +8 -0
  188. package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +3 -0
  189. package/deps/rocksdb/rocksdb/utilities/persistent_cache/hash_table_bench.cc +6 -4
  190. package/deps/rocksdb/rocksdb/utilities/persistent_cache/volatile_tier_impl.h +2 -3
  191. package/deps/rocksdb/rocksdb/utilities/transactions/optimistic_transaction_test.cc +34 -21
  192. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.cc +31 -7
  193. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction.h +1 -0
  194. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +63 -0
  195. package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +40 -0
  196. package/deps/rocksdb/rocksdb/utilities/transactions/timestamped_snapshot_test.cc +426 -0
  197. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.cc +37 -0
  198. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_base.h +6 -0
  199. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +16 -18
  200. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +18 -0
  201. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +61 -0
  202. package/deps/rocksdb/rocksdb.gyp +1 -0
  203. package/index.js +5 -2
  204. package/package.json +1 -1
  205. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  206. package/prebuilds/darwin-x64/node.napi.node +0 -0
  207. package/prebuilds/linux-x64/node.napi.node +0 -0
@@ -0,0 +1,292 @@
1
+ // Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ // This source code is licensed under both the GPLv2 (found in the
3
+ // COPYING file in the root directory) and Apache 2.0 License
4
+ // (found in the LICENSE.Apache file in the root directory).
5
+
6
+ #include "db/wide/wide_column_serialization.h"
7
+
8
+ #include "test_util/testharness.h"
9
+ #include "util/coding.h"
10
+
11
+ namespace ROCKSDB_NAMESPACE {
12
+
13
+ TEST(WideColumnSerializationTest, Construct) {
14
+ constexpr char foo[] = "foo";
15
+ constexpr char bar[] = "bar";
16
+
17
+ const std::string foo_str(foo);
18
+ const std::string bar_str(bar);
19
+
20
+ const Slice foo_slice(foo_str);
21
+ const Slice bar_slice(bar_str);
22
+
23
+ {
24
+ WideColumn column(foo, bar);
25
+ ASSERT_EQ(column.name(), foo);
26
+ ASSERT_EQ(column.value(), bar);
27
+ }
28
+
29
+ {
30
+ WideColumn column(foo_str, bar);
31
+ ASSERT_EQ(column.name(), foo_str);
32
+ ASSERT_EQ(column.value(), bar);
33
+ }
34
+
35
+ {
36
+ WideColumn column(foo_slice, bar);
37
+ ASSERT_EQ(column.name(), foo_slice);
38
+ ASSERT_EQ(column.value(), bar);
39
+ }
40
+
41
+ {
42
+ WideColumn column(foo, bar_str);
43
+ ASSERT_EQ(column.name(), foo);
44
+ ASSERT_EQ(column.value(), bar_str);
45
+ }
46
+
47
+ {
48
+ WideColumn column(foo_str, bar_str);
49
+ ASSERT_EQ(column.name(), foo_str);
50
+ ASSERT_EQ(column.value(), bar_str);
51
+ }
52
+
53
+ {
54
+ WideColumn column(foo_slice, bar_str);
55
+ ASSERT_EQ(column.name(), foo_slice);
56
+ ASSERT_EQ(column.value(), bar_str);
57
+ }
58
+
59
+ {
60
+ WideColumn column(foo, bar_slice);
61
+ ASSERT_EQ(column.name(), foo);
62
+ ASSERT_EQ(column.value(), bar_slice);
63
+ }
64
+
65
+ {
66
+ WideColumn column(foo_str, bar_slice);
67
+ ASSERT_EQ(column.name(), foo_str);
68
+ ASSERT_EQ(column.value(), bar_slice);
69
+ }
70
+
71
+ {
72
+ WideColumn column(foo_slice, bar_slice);
73
+ ASSERT_EQ(column.name(), foo_slice);
74
+ ASSERT_EQ(column.value(), bar_slice);
75
+ }
76
+
77
+ {
78
+ constexpr char foo_name[] = "foo_name";
79
+ constexpr char bar_value[] = "bar_value";
80
+
81
+ WideColumn column(std::piecewise_construct,
82
+ std::forward_as_tuple(foo_name, sizeof(foo) - 1),
83
+ std::forward_as_tuple(bar_value, sizeof(bar) - 1));
84
+ ASSERT_EQ(column.name(), foo);
85
+ ASSERT_EQ(column.value(), bar);
86
+ }
87
+ }
88
+
89
+ TEST(WideColumnSerializationTest, SerializeDeserialize) {
90
+ WideColumns columns{{"foo", "bar"}, {"hello", "world"}};
91
+ std::string output;
92
+
93
+ ASSERT_OK(WideColumnSerialization::Serialize(columns, output));
94
+
95
+ Slice input(output);
96
+ WideColumns deserialized_columns;
97
+
98
+ ASSERT_OK(WideColumnSerialization::Deserialize(input, deserialized_columns));
99
+ ASSERT_EQ(columns, deserialized_columns);
100
+
101
+ {
102
+ const auto it = WideColumnSerialization::Find(deserialized_columns, "foo");
103
+ ASSERT_NE(it, deserialized_columns.cend());
104
+ ASSERT_EQ(*it, deserialized_columns.front());
105
+ }
106
+
107
+ {
108
+ const auto it =
109
+ WideColumnSerialization::Find(deserialized_columns, "hello");
110
+ ASSERT_NE(it, deserialized_columns.cend());
111
+ ASSERT_EQ(*it, deserialized_columns.back());
112
+ }
113
+
114
+ {
115
+ const auto it =
116
+ WideColumnSerialization::Find(deserialized_columns, "fubar");
117
+ ASSERT_EQ(it, deserialized_columns.cend());
118
+ }
119
+
120
+ {
121
+ const auto it =
122
+ WideColumnSerialization::Find(deserialized_columns, "snafu");
123
+ ASSERT_EQ(it, deserialized_columns.cend());
124
+ }
125
+ }
126
+
127
+ TEST(WideColumnSerializationTest, DeserializeVersionError) {
128
+ // Can't decode version
129
+
130
+ std::string buf;
131
+
132
+ Slice input(buf);
133
+ WideColumns columns;
134
+
135
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
136
+ ASSERT_TRUE(s.IsCorruption());
137
+ ASSERT_TRUE(std::strstr(s.getState(), "version"));
138
+ }
139
+
140
+ TEST(WideColumnSerializationTest, DeserializeUnsupportedVersion) {
141
+ // Unsupported version
142
+ constexpr uint32_t future_version = 1000;
143
+
144
+ std::string buf;
145
+ PutVarint32(&buf, future_version);
146
+
147
+ Slice input(buf);
148
+ WideColumns columns;
149
+
150
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
151
+ ASSERT_TRUE(s.IsNotSupported());
152
+ ASSERT_TRUE(std::strstr(s.getState(), "version"));
153
+ }
154
+
155
+ TEST(WideColumnSerializationTest, DeserializeNumberOfColumnsError) {
156
+ // Can't decode number of columns
157
+
158
+ std::string buf;
159
+ PutVarint32(&buf, WideColumnSerialization::kCurrentVersion);
160
+
161
+ Slice input(buf);
162
+ WideColumns columns;
163
+
164
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
165
+ ASSERT_TRUE(s.IsCorruption());
166
+ ASSERT_TRUE(std::strstr(s.getState(), "number"));
167
+ }
168
+
169
+ TEST(WideColumnSerializationTest, DeserializeColumnsError) {
170
+ std::string buf;
171
+
172
+ PutVarint32(&buf, WideColumnSerialization::kCurrentVersion);
173
+
174
+ constexpr uint32_t num_columns = 2;
175
+ PutVarint32(&buf, num_columns);
176
+
177
+ // Can't decode the first column name
178
+ {
179
+ Slice input(buf);
180
+ WideColumns columns;
181
+
182
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
183
+ ASSERT_TRUE(s.IsCorruption());
184
+ ASSERT_TRUE(std::strstr(s.getState(), "name"));
185
+ }
186
+
187
+ constexpr char first_column_name[] = "foo";
188
+ PutLengthPrefixedSlice(&buf, first_column_name);
189
+
190
+ // Can't decode the size of the first column value
191
+ {
192
+ Slice input(buf);
193
+ WideColumns columns;
194
+
195
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
196
+ ASSERT_TRUE(s.IsCorruption());
197
+ ASSERT_TRUE(std::strstr(s.getState(), "value size"));
198
+ }
199
+
200
+ constexpr uint32_t first_value_size = 16;
201
+ PutVarint32(&buf, first_value_size);
202
+
203
+ // Can't decode the second column name
204
+ {
205
+ Slice input(buf);
206
+ WideColumns columns;
207
+
208
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
209
+ ASSERT_TRUE(s.IsCorruption());
210
+ ASSERT_TRUE(std::strstr(s.getState(), "name"));
211
+ }
212
+
213
+ constexpr char second_column_name[] = "hello";
214
+ PutLengthPrefixedSlice(&buf, second_column_name);
215
+
216
+ // Can't decode the size of the second column value
217
+ {
218
+ Slice input(buf);
219
+ WideColumns columns;
220
+
221
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
222
+ ASSERT_TRUE(s.IsCorruption());
223
+ ASSERT_TRUE(std::strstr(s.getState(), "value size"));
224
+ }
225
+
226
+ constexpr uint32_t second_value_size = 64;
227
+ PutVarint32(&buf, second_value_size);
228
+
229
+ // Can't decode the payload of the first column
230
+ {
231
+ Slice input(buf);
232
+ WideColumns columns;
233
+
234
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
235
+ ASSERT_TRUE(s.IsCorruption());
236
+ ASSERT_TRUE(std::strstr(s.getState(), "payload"));
237
+ }
238
+
239
+ buf.append(first_value_size, '0');
240
+
241
+ // Can't decode the payload of the second column
242
+ {
243
+ Slice input(buf);
244
+ WideColumns columns;
245
+
246
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
247
+ ASSERT_TRUE(s.IsCorruption());
248
+ ASSERT_TRUE(std::strstr(s.getState(), "payload"));
249
+ }
250
+
251
+ buf.append(second_value_size, 'x');
252
+
253
+ // Success
254
+ {
255
+ Slice input(buf);
256
+ WideColumns columns;
257
+
258
+ ASSERT_OK(WideColumnSerialization::Deserialize(input, columns));
259
+ }
260
+ }
261
+
262
+ TEST(WideColumnSerializationTest, DeserializeColumnsOutOfOrder) {
263
+ std::string buf;
264
+
265
+ PutVarint32(&buf, WideColumnSerialization::kCurrentVersion);
266
+
267
+ constexpr uint32_t num_columns = 2;
268
+ PutVarint32(&buf, num_columns);
269
+
270
+ constexpr char first_column_name[] = "b";
271
+ PutLengthPrefixedSlice(&buf, first_column_name);
272
+
273
+ constexpr uint32_t first_value_size = 16;
274
+ PutVarint32(&buf, first_value_size);
275
+
276
+ constexpr char second_column_name[] = "a";
277
+ PutLengthPrefixedSlice(&buf, second_column_name);
278
+
279
+ Slice input(buf);
280
+ WideColumns columns;
281
+
282
+ const Status s = WideColumnSerialization::Deserialize(input, columns);
283
+ ASSERT_TRUE(s.IsCorruption());
284
+ ASSERT_TRUE(std::strstr(s.getState(), "order"));
285
+ }
286
+
287
+ } // namespace ROCKSDB_NAMESPACE
288
+
289
+ int main(int argc, char** argv) {
290
+ ::testing::InitGoogleTest(&argc, argv);
291
+ return RUN_ALL_TESTS();
292
+ }
@@ -15,6 +15,7 @@
15
15
  #include <vector>
16
16
 
17
17
  #include "db/dbformat.h"
18
+ #include "db/post_memtable_callback.h"
18
19
  #include "db/pre_release_callback.h"
19
20
  #include "db/write_callback.h"
20
21
  #include "monitoring/instrumented_mutex.h"
@@ -122,6 +123,7 @@ class WriteThread {
122
123
  size_t batch_cnt; // if non-zero, number of sub-batches in the write batch
123
124
  size_t protection_bytes_per_key;
124
125
  PreReleaseCallback* pre_release_callback;
126
+ PostMemTableCallback* post_memtable_callback;
125
127
  uint64_t log_used; // log number that this batch was inserted into
126
128
  uint64_t log_ref; // log number that memtable insert should reference
127
129
  WriteCallback* callback;
@@ -147,6 +149,7 @@ class WriteThread {
147
149
  batch_cnt(0),
148
150
  protection_bytes_per_key(0),
149
151
  pre_release_callback(nullptr),
152
+ post_memtable_callback(nullptr),
150
153
  log_used(0),
151
154
  log_ref(0),
152
155
  callback(nullptr),
@@ -160,7 +163,8 @@ class WriteThread {
160
163
  Writer(const WriteOptions& write_options, WriteBatch* _batch,
161
164
  WriteCallback* _callback, uint64_t _log_ref, bool _disable_memtable,
162
165
  size_t _batch_cnt = 0,
163
- PreReleaseCallback* _pre_release_callback = nullptr)
166
+ PreReleaseCallback* _pre_release_callback = nullptr,
167
+ PostMemTableCallback* _post_memtable_callback = nullptr)
164
168
  : batch(_batch),
165
169
  sync(write_options.sync),
166
170
  no_slowdown(write_options.no_slowdown),
@@ -170,6 +174,7 @@ class WriteThread {
170
174
  batch_cnt(_batch_cnt),
171
175
  protection_bytes_per_key(_batch->GetProtectionBytesPerKey()),
172
176
  pre_release_callback(_pre_release_callback),
177
+ post_memtable_callback(_post_memtable_callback),
173
178
  log_used(0),
174
179
  log_ref(_log_ref),
175
180
  callback(_callback),
@@ -340,6 +340,8 @@ class BatchedOpsStressTest : public StressTest {
340
340
  }
341
341
 
342
342
  void VerifyDb(ThreadState* /* thread */) const override {}
343
+
344
+ void ContinuouslyVerifyDb(ThreadState* /* thread */) const override {}
343
345
  };
344
346
 
345
347
  StressTest* CreateBatchedOpsStressTest() { return new BatchedOpsStressTest(); }
@@ -448,21 +448,24 @@ class CfConsistencyStressTest : public StressTest {
448
448
 
449
449
  DB* db_ptr = cmp_db_ ? cmp_db_ : db_;
450
450
  const auto& cfhs = cmp_db_ ? cmp_cfhs_ : column_families_;
451
- const auto ss_deleter = [&](const Snapshot* ss) {
452
- db_ptr->ReleaseSnapshot(ss);
453
- };
454
- std::unique_ptr<const Snapshot, decltype(ss_deleter)> snapshot_guard(
455
- db_ptr->GetSnapshot(), ss_deleter);
456
- if (cmp_db_) {
457
- status = cmp_db_->TryCatchUpWithPrimary();
458
- }
451
+
452
+ // Take a snapshot to preserve the state of primary db.
453
+ ManagedSnapshot snapshot_guard(db_);
454
+
459
455
  SharedState* shared = thread->shared;
460
456
  assert(shared);
461
- if (!status.ok()) {
462
- shared->SetShouldStopTest();
463
- return;
457
+
458
+ if (cmp_db_) {
459
+ status = cmp_db_->TryCatchUpWithPrimary();
460
+ if (!status.ok()) {
461
+ fprintf(stderr, "TryCatchUpWithPrimary: %s\n",
462
+ status.ToString().c_str());
463
+ shared->SetShouldStopTest();
464
+ assert(false);
465
+ return;
466
+ }
464
467
  }
465
- assert(cmp_db_ || snapshot_guard.get());
468
+
466
469
  const auto checksum_column_family = [](Iterator* iter,
467
470
  uint32_t* checksum) -> Status {
468
471
  assert(nullptr != checksum);
@@ -476,17 +479,29 @@ class CfConsistencyStressTest : public StressTest {
476
479
  };
477
480
  // This `ReadOptions` is for validation purposes. Ignore
478
481
  // `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
479
- ReadOptions ropts;
482
+ ReadOptions ropts(FLAGS_verify_checksum, true);
480
483
  ropts.total_order_seek = true;
481
- ropts.snapshot = snapshot_guard.get();
484
+ if (nullptr == cmp_db_) {
485
+ ropts.snapshot = snapshot_guard.snapshot();
486
+ }
482
487
  uint32_t crc = 0;
483
488
  {
484
489
  // Compute crc for all key-values of default column family.
485
490
  std::unique_ptr<Iterator> it(db_ptr->NewIterator(ropts));
486
491
  status = checksum_column_family(it.get(), &crc);
492
+ if (!status.ok()) {
493
+ fprintf(stderr, "Computing checksum of default cf: %s\n",
494
+ status.ToString().c_str());
495
+ assert(false);
496
+ }
487
497
  }
488
- uint32_t tmp_crc = 0;
489
- if (status.ok()) {
498
+ // Since we currently intentionally disallow reading from the secondary
499
+ // instance with snapshot, we cannot achieve cross-cf consistency if WAL is
500
+ // enabled because there is no guarantee that secondary instance replays
501
+ // the primary's WAL to a consistent point where all cfs have the same
502
+ // data.
503
+ if (status.ok() && FLAGS_disable_wal) {
504
+ uint32_t tmp_crc = 0;
490
505
  for (ColumnFamilyHandle* cfh : cfhs) {
491
506
  if (cfh == db_ptr->DefaultColumnFamily()) {
492
507
  continue;
@@ -497,11 +512,19 @@ class CfConsistencyStressTest : public StressTest {
497
512
  break;
498
513
  }
499
514
  }
500
- }
501
- if (!status.ok() || tmp_crc != crc) {
502
- shared->SetShouldStopTest();
515
+ if (!status.ok()) {
516
+ fprintf(stderr, "status: %s\n", status.ToString().c_str());
517
+ shared->SetShouldStopTest();
518
+ assert(false);
519
+ } else if (tmp_crc != crc) {
520
+ fprintf(stderr, "tmp_crc=%" PRIu32 " crc=%" PRIu32 "\n", tmp_crc, crc);
521
+ shared->SetShouldStopTest();
522
+ assert(false);
523
+ }
503
524
  }
504
525
  }
526
+ #else // ROCKSDB_LITE
527
+ void ContinuouslyVerifyDb(ThreadState* /*thread*/) const override {}
505
528
  #endif // !ROCKSDB_LITE
506
529
 
507
530
  std::vector<int> GenerateColumnFamilies(
@@ -148,6 +148,34 @@ void DbVerificationThread(void* v) {
148
148
  }
149
149
  }
150
150
 
151
+ void SnapshotGcThread(void* v) {
152
+ assert(FLAGS_create_timestamped_snapshot_one_in > 0);
153
+ auto* thread = reinterpret_cast<ThreadState*>(v);
154
+ assert(thread);
155
+ SharedState* shared = thread->shared;
156
+ assert(shared);
157
+ StressTest* stress_test = shared->GetStressTest();
158
+ assert(stress_test);
159
+ while (true) {
160
+ {
161
+ MutexLock l(shared->GetMutex());
162
+ if (shared->ShouldStopBgThread()) {
163
+ shared->IncBgThreadsFinished();
164
+ if (shared->BgThreadsFinished()) {
165
+ shared->GetCondVar()->SignalAll();
166
+ }
167
+ return;
168
+ }
169
+ }
170
+
171
+ uint64_t now = db_stress_env->NowNanos();
172
+ constexpr uint64_t time_diff = static_cast<uint64_t>(1000) * 1000 * 1000;
173
+ stress_test->ReleaseOldTimestampedSnapshots(now - time_diff);
174
+
175
+ db_stress_env->SleepForMicroseconds(1000 * 1000);
176
+ }
177
+ }
178
+
151
179
  void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
152
180
  if (!FLAGS_verbose) {
153
181
  return;
@@ -142,7 +142,7 @@ DECLARE_bool(charge_table_reader);
142
142
  DECLARE_int32(top_level_index_pinning);
143
143
  DECLARE_int32(partition_pinning);
144
144
  DECLARE_int32(unpartitioned_pinning);
145
- DECLARE_bool(use_clock_cache);
145
+ DECLARE_string(cache_type);
146
146
  DECLARE_uint64(subcompactions);
147
147
  DECLARE_uint64(periodic_compaction_seconds);
148
148
  DECLARE_uint64(compaction_ttl);
@@ -229,7 +229,6 @@ DECLARE_uint64(ops_per_thread);
229
229
  DECLARE_uint64(log2_keys_per_lock);
230
230
  DECLARE_uint64(max_manifest_file_size);
231
231
  DECLARE_bool(in_place_update);
232
- DECLARE_int32(secondary_catch_up_one_in);
233
232
  DECLARE_string(memtablerep);
234
233
  DECLARE_int32(prefix_size);
235
234
  DECLARE_bool(use_merge);
@@ -266,6 +265,7 @@ DECLARE_bool(enable_blob_garbage_collection);
266
265
  DECLARE_double(blob_garbage_collection_age_cutoff);
267
266
  DECLARE_double(blob_garbage_collection_force_threshold);
268
267
  DECLARE_uint64(blob_compaction_readahead_size);
268
+ DECLARE_int32(blob_file_starting_level);
269
269
 
270
270
  DECLARE_int32(approximate_size_one_in);
271
271
  DECLARE_bool(sync_fault_injection);
@@ -295,6 +295,8 @@ DECLARE_bool(async_io);
295
295
  DECLARE_string(wal_compression);
296
296
  DECLARE_bool(verify_sst_unique_id_in_manifest);
297
297
 
298
+ DECLARE_int32(create_timestamped_snapshot_one_in);
299
+
298
300
  constexpr long KB = 1024;
299
301
  constexpr int kRandomValueMaxFactor = 3;
300
302
  constexpr int kValueMaxLen = 100;
@@ -583,6 +585,8 @@ extern void PoolSizeChangeThread(void* v);
583
585
 
584
586
  extern void DbVerificationThread(void* v);
585
587
 
588
+ extern void SnapshotGcThread(void* v);
589
+
586
590
  extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz);
587
591
 
588
592
  extern int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration);
@@ -84,6 +84,10 @@ bool RunStressTest(StressTest* stress) {
84
84
  shared.IncBgThreads();
85
85
  }
86
86
 
87
+ if (FLAGS_create_timestamped_snapshot_one_in > 0) {
88
+ shared.IncBgThreads();
89
+ }
90
+
87
91
  std::vector<ThreadState*> threads(n);
88
92
  for (uint32_t i = 0; i < n; i++) {
89
93
  threads[i] = new ThreadState(i, &shared);
@@ -101,6 +105,11 @@ bool RunStressTest(StressTest* stress) {
101
105
  &continuous_verification_thread);
102
106
  }
103
107
 
108
+ ThreadState snapshots_gc_thread(0, &shared);
109
+ if (FLAGS_create_timestamped_snapshot_one_in > 0) {
110
+ db_stress_env->StartThread(SnapshotGcThread, &snapshots_gc_thread);
111
+ }
112
+
104
113
  // Each thread goes through the following states:
105
114
  // initializing -> wait for others to init -> read/populate/depopulate
106
115
  // wait for others to operate -> verify -> done
@@ -169,7 +178,8 @@ bool RunStressTest(StressTest* stress) {
169
178
  stress->PrintStatistics();
170
179
 
171
180
  if (FLAGS_compaction_thread_pool_adjust_interval > 0 ||
172
- FLAGS_continuous_verification_interval > 0) {
181
+ FLAGS_continuous_verification_interval > 0 ||
182
+ FLAGS_create_timestamped_snapshot_one_in > 0) {
173
183
  MutexLock l(shared.GetMutex());
174
184
  shared.SetShouldStopBgThread();
175
185
  while (!shared.BgThreadsFinished()) {
@@ -177,10 +187,6 @@ bool RunStressTest(StressTest* stress) {
177
187
  }
178
188
  }
179
189
 
180
- if (!stress->VerifySecondaries()) {
181
- return false;
182
- }
183
-
184
190
  if (shared.HasVerificationFailedYet()) {
185
191
  fprintf(stderr, "Verification failed :(\n");
186
192
  return false;
@@ -39,8 +39,8 @@ DEFINE_string(key_len_percent_dist, "",
39
39
  "24 bytes. If not specified, it will be evenly distributed");
40
40
 
41
41
  DEFINE_int32(key_window_scale_factor, 10,
42
- "This value will be multiplied by 100 to come up with a window "
43
- "size for varying the key length");
42
+ "This value will be multiplied by 100 to come up with a window "
43
+ "size for varying the key length");
44
44
 
45
45
  DEFINE_int32(column_families, 10, "Number of column families");
46
46
 
@@ -343,8 +343,7 @@ DEFINE_int32(
343
343
  "Type of pinning for unpartitioned metadata blocks (see `enum PinningTier` "
344
344
  "in table.h)");
345
345
 
346
- DEFINE_bool(use_clock_cache, false,
347
- "Replace default LRU block cache with clock cache.");
346
+ DEFINE_string(cache_type, "lru_cache", "Type of block cache.");
348
347
 
349
348
  DEFINE_uint64(subcompactions, 1,
350
349
  "Maximum number of subcompactions to divide L0-L1 compactions "
@@ -440,6 +439,12 @@ DEFINE_uint64(blob_compaction_readahead_size,
440
439
  .blob_compaction_readahead_size,
441
440
  "[Integrated BlobDB] Compaction readahead for blob files.");
442
441
 
442
+ DEFINE_int32(
443
+ blob_file_starting_level,
444
+ ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().blob_file_starting_level,
445
+ "[Integrated BlobDB] Enable writing blob files during flushes and "
446
+ "compactions starting from the specified level.");
447
+
443
448
  static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
444
449
  RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);
445
450
 
@@ -495,7 +500,9 @@ DEFINE_string(db, "", "Use the db with the following name.");
495
500
  DEFINE_string(secondaries_base, "",
496
501
  "Use this path as the base path for secondary instances.");
497
502
 
498
- DEFINE_bool(test_secondary, false, "Test secondary instance.");
503
+ DEFINE_bool(test_secondary, false,
504
+ "If true, start an additional secondary instance which can be used "
505
+ "for verification.");
499
506
 
500
507
  DEFINE_string(
501
508
  expected_values_dir, "",
@@ -631,7 +638,7 @@ DEFINE_int32(ingest_external_file_one_in, 0,
631
638
  "every N operations on average. 0 indicates IngestExternalFile() "
632
639
  "is disabled.");
633
640
 
634
- DEFINE_int32(ingest_external_file_width, 1000,
641
+ DEFINE_int32(ingest_external_file_width, 100,
635
642
  "The width of the ingested external files.");
636
643
 
637
644
  DEFINE_int32(compact_files_one_in, 0,
@@ -785,11 +792,6 @@ DEFINE_uint64(max_manifest_file_size, 16384, "Maximum size of a MANIFEST file");
785
792
 
786
793
  DEFINE_bool(in_place_update, false, "On true, does inplace update in memtable");
787
794
 
788
- DEFINE_int32(secondary_catch_up_one_in, 0,
789
- "If non-zero, the secondaries attemp to catch up with the primary "
790
- "once for every N operations on average. 0 indicates the "
791
- "secondaries do not try to catch up after open.");
792
-
793
795
  DEFINE_string(memtablerep, "skip_list", "");
794
796
 
795
797
  inline static bool ValidatePrefixSize(const char* flagname, int32_t value) {
@@ -858,7 +860,7 @@ DEFINE_int32(approximate_size_one_in, 64,
858
860
  " random key ranges.");
859
861
 
860
862
  DEFINE_int32(read_fault_one_in, 1000,
861
- "On non-zero, enables fault injection on read");
863
+ "On non-zero, enables fault injection on read");
862
864
 
863
865
  DEFINE_int32(get_property_one_in, 1000,
864
866
  "If non-zero, then DB::GetProperty() will be called to get various"
@@ -965,4 +967,8 @@ DEFINE_bool(
965
967
  "DB-open try verifying the SST unique id between MANIFEST and SST "
966
968
  "properties.");
967
969
 
970
+ DEFINE_int32(
971
+ create_timestamped_snapshot_one_in, 0,
972
+ "On non-zero, create timestamped snapshots upon transaction commits.");
973
+
968
974
  #endif // GFLAGS