@nxtedition/rocksdb 15.1.2 → 15.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/binding.cc +79 -38
  2. package/build.sh +1 -2
  3. package/deps/rocksdb/rocksdb/BUCK +10 -8
  4. package/deps/rocksdb/rocksdb/CMakeLists.txt +27 -2
  5. package/deps/rocksdb/rocksdb/Makefile +27 -116
  6. package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +1 -1
  7. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +101 -124
  8. package/deps/rocksdb/rocksdb/cache/clock_cache.h +47 -30
  9. package/deps/rocksdb/rocksdb/db/c.cc +793 -131
  10. package/deps/rocksdb/rocksdb/db/c_test.c +571 -0
  11. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +226 -0
  12. package/deps/rocksdb/rocksdb/db/compaction/compaction.h +4 -0
  13. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +95 -59
  14. package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -2
  15. package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +45 -35
  16. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +8 -4
  17. package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +1 -1
  18. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +11 -6
  19. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +8 -2
  20. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +47 -0
  21. package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +12 -2
  22. package/deps/rocksdb/rocksdb/db/compaction/compaction_service_test.cc +82 -0
  23. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.cc +2 -2
  24. package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +1 -1
  25. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +69 -24
  26. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +9 -1
  27. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +65 -0
  28. package/deps/rocksdb/rocksdb/db/db_etc3_test.cc +161 -0
  29. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -0
  30. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +20 -7
  31. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +13 -0
  32. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +114 -39
  33. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +3 -0
  34. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_follower.cc +3 -3
  35. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +1 -1
  36. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_secondary.cc +39 -25
  37. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +361 -0
  38. package/deps/rocksdb/rocksdb/db/db_options_test.cc +35 -0
  39. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +83 -0
  40. package/deps/rocksdb/rocksdb/db/db_test.cc +249 -4
  41. package/deps/rocksdb/rocksdb/db/db_test2.cc +3 -0
  42. package/deps/rocksdb/rocksdb/db/db_test_util.cc +2 -1
  43. package/deps/rocksdb/rocksdb/db/db_wal_test.cc +3 -2
  44. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +7 -7
  45. package/deps/rocksdb/rocksdb/db/listener_test.cc +7 -17
  46. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +4 -2
  47. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +41 -0
  48. package/deps/rocksdb/rocksdb/db/repair.cc +2 -2
  49. package/deps/rocksdb/rocksdb/db/version_edit.h +7 -4
  50. package/deps/rocksdb/rocksdb/db/version_set.cc +299 -90
  51. package/deps/rocksdb/rocksdb/db/version_set.h +56 -9
  52. package/deps/rocksdb/rocksdb/db/version_set_test.cc +41 -39
  53. package/deps/rocksdb/rocksdb/db/version_util.h +3 -2
  54. package/deps/rocksdb/rocksdb/db/wal_manager.cc +7 -1
  55. package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +48 -10
  56. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +1 -0
  57. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +5 -1
  58. package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +16 -5
  59. package/deps/rocksdb/rocksdb/env/env_test.cc +126 -41
  60. package/deps/rocksdb/rocksdb/env/fs_posix.cc +14 -7
  61. package/deps/rocksdb/rocksdb/env/io_posix.cc +304 -112
  62. package/deps/rocksdb/rocksdb/env/io_posix.h +16 -4
  63. package/deps/rocksdb/rocksdb/env/io_posix_test.cc +43 -0
  64. package/deps/rocksdb/rocksdb/folly.mk +148 -0
  65. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_compression.h +29 -3
  66. package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h +73 -0
  67. package/deps/rocksdb/rocksdb/include/rocksdb/c.h +246 -0
  68. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +0 -2
  69. package/deps/rocksdb/rocksdb/include/rocksdb/data_structure.h +15 -9
  70. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +19 -9
  71. package/deps/rocksdb/rocksdb/include/rocksdb/env.h +1 -1
  72. package/deps/rocksdb/rocksdb/include/rocksdb/file_system.h +6 -4
  73. package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +14 -0
  74. package/deps/rocksdb/rocksdb/include/rocksdb/options.h +67 -6
  75. package/deps/rocksdb/rocksdb/include/rocksdb/sst_file_writer.h +1 -7
  76. package/deps/rocksdb/rocksdb/include/rocksdb/statistics.h +3 -0
  77. package/deps/rocksdb/rocksdb/include/rocksdb/thread_status.h +6 -14
  78. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +8 -1
  79. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/env_mirror.h +2 -2
  80. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/ldb_cmd_execute_result.h +0 -4
  81. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/option_change_migration.h +33 -5
  82. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +6 -0
  83. package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
  84. package/deps/rocksdb/rocksdb/monitoring/statistics.cc +2 -0
  85. package/deps/rocksdb/rocksdb/monitoring/thread_status_impl.cc +5 -2
  86. package/deps/rocksdb/rocksdb/monitoring/thread_status_updater.cc +2 -2
  87. package/deps/rocksdb/rocksdb/monitoring/thread_status_updater.h +6 -6
  88. package/deps/rocksdb/rocksdb/monitoring/thread_status_updater_debug.cc +2 -2
  89. package/deps/rocksdb/rocksdb/monitoring/thread_status_util.cc +10 -5
  90. package/deps/rocksdb/rocksdb/monitoring/thread_status_util.h +2 -2
  91. package/deps/rocksdb/rocksdb/options/cf_options.cc +15 -3
  92. package/deps/rocksdb/rocksdb/options/cf_options.h +7 -0
  93. package/deps/rocksdb/rocksdb/options/db_options.cc +27 -36
  94. package/deps/rocksdb/rocksdb/options/db_options.h +3 -2
  95. package/deps/rocksdb/rocksdb/options/options.cc +4 -0
  96. package/deps/rocksdb/rocksdb/options/options_helper.cc +8 -2
  97. package/deps/rocksdb/rocksdb/options/options_settable_test.cc +4 -1
  98. package/deps/rocksdb/rocksdb/options/options_test.cc +19 -3
  99. package/deps/rocksdb/rocksdb/src.mk +1 -1
  100. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.cc +155 -32
  101. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_builder.h +7 -3
  102. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.cc +169 -125
  103. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_iterator.h +22 -7
  104. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +43 -24
  105. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +9 -5
  106. package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +9 -8
  107. package/deps/rocksdb/rocksdb/table/block_based/filter_block.h +17 -0
  108. package/deps/rocksdb/rocksdb/table/block_based/filter_policy.cc +15 -5
  109. package/deps/rocksdb/rocksdb/table/block_based/filter_policy_internal.h +13 -18
  110. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.cc +29 -0
  111. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block.h +6 -0
  112. package/deps/rocksdb/rocksdb/table/block_based/full_filter_block_test.cc +15 -0
  113. package/deps/rocksdb/rocksdb/table/block_based/index_builder.cc +79 -19
  114. package/deps/rocksdb/rocksdb/table/block_based/index_builder.h +48 -20
  115. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc +51 -0
  116. package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.h +19 -0
  117. package/deps/rocksdb/rocksdb/table/block_based/user_defined_index_wrapper.h +1 -1
  118. package/deps/rocksdb/rocksdb/table/external_table.cc +2 -2
  119. package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +3 -2
  120. package/deps/rocksdb/rocksdb/table/sst_file_dumper.h +3 -1
  121. package/deps/rocksdb/rocksdb/table/table_builder.h +5 -0
  122. package/deps/rocksdb/rocksdb/table/table_reader.h +4 -2
  123. package/deps/rocksdb/rocksdb/table/table_test.cc +48 -39
  124. package/deps/rocksdb/rocksdb/test_util/sync_point.cc +4 -0
  125. package/deps/rocksdb/rocksdb/test_util/sync_point.h +32 -0
  126. package/deps/rocksdb/rocksdb/test_util/testutil.h +6 -2
  127. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +14 -4
  128. package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc +8 -5
  129. package/deps/rocksdb/rocksdb/tools/ldb_cmd_test.cc +3 -2
  130. package/deps/rocksdb/rocksdb/tools/sst_dump_tool.cc +63 -12
  131. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.cc +16 -1
  132. package/deps/rocksdb/rocksdb/util/auto_tune_compressor.h +5 -1
  133. package/deps/rocksdb/rocksdb/util/bit_fields.h +133 -23
  134. package/deps/rocksdb/rocksdb/util/bloom_test.cc +2 -5
  135. package/deps/rocksdb/rocksdb/util/compression.cc +51 -23
  136. package/deps/rocksdb/rocksdb/util/compression_test.cc +525 -270
  137. package/deps/rocksdb/rocksdb/util/filter_bench.cc +3 -4
  138. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.cc +11 -2
  139. package/deps/rocksdb/rocksdb/util/simple_mixed_compressor.h +4 -1
  140. package/deps/rocksdb/rocksdb/util/slice_test.cc +92 -0
  141. package/deps/rocksdb/rocksdb/util/thread_list_test.cc +2 -2
  142. package/deps/rocksdb/rocksdb/util/thread_operation.h +2 -2
  143. package/deps/rocksdb/rocksdb/util/threadpool_imp.cc +2 -2
  144. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +19 -2
  145. package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +75 -0
  146. package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_test.cc +1 -0
  147. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration.cc +303 -111
  148. package/deps/rocksdb/rocksdb/utilities/option_change_migration/option_change_migration_test.cc +379 -0
  149. package/deps/rocksdb/rocksdb.gyp +6 -4
  150. package/iterator.js +66 -70
  151. package/package.json +6 -6
  152. package/prebuilds/darwin-arm64/@nxtedition+rocksdb.node +0 -0
  153. package/deps/rocksdb/rocksdb/table/block_based/index_builder_test.cc +0 -183
@@ -35,12 +35,6 @@
35
35
  #include "rocksdb/version.h"
36
36
  #include "rocksdb/wide_columns.h"
37
37
 
38
- #if defined(__GNUC__) || defined(__clang__)
39
- #define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__))
40
- #elif _WIN32
41
- #define ROCKSDB_DEPRECATED_FUNC __declspec(deprecated)
42
- #endif
43
-
44
38
  namespace ROCKSDB_NAMESPACE {
45
39
 
46
40
  struct ColumnFamilyOptions;
@@ -1111,9 +1105,12 @@ class DB {
1111
1105
  // details. For optimal performance, ensure that either all entries in
1112
1106
  // scan_opts specify the range limit, or none of them do.
1113
1107
  //
1114
- // NOTE: iterate_upper_bound in ReadOptions will be ignored. Instead, the
1115
- // range.limit in ScanOptions is consulted to determine the upper bound key,
1116
- // if specified.
1108
+ // NOTE: NOT YET SUPPORTED in DBs using user timestamp (see
1109
+ // Comparator::timestamp_size())
1110
+ //
1111
+ // NOTE: iterate_upper_bound in ReadOptions will
1112
+ // be ignored. Instead, the range.limit in ScanOptions is consulted to
1113
+ // determine the upper bound key, if specified.
1117
1114
  //
1118
1115
  // Example usage -
1119
1116
  // std::vector<ScanOptions> scans{{.start = Slice("bar")},
@@ -1935,11 +1932,24 @@ class DB {
1935
1932
  virtual void GetColumnFamilyMetaData(ColumnFamilyHandle* /*column_family*/,
1936
1933
  ColumnFamilyMetaData* /*metadata*/) {}
1937
1934
 
1935
+ // Obtains the LSM-tree meta data of the specified column family of the DB
1936
+ // with optional filtering by key range and level.
1937
+ virtual void GetColumnFamilyMetaData(
1938
+ ColumnFamilyHandle* /*column_family*/,
1939
+ const GetColumnFamilyMetaDataOptions& /*options*/,
1940
+ ColumnFamilyMetaData* /*metadata*/) {}
1941
+
1938
1942
  // Get the metadata of the default column family.
1939
1943
  void GetColumnFamilyMetaData(ColumnFamilyMetaData* metadata) {
1940
1944
  GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
1941
1945
  }
1942
1946
 
1947
+ // Get the metadata of the default column family with optional filtering.
1948
+ void GetColumnFamilyMetaData(const GetColumnFamilyMetaDataOptions& options,
1949
+ ColumnFamilyMetaData* metadata) {
1950
+ GetColumnFamilyMetaData(DefaultColumnFamily(), options, metadata);
1951
+ }
1952
+
1943
1953
  // Obtains the LSM-tree meta data of all column families of the DB, including
1944
1954
  // metadata for each live table (SST) file and each blob file in the DB.
1945
1955
  virtual void GetAllColumnFamilyMetaData(
@@ -39,7 +39,7 @@
39
39
  #undef LoadLibrary
40
40
  #endif
41
41
 
42
- #if defined(__GNUC__) || defined(__clang__)
42
+ #if defined(__GNUC__) || defined(__clang__) // ODR-SAFE (essentially)
43
43
  #define ROCKSDB_PRINTF_FORMAT_ATTR(format_param, dots_param) \
44
44
  __attribute__((__format__(__printf__, format_param, dots_param)))
45
45
  #else
@@ -558,7 +558,7 @@ class FileSystem : public Customizable {
558
558
  }
559
559
 
560
560
  // This seems to clash with a macro on Windows, so #undef it here
561
- #ifdef DeleteFile
561
+ #ifdef DeleteFile // ODR-SAFE
562
562
  #undef DeleteFile
563
563
  #endif
564
564
  // Delete the named file.
@@ -719,7 +719,7 @@ class FileSystem : public Customizable {
719
719
  const ImmutableDBOptions& db_options) const;
720
720
 
721
721
  // This seems to clash with a macro on Windows, so #undef it here
722
- #ifdef GetFreeSpace
722
+ #ifdef GetFreeSpace // ODR-SAFE
723
723
  #undef GetFreeSpace
724
724
  #endif
725
725
 
@@ -1166,8 +1166,10 @@ class FSWritableFile {
1166
1166
 
1167
1167
  // Truncate is necessary to trim the file to the correct size
1168
1168
  // before closing. It is not always possible to keep track of the file
1169
- // size due to whole pages writes. The behavior is undefined if called
1170
- // with other writes to follow.
1169
+ // size due to whole pages writes. If called with other writes to follow,
1170
+ // the behavior is file system specific. Posix will reseek to the new EOF.
1171
+ // Other file systems may behave differently. Its the caller's
1172
+ // responsibility to check the file system contract.
1171
1173
  virtual IOStatus Truncate(uint64_t /*size*/, const IOOptions& /*options*/,
1172
1174
  IODebugContext* /*dbg*/) {
1173
1175
  return IOStatus::OK();
@@ -224,6 +224,20 @@ struct LevelMetaData {
224
224
  const std::vector<SstFileMetaData> files;
225
225
  };
226
226
 
227
+ // Options for filtering column family metadata by key range.
228
+ struct GetColumnFamilyMetaDataOptions {
229
+ RangeOpt range;
230
+
231
+ // The level to filter on. If -1, all levels are included.
232
+ int level = -1;
233
+
234
+ GetColumnFamilyMetaDataOptions() = default;
235
+
236
+ GetColumnFamilyMetaDataOptions(const OptSlice& _start_key,
237
+ const OptSlice& _end_key, int _level = -1)
238
+ : range(_start_key, _end_key), level(_level) {}
239
+ };
240
+
227
241
  // The metadata that describes a column family.
228
242
  struct ColumnFamilyMetaData {
229
243
  ColumnFamilyMetaData() : size(0), file_count(0), name("") {}
@@ -32,7 +32,7 @@
32
32
  #include "rocksdb/version.h"
33
33
  #include "rocksdb/write_buffer_manager.h"
34
34
 
35
- #ifdef max
35
+ #ifdef max // ODR-SAFE
36
36
  #undef max
37
37
  #endif
38
38
 
@@ -958,12 +958,67 @@ struct DBOptions {
958
958
  // Default: 0
959
959
  size_t recycle_log_file_num = 0;
960
960
 
961
- // manifest file is rolled over on reaching this limit.
962
- // The older manifest file be deleted.
963
- // The default value is 1GB so that the manifest file can grow, but not
964
- // reach the limit of storage capacity.
961
+ // The manifest file is rolled over on reaching this limit AND the
962
+ // space amp limit described in max_manifest_space_amp_pct. More trade-off
963
+ // details there.
964
+ //
965
+ // NOTE: this option used to be a hard limit, but that made this a dangerous
966
+ // tuning parameter for optimizing manifest file size because the best
967
+ // size really depends on the DB size and average SST file size (and other
968
+ // settings). Now it is essentially a minimum for the auto-tuned max manifest
969
+ // file size.
970
+ //
971
+ // Until the max_manifest_space_amp_pct feature is fully validated to show a
972
+ // smaller default here like 1MB is appropriate, the default value is 1GB to
973
+ // match historical behavior (without it being a hard limit in case of giant
974
+ // compacted manifest size).
975
+ //
976
+ // This option is mutable with SetDBOptions(), taking effect on the next
977
+ // manifest write (e.g. completed DB compaction or flush).
965
978
  uint64_t max_manifest_file_size = 1024 * 1024 * 1024;
966
979
 
980
+ // This option mostly replaces max_manifest_file_size to control an auto-tuned
981
+ // balance of manifest write amplification and space amplification. A new
982
+ // manifest file is created with the "compacted" contents of the old one when
983
+ // current_manifest_size
984
+ // >
985
+ // max(max_manifest_file_size,
986
+ // est_compacted_manifest_size * (1 + max_manifest_space_amp_pct/100))
987
+ //
988
+ // where est_compacted_manifest_size is an estimate of how big a new compacted
989
+ // version of the current manifest would be. Currently, the estimate used is
990
+ // the last newly-written manifest, in its "compacted" form.
991
+ //
992
+ // Space amplification in the manifest file might be less of a concern for
993
+ // primary storage space and more of a concern for DB recover time and size of
994
+ // backup files that aren't incremental between backups. To minimize manifest
995
+ // churn on initial DB population, setting max_manifest_file_size to something
996
+ // not too small, like 1MB, should suffice. Similarly, write amp on the
997
+ // manifest file is likely not a direct concern but completed compactions and
998
+ // flushes cannot (currently) be committed while the (relatively small)
999
+ // manifest file is being compacted. Manifest compactions should not
1000
+ // interfere with user write latency or throughput unless the DB is
1001
+ // chronically stalling or close to stalling writes already.
1002
+ //
1003
+ // For this option to have a meaningful effect, it is recommended to set
1004
+ // max_manifest_file_size to something modest like 1MB. Then we can interpret
1005
+ // values for this option as follows, starting with minimum space amp and
1006
+ // maximum write amp:
1007
+ // * 0 - Every manifest write (flush, compaction, etc.) generates a whole new
1008
+ // manifest. Only useful for testing.
1009
+ // * very small - Doesn't take many manifest writes to generate a whole new
1010
+ // manifest.
1011
+ // * 100 - In a DB with pretty consistent number of SST files, etc., achieves
1012
+ // about 1.0 write amp (writing about 2x the theoretical minimum) and a max of
1013
+ // about 1.0 space amp (manifest up to 2x the compacted size).
1014
+ // * 500 - Recommended and default: 0.2 write amp and up to roughly 5.0 space
1015
+ // amp.
1016
+ // * 10000 - 0.01 write amp and up to 100 space amp on the manifest.
1017
+ //
1018
+ // This option is mutable with SetDBOptions(), taking effect on the next
1019
+ // manifest write (e.g. completed DB compaction or flush).
1020
+ int max_manifest_space_amp_pct = 500;
1021
+
967
1022
  // Number of shards used for table cache.
968
1023
  int table_cache_numshardbits = 6;
969
1024
 
@@ -2383,11 +2438,17 @@ struct CompactionOptions {
2383
2438
  // "default_write_temperature"
2384
2439
  Temperature output_temperature_override = Temperature::kUnknown;
2385
2440
 
2441
+ // Option to optimize the manual compaction by enabling trivial move for non
2442
+ // overlapping files.
2443
+ // Default: false
2444
+ bool allow_trivial_move;
2445
+
2386
2446
  CompactionOptions()
2387
2447
  : compression(kDisableCompressionOption),
2388
2448
  output_file_size_limit(std::numeric_limits<uint64_t>::max()),
2389
2449
  max_subcompactions(0),
2390
- canceled(nullptr) {}
2450
+ canceled(nullptr),
2451
+ allow_trivial_move(false) {}
2391
2452
  };
2392
2453
 
2393
2454
  // For level based compaction, we can configure if we want to skip/force
@@ -15,12 +15,6 @@
15
15
  #include "rocksdb/types.h"
16
16
  #include "rocksdb/wide_columns.h"
17
17
 
18
- #if defined(__GNUC__) || defined(__clang__)
19
- #define ROCKSDB_DEPRECATED_FUNC __attribute__((__deprecated__))
20
- #elif _WIN32
21
- #define ROCKSDB_DEPRECATED_FUNC __declspec(deprecated)
22
- #endif
23
-
24
18
  namespace ROCKSDB_NAMESPACE {
25
19
 
26
20
  class Comparator;
@@ -117,7 +111,7 @@ class SstFileWriter {
117
111
  // REQUIRES: user_key is after any previously added point (Put/Merge/Delete)
118
112
  // key according to the comparator.
119
113
  // REQUIRES: comparator is *not* timestamp-aware.
120
- ROCKSDB_DEPRECATED_FUNC Status Add(const Slice& user_key, const Slice& value);
114
+ [[deprecated]] Status Add(const Slice& user_key, const Slice& value);
121
115
 
122
116
  // Add a Put key with value to currently opened file
123
117
  // REQUIRES: user_key is after any previously added point (Put/Merge/Delete)
@@ -692,6 +692,9 @@ enum Histograms : uint32_t {
692
692
  // Number of operations per transaction.
693
693
  NUM_OP_PER_TRANSACTION,
694
694
 
695
+ // MultiScan Prefill iterator Prepare cost
696
+ MULTISCAN_PREPARE_ITERATORS,
697
+
695
698
  HISTOGRAM_ENUM_MAX
696
699
  };
697
700
 
@@ -22,24 +22,16 @@
22
22
 
23
23
  #include "rocksdb/rocksdb_namespace.h"
24
24
 
25
- #if !defined(NROCKSDB_THREAD_STATUS)
26
- #define ROCKSDB_USING_THREAD_STATUS
27
- #endif
28
-
29
25
  namespace ROCKSDB_NAMESPACE {
30
26
 
31
- // TODO(yhchiang): remove this function once c++14 is available
32
- // as std::max will be able to cover this.
33
- // Current MS compiler does not support constexpr
34
- template <int A, int B>
35
- struct constexpr_max {
36
- static const int result = (A > B) ? A : B;
37
- };
38
-
39
27
  // A structure that describes the current status of a thread.
40
28
  // The status of active threads can be fetched using
41
29
  // ROCKSDB_NAMESPACE::GetThreadList().
42
30
  struct ThreadStatus {
31
+ // Whether RocksDB was built with !NROCKSDB_THREAD_STATUS for
32
+ // ROCKSDB_NAMESPACE::GetThreadList() to be supported.
33
+ static const bool kEnabled;
34
+
43
35
  // The type of a thread.
44
36
  enum ThreadType : int {
45
37
  HIGH_PRIORITY = 0, // RocksDB BG thread in high-pri thread pool
@@ -102,8 +94,8 @@ struct ThreadStatus {
102
94
 
103
95
  // The maximum number of properties of an operation.
104
96
  // This number should be set to the biggest NUM_XXX_PROPERTIES.
105
- static const int kNumOperationProperties =
106
- constexpr_max<NUM_COMPACTION_PROPERTIES, NUM_FLUSH_PROPERTIES>::result;
97
+ static constexpr int kNumOperationProperties =
98
+ std::max(int{NUM_COMPACTION_PROPERTIES}, int{NUM_FLUSH_PROPERTIES});
107
99
 
108
100
  // The type used to refer to a thread state.
109
101
  // A state describes lower-level action of a thread
@@ -621,7 +621,14 @@ class BackupEngineAppendOnlyBase {
621
621
  // The backup will stop ASAP and the call to CreateNewBackup will
622
622
  // return Status::Incomplete(). It will not clean up after itself, but
623
623
  // the state will remain consistent. The state will be cleaned up the
624
- // next time you call CreateNewBackup or GarbageCollect.
624
+ // next time you call CreateNewBackup or GarbageCollect for the same backup
625
+ // directory on a new BackupEngine object.
626
+ //
627
+ // NOTE: This is a one-way operation. Once StopBackup() is called on a
628
+ // BackupEngine instance, all subsequent backup requests (CreateNewBackup,
629
+ // CreateNewBackupWithMetadata) will fail with Status::Incomplete().
630
+ // To create new backups after calling StopBackup(), you must open a new
631
+ // BackupEngine instance.
625
632
  virtual void StopBackup() = 0;
626
633
 
627
634
  // Will delete any files left over from incomplete creation or deletion of
@@ -68,7 +68,7 @@ class EnvMirror : public EnvWrapper {
68
68
  assert(as == bs);
69
69
  return as;
70
70
  }
71
- #if defined(_MSC_VER)
71
+ #if defined(_MSC_VER) // ODR-SAFE
72
72
  #pragma warning(push)
73
73
  // logical operation on address of string constant
74
74
  #pragma warning(disable : 4130)
@@ -87,7 +87,7 @@ class EnvMirror : public EnvWrapper {
87
87
  *r = ar;
88
88
  return as;
89
89
  }
90
- #if defined(_MSC_VER)
90
+ #if defined(_MSC_VER) // ODR-SAFE
91
91
  #pragma warning(pop)
92
92
  #endif
93
93
  Status DeleteFile(const std::string& f) override {
@@ -9,10 +9,6 @@
9
9
 
10
10
  #include "rocksdb/rocksdb_namespace.h"
11
11
 
12
- #ifdef FAILED
13
- #undef FAILED
14
- #endif
15
-
16
12
  namespace ROCKSDB_NAMESPACE {
17
13
 
18
14
  class LDBCommandExecuteResult {
@@ -6,19 +6,47 @@
6
6
  #pragma once
7
7
 
8
8
  #include <string>
9
+ #include <vector>
9
10
 
11
+ #include "rocksdb/db.h"
10
12
  #include "rocksdb/options.h"
11
13
  #include "rocksdb/status.h"
12
14
 
13
15
  namespace ROCKSDB_NAMESPACE {
14
- // Try to migrate DB created with old_opts to be use new_opts.
15
- // Multiple column families is not supported.
16
- // It is best-effort. No guarantee to succeed.
17
- // A full compaction may be executed.
16
+ // Prepares a database to be compatible with new_opts after using old_opts.
17
+ // Restructures the LSM tree but does NOT apply new_opts - you must call
18
+ // DB::Open(new_opts, dbname) afterward to actually use the new configuration.
19
+ // It is best-effort with no guarantee to succeed. A full compaction may be
20
+ // executed.
21
+ //
22
+ // Limitations: single column family only
23
+ //
18
24
  // WARNING: using this to migrate from non-FIFO to FIFO compaction
19
25
  // with `Options::compaction_options_fifo.max_table_files_size` > 0 can cause
20
26
  // the whole DB to be dropped right after migration if the migrated data is
21
27
  // larger than `max_table_files_size`
22
- Status OptionChangeMigration(std::string dbname, const Options& old_opts,
28
+ Status OptionChangeMigration(const std::string& dbname, const Options& old_opts,
23
29
  const Options& new_opts);
30
+
31
+ // Multi-CF version: Prepares a database with multiple column families to be
32
+ // compatible with new options after using old options.
33
+ //
34
+ // REQUIREMENTS:
35
+ // - old_cf_descs and new_cf_descs MUST have the same number of CFs
36
+ // - old_cf_descs and new_cf_descs MUST have the same CF names IN THE SAME ORDER
37
+ // - Adding or dropping CFs is NOT supported - use CreateColumnFamily() or
38
+ // DropColumnFamily() separately before/after migration
39
+ //
40
+ // The function will return InvalidArgument status if these requirements are
41
+ // violated.
42
+ //
43
+ // WARNING: using this to migrate from non-FIFO to FIFO compaction
44
+ // with `max_table_files_size` > 0 can cause the whole DB to be dropped right
45
+ // after migration if the migrated data is larger than `max_table_files_size`
46
+ Status OptionChangeMigration(
47
+ const std::string& dbname, const DBOptions& old_db_opts,
48
+ const std::vector<ColumnFamilyDescriptor>& old_cf_descs,
49
+ const DBOptions& new_db_opts,
50
+ const std::vector<ColumnFamilyDescriptor>& new_cf_descs);
51
+
24
52
  } // namespace ROCKSDB_NAMESPACE
@@ -456,6 +456,12 @@ class StackableDB : public DB {
456
456
  db_->GetColumnFamilyMetaData(column_family, cf_meta);
457
457
  }
458
458
 
459
+ void GetColumnFamilyMetaData(ColumnFamilyHandle* column_family,
460
+ const GetColumnFamilyMetaDataOptions& options,
461
+ ColumnFamilyMetaData* metadata) override {
462
+ db_->GetColumnFamilyMetaData(column_family, options, metadata);
463
+ }
464
+
459
465
  using DB::StartBlockCacheTrace;
460
466
  Status StartBlockCacheTrace(
461
467
  const TraceOptions& trace_options,
@@ -12,8 +12,8 @@
12
12
  // NOTE: in 'main' development branch, this should be the *next*
13
13
  // minor or major version number planned for release.
14
14
  #define ROCKSDB_MAJOR 10
15
- #define ROCKSDB_MINOR 8
16
- #define ROCKSDB_PATCH 0
15
+ #define ROCKSDB_MINOR 10
16
+ #define ROCKSDB_PATCH 1
17
17
 
18
18
  // Make it easy to do conditional compilation based on version checks, i.e.
19
19
  // #if ROCKSDB_VERSION_GE(4, 5, 6)
@@ -352,6 +352,8 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
352
352
  {TABLE_OPEN_PREFETCH_TAIL_READ_BYTES,
353
353
  "rocksdb.table.open.prefetch.tail.read.bytes"},
354
354
  {NUM_OP_PER_TRANSACTION, "rocksdb.num.op.per.transaction"},
355
+ {MULTISCAN_PREPARE_ITERATORS,
356
+ "rocksdb.multiscan.op.prepare.iterators.micros"},
355
357
  };
356
358
 
357
359
  std::shared_ptr<Statistics> CreateDBStatistics() {
@@ -13,7 +13,9 @@
13
13
 
14
14
  namespace ROCKSDB_NAMESPACE {
15
15
 
16
- #ifdef ROCKSDB_USING_THREAD_STATUS
16
+ #ifndef NROCKSDB_THREAD_STATUS
17
+ const bool ThreadStatus::kEnabled = true;
18
+
17
19
  std::string ThreadStatus::GetThreadTypeName(
18
20
  ThreadStatus::ThreadType thread_type) {
19
21
  switch (thread_type) {
@@ -117,6 +119,7 @@ std::map<std::string, uint64_t> ThreadStatus::InterpretOperationProperties(
117
119
  }
118
120
 
119
121
  #else
122
+ const bool ThreadStatus::kEnabled = false;
120
123
 
121
124
  std::string ThreadStatus::GetThreadTypeName(
122
125
  ThreadStatus::ThreadType /*thread_type*/) {
@@ -159,5 +162,5 @@ std::map<std::string, uint64_t> ThreadStatus::InterpretOperationProperties(
159
162
  return std::map<std::string, uint64_t>();
160
163
  }
161
164
 
162
- #endif // ROCKSDB_USING_THREAD_STATUS
165
+ #endif // !NROCKSDB_THREAD_STATUS
163
166
  } // namespace ROCKSDB_NAMESPACE
@@ -14,7 +14,7 @@
14
14
 
15
15
  namespace ROCKSDB_NAMESPACE {
16
16
 
17
- #ifdef ROCKSDB_USING_THREAD_STATUS
17
+ #ifndef NROCKSDB_THREAD_STATUS
18
18
 
19
19
  thread_local ThreadStatusData* ThreadStatusUpdater::thread_status_data_ =
20
20
  nullptr;
@@ -324,5 +324,5 @@ void ThreadStatusUpdater::SetThreadOperationProperty(int /*i*/,
324
324
  void ThreadStatusUpdater::IncreaseThreadOperationProperty(int /*i*/,
325
325
  uint64_t /*delta*/) {}
326
326
 
327
- #endif // ROCKSDB_USING_THREAD_STATUS
327
+ #endif // !NROCKSDB_THREAD_STATUS
328
328
  } // namespace ROCKSDB_NAMESPACE
@@ -47,7 +47,7 @@ class ColumnFamilyHandle;
47
47
 
48
48
  // The structure that keeps constant information about a column family.
49
49
  struct ConstantColumnFamilyInfo {
50
- #ifdef ROCKSDB_USING_THREAD_STATUS
50
+ #ifndef NROCKSDB_THREAD_STATUS
51
51
  public:
52
52
  ConstantColumnFamilyInfo(const void* _db_key, const std::string& _db_name,
53
53
  const std::string& _cf_name)
@@ -55,13 +55,13 @@ struct ConstantColumnFamilyInfo {
55
55
  const void* db_key;
56
56
  const std::string db_name;
57
57
  const std::string cf_name;
58
- #endif // ROCKSDB_USING_THREAD_STATUS
58
+ #endif // !NROCKSDB_THREAD_STATUS
59
59
  };
60
60
 
61
61
  // the internal data-structure that is used to reflect the current
62
62
  // status of a thread using a set of atomic pointers.
63
63
  struct ThreadStatusData {
64
- #ifdef ROCKSDB_USING_THREAD_STATUS
64
+ #ifndef NROCKSDB_THREAD_STATUS
65
65
  explicit ThreadStatusData() {
66
66
  enable_tracking.store(false);
67
67
  thread_id.store(0);
@@ -86,7 +86,7 @@ struct ThreadStatusData {
86
86
  std::atomic<ThreadStatus::OperationStage> operation_stage;
87
87
  std::atomic<uint64_t> op_properties[ThreadStatus::kNumOperationProperties];
88
88
  std::atomic<ThreadStatus::StateType> state_type;
89
- #endif // ROCKSDB_USING_THREAD_STATUS
89
+ #endif // !NROCKSDB_THREAD_STATUS
90
90
  };
91
91
 
92
92
  // The class that stores and updates the status of the current thread
@@ -190,7 +190,7 @@ class ThreadStatusUpdater {
190
190
  const std::vector<ColumnFamilyHandle*>& handles, bool check_exist);
191
191
 
192
192
  protected:
193
- #ifdef ROCKSDB_USING_THREAD_STATUS
193
+ #ifndef NROCKSDB_THREAD_STATUS
194
194
  // The thread-local variable for storing thread status.
195
195
  static thread_local ThreadStatusData* thread_status_data_;
196
196
 
@@ -220,7 +220,7 @@ class ThreadStatusUpdater {
220
220
 
221
221
  #else
222
222
  static ThreadStatusData* thread_status_data_;
223
- #endif // ROCKSDB_USING_THREAD_STATUS
223
+ #endif // !NROCKSDB_THREAD_STATUS
224
224
  };
225
225
 
226
226
  } // namespace ROCKSDB_NAMESPACE
@@ -12,7 +12,7 @@
12
12
  namespace ROCKSDB_NAMESPACE {
13
13
 
14
14
  #ifndef NDEBUG
15
- #ifdef ROCKSDB_USING_THREAD_STATUS
15
+ #ifndef NROCKSDB_THREAD_STATUS
16
16
  void ThreadStatusUpdater::TEST_VerifyColumnFamilyInfoMap(
17
17
  const std::vector<ColumnFamilyHandle*>& handles, bool check_exist) {
18
18
  std::unique_lock<std::mutex> lock(thread_list_mutex_);
@@ -37,7 +37,7 @@ void ThreadStatusUpdater::TEST_VerifyColumnFamilyInfoMap(
37
37
  const std::vector<ColumnFamilyHandle*>& /*handles*/, bool /*check_exist*/) {
38
38
  }
39
39
 
40
- #endif // ROCKSDB_USING_THREAD_STATUS
40
+ #endif // !NROCKSDB_THREAD_STATUS
41
41
  #endif // !NDEBUG
42
42
 
43
43
  } // namespace ROCKSDB_NAMESPACE
@@ -11,7 +11,7 @@
11
11
 
12
12
  namespace ROCKSDB_NAMESPACE {
13
13
 
14
- #ifdef ROCKSDB_USING_THREAD_STATUS
14
+ #ifndef NROCKSDB_THREAD_STATUS
15
15
  thread_local ThreadStatusUpdater*
16
16
  ThreadStatusUtil::thread_updater_local_cache_ = nullptr;
17
17
  thread_local bool ThreadStatusUtil::thread_updater_initialized_ = false;
@@ -171,9 +171,10 @@ AutoThreadOperationStageUpdater::~AutoThreadOperationStageUpdater() {
171
171
  ThreadStatusUpdater* ThreadStatusUtil::thread_updater_local_cache_ = nullptr;
172
172
  bool ThreadStatusUtil::thread_updater_initialized_ = false;
173
173
 
174
- bool ThreadStatusUtil::MaybeInitThreadLocalUpdater(const Env* /*env*/) {
175
- return false;
176
- }
174
+ void ThreadStatusUtil::RegisterThread(
175
+ const Env* /*env*/, ThreadStatus::ThreadType /*thread_type*/) {}
176
+
177
+ void ThreadStatusUtil::UnregisterThread() {}
177
178
 
178
179
  void ThreadStatusUtil::SetEnableTracking(bool /*enable_tracking*/) {}
179
180
 
@@ -204,11 +205,15 @@ void ThreadStatusUtil::EraseDatabaseInfo(const DB* /*db*/) {}
204
205
 
205
206
  void ThreadStatusUtil::ResetThreadStatus() {}
206
207
 
208
+ bool ThreadStatusUtil::MaybeInitThreadLocalUpdater(const Env* /*env*/) {
209
+ return false;
210
+ }
211
+
207
212
  AutoThreadOperationStageUpdater::AutoThreadOperationStageUpdater(
208
213
  ThreadStatus::OperationStage /*stage*/) {}
209
214
 
210
215
  AutoThreadOperationStageUpdater::~AutoThreadOperationStageUpdater() {}
211
216
 
212
- #endif // ROCKSDB_USING_THREAD_STATUS
217
+ #endif // !NROCKSDB_THREAD_STATUS
213
218
 
214
219
  } // namespace ROCKSDB_NAMESPACE
@@ -90,7 +90,7 @@ class ThreadStatusUtil {
90
90
  // a non-null pointer.
91
91
  static bool MaybeInitThreadLocalUpdater(const Env* env);
92
92
 
93
- #ifdef ROCKSDB_USING_THREAD_STATUS
93
+ #ifndef NROCKSDB_THREAD_STATUS
94
94
  // A boolean flag indicating whether thread_updater_local_cache_
95
95
  // is initialized. It is set to true when an Env uses any
96
96
  // ThreadStatusUtil functions using the current thread other
@@ -130,7 +130,7 @@ class AutoThreadOperationStageUpdater {
130
130
  explicit AutoThreadOperationStageUpdater(ThreadStatus::OperationStage stage);
131
131
  ~AutoThreadOperationStageUpdater();
132
132
 
133
- #ifdef ROCKSDB_USING_THREAD_STATUS
133
+ #ifndef NROCKSDB_THREAD_STATUS
134
134
  private:
135
135
  ThreadStatus::OperationStage prev_stage_;
136
136
  #endif
@@ -395,6 +395,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
395
395
  {offsetof(struct MutableCFOptions, paranoid_file_checks),
396
396
  OptionType::kBoolean, OptionVerificationType::kNormal,
397
397
  OptionTypeFlags::kMutable}},
398
+ {"verify_output_flags",
399
+ {offsetof(struct MutableCFOptions, verify_output_flags),
400
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
401
+ OptionTypeFlags::kMutable}},
398
402
  {"verify_checksums_in_compaction",
399
403
  {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
400
404
  OptionTypeFlags::kMutable}},
@@ -450,6 +454,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
450
454
  {offsetof(struct MutableCFOptions, target_file_size_multiplier),
451
455
  OptionType::kInt, OptionVerificationType::kNormal,
452
456
  OptionTypeFlags::kMutable}},
457
+ {"target_file_size_is_upper_bound",
458
+ {offsetof(struct MutableCFOptions, target_file_size_is_upper_bound),
459
+ OptionType::kBoolean, OptionVerificationType::kNormal,
460
+ OptionTypeFlags::kMutable}},
453
461
  {"arena_block_size",
454
462
  {offsetof(struct MutableCFOptions, arena_block_size),
455
463
  OptionType::kSizeT, OptionVerificationType::kNormal,
@@ -1070,10 +1078,12 @@ uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
1070
1078
  if (op1 == 0 || op2 <= 0) {
1071
1079
  return 0;
1072
1080
  }
1073
- if (std::numeric_limits<uint64_t>::max() / op1 < op2) {
1074
- return op1;
1081
+
1082
+ if (op1 * op2 < static_cast<double>(std::numeric_limits<uint64_t>::max())) {
1083
+ return static_cast<uint64_t>(op1 * op2);
1075
1084
  }
1076
- return static_cast<uint64_t>(op1 * op2);
1085
+
1086
+ return op1;
1077
1087
  }
1078
1088
 
1079
1089
  // when level_compaction_dynamic_level_bytes is true and leveled compaction
@@ -1168,6 +1178,8 @@ void MutableCFOptions::Dump(Logger* log) const {
1168
1178
  target_file_size_base);
1169
1179
  ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
1170
1180
  target_file_size_multiplier);
1181
+ ROCKS_LOG_INFO(log, " target_file_size_is_upper_bound: %d",
1182
+ target_file_size_is_upper_bound);
1171
1183
  ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
1172
1184
  max_bytes_for_level_base);
1173
1185
  ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",