@nxtedition/rocksdb 7.1.33 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/BUILDING.md +2 -2
  2. package/binding.cc +0 -147
  3. package/deps/rocksdb/rocksdb/cache/clock_cache.cc +402 -345
  4. package/deps/rocksdb/rocksdb/cache/clock_cache.h +121 -64
  5. package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +28 -18
  6. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +1 -0
  7. package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +2 -0
  8. package/deps/rocksdb/rocksdb/db/builder.cc +2 -1
  9. package/deps/rocksdb/rocksdb/db/c.cc +563 -673
  10. package/deps/rocksdb/rocksdb/db/c_test.c +168 -169
  11. package/deps/rocksdb/rocksdb/db/column_family.cc +16 -15
  12. package/deps/rocksdb/rocksdb/db/column_family.h +7 -7
  13. package/deps/rocksdb/rocksdb/db/column_family_test.cc +17 -28
  14. package/deps/rocksdb/rocksdb/db/compact_files_test.cc +4 -9
  15. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +8 -3
  16. package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +114 -0
  17. package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +2 -3
  18. package/deps/rocksdb/rocksdb/db/convenience.cc +3 -5
  19. package/deps/rocksdb/rocksdb/db/corruption_test.cc +10 -14
  20. package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +9 -13
  21. package/deps/rocksdb/rocksdb/db/db_basic_test.cc +2 -2
  22. package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +2 -2
  23. package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +14 -16
  24. package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +52 -72
  25. package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +2 -2
  26. package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +12 -12
  27. package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -2
  28. package/deps/rocksdb/rocksdb/db/db_flush_test.cc +3 -3
  29. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +1 -12
  30. package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +3 -0
  31. package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +26 -0
  32. package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +1 -0
  33. package/deps/rocksdb/rocksdb/db/db_iter.cc +12 -6
  34. package/deps/rocksdb/rocksdb/db/db_iter.h +1 -0
  35. package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +6 -7
  36. package/deps/rocksdb/rocksdb/db/db_iter_test.cc +10 -8
  37. package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +15 -13
  38. package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -9
  39. package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +4 -4
  40. package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +1 -1
  41. package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +2 -4
  42. package/deps/rocksdb/rocksdb/db/db_options_test.cc +4 -4
  43. package/deps/rocksdb/rocksdb/db/db_properties_test.cc +7 -4
  44. package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +7 -5
  45. package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +1 -1
  46. package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +8 -6
  47. package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +18 -23
  48. package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +3 -5
  49. package/deps/rocksdb/rocksdb/db/db_test.cc +10 -5
  50. package/deps/rocksdb/rocksdb/db/db_test2.cc +172 -169
  51. package/deps/rocksdb/rocksdb/db/db_test_util.cc +68 -66
  52. package/deps/rocksdb/rocksdb/db/db_test_util.h +1 -3
  53. package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +31 -39
  54. package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +182 -2
  55. package/deps/rocksdb/rocksdb/db/db_write_test.cc +43 -40
  56. package/deps/rocksdb/rocksdb/db/dbformat.h +15 -0
  57. package/deps/rocksdb/rocksdb/db/dbformat_test.cc +35 -34
  58. package/deps/rocksdb/rocksdb/db/deletefile_test.cc +10 -11
  59. package/deps/rocksdb/rocksdb/db/error_handler.cc +6 -6
  60. package/deps/rocksdb/rocksdb/db/error_handler.h +93 -94
  61. package/deps/rocksdb/rocksdb/db/event_helpers.cc +1 -1
  62. package/deps/rocksdb/rocksdb/db/event_helpers.h +3 -3
  63. package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +16 -17
  64. package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -2
  65. package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +1 -2
  66. package/deps/rocksdb/rocksdb/db/file_indexer.cc +2 -0
  67. package/deps/rocksdb/rocksdb/db/file_indexer.h +2 -1
  68. package/deps/rocksdb/rocksdb/db/file_indexer_test.cc +4 -2
  69. package/deps/rocksdb/rocksdb/db/filename_test.cc +27 -29
  70. package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -13
  71. package/deps/rocksdb/rocksdb/db/flush_job_test.cc +2 -2
  72. package/deps/rocksdb/rocksdb/db/forward_iterator.cc +15 -21
  73. package/deps/rocksdb/rocksdb/db/forward_iterator.h +7 -6
  74. package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +4 -2
  75. package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +2 -2
  76. package/deps/rocksdb/rocksdb/db/internal_stats.cc +59 -14
  77. package/deps/rocksdb/rocksdb/db/internal_stats.h +27 -11
  78. package/deps/rocksdb/rocksdb/db/job_context.h +5 -6
  79. package/deps/rocksdb/rocksdb/db/listener_test.cc +21 -23
  80. package/deps/rocksdb/rocksdb/db/log_reader.cc +7 -11
  81. package/deps/rocksdb/rocksdb/db/log_reader.h +4 -6
  82. package/deps/rocksdb/rocksdb/db/log_test.cc +6 -12
  83. package/deps/rocksdb/rocksdb/db/log_writer.h +1 -1
  84. package/deps/rocksdb/rocksdb/db/logs_with_prep_tracker.h +0 -1
  85. package/deps/rocksdb/rocksdb/db/lookup_key.h +4 -1
  86. package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -1
  87. package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +3 -5
  88. package/deps/rocksdb/rocksdb/db/memtable.cc +34 -22
  89. package/deps/rocksdb/rocksdb/db/memtable.h +4 -6
  90. package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -0
  91. package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +37 -13
  92. package/deps/rocksdb/rocksdb/db/merge_context.h +1 -0
  93. package/deps/rocksdb/rocksdb/db/merge_helper.cc +128 -14
  94. package/deps/rocksdb/rocksdb/db/merge_helper.h +15 -7
  95. package/deps/rocksdb/rocksdb/db/merge_helper_test.cc +2 -1
  96. package/deps/rocksdb/rocksdb/db/merge_operator.cc +5 -6
  97. package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +4 -3
  98. package/deps/rocksdb/rocksdb/db/options_file_test.cc +1 -1
  99. package/deps/rocksdb/rocksdb/db/perf_context_test.cc +55 -43
  100. package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +288 -299
  101. package/deps/rocksdb/rocksdb/db/prefix_test.cc +22 -27
  102. package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +1 -1
  103. package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +1 -1
  104. package/deps/rocksdb/rocksdb/db/repair.cc +7 -8
  105. package/deps/rocksdb/rocksdb/db/repair_test.cc +3 -4
  106. package/deps/rocksdb/rocksdb/db/snapshot_impl.cc +4 -5
  107. package/deps/rocksdb/rocksdb/db/snapshot_impl.h +10 -4
  108. package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -4
  109. package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +6 -7
  110. package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +22 -22
  111. package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +12 -12
  112. package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +6 -8
  113. package/deps/rocksdb/rocksdb/db/trim_history_scheduler.h +2 -0
  114. package/deps/rocksdb/rocksdb/db/version_builder_test.cc +3 -3
  115. package/deps/rocksdb/rocksdb/db/version_edit.cc +2 -5
  116. package/deps/rocksdb/rocksdb/db/version_edit.h +8 -12
  117. package/deps/rocksdb/rocksdb/db/version_set.cc +74 -102
  118. package/deps/rocksdb/rocksdb/db/version_set.h +8 -10
  119. package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +0 -5
  120. package/deps/rocksdb/rocksdb/db/version_set_test.cc +47 -45
  121. package/deps/rocksdb/rocksdb/db/wal_manager.cc +6 -5
  122. package/deps/rocksdb/rocksdb/db/wal_manager.h +2 -2
  123. package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +4 -3
  124. package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +144 -61
  125. package/deps/rocksdb/rocksdb/db/write_batch.cc +41 -24
  126. package/deps/rocksdb/rocksdb/db/write_batch_internal.h +2 -7
  127. package/deps/rocksdb/rocksdb/db/write_batch_test.cc +105 -104
  128. package/deps/rocksdb/rocksdb/db/write_callback_test.cc +5 -4
  129. package/deps/rocksdb/rocksdb/db/write_controller.h +1 -0
  130. package/deps/rocksdb/rocksdb/db/write_controller_test.cc +1 -1
  131. package/deps/rocksdb/rocksdb/db/write_thread.cc +8 -6
  132. package/deps/rocksdb/rocksdb/env/io_posix.h +6 -0
  133. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +134 -65
  134. package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +29 -0
  135. package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -0
  136. package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -4
  137. package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +1 -0
  138. package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +4 -0
  139. package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +14 -4
  140. package/deps/rocksdb/rocksdb/table/get_context.cc +52 -7
  141. package/deps/rocksdb/rocksdb/table/get_context.h +1 -2
  142. package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +13 -0
  143. package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +36 -4
  144. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +6 -6
  145. package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +23 -28
  146. package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +11 -1
  147. package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +19 -17
  148. package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +10 -7
  149. package/index.js +3 -195
  150. package/package.json +2 -4
  151. package/prebuilds/darwin-arm64/node.napi.node +0 -0
  152. package/prebuilds/linux-x64/node.napi.node +0 -0
  153. package/common.js +0 -7
@@ -106,9 +106,8 @@ Status ExternalSstFileIngestionJob::Prepare(
106
106
  for (IngestedFileInfo& f : files_to_ingest_) {
107
107
  f.copy_file = false;
108
108
  const std::string path_outside_db = f.external_file_path;
109
- const std::string path_inside_db =
110
- TableFileName(cfd_->ioptions()->cf_paths, f.fd.GetNumber(),
111
- f.fd.GetPathId());
109
+ const std::string path_inside_db = TableFileName(
110
+ cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
112
111
  if (ingestion_options_.move_files) {
113
112
  status =
114
113
  fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
@@ -491,7 +490,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
491
490
  stream.StartArray();
492
491
 
493
492
  for (IngestedFileInfo& f : files_to_ingest_) {
494
- InternalStats::CompactionStats stats(CompactionReason::kExternalSstIngestion, 1);
493
+ InternalStats::CompactionStats stats(
494
+ CompactionReason::kExternalSstIngestion, 1);
495
495
  stats.micros = total_time;
496
496
  // If actual copy occurred for this file, then we need to count the file
497
497
  // size as the actual bytes written. If the file was linked, then we ignore
@@ -591,8 +591,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
591
591
  std::unique_ptr<FSRandomAccessFile> sst_file;
592
592
  std::unique_ptr<RandomAccessFileReader> sst_file_reader;
593
593
 
594
- status = fs_->NewRandomAccessFile(external_file, env_options_,
595
- &sst_file, nullptr);
594
+ status =
595
+ fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
596
596
  if (!status.ok()) {
597
597
  return status;
598
598
  }
@@ -658,9 +658,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
658
658
  assert(seqno_iter == uprops.end());
659
659
  file_to_ingest->original_seqno = 0;
660
660
  if (ingestion_options_.allow_blocking_flush ||
661
- ingestion_options_.allow_global_seqno) {
661
+ ingestion_options_.allow_global_seqno) {
662
662
  return Status::InvalidArgument(
663
- "External SST file V1 does not support global seqno");
663
+ "External SST file V1 does not support global seqno");
664
664
  }
665
665
  } else {
666
666
  return Status::InvalidArgument("External file version is not supported");
@@ -857,7 +857,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
857
857
  return status;
858
858
  }
859
859
 
860
- TEST_SYNC_POINT_CALLBACK(
860
+ TEST_SYNC_POINT_CALLBACK(
861
861
  "ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
862
862
  &overlap_with_db);
863
863
  file_to_ingest->picked_level = target_level;
@@ -872,10 +872,10 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
872
872
  auto* vstorage = cfd_->current()->storage_info();
873
873
  // first check if new files fit in the bottommost level
874
874
  int bottom_lvl = cfd_->NumberLevels() - 1;
875
- if(!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
875
+ if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
876
876
  return Status::InvalidArgument(
877
- "Can't ingest_behind file as it doesn't fit "
878
- "at the bottommost level!");
877
+ "Can't ingest_behind file as it doesn't fit "
878
+ "at the bottommost level!");
879
879
  }
880
880
 
881
881
  // second check if despite allow_ingest_behind=true we still have 0 seqnums
@@ -884,8 +884,8 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
884
884
  for (auto file : vstorage->LevelFiles(lvl)) {
885
885
  if (file->fd.smallest_seqno == 0) {
886
886
  return Status::InvalidArgument(
887
- "Can't ingest_behind file as despite allow_ingest_behind=true "
888
- "there are files with 0 seqno in database at upper levels!");
887
+ "Can't ingest_behind file as despite allow_ingest_behind=true "
888
+ "there are files with 0 seqno in database at upper levels!");
889
889
  }
890
890
  }
891
891
  }
@@ -912,9 +912,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
912
912
  // If the file system does not support random write, then we should not.
913
913
  // Otherwise we should.
914
914
  std::unique_ptr<FSRandomRWFile> rwfile;
915
- Status status =
916
- fs_->NewRandomRWFile(file_to_ingest->internal_file_path, env_options_,
917
- &rwfile, nullptr);
915
+ Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path,
916
+ env_options_, &rwfile, nullptr);
918
917
  TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
919
918
  &status);
920
919
  if (status.ok()) {
@@ -301,7 +301,8 @@ TEST_F(ExternalSSTFileTest, Basic) {
301
301
 
302
302
  SstFileWriter sst_file_writer(EnvOptions(), options);
303
303
 
304
- // Current file size should be 0 after sst_file_writer init and before open a file.
304
+ // Current file size should be 0 after sst_file_writer init and before open
305
+ // a file.
305
306
  ASSERT_EQ(sst_file_writer.FileSize(), 0);
306
307
 
307
308
  // file1.sst (0 => 99)
@@ -2318,7 +2319,6 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
2318
2319
  table_options.cache_index_and_filter_blocks = true;
2319
2320
  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
2320
2321
 
2321
-
2322
2322
  // Create external SST file and include bloom filters
2323
2323
  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
2324
2324
  DestroyAndReopen(options);
@@ -338,8 +338,7 @@ class FaultInjectionTest
338
338
  FaultInjectionTest::kValExpectNoError));
339
339
  }
340
340
 
341
- void NoWriteTestPreFault() {
342
- }
341
+ void NoWriteTestPreFault() {}
343
342
 
344
343
  void NoWriteTestReopenWithFault(ResetMethod reset_method) {
345
344
  CloseDB();
@@ -8,8 +8,10 @@
8
8
  // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
9
 
10
10
  #include "db/file_indexer.h"
11
+
11
12
  #include <algorithm>
12
13
  #include <functional>
14
+
13
15
  #include "db/version_edit.h"
14
16
  #include "rocksdb/comparator.h"
15
17
 
@@ -12,6 +12,7 @@
12
12
  #include <functional>
13
13
  #include <limits>
14
14
  #include <vector>
15
+
15
16
  #include "memory/arena.h"
16
17
  #include "port/port.h"
17
18
  #include "util/autovector.h"
@@ -66,7 +67,7 @@ class FileIndexer {
66
67
 
67
68
  struct IndexUnit {
68
69
  IndexUnit()
69
- : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
70
+ : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
70
71
  // During file search, a key is compared against smallest and largest
71
72
  // from a FileMetaData. It can have 3 possible outcomes:
72
73
  // (1) key is smaller than smallest, implying it is also smaller than
@@ -8,7 +8,9 @@
8
8
  // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
9
 
10
10
  #include "db/file_indexer.h"
11
+
11
12
  #include <string>
13
+
12
14
  #include "db/dbformat.h"
13
15
  #include "db/version_edit.h"
14
16
  #include "port/stack_trace.h"
@@ -73,8 +75,8 @@ class FileIndexerTest : public testing::Test {
73
75
  }
74
76
 
75
77
  void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
76
- const int cmp_smallest, const int cmp_largest, int32_t* left_index,
77
- int32_t* right_index) {
78
+ const int cmp_smallest, const int cmp_largest,
79
+ int32_t* left_index, int32_t* right_index) {
78
80
  *left_index = 100;
79
81
  *right_index = 100;
80
82
  indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
@@ -69,35 +69,33 @@ TEST_F(FileNameTest, Parse) {
69
69
  }
70
70
 
71
71
  // Errors
72
- static const char* errors[] = {
73
- "",
74
- "foo",
75
- "foo-dx-100.log",
76
- ".log",
77
- "",
78
- "manifest",
79
- "CURREN",
80
- "CURRENTX",
81
- "MANIFES",
82
- "MANIFEST",
83
- "MANIFEST-",
84
- "XMANIFEST-3",
85
- "MANIFEST-3x",
86
- "META",
87
- "METADB",
88
- "METADB-",
89
- "XMETADB-3",
90
- "METADB-3x",
91
- "LOC",
92
- "LOCKx",
93
- "LO",
94
- "LOGx",
95
- "18446744073709551616.log",
96
- "184467440737095516150.log",
97
- "100",
98
- "100.",
99
- "100.lop"
100
- };
72
+ static const char* errors[] = {"",
73
+ "foo",
74
+ "foo-dx-100.log",
75
+ ".log",
76
+ "",
77
+ "manifest",
78
+ "CURREN",
79
+ "CURRENTX",
80
+ "MANIFES",
81
+ "MANIFEST",
82
+ "MANIFEST-",
83
+ "XMANIFEST-3",
84
+ "MANIFEST-3x",
85
+ "META",
86
+ "METADB",
87
+ "METADB-",
88
+ "XMETADB-3",
89
+ "METADB-3x",
90
+ "LOC",
91
+ "LOCKx",
92
+ "LO",
93
+ "LOGx",
94
+ "18446744073709551616.log",
95
+ "184467440737095516150.log",
96
+ "100",
97
+ "100.",
98
+ "100.lop"};
101
99
  for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
102
100
  std::string f = errors[i];
103
101
  ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
@@ -48,7 +48,7 @@
48
48
 
49
49
  namespace ROCKSDB_NAMESPACE {
50
50
 
51
- const char* GetFlushReasonString (FlushReason flush_reason) {
51
+ const char* GetFlushReasonString(FlushReason flush_reason) {
52
52
  switch (flush_reason) {
53
53
  case FlushReason::kOthers:
54
54
  return "Other Reasons";
@@ -136,17 +136,14 @@ FlushJob::FlushJob(
136
136
  TEST_SYNC_POINT("FlushJob::FlushJob()");
137
137
  }
138
138
 
139
- FlushJob::~FlushJob() {
140
- ThreadStatusUtil::ResetThreadStatus();
141
- }
139
+ FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); }
142
140
 
143
141
  void FlushJob::ReportStartedFlush() {
144
142
  ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
145
143
  db_options_.enable_thread_tracking);
146
144
  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
147
- ThreadStatusUtil::SetThreadOperationProperty(
148
- ThreadStatus::COMPACTION_JOB_ID,
149
- job_context_->job_id);
145
+ ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
146
+ job_context_->job_id);
150
147
  IOSTATS_RESET(bytes_written);
151
148
  }
152
149
 
@@ -156,8 +153,7 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
156
153
  input_size += mem->ApproximateMemoryUsage();
157
154
  }
158
155
  ThreadStatusUtil::IncreaseThreadOperationProperty(
159
- ThreadStatus::FLUSH_BYTES_MEMTABLES,
160
- input_size);
156
+ ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size);
161
157
  }
162
158
 
163
159
  void FlushJob::RecordFlushIOStats() {
@@ -220,8 +216,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
220
216
  double mempurge_threshold =
221
217
  mutable_cf_options_.experimental_mempurge_threshold;
222
218
 
223
- AutoThreadOperationStageUpdater stage_run(
224
- ThreadStatus::STAGE_FLUSH_RUN);
219
+ AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN);
225
220
  if (mems_.empty()) {
226
221
  ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
227
222
  cfd_->GetName().c_str());
@@ -906,8 +901,7 @@ Status FlushJob::WriteLevel0Table() {
906
901
  }
907
902
  const uint64_t current_time = static_cast<uint64_t>(_current_time);
908
903
 
909
- uint64_t oldest_key_time =
910
- mems_.front()->ApproximateOldestKeyTime();
904
+ uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
911
905
 
912
906
  // It's not clear whether oldest_key_time is always available. In case
913
907
  // it is not available, use current_time.
@@ -214,8 +214,8 @@ TEST_F(FlushJobTest, NonEmpty) {
214
214
  // Note: the first two blob references will not be considered when resolving
215
215
  // the oldest blob file referenced (the first one is inlined TTL, while the
216
216
  // second one is TTL and thus points to a TTL blob file).
217
- constexpr std::array<uint64_t, 6> blob_file_numbers{{
218
- kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
217
+ constexpr std::array<uint64_t, 6> blob_file_numbers{
218
+ {kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
219
219
  for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
220
220
  std::string key(std::to_string(i + 10001));
221
221
  std::string blob_index;
@@ -104,9 +104,7 @@ class ForwardLevelIterator : public InternalIterator {
104
104
  status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
105
105
  valid_ = false;
106
106
  }
107
- bool Valid() const override {
108
- return valid_;
109
- }
107
+ bool Valid() const override { return valid_; }
110
108
  void SeekToFirst() override {
111
109
  assert(file_iter_ != nullptr);
112
110
  if (!status_.ok()) {
@@ -249,9 +247,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
249
247
  immutable_status_.PermitUncheckedError();
250
248
  }
251
249
 
252
- ForwardIterator::~ForwardIterator() {
253
- Cleanup(true);
254
- }
250
+ ForwardIterator::~ForwardIterator() { Cleanup(true); }
255
251
 
256
252
  void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
257
253
  bool background_purge_on_iterator_cleanup) {
@@ -284,13 +280,13 @@ struct SVCleanupParams {
284
280
  SuperVersion* sv;
285
281
  bool background_purge_on_iterator_cleanup;
286
282
  };
287
- }
283
+ } // anonymous namespace
288
284
 
289
285
  // Used in PinnedIteratorsManager to release pinned SuperVersion
290
286
  void ForwardIterator::DeferredSVCleanup(void* arg) {
291
287
  auto d = reinterpret_cast<SVCleanupParams*>(arg);
292
- ForwardIterator::SVCleanup(
293
- d->db, d->sv, d->background_purge_on_iterator_cleanup);
288
+ ForwardIterator::SVCleanup(d->db, d->sv,
289
+ d->background_purge_on_iterator_cleanup);
294
290
  delete d;
295
291
  }
296
292
 
@@ -547,8 +543,7 @@ void ForwardIterator::Next() {
547
543
  assert(valid_);
548
544
  bool update_prev_key = false;
549
545
 
550
- if (sv_ == nullptr ||
551
- sv_->version_number != cfd_->GetSuperVersionNumber()) {
546
+ if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) {
552
547
  std::string current_key = key().ToString();
553
548
  Slice old_key(current_key.data(), current_key.size());
554
549
 
@@ -578,7 +573,6 @@ void ForwardIterator::Next() {
578
573
  update_prev_key = true;
579
574
  }
580
575
 
581
-
582
576
  if (update_prev_key) {
583
577
  prev_key_.SetInternalKey(current_->key());
584
578
  is_prev_set_ = true;
@@ -635,7 +629,7 @@ bool ForwardIterator::PrepareValue() {
635
629
 
636
630
  assert(!current_->Valid());
637
631
  assert(!current_->status().ok());
638
- assert(current_ != mutable_iter_); // memtable iterator can't fail
632
+ assert(current_ != mutable_iter_); // memtable iterator can't fail
639
633
  assert(immutable_status_.ok());
640
634
 
641
635
  valid_ = false;
@@ -950,11 +944,11 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
950
944
  }
951
945
  Slice prev_key = prev_key_.GetInternalKey();
952
946
  if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
953
- prefix_extractor_->Transform(prev_key)) != 0) {
947
+ prefix_extractor_->Transform(prev_key)) != 0) {
954
948
  return true;
955
949
  }
956
950
  if (cfd_->internal_comparator().InternalKeyComparator::Compare(
957
- prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
951
+ prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
958
952
  return true;
959
953
  }
960
954
 
@@ -963,8 +957,8 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
963
957
  return false;
964
958
  }
965
959
  if (cfd_->internal_comparator().InternalKeyComparator::Compare(
966
- target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
967
- : current_->key()) > 0) {
960
+ target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
961
+ : current_->key()) > 0) {
968
962
  return true;
969
963
  }
970
964
  return false;
@@ -1040,11 +1034,11 @@ uint32_t ForwardIterator::FindFileInRange(
1040
1034
  uint32_t left, uint32_t right) {
1041
1035
  auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool {
1042
1036
  return cfd_->internal_comparator().InternalKeyComparator::Compare(
1043
- f->largest.Encode(), k) < 0;
1037
+ f->largest.Encode(), k) < 0;
1044
1038
  };
1045
- const auto &b = files.begin();
1046
- return static_cast<uint32_t>(std::lower_bound(b + left,
1047
- b + right, internal_key, cmp) - b);
1039
+ const auto& b = files.begin();
1040
+ return static_cast<uint32_t>(
1041
+ std::lower_bound(b + left, b + right, internal_key, cmp) - b);
1048
1042
  }
1049
1043
 
1050
1044
  void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
@@ -7,9 +7,9 @@
7
7
  #include "rocksdb/comparator.h"
8
8
  #ifndef ROCKSDB_LITE
9
9
 
10
+ #include <queue>
10
11
  #include <string>
11
12
  #include <vector>
12
- #include <queue>
13
13
 
14
14
  #include "memory/arena.h"
15
15
  #include "rocksdb/db.h"
@@ -35,6 +35,7 @@ class MinIterComparator {
35
35
  bool operator()(InternalIterator* a, InternalIterator* b) {
36
36
  return comparator_->Compare(a->key(), b->key()) > 0;
37
37
  }
38
+
38
39
  private:
39
40
  const CompareInterface* comparator_;
40
41
  };
@@ -92,8 +93,8 @@ class ForwardIterator : public InternalIterator {
92
93
  // either done immediately or deferred until this iterator is unpinned by
93
94
  // PinnedIteratorsManager.
94
95
  void SVCleanup();
95
- static void SVCleanup(
96
- DBImpl* db, SuperVersion* sv, bool background_purge_on_iterator_cleanup);
96
+ static void SVCleanup(DBImpl* db, SuperVersion* sv,
97
+ bool background_purge_on_iterator_cleanup);
97
98
  static void DeferredSVCleanup(void* arg);
98
99
 
99
100
  void RebuildIterators(bool refresh_sv);
@@ -107,9 +108,9 @@ class ForwardIterator : public InternalIterator {
107
108
  void UpdateCurrent();
108
109
  bool NeedToSeekImmutable(const Slice& internal_key);
109
110
  void DeleteCurrentIter();
110
- uint32_t FindFileInRange(
111
- const std::vector<FileMetaData*>& files, const Slice& internal_key,
112
- uint32_t left, uint32_t right);
111
+ uint32_t FindFileInRange(const std::vector<FileMetaData*>& files,
112
+ const Slice& internal_key, uint32_t left,
113
+ uint32_t right);
113
114
 
114
115
  bool IsOverUpperBound(const Slice& internal_key) const;
115
116
 
@@ -14,6 +14,7 @@ int main() {
14
14
  int main() { return 0; }
15
15
  #else
16
16
  #include <semaphore.h>
17
+
17
18
  #include <atomic>
18
19
  #include <bitset>
19
20
  #include <chrono>
@@ -281,8 +282,9 @@ struct StatsThread {
281
282
  }
282
283
  auto now = std::chrono::steady_clock::now();
283
284
  double elapsed =
284
- std::chrono::duration_cast<std::chrono::duration<double> >(
285
- now - tlast).count();
285
+ std::chrono::duration_cast<std::chrono::duration<double> >(now -
286
+ tlast)
287
+ .count();
286
288
  uint64_t w = ::stats.written.load();
287
289
  uint64_t r = ::stats.read.load();
288
290
  fprintf(stderr,
@@ -228,8 +228,8 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
228
228
  std::unique_ptr<FSRandomAccessFile> sst_file;
229
229
  std::unique_ptr<RandomAccessFileReader> sst_file_reader;
230
230
 
231
- status = fs_->NewRandomAccessFile(external_file, env_options_,
232
- &sst_file, nullptr);
231
+ status =
232
+ fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
233
233
  if (!status.ok()) {
234
234
  return status;
235
235
  }
@@ -423,6 +423,10 @@ const std::string DB::Properties::kBlobCacheUsage =
423
423
  const std::string DB::Properties::kBlobCachePinnedUsage =
424
424
  rocksdb_prefix + blob_cache_pinned_usage;
425
425
 
426
+ const std::string InternalStats::kPeriodicCFStats =
427
+ DB::Properties::kCFStats + ".periodic";
428
+ const int InternalStats::kMaxNoChangePeriodSinceDump = 8;
429
+
426
430
  const UnorderedMap<std::string, DBPropertyInfo>
427
431
  InternalStats::ppt_name_to_info = {
428
432
  {DB::Properties::kNumFilesAtLevelPrefix,
@@ -438,6 +442,9 @@ const UnorderedMap<std::string, DBPropertyInfo>
438
442
  {DB::Properties::kCFStats,
439
443
  {false, &InternalStats::HandleCFStats, nullptr,
440
444
  &InternalStats::HandleCFMapStats, nullptr}},
445
+ {InternalStats::kPeriodicCFStats,
446
+ {false, &InternalStats::HandleCFStatsPeriodic, nullptr, nullptr,
447
+ nullptr}},
441
448
  {DB::Properties::kCFStatsNoFileHistogram,
442
449
  {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr, nullptr,
443
450
  nullptr}},
@@ -605,6 +612,7 @@ InternalStats::InternalStats(int num_levels, SystemClock* clock,
605
612
  comp_stats_(num_levels),
606
613
  comp_stats_by_pri_(Env::Priority::TOTAL),
607
614
  file_read_latency_(num_levels),
615
+ has_cf_change_since_dump_(true),
608
616
  bg_error_count_(0),
609
617
  number_levels_(num_levels),
610
618
  clock_(clock),
@@ -1041,9 +1049,41 @@ bool InternalStats::HandleCFStats(std::string* value, Slice /*suffix*/) {
1041
1049
  return true;
1042
1050
  }
1043
1051
 
1052
+ bool InternalStats::HandleCFStatsPeriodic(std::string* value,
1053
+ Slice /*suffix*/) {
1054
+ bool has_change = has_cf_change_since_dump_;
1055
+ if (!has_change) {
1056
+ // If file histogram changes, there is activity in this period too.
1057
+ uint64_t new_histogram_num = 0;
1058
+ for (int level = 0; level < number_levels_; level++) {
1059
+ new_histogram_num += file_read_latency_[level].num();
1060
+ }
1061
+ new_histogram_num += blob_file_read_latency_.num();
1062
+ if (new_histogram_num != last_histogram_num) {
1063
+ has_change = true;
1064
+ last_histogram_num = new_histogram_num;
1065
+ }
1066
+ }
1067
+ if (has_change) {
1068
+ no_cf_change_period_since_dump_ = 0;
1069
+ has_cf_change_since_dump_ = false;
1070
+ } else if (no_cf_change_period_since_dump_++ > 0) {
1071
+ // Not ready to sync
1072
+ if (no_cf_change_period_since_dump_ == kMaxNoChangePeriodSinceDump) {
1073
+ // Next periodic, we need to dump stats even if there is no change.
1074
+ no_cf_change_period_since_dump_ = 0;
1075
+ }
1076
+ return true;
1077
+ }
1078
+
1079
+ DumpCFStatsNoFileHistogram(/*is_periodic=*/true, value);
1080
+ DumpCFFileHistogram(value);
1081
+ return true;
1082
+ }
1083
+
1044
1084
  bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
1045
1085
  Slice /*suffix*/) {
1046
- DumpCFStatsNoFileHistogram(value);
1086
+ DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value);
1047
1087
  return true;
1048
1088
  }
1049
1089
 
@@ -1708,11 +1748,12 @@ void InternalStats::DumpCFMapStatsIOStalls(
1708
1748
  }
1709
1749
 
1710
1750
  void InternalStats::DumpCFStats(std::string* value) {
1711
- DumpCFStatsNoFileHistogram(value);
1751
+ DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value);
1712
1752
  DumpCFFileHistogram(value);
1713
1753
  }
1714
1754
 
1715
- void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
1755
+ void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic,
1756
+ std::string* value) {
1716
1757
  char buf[2000];
1717
1758
  // Per-ColumnFamily stats
1718
1759
  PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Level");
@@ -1864,9 +1905,11 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
1864
1905
  interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
1865
1906
  interval_compact_micros / kMicrosInSec);
1866
1907
  value->append(buf);
1867
- cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
1868
- cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
1869
- cf_stats_snapshot_.compact_micros = compact_micros;
1908
+ if (is_periodic) {
1909
+ cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
1910
+ cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
1911
+ cf_stats_snapshot_.compact_micros = compact_micros;
1912
+ }
1870
1913
 
1871
1914
  snprintf(buf, sizeof(buf),
1872
1915
  "Stalls(count): %" PRIu64
@@ -1897,14 +1940,16 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
1897
1940
  total_stall_count - cf_stats_snapshot_.stall_count);
1898
1941
  value->append(buf);
1899
1942
 
1900
- cf_stats_snapshot_.seconds_up = seconds_up;
1901
- cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
1902
- cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
1903
- cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
1904
- cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
1905
- cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
1906
- cf_stats_snapshot_.comp_stats = compaction_stats_sum;
1907
- cf_stats_snapshot_.stall_count = total_stall_count;
1943
+ if (is_periodic) {
1944
+ cf_stats_snapshot_.seconds_up = seconds_up;
1945
+ cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
1946
+ cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
1947
+ cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
1948
+ cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
1949
+ cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
1950
+ cf_stats_snapshot_.comp_stats = compaction_stats_sum;
1951
+ cf_stats_snapshot_.stall_count = total_stall_count;
1952
+ }
1908
1953
 
1909
1954
  // Do not gather cache entry stats during CFStats because DB
1910
1955
  // mutex is held. Only dump last cached collection (rely on DB