@nxtedition/rocksdb 7.1.33 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BUILDING.md +2 -2
- package/binding.cc +0 -147
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +402 -345
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +121 -64
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +28 -18
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +1 -0
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +2 -0
- package/deps/rocksdb/rocksdb/db/builder.cc +2 -1
- package/deps/rocksdb/rocksdb/db/c.cc +563 -673
- package/deps/rocksdb/rocksdb/db/c_test.c +168 -169
- package/deps/rocksdb/rocksdb/db/column_family.cc +16 -15
- package/deps/rocksdb/rocksdb/db/column_family.h +7 -7
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +17 -28
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +4 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +8 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +114 -0
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/convenience.cc +3 -5
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +10 -14
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +9 -13
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +52 -72
- package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +12 -12
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -2
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +1 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +3 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +26 -0
- package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter.cc +12 -6
- package/deps/rocksdb/rocksdb/db/db_iter.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +6 -7
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +10 -8
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +15 -13
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +2 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +7 -4
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +8 -6
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +18 -23
- package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/db_test.cc +10 -5
- package/deps/rocksdb/rocksdb/db/db_test2.cc +172 -169
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +68 -66
- package/deps/rocksdb/rocksdb/db/db_test_util.h +1 -3
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +31 -39
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +182 -2
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +43 -40
- package/deps/rocksdb/rocksdb/db/dbformat.h +15 -0
- package/deps/rocksdb/rocksdb/db/dbformat_test.cc +35 -34
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +10 -11
- package/deps/rocksdb/rocksdb/db/error_handler.cc +6 -6
- package/deps/rocksdb/rocksdb/db/error_handler.h +93 -94
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +1 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.h +3 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +16 -17
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +1 -2
- package/deps/rocksdb/rocksdb/db/file_indexer.cc +2 -0
- package/deps/rocksdb/rocksdb/db/file_indexer.h +2 -1
- package/deps/rocksdb/rocksdb/db/file_indexer_test.cc +4 -2
- package/deps/rocksdb/rocksdb/db/filename_test.cc +27 -29
- package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -13
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +15 -21
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +7 -6
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +2 -2
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +59 -14
- package/deps/rocksdb/rocksdb/db/internal_stats.h +27 -11
- package/deps/rocksdb/rocksdb/db/job_context.h +5 -6
- package/deps/rocksdb/rocksdb/db/listener_test.cc +21 -23
- package/deps/rocksdb/rocksdb/db/log_reader.cc +7 -11
- package/deps/rocksdb/rocksdb/db/log_reader.h +4 -6
- package/deps/rocksdb/rocksdb/db/log_test.cc +6 -12
- package/deps/rocksdb/rocksdb/db/log_writer.h +1 -1
- package/deps/rocksdb/rocksdb/db/logs_with_prep_tracker.h +0 -1
- package/deps/rocksdb/rocksdb/db/lookup_key.h +4 -1
- package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -1
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/memtable.cc +34 -22
- package/deps/rocksdb/rocksdb/db/memtable.h +4 -6
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +37 -13
- package/deps/rocksdb/rocksdb/db/merge_context.h +1 -0
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +128 -14
- package/deps/rocksdb/rocksdb/db/merge_helper.h +15 -7
- package/deps/rocksdb/rocksdb/db/merge_helper_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/merge_operator.cc +5 -6
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +55 -43
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +288 -299
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +22 -27
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +7 -8
- package/deps/rocksdb/rocksdb/db/repair_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/snapshot_impl.cc +4 -5
- package/deps/rocksdb/rocksdb/db/snapshot_impl.h +10 -4
- package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -4
- package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +6 -7
- package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +22 -22
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +12 -12
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +6 -8
- package/deps/rocksdb/rocksdb/db/trim_history_scheduler.h +2 -0
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/version_edit.cc +2 -5
- package/deps/rocksdb/rocksdb/db/version_edit.h +8 -12
- package/deps/rocksdb/rocksdb/db/version_set.cc +74 -102
- package/deps/rocksdb/rocksdb/db/version_set.h +8 -10
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +0 -5
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +47 -45
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +6 -5
- package/deps/rocksdb/rocksdb/db/wal_manager.h +2 -2
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +144 -61
- package/deps/rocksdb/rocksdb/db/write_batch.cc +41 -24
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +2 -7
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +105 -104
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +5 -4
- package/deps/rocksdb/rocksdb/db/write_controller.h +1 -0
- package/deps/rocksdb/rocksdb/db/write_controller_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/write_thread.cc +8 -6
- package/deps/rocksdb/rocksdb/env/io_posix.h +6 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +134 -65
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +29 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +14 -4
- package/deps/rocksdb/rocksdb/table/get_context.cc +52 -7
- package/deps/rocksdb/rocksdb/table/get_context.h +1 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +13 -0
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +36 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +23 -28
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +11 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +19 -17
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +10 -7
- package/index.js +3 -195
- package/package.json +2 -4
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/common.js +0 -7
|
@@ -106,9 +106,8 @@ Status ExternalSstFileIngestionJob::Prepare(
|
|
|
106
106
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
|
107
107
|
f.copy_file = false;
|
|
108
108
|
const std::string path_outside_db = f.external_file_path;
|
|
109
|
-
const std::string path_inside_db =
|
|
110
|
-
|
|
111
|
-
f.fd.GetPathId());
|
|
109
|
+
const std::string path_inside_db = TableFileName(
|
|
110
|
+
cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
|
|
112
111
|
if (ingestion_options_.move_files) {
|
|
113
112
|
status =
|
|
114
113
|
fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
|
|
@@ -491,7 +490,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
|
|
|
491
490
|
stream.StartArray();
|
|
492
491
|
|
|
493
492
|
for (IngestedFileInfo& f : files_to_ingest_) {
|
|
494
|
-
InternalStats::CompactionStats stats(
|
|
493
|
+
InternalStats::CompactionStats stats(
|
|
494
|
+
CompactionReason::kExternalSstIngestion, 1);
|
|
495
495
|
stats.micros = total_time;
|
|
496
496
|
// If actual copy occurred for this file, then we need to count the file
|
|
497
497
|
// size as the actual bytes written. If the file was linked, then we ignore
|
|
@@ -591,8 +591,8 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
|
|
591
591
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
|
592
592
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
|
593
593
|
|
|
594
|
-
status =
|
|
595
|
-
|
|
594
|
+
status =
|
|
595
|
+
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
|
596
596
|
if (!status.ok()) {
|
|
597
597
|
return status;
|
|
598
598
|
}
|
|
@@ -658,9 +658,9 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
|
|
658
658
|
assert(seqno_iter == uprops.end());
|
|
659
659
|
file_to_ingest->original_seqno = 0;
|
|
660
660
|
if (ingestion_options_.allow_blocking_flush ||
|
|
661
|
-
|
|
661
|
+
ingestion_options_.allow_global_seqno) {
|
|
662
662
|
return Status::InvalidArgument(
|
|
663
|
-
|
|
663
|
+
"External SST file V1 does not support global seqno");
|
|
664
664
|
}
|
|
665
665
|
} else {
|
|
666
666
|
return Status::InvalidArgument("External file version is not supported");
|
|
@@ -857,7 +857,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
|
|
|
857
857
|
return status;
|
|
858
858
|
}
|
|
859
859
|
|
|
860
|
-
|
|
860
|
+
TEST_SYNC_POINT_CALLBACK(
|
|
861
861
|
"ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
|
|
862
862
|
&overlap_with_db);
|
|
863
863
|
file_to_ingest->picked_level = target_level;
|
|
@@ -872,10 +872,10 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
|
|
|
872
872
|
auto* vstorage = cfd_->current()->storage_info();
|
|
873
873
|
// first check if new files fit in the bottommost level
|
|
874
874
|
int bottom_lvl = cfd_->NumberLevels() - 1;
|
|
875
|
-
if(!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
|
|
875
|
+
if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
|
|
876
876
|
return Status::InvalidArgument(
|
|
877
|
-
|
|
878
|
-
|
|
877
|
+
"Can't ingest_behind file as it doesn't fit "
|
|
878
|
+
"at the bottommost level!");
|
|
879
879
|
}
|
|
880
880
|
|
|
881
881
|
// second check if despite allow_ingest_behind=true we still have 0 seqnums
|
|
@@ -884,8 +884,8 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
|
|
|
884
884
|
for (auto file : vstorage->LevelFiles(lvl)) {
|
|
885
885
|
if (file->fd.smallest_seqno == 0) {
|
|
886
886
|
return Status::InvalidArgument(
|
|
887
|
-
|
|
888
|
-
|
|
887
|
+
"Can't ingest_behind file as despite allow_ingest_behind=true "
|
|
888
|
+
"there are files with 0 seqno in database at upper levels!");
|
|
889
889
|
}
|
|
890
890
|
}
|
|
891
891
|
}
|
|
@@ -912,9 +912,8 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
|
|
|
912
912
|
// If the file system does not support random write, then we should not.
|
|
913
913
|
// Otherwise we should.
|
|
914
914
|
std::unique_ptr<FSRandomRWFile> rwfile;
|
|
915
|
-
Status status =
|
|
916
|
-
|
|
917
|
-
&rwfile, nullptr);
|
|
915
|
+
Status status = fs_->NewRandomRWFile(file_to_ingest->internal_file_path,
|
|
916
|
+
env_options_, &rwfile, nullptr);
|
|
918
917
|
TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::NewRandomRWFile",
|
|
919
918
|
&status);
|
|
920
919
|
if (status.ok()) {
|
|
@@ -301,7 +301,8 @@ TEST_F(ExternalSSTFileTest, Basic) {
|
|
|
301
301
|
|
|
302
302
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
|
303
303
|
|
|
304
|
-
// Current file size should be 0 after sst_file_writer init and before open
|
|
304
|
+
// Current file size should be 0 after sst_file_writer init and before open
|
|
305
|
+
// a file.
|
|
305
306
|
ASSERT_EQ(sst_file_writer.FileSize(), 0);
|
|
306
307
|
|
|
307
308
|
// file1.sst (0 => 99)
|
|
@@ -2318,7 +2319,6 @@ TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
|
|
|
2318
2319
|
table_options.cache_index_and_filter_blocks = true;
|
|
2319
2320
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
2320
2321
|
|
|
2321
|
-
|
|
2322
2322
|
// Create external SST file and include bloom filters
|
|
2323
2323
|
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
2324
2324
|
DestroyAndReopen(options);
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
#include <functional>
|
|
13
13
|
#include <limits>
|
|
14
14
|
#include <vector>
|
|
15
|
+
|
|
15
16
|
#include "memory/arena.h"
|
|
16
17
|
#include "port/port.h"
|
|
17
18
|
#include "util/autovector.h"
|
|
@@ -66,7 +67,7 @@ class FileIndexer {
|
|
|
66
67
|
|
|
67
68
|
struct IndexUnit {
|
|
68
69
|
IndexUnit()
|
|
69
|
-
|
|
70
|
+
: smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
|
|
70
71
|
// During file search, a key is compared against smallest and largest
|
|
71
72
|
// from a FileMetaData. It can have 3 possible outcomes:
|
|
72
73
|
// (1) key is smaller than smallest, implying it is also smaller than
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
9
9
|
|
|
10
10
|
#include "db/file_indexer.h"
|
|
11
|
+
|
|
11
12
|
#include <string>
|
|
13
|
+
|
|
12
14
|
#include "db/dbformat.h"
|
|
13
15
|
#include "db/version_edit.h"
|
|
14
16
|
#include "port/stack_trace.h"
|
|
@@ -73,8 +75,8 @@ class FileIndexerTest : public testing::Test {
|
|
|
73
75
|
}
|
|
74
76
|
|
|
75
77
|
void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
|
|
76
|
-
|
|
77
|
-
|
|
78
|
+
const int cmp_smallest, const int cmp_largest,
|
|
79
|
+
int32_t* left_index, int32_t* right_index) {
|
|
78
80
|
*left_index = 100;
|
|
79
81
|
*right_index = 100;
|
|
80
82
|
indexer->GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
|
|
@@ -69,35 +69,33 @@ TEST_F(FileNameTest, Parse) {
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
// Errors
|
|
72
|
-
static const char* errors[] = {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
"100.lop"
|
|
100
|
-
};
|
|
72
|
+
static const char* errors[] = {"",
|
|
73
|
+
"foo",
|
|
74
|
+
"foo-dx-100.log",
|
|
75
|
+
".log",
|
|
76
|
+
"",
|
|
77
|
+
"manifest",
|
|
78
|
+
"CURREN",
|
|
79
|
+
"CURRENTX",
|
|
80
|
+
"MANIFES",
|
|
81
|
+
"MANIFEST",
|
|
82
|
+
"MANIFEST-",
|
|
83
|
+
"XMANIFEST-3",
|
|
84
|
+
"MANIFEST-3x",
|
|
85
|
+
"META",
|
|
86
|
+
"METADB",
|
|
87
|
+
"METADB-",
|
|
88
|
+
"XMETADB-3",
|
|
89
|
+
"METADB-3x",
|
|
90
|
+
"LOC",
|
|
91
|
+
"LOCKx",
|
|
92
|
+
"LO",
|
|
93
|
+
"LOGx",
|
|
94
|
+
"18446744073709551616.log",
|
|
95
|
+
"184467440737095516150.log",
|
|
96
|
+
"100",
|
|
97
|
+
"100.",
|
|
98
|
+
"100.lop"};
|
|
101
99
|
for (unsigned int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
|
102
100
|
std::string f = errors[i];
|
|
103
101
|
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
|
|
49
49
|
namespace ROCKSDB_NAMESPACE {
|
|
50
50
|
|
|
51
|
-
const char* GetFlushReasonString
|
|
51
|
+
const char* GetFlushReasonString(FlushReason flush_reason) {
|
|
52
52
|
switch (flush_reason) {
|
|
53
53
|
case FlushReason::kOthers:
|
|
54
54
|
return "Other Reasons";
|
|
@@ -136,17 +136,14 @@ FlushJob::FlushJob(
|
|
|
136
136
|
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
-
FlushJob::~FlushJob() {
|
|
140
|
-
ThreadStatusUtil::ResetThreadStatus();
|
|
141
|
-
}
|
|
139
|
+
FlushJob::~FlushJob() { ThreadStatusUtil::ResetThreadStatus(); }
|
|
142
140
|
|
|
143
141
|
void FlushJob::ReportStartedFlush() {
|
|
144
142
|
ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
|
|
145
143
|
db_options_.enable_thread_tracking);
|
|
146
144
|
ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
|
|
147
|
-
ThreadStatusUtil::SetThreadOperationProperty(
|
|
148
|
-
|
|
149
|
-
job_context_->job_id);
|
|
145
|
+
ThreadStatusUtil::SetThreadOperationProperty(ThreadStatus::COMPACTION_JOB_ID,
|
|
146
|
+
job_context_->job_id);
|
|
150
147
|
IOSTATS_RESET(bytes_written);
|
|
151
148
|
}
|
|
152
149
|
|
|
@@ -156,8 +153,7 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
|
|
|
156
153
|
input_size += mem->ApproximateMemoryUsage();
|
|
157
154
|
}
|
|
158
155
|
ThreadStatusUtil::IncreaseThreadOperationProperty(
|
|
159
|
-
ThreadStatus::FLUSH_BYTES_MEMTABLES,
|
|
160
|
-
input_size);
|
|
156
|
+
ThreadStatus::FLUSH_BYTES_MEMTABLES, input_size);
|
|
161
157
|
}
|
|
162
158
|
|
|
163
159
|
void FlushJob::RecordFlushIOStats() {
|
|
@@ -220,8 +216,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
|
|
|
220
216
|
double mempurge_threshold =
|
|
221
217
|
mutable_cf_options_.experimental_mempurge_threshold;
|
|
222
218
|
|
|
223
|
-
AutoThreadOperationStageUpdater stage_run(
|
|
224
|
-
ThreadStatus::STAGE_FLUSH_RUN);
|
|
219
|
+
AutoThreadOperationStageUpdater stage_run(ThreadStatus::STAGE_FLUSH_RUN);
|
|
225
220
|
if (mems_.empty()) {
|
|
226
221
|
ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
|
|
227
222
|
cfd_->GetName().c_str());
|
|
@@ -906,8 +901,7 @@ Status FlushJob::WriteLevel0Table() {
|
|
|
906
901
|
}
|
|
907
902
|
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
|
908
903
|
|
|
909
|
-
uint64_t oldest_key_time =
|
|
910
|
-
mems_.front()->ApproximateOldestKeyTime();
|
|
904
|
+
uint64_t oldest_key_time = mems_.front()->ApproximateOldestKeyTime();
|
|
911
905
|
|
|
912
906
|
// It's not clear whether oldest_key_time is always available. In case
|
|
913
907
|
// it is not available, use current_time.
|
|
@@ -214,8 +214,8 @@ TEST_F(FlushJobTest, NonEmpty) {
|
|
|
214
214
|
// Note: the first two blob references will not be considered when resolving
|
|
215
215
|
// the oldest blob file referenced (the first one is inlined TTL, while the
|
|
216
216
|
// second one is TTL and thus points to a TTL blob file).
|
|
217
|
-
constexpr std::array<uint64_t, 6> blob_file_numbers{
|
|
218
|
-
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
|
217
|
+
constexpr std::array<uint64_t, 6> blob_file_numbers{
|
|
218
|
+
{kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
|
219
219
|
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
|
220
220
|
std::string key(std::to_string(i + 10001));
|
|
221
221
|
std::string blob_index;
|
|
@@ -104,9 +104,7 @@ class ForwardLevelIterator : public InternalIterator {
|
|
|
104
104
|
status_ = Status::NotSupported("ForwardLevelIterator::Prev()");
|
|
105
105
|
valid_ = false;
|
|
106
106
|
}
|
|
107
|
-
bool Valid() const override {
|
|
108
|
-
return valid_;
|
|
109
|
-
}
|
|
107
|
+
bool Valid() const override { return valid_; }
|
|
110
108
|
void SeekToFirst() override {
|
|
111
109
|
assert(file_iter_ != nullptr);
|
|
112
110
|
if (!status_.ok()) {
|
|
@@ -249,9 +247,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
|
|
|
249
247
|
immutable_status_.PermitUncheckedError();
|
|
250
248
|
}
|
|
251
249
|
|
|
252
|
-
ForwardIterator::~ForwardIterator() {
|
|
253
|
-
Cleanup(true);
|
|
254
|
-
}
|
|
250
|
+
ForwardIterator::~ForwardIterator() { Cleanup(true); }
|
|
255
251
|
|
|
256
252
|
void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
|
|
257
253
|
bool background_purge_on_iterator_cleanup) {
|
|
@@ -284,13 +280,13 @@ struct SVCleanupParams {
|
|
|
284
280
|
SuperVersion* sv;
|
|
285
281
|
bool background_purge_on_iterator_cleanup;
|
|
286
282
|
};
|
|
287
|
-
}
|
|
283
|
+
} // anonymous namespace
|
|
288
284
|
|
|
289
285
|
// Used in PinnedIteratorsManager to release pinned SuperVersion
|
|
290
286
|
void ForwardIterator::DeferredSVCleanup(void* arg) {
|
|
291
287
|
auto d = reinterpret_cast<SVCleanupParams*>(arg);
|
|
292
|
-
ForwardIterator::SVCleanup(
|
|
293
|
-
|
|
288
|
+
ForwardIterator::SVCleanup(d->db, d->sv,
|
|
289
|
+
d->background_purge_on_iterator_cleanup);
|
|
294
290
|
delete d;
|
|
295
291
|
}
|
|
296
292
|
|
|
@@ -547,8 +543,7 @@ void ForwardIterator::Next() {
|
|
|
547
543
|
assert(valid_);
|
|
548
544
|
bool update_prev_key = false;
|
|
549
545
|
|
|
550
|
-
if (sv_ == nullptr ||
|
|
551
|
-
sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
|
546
|
+
if (sv_ == nullptr || sv_->version_number != cfd_->GetSuperVersionNumber()) {
|
|
552
547
|
std::string current_key = key().ToString();
|
|
553
548
|
Slice old_key(current_key.data(), current_key.size());
|
|
554
549
|
|
|
@@ -578,7 +573,6 @@ void ForwardIterator::Next() {
|
|
|
578
573
|
update_prev_key = true;
|
|
579
574
|
}
|
|
580
575
|
|
|
581
|
-
|
|
582
576
|
if (update_prev_key) {
|
|
583
577
|
prev_key_.SetInternalKey(current_->key());
|
|
584
578
|
is_prev_set_ = true;
|
|
@@ -635,7 +629,7 @@ bool ForwardIterator::PrepareValue() {
|
|
|
635
629
|
|
|
636
630
|
assert(!current_->Valid());
|
|
637
631
|
assert(!current_->status().ok());
|
|
638
|
-
assert(current_ != mutable_iter_);
|
|
632
|
+
assert(current_ != mutable_iter_); // memtable iterator can't fail
|
|
639
633
|
assert(immutable_status_.ok());
|
|
640
634
|
|
|
641
635
|
valid_ = false;
|
|
@@ -950,11 +944,11 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
|
|
|
950
944
|
}
|
|
951
945
|
Slice prev_key = prev_key_.GetInternalKey();
|
|
952
946
|
if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
|
|
953
|
-
|
|
947
|
+
prefix_extractor_->Transform(prev_key)) != 0) {
|
|
954
948
|
return true;
|
|
955
949
|
}
|
|
956
950
|
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
|
957
|
-
|
|
951
|
+
prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
|
|
958
952
|
return true;
|
|
959
953
|
}
|
|
960
954
|
|
|
@@ -963,8 +957,8 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
|
|
|
963
957
|
return false;
|
|
964
958
|
}
|
|
965
959
|
if (cfd_->internal_comparator().InternalKeyComparator::Compare(
|
|
966
|
-
|
|
967
|
-
|
|
960
|
+
target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
|
|
961
|
+
: current_->key()) > 0) {
|
|
968
962
|
return true;
|
|
969
963
|
}
|
|
970
964
|
return false;
|
|
@@ -1040,11 +1034,11 @@ uint32_t ForwardIterator::FindFileInRange(
|
|
|
1040
1034
|
uint32_t left, uint32_t right) {
|
|
1041
1035
|
auto cmp = [&](const FileMetaData* f, const Slice& k) -> bool {
|
|
1042
1036
|
return cfd_->internal_comparator().InternalKeyComparator::Compare(
|
|
1043
|
-
|
|
1037
|
+
f->largest.Encode(), k) < 0;
|
|
1044
1038
|
};
|
|
1045
|
-
const auto
|
|
1046
|
-
return static_cast<uint32_t>(
|
|
1047
|
-
|
|
1039
|
+
const auto& b = files.begin();
|
|
1040
|
+
return static_cast<uint32_t>(
|
|
1041
|
+
std::lower_bound(b + left, b + right, internal_key, cmp) - b);
|
|
1048
1042
|
}
|
|
1049
1043
|
|
|
1050
1044
|
void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
|
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
#include "rocksdb/comparator.h"
|
|
8
8
|
#ifndef ROCKSDB_LITE
|
|
9
9
|
|
|
10
|
+
#include <queue>
|
|
10
11
|
#include <string>
|
|
11
12
|
#include <vector>
|
|
12
|
-
#include <queue>
|
|
13
13
|
|
|
14
14
|
#include "memory/arena.h"
|
|
15
15
|
#include "rocksdb/db.h"
|
|
@@ -35,6 +35,7 @@ class MinIterComparator {
|
|
|
35
35
|
bool operator()(InternalIterator* a, InternalIterator* b) {
|
|
36
36
|
return comparator_->Compare(a->key(), b->key()) > 0;
|
|
37
37
|
}
|
|
38
|
+
|
|
38
39
|
private:
|
|
39
40
|
const CompareInterface* comparator_;
|
|
40
41
|
};
|
|
@@ -92,8 +93,8 @@ class ForwardIterator : public InternalIterator {
|
|
|
92
93
|
// either done immediately or deferred until this iterator is unpinned by
|
|
93
94
|
// PinnedIteratorsManager.
|
|
94
95
|
void SVCleanup();
|
|
95
|
-
static void SVCleanup(
|
|
96
|
-
|
|
96
|
+
static void SVCleanup(DBImpl* db, SuperVersion* sv,
|
|
97
|
+
bool background_purge_on_iterator_cleanup);
|
|
97
98
|
static void DeferredSVCleanup(void* arg);
|
|
98
99
|
|
|
99
100
|
void RebuildIterators(bool refresh_sv);
|
|
@@ -107,9 +108,9 @@ class ForwardIterator : public InternalIterator {
|
|
|
107
108
|
void UpdateCurrent();
|
|
108
109
|
bool NeedToSeekImmutable(const Slice& internal_key);
|
|
109
110
|
void DeleteCurrentIter();
|
|
110
|
-
uint32_t FindFileInRange(
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
uint32_t FindFileInRange(const std::vector<FileMetaData*>& files,
|
|
112
|
+
const Slice& internal_key, uint32_t left,
|
|
113
|
+
uint32_t right);
|
|
113
114
|
|
|
114
115
|
bool IsOverUpperBound(const Slice& internal_key) const;
|
|
115
116
|
|
|
@@ -14,6 +14,7 @@ int main() {
|
|
|
14
14
|
int main() { return 0; }
|
|
15
15
|
#else
|
|
16
16
|
#include <semaphore.h>
|
|
17
|
+
|
|
17
18
|
#include <atomic>
|
|
18
19
|
#include <bitset>
|
|
19
20
|
#include <chrono>
|
|
@@ -281,8 +282,9 @@ struct StatsThread {
|
|
|
281
282
|
}
|
|
282
283
|
auto now = std::chrono::steady_clock::now();
|
|
283
284
|
double elapsed =
|
|
284
|
-
std::chrono::duration_cast<std::chrono::duration<double> >(
|
|
285
|
-
|
|
285
|
+
std::chrono::duration_cast<std::chrono::duration<double> >(now -
|
|
286
|
+
tlast)
|
|
287
|
+
.count();
|
|
286
288
|
uint64_t w = ::stats.written.load();
|
|
287
289
|
uint64_t r = ::stats.read.load();
|
|
288
290
|
fprintf(stderr,
|
|
@@ -228,8 +228,8 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
|
|
|
228
228
|
std::unique_ptr<FSRandomAccessFile> sst_file;
|
|
229
229
|
std::unique_ptr<RandomAccessFileReader> sst_file_reader;
|
|
230
230
|
|
|
231
|
-
status =
|
|
232
|
-
|
|
231
|
+
status =
|
|
232
|
+
fs_->NewRandomAccessFile(external_file, env_options_, &sst_file, nullptr);
|
|
233
233
|
if (!status.ok()) {
|
|
234
234
|
return status;
|
|
235
235
|
}
|
|
@@ -423,6 +423,10 @@ const std::string DB::Properties::kBlobCacheUsage =
|
|
|
423
423
|
const std::string DB::Properties::kBlobCachePinnedUsage =
|
|
424
424
|
rocksdb_prefix + blob_cache_pinned_usage;
|
|
425
425
|
|
|
426
|
+
const std::string InternalStats::kPeriodicCFStats =
|
|
427
|
+
DB::Properties::kCFStats + ".periodic";
|
|
428
|
+
const int InternalStats::kMaxNoChangePeriodSinceDump = 8;
|
|
429
|
+
|
|
426
430
|
const UnorderedMap<std::string, DBPropertyInfo>
|
|
427
431
|
InternalStats::ppt_name_to_info = {
|
|
428
432
|
{DB::Properties::kNumFilesAtLevelPrefix,
|
|
@@ -438,6 +442,9 @@ const UnorderedMap<std::string, DBPropertyInfo>
|
|
|
438
442
|
{DB::Properties::kCFStats,
|
|
439
443
|
{false, &InternalStats::HandleCFStats, nullptr,
|
|
440
444
|
&InternalStats::HandleCFMapStats, nullptr}},
|
|
445
|
+
{InternalStats::kPeriodicCFStats,
|
|
446
|
+
{false, &InternalStats::HandleCFStatsPeriodic, nullptr, nullptr,
|
|
447
|
+
nullptr}},
|
|
441
448
|
{DB::Properties::kCFStatsNoFileHistogram,
|
|
442
449
|
{false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr, nullptr,
|
|
443
450
|
nullptr}},
|
|
@@ -605,6 +612,7 @@ InternalStats::InternalStats(int num_levels, SystemClock* clock,
|
|
|
605
612
|
comp_stats_(num_levels),
|
|
606
613
|
comp_stats_by_pri_(Env::Priority::TOTAL),
|
|
607
614
|
file_read_latency_(num_levels),
|
|
615
|
+
has_cf_change_since_dump_(true),
|
|
608
616
|
bg_error_count_(0),
|
|
609
617
|
number_levels_(num_levels),
|
|
610
618
|
clock_(clock),
|
|
@@ -1041,9 +1049,41 @@ bool InternalStats::HandleCFStats(std::string* value, Slice /*suffix*/) {
|
|
|
1041
1049
|
return true;
|
|
1042
1050
|
}
|
|
1043
1051
|
|
|
1052
|
+
bool InternalStats::HandleCFStatsPeriodic(std::string* value,
|
|
1053
|
+
Slice /*suffix*/) {
|
|
1054
|
+
bool has_change = has_cf_change_since_dump_;
|
|
1055
|
+
if (!has_change) {
|
|
1056
|
+
// If file histogram changes, there is activity in this period too.
|
|
1057
|
+
uint64_t new_histogram_num = 0;
|
|
1058
|
+
for (int level = 0; level < number_levels_; level++) {
|
|
1059
|
+
new_histogram_num += file_read_latency_[level].num();
|
|
1060
|
+
}
|
|
1061
|
+
new_histogram_num += blob_file_read_latency_.num();
|
|
1062
|
+
if (new_histogram_num != last_histogram_num) {
|
|
1063
|
+
has_change = true;
|
|
1064
|
+
last_histogram_num = new_histogram_num;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
if (has_change) {
|
|
1068
|
+
no_cf_change_period_since_dump_ = 0;
|
|
1069
|
+
has_cf_change_since_dump_ = false;
|
|
1070
|
+
} else if (no_cf_change_period_since_dump_++ > 0) {
|
|
1071
|
+
// Not ready to sync
|
|
1072
|
+
if (no_cf_change_period_since_dump_ == kMaxNoChangePeriodSinceDump) {
|
|
1073
|
+
// Next periodic, we need to dump stats even if there is no change.
|
|
1074
|
+
no_cf_change_period_since_dump_ = 0;
|
|
1075
|
+
}
|
|
1076
|
+
return true;
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
DumpCFStatsNoFileHistogram(/*is_periodic=*/true, value);
|
|
1080
|
+
DumpCFFileHistogram(value);
|
|
1081
|
+
return true;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1044
1084
|
bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
|
|
1045
1085
|
Slice /*suffix*/) {
|
|
1046
|
-
DumpCFStatsNoFileHistogram(value);
|
|
1086
|
+
DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value);
|
|
1047
1087
|
return true;
|
|
1048
1088
|
}
|
|
1049
1089
|
|
|
@@ -1708,11 +1748,12 @@ void InternalStats::DumpCFMapStatsIOStalls(
|
|
|
1708
1748
|
}
|
|
1709
1749
|
|
|
1710
1750
|
void InternalStats::DumpCFStats(std::string* value) {
|
|
1711
|
-
DumpCFStatsNoFileHistogram(value);
|
|
1751
|
+
DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value);
|
|
1712
1752
|
DumpCFFileHistogram(value);
|
|
1713
1753
|
}
|
|
1714
1754
|
|
|
1715
|
-
void InternalStats::DumpCFStatsNoFileHistogram(
|
|
1755
|
+
void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic,
|
|
1756
|
+
std::string* value) {
|
|
1716
1757
|
char buf[2000];
|
|
1717
1758
|
// Per-ColumnFamily stats
|
|
1718
1759
|
PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Level");
|
|
@@ -1864,9 +1905,11 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
|
|
|
1864
1905
|
interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
|
|
1865
1906
|
interval_compact_micros / kMicrosInSec);
|
|
1866
1907
|
value->append(buf);
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1908
|
+
if (is_periodic) {
|
|
1909
|
+
cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
|
|
1910
|
+
cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
|
|
1911
|
+
cf_stats_snapshot_.compact_micros = compact_micros;
|
|
1912
|
+
}
|
|
1870
1913
|
|
|
1871
1914
|
snprintf(buf, sizeof(buf),
|
|
1872
1915
|
"Stalls(count): %" PRIu64
|
|
@@ -1897,14 +1940,16 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
|
|
|
1897
1940
|
total_stall_count - cf_stats_snapshot_.stall_count);
|
|
1898
1941
|
value->append(buf);
|
|
1899
1942
|
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1943
|
+
if (is_periodic) {
|
|
1944
|
+
cf_stats_snapshot_.seconds_up = seconds_up;
|
|
1945
|
+
cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
|
|
1946
|
+
cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
|
|
1947
|
+
cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
|
|
1948
|
+
cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
|
|
1949
|
+
cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
|
|
1950
|
+
cf_stats_snapshot_.comp_stats = compaction_stats_sum;
|
|
1951
|
+
cf_stats_snapshot_.stall_count = total_stall_count;
|
|
1952
|
+
}
|
|
1908
1953
|
|
|
1909
1954
|
// Do not gather cache entry stats during CFStats because DB
|
|
1910
1955
|
// mutex is held. Only dump last cached collection (rely on DB
|