@nxtedition/rocksdb 8.1.2 → 8.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deps/rocksdb/rocksdb/CMakeLists.txt +0 -9
- package/deps/rocksdb/rocksdb/Makefile +1 -13
- package/deps/rocksdb/rocksdb/TARGETS +0 -4
- package/deps/rocksdb/rocksdb/db/blob/blob_counting_iterator.h +0 -11
- package/deps/rocksdb/rocksdb/db/builder.cc +4 -13
- package/deps/rocksdb/rocksdb/db/builder.h +1 -2
- package/deps/rocksdb/rocksdb/db/c.cc +0 -15
- package/deps/rocksdb/rocksdb/db/c_test.c +0 -3
- package/deps/rocksdb/rocksdb/db/column_family.cc +5 -11
- package/deps/rocksdb/rocksdb/db/column_family.h +0 -20
- package/deps/rocksdb/rocksdb/db/compaction/clipping_iterator.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +37 -31
- package/deps/rocksdb/rocksdb/db/compaction/compaction.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +7 -24
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.h +1 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +1 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +2 -4
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job_test.cc +6 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.cc +26 -104
- package/deps/rocksdb/rocksdb/db/compaction/compaction_outputs.h +0 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +18 -11
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +16 -17
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +6 -19
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +22 -22
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +52 -81
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.cc +1 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_universal.h +5 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +2 -8
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +0 -8
- package/deps/rocksdb/rocksdb/db/compaction/tiered_compaction_test.cc +138 -266
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +1 -86
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +3 -2
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +123 -897
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +20 -31
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +9 -5
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +28 -121
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_experimental.cc +2 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +4 -8
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +0 -10
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +4 -325
- package/deps/rocksdb/rocksdb/db/db_test.cc +0 -3
- package/deps/rocksdb/rocksdb/db/db_test2.cc +8 -233
- package/deps/rocksdb/rocksdb/db/db_test_util.h +0 -3
- package/deps/rocksdb/rocksdb/db/db_wal_test.cc +0 -129
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_compaction_test.cc +0 -21
- package/deps/rocksdb/rocksdb/db/dbformat.cc +0 -25
- package/deps/rocksdb/rocksdb/db/dbformat.h +0 -2
- package/deps/rocksdb/rocksdb/db/experimental.cc +2 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_basic_test.cc +0 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +13 -92
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.h +1 -38
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +110 -14
- package/deps/rocksdb/rocksdb/db/flush_job.cc +4 -6
- package/deps/rocksdb/rocksdb/db/history_trimming_iterator.h +0 -4
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +53 -56
- package/deps/rocksdb/rocksdb/db/import_column_family_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/memtable.cc +1 -1
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +0 -4
- package/deps/rocksdb/rocksdb/db/periodic_task_scheduler_test.cc +10 -10
- package/deps/rocksdb/rocksdb/db/range_tombstone_fragmenter_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +22 -65
- package/deps/rocksdb/rocksdb/db/repair_test.cc +0 -54
- package/deps/rocksdb/rocksdb/db/seqno_time_test.cc +26 -26
- package/deps/rocksdb/rocksdb/db/table_properties_collector.h +1 -3
- package/deps/rocksdb/rocksdb/db/version_builder.cc +43 -90
- package/deps/rocksdb/rocksdb/db/version_builder.h +0 -20
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +93 -218
- package/deps/rocksdb/rocksdb/db/version_edit.cc +1 -27
- package/deps/rocksdb/rocksdb/db/version_edit.h +9 -33
- package/deps/rocksdb/rocksdb/db/version_edit_handler.cc +6 -13
- package/deps/rocksdb/rocksdb/db/version_edit_handler.h +6 -17
- package/deps/rocksdb/rocksdb/db/version_edit_test.cc +17 -19
- package/deps/rocksdb/rocksdb/db/version_set.cc +26 -166
- package/deps/rocksdb/rocksdb/db/version_set.h +2 -32
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +31 -65
- package/deps/rocksdb/rocksdb/db/write_thread.cc +2 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.h +0 -1
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +32 -31
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.h +1 -2
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_env_wrapper.h +6 -8
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_gflags.cc +0 -4
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +4 -11
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_tool.cc +15 -16
- package/deps/rocksdb/rocksdb/db_stress_tool/no_batched_ops_stress.cc +1 -13
- package/deps/rocksdb/rocksdb/file/prefetch_test.cc +75 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/c.h +0 -8
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +0 -6
- package/deps/rocksdb/rocksdb/include/rocksdb/listener.h +1 -7
- package/deps/rocksdb/rocksdb/include/rocksdb/metadata.h +3 -9
- package/deps/rocksdb/rocksdb/include/rocksdb/options.h +1 -2
- package/deps/rocksdb/rocksdb/include/rocksdb/status.h +0 -3
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/backup_engine.h +9 -69
- package/deps/rocksdb/rocksdb/include/rocksdb/version.h +2 -2
- package/deps/rocksdb/rocksdb/memory/arena.cc +87 -23
- package/deps/rocksdb/rocksdb/memory/arena.h +31 -25
- package/deps/rocksdb/rocksdb/memory/arena_test.cc +0 -90
- package/deps/rocksdb/rocksdb/monitoring/stats_history_test.cc +26 -26
- package/deps/rocksdb/rocksdb/src.mk +0 -2
- package/deps/rocksdb/rocksdb/table/adaptive/adaptive_table_factory.cc +2 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +2 -3
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_impl.h +2 -6
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_test.cc +1 -1
- package/deps/rocksdb/rocksdb/table/block_fetcher_test.cc +3 -3
- package/deps/rocksdb/rocksdb/table/format.cc +20 -24
- package/deps/rocksdb/rocksdb/table/format.h +2 -5
- package/deps/rocksdb/rocksdb/table/merging_iterator.cc +105 -54
- package/deps/rocksdb/rocksdb/table/merging_iterator.h +0 -80
- package/deps/rocksdb/rocksdb/table/meta_blocks.cc +2 -2
- package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc +1 -1
- package/deps/rocksdb/rocksdb/table/table_test.cc +6 -7
- package/deps/rocksdb/rocksdb/test_util/testutil.h +0 -10
- package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h +2 -2
- package/deps/rocksdb/rocksdb/util/bloom_test.cc +1 -1
- package/deps/rocksdb/rocksdb/util/status.cc +0 -7
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc +74 -250
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +4 -199
- package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc +0 -1
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.cc +0 -39
- package/deps/rocksdb/rocksdb/utilities/fault_injection_fs.h +0 -9
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +0 -59
- package/deps/rocksdb/rocksdb.gyp +0 -3
- package/index.js +2 -2
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
- package/deps/rocksdb/rocksdb/port/mmap.cc +0 -98
- package/deps/rocksdb/rocksdb/port/mmap.h +0 -70
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.cc +0 -142
- package/deps/rocksdb/rocksdb/table/compaction_merging_iterator.h +0 -241
|
@@ -39,8 +39,6 @@
|
|
|
39
39
|
#include "db/table_cache.h"
|
|
40
40
|
#include "db/version_builder.h"
|
|
41
41
|
#include "db/version_edit_handler.h"
|
|
42
|
-
#include "table/compaction_merging_iterator.h"
|
|
43
|
-
|
|
44
42
|
#if USE_COROUTINES
|
|
45
43
|
#include "folly/experimental/coro/BlockingWait.h"
|
|
46
44
|
#include "folly/experimental/coro/Collect.h"
|
|
@@ -1773,8 +1771,8 @@ void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
|
|
|
1773
1771
|
file->stats.num_reads_sampled.load(std::memory_order_relaxed),
|
|
1774
1772
|
file->being_compacted, file->temperature,
|
|
1775
1773
|
file->oldest_blob_file_number, file->TryGetOldestAncesterTime(),
|
|
1776
|
-
file->TryGetFileCreationTime(), file->
|
|
1777
|
-
file->
|
|
1774
|
+
file->TryGetFileCreationTime(), file->file_checksum,
|
|
1775
|
+
file->file_checksum_func_name);
|
|
1778
1776
|
files.back().num_entries = file->num_entries;
|
|
1779
1777
|
files.back().num_deletions = file->num_deletions;
|
|
1780
1778
|
level_size += file->fd.GetFileSize();
|
|
@@ -2038,8 +2036,7 @@ VersionStorageInfo::VersionStorageInfo(
|
|
|
2038
2036
|
const InternalKeyComparator* internal_comparator,
|
|
2039
2037
|
const Comparator* user_comparator, int levels,
|
|
2040
2038
|
CompactionStyle compaction_style, VersionStorageInfo* ref_vstorage,
|
|
2041
|
-
bool _force_consistency_checks
|
|
2042
|
-
EpochNumberRequirement epoch_number_requirement)
|
|
2039
|
+
bool _force_consistency_checks)
|
|
2043
2040
|
: internal_comparator_(internal_comparator),
|
|
2044
2041
|
user_comparator_(user_comparator),
|
|
2045
2042
|
// cfd is nullptr if Version is dummy
|
|
@@ -2067,8 +2064,7 @@ VersionStorageInfo::VersionStorageInfo(
|
|
|
2067
2064
|
current_num_samples_(0),
|
|
2068
2065
|
estimated_compaction_needed_bytes_(0),
|
|
2069
2066
|
finalized_(false),
|
|
2070
|
-
force_consistency_checks_(_force_consistency_checks)
|
|
2071
|
-
epoch_number_requirement_(epoch_number_requirement) {
|
|
2067
|
+
force_consistency_checks_(_force_consistency_checks) {
|
|
2072
2068
|
if (ref_vstorage != nullptr) {
|
|
2073
2069
|
accumulated_file_size_ = ref_vstorage->accumulated_file_size_;
|
|
2074
2070
|
accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_;
|
|
@@ -2089,8 +2085,7 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
|
|
|
2089
2085
|
const FileOptions& file_opt,
|
|
2090
2086
|
const MutableCFOptions mutable_cf_options,
|
|
2091
2087
|
const std::shared_ptr<IOTracer>& io_tracer,
|
|
2092
|
-
uint64_t version_number
|
|
2093
|
-
EpochNumberRequirement epoch_number_requirement)
|
|
2088
|
+
uint64_t version_number)
|
|
2094
2089
|
: env_(vset->env_),
|
|
2095
2090
|
clock_(vset->clock_),
|
|
2096
2091
|
cfd_(column_family_data),
|
|
@@ -2109,8 +2104,7 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
|
|
|
2109
2104
|
(cfd_ == nullptr || cfd_->current() == nullptr)
|
|
2110
2105
|
? nullptr
|
|
2111
2106
|
: cfd_->current()->storage_info(),
|
|
2112
|
-
cfd_ == nullptr ? false : cfd_->ioptions()->force_consistency_checks,
|
|
2113
|
-
epoch_number_requirement),
|
|
2107
|
+
cfd_ == nullptr ? false : cfd_->ioptions()->force_consistency_checks),
|
|
2114
2108
|
vset_(vset),
|
|
2115
2109
|
next_(this),
|
|
2116
2110
|
prev_(this),
|
|
@@ -2960,7 +2954,7 @@ bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
|
|
|
2960
2954
|
file_meta->num_deletions = tp->num_deletions;
|
|
2961
2955
|
file_meta->raw_value_size = tp->raw_value_size;
|
|
2962
2956
|
file_meta->raw_key_size = tp->raw_key_size;
|
|
2963
|
-
|
|
2957
|
+
|
|
2964
2958
|
return true;
|
|
2965
2959
|
}
|
|
2966
2960
|
|
|
@@ -3062,15 +3056,11 @@ void VersionStorageInfo::ComputeCompensatedSizes() {
|
|
|
3062
3056
|
// size of deletion entries in a stable workload, the deletion
|
|
3063
3057
|
// compensation logic might introduce unwanted effet which changes the
|
|
3064
3058
|
// shape of LSM tree.
|
|
3065
|
-
if (
|
|
3066
|
-
file_meta->num_entries) {
|
|
3059
|
+
if (file_meta->num_deletions * 2 >= file_meta->num_entries) {
|
|
3067
3060
|
file_meta->compensated_file_size +=
|
|
3068
|
-
(
|
|
3069
|
-
file_meta->num_entries) *
|
|
3061
|
+
(file_meta->num_deletions * 2 - file_meta->num_entries) *
|
|
3070
3062
|
average_value_size * kDeletionWeightOnCompaction;
|
|
3071
3063
|
}
|
|
3072
|
-
file_meta->compensated_file_size +=
|
|
3073
|
-
file_meta->compensated_range_deletion_size;
|
|
3074
3064
|
}
|
|
3075
3065
|
}
|
|
3076
3066
|
}
|
|
@@ -4290,74 +4280,6 @@ const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch,
|
|
|
4290
4280
|
return scratch->buffer;
|
|
4291
4281
|
}
|
|
4292
4282
|
|
|
4293
|
-
bool VersionStorageInfo::HasMissingEpochNumber() const {
|
|
4294
|
-
for (int level = 0; level < num_levels_; ++level) {
|
|
4295
|
-
for (const FileMetaData* f : files_[level]) {
|
|
4296
|
-
if (f->epoch_number == kUnknownEpochNumber) {
|
|
4297
|
-
return true;
|
|
4298
|
-
}
|
|
4299
|
-
}
|
|
4300
|
-
}
|
|
4301
|
-
return false;
|
|
4302
|
-
}
|
|
4303
|
-
|
|
4304
|
-
uint64_t VersionStorageInfo::GetMaxEpochNumberOfFiles() const {
|
|
4305
|
-
uint64_t max_epoch_number = kUnknownEpochNumber;
|
|
4306
|
-
for (int level = 0; level < num_levels_; ++level) {
|
|
4307
|
-
for (const FileMetaData* f : files_[level]) {
|
|
4308
|
-
max_epoch_number = std::max(max_epoch_number, f->epoch_number);
|
|
4309
|
-
}
|
|
4310
|
-
}
|
|
4311
|
-
return max_epoch_number;
|
|
4312
|
-
}
|
|
4313
|
-
|
|
4314
|
-
void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd) {
|
|
4315
|
-
cfd->ResetNextEpochNumber();
|
|
4316
|
-
|
|
4317
|
-
bool reserve_epoch_num_for_file_ingested_behind =
|
|
4318
|
-
cfd->ioptions()->allow_ingest_behind;
|
|
4319
|
-
if (reserve_epoch_num_for_file_ingested_behind) {
|
|
4320
|
-
uint64_t reserved_epoch_number = cfd->NewEpochNumber();
|
|
4321
|
-
assert(reserved_epoch_number == kReservedEpochNumberForFileIngestedBehind);
|
|
4322
|
-
ROCKS_LOG_INFO(cfd->ioptions()->info_log.get(),
|
|
4323
|
-
"[%s]CF has reserved epoch number %" PRIu64
|
|
4324
|
-
" for files ingested "
|
|
4325
|
-
"behind since `Options::allow_ingest_behind` is true",
|
|
4326
|
-
cfd->GetName().c_str(), reserved_epoch_number);
|
|
4327
|
-
}
|
|
4328
|
-
|
|
4329
|
-
if (HasMissingEpochNumber()) {
|
|
4330
|
-
assert(epoch_number_requirement_ == EpochNumberRequirement::kMightMissing);
|
|
4331
|
-
assert(num_levels_ >= 1);
|
|
4332
|
-
|
|
4333
|
-
for (int level = num_levels_ - 1; level >= 1; --level) {
|
|
4334
|
-
auto& files_at_level = files_[level];
|
|
4335
|
-
if (files_at_level.empty()) {
|
|
4336
|
-
continue;
|
|
4337
|
-
}
|
|
4338
|
-
uint64_t next_epoch_number = cfd->NewEpochNumber();
|
|
4339
|
-
for (FileMetaData* f : files_at_level) {
|
|
4340
|
-
f->epoch_number = next_epoch_number;
|
|
4341
|
-
}
|
|
4342
|
-
}
|
|
4343
|
-
|
|
4344
|
-
for (auto file_meta_iter = files_[0].rbegin();
|
|
4345
|
-
file_meta_iter != files_[0].rend(); file_meta_iter++) {
|
|
4346
|
-
FileMetaData* f = *file_meta_iter;
|
|
4347
|
-
f->epoch_number = cfd->NewEpochNumber();
|
|
4348
|
-
}
|
|
4349
|
-
|
|
4350
|
-
ROCKS_LOG_WARN(cfd->ioptions()->info_log.get(),
|
|
4351
|
-
"[%s]CF's epoch numbers are inferred based on seqno",
|
|
4352
|
-
cfd->GetName().c_str());
|
|
4353
|
-
epoch_number_requirement_ = EpochNumberRequirement::kMustPresent;
|
|
4354
|
-
} else {
|
|
4355
|
-
assert(epoch_number_requirement_ == EpochNumberRequirement::kMustPresent);
|
|
4356
|
-
cfd->SetNextEpochNumber(
|
|
4357
|
-
std::max(GetMaxEpochNumberOfFiles() + 1, cfd->GetNextEpochNumber()));
|
|
4358
|
-
}
|
|
4359
|
-
}
|
|
4360
|
-
|
|
4361
4283
|
uint64_t VersionStorageInfo::MaxNextLevelOverlappingBytes() {
|
|
4362
4284
|
uint64_t result = 0;
|
|
4363
4285
|
std::vector<FileMetaData*> overlaps;
|
|
@@ -5055,15 +4977,10 @@ Status VersionSet::ProcessManifestWrites(
|
|
|
5055
4977
|
if (!descriptor_log_ ||
|
|
5056
4978
|
manifest_file_size_ > db_options_->max_manifest_file_size) {
|
|
5057
4979
|
TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeNewManifest");
|
|
5058
|
-
TEST_SYNC_POINT_CALLBACK(
|
|
5059
|
-
"VersionSet::ProcessManifestWrites:BeforeNewManifest", nullptr);
|
|
5060
4980
|
new_descriptor_log = true;
|
|
5061
4981
|
} else {
|
|
5062
4982
|
pending_manifest_file_number_ = manifest_file_number_;
|
|
5063
4983
|
}
|
|
5064
|
-
TEST_SYNC_POINT_CALLBACK(
|
|
5065
|
-
"VersionSet::ProcessManifestWrites:PostDecidingCreateNewManifestOrNot",
|
|
5066
|
-
&new_descriptor_log);
|
|
5067
4984
|
|
|
5068
4985
|
// Local cached copy of state variable(s). WriteCurrentStateToManifest()
|
|
5069
4986
|
// reads its content after releasing db mutex to avoid race with
|
|
@@ -5192,7 +5109,6 @@ Status VersionSet::ProcessManifestWrites(
|
|
|
5192
5109
|
break;
|
|
5193
5110
|
}
|
|
5194
5111
|
}
|
|
5195
|
-
|
|
5196
5112
|
if (s.ok()) {
|
|
5197
5113
|
io_s = SyncManifest(db_options_, descriptor_log_->file());
|
|
5198
5114
|
manifest_io_status = io_s;
|
|
@@ -5600,8 +5516,7 @@ Status VersionSet::GetCurrentManifestPath(const std::string& dbname,
|
|
|
5600
5516
|
Status VersionSet::Recover(
|
|
5601
5517
|
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
|
5602
5518
|
std::string* db_id, bool no_error_if_files_missing) {
|
|
5603
|
-
// Read "CURRENT" file, which contains a pointer to the current manifest
|
|
5604
|
-
// file
|
|
5519
|
+
// Read "CURRENT" file, which contains a pointer to the current manifest file
|
|
5605
5520
|
std::string manifest_path;
|
|
5606
5521
|
Status s = GetCurrentManifestPath(dbname_, fs_.get(), &manifest_path,
|
|
5607
5522
|
&manifest_file_number_);
|
|
@@ -5635,8 +5550,7 @@ Status VersionSet::Recover(
|
|
|
5635
5550
|
true /* checksum */, 0 /* log_number */);
|
|
5636
5551
|
VersionEditHandler handler(
|
|
5637
5552
|
read_only, column_families, const_cast<VersionSet*>(this),
|
|
5638
|
-
/*track_missing_files=*/false, no_error_if_files_missing, io_tracer_
|
|
5639
|
-
EpochNumberRequirement::kMightMissing);
|
|
5553
|
+
/*track_missing_files=*/false, no_error_if_files_missing, io_tracer_);
|
|
5640
5554
|
handler.Iterate(reader, &log_read_status);
|
|
5641
5555
|
s = handler.status();
|
|
5642
5556
|
if (s.ok()) {
|
|
@@ -5645,9 +5559,6 @@ Status VersionSet::Recover(
|
|
|
5645
5559
|
assert(current_manifest_file_size != 0);
|
|
5646
5560
|
handler.GetDbId(db_id);
|
|
5647
5561
|
}
|
|
5648
|
-
if (s.ok()) {
|
|
5649
|
-
RecoverEpochNumbers();
|
|
5650
|
-
}
|
|
5651
5562
|
}
|
|
5652
5563
|
|
|
5653
5564
|
if (s.ok()) {
|
|
@@ -5807,8 +5718,7 @@ Status VersionSet::TryRecoverFromOneManifest(
|
|
|
5807
5718
|
log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter,
|
|
5808
5719
|
/*checksum=*/true, /*log_num=*/0);
|
|
5809
5720
|
VersionEditHandlerPointInTime handler_pit(
|
|
5810
|
-
read_only, column_families, const_cast<VersionSet*>(this), io_tracer_
|
|
5811
|
-
EpochNumberRequirement::kMightMissing);
|
|
5721
|
+
read_only, column_families, const_cast<VersionSet*>(this), io_tracer_);
|
|
5812
5722
|
|
|
5813
5723
|
handler_pit.Iterate(reader, &s);
|
|
5814
5724
|
|
|
@@ -5817,21 +5727,7 @@ Status VersionSet::TryRecoverFromOneManifest(
|
|
|
5817
5727
|
assert(nullptr != has_missing_table_file);
|
|
5818
5728
|
*has_missing_table_file = handler_pit.HasMissingFiles();
|
|
5819
5729
|
|
|
5820
|
-
|
|
5821
|
-
if (s.ok()) {
|
|
5822
|
-
RecoverEpochNumbers();
|
|
5823
|
-
}
|
|
5824
|
-
return s;
|
|
5825
|
-
}
|
|
5826
|
-
|
|
5827
|
-
void VersionSet::RecoverEpochNumbers() {
|
|
5828
|
-
for (auto cfd : *column_family_set_) {
|
|
5829
|
-
if (cfd->IsDropped()) {
|
|
5830
|
-
continue;
|
|
5831
|
-
}
|
|
5832
|
-
assert(cfd->initialized());
|
|
5833
|
-
cfd->RecoverEpochNumbers();
|
|
5834
|
-
}
|
|
5730
|
+
return handler_pit.status();
|
|
5835
5731
|
}
|
|
5836
5732
|
|
|
5837
5733
|
Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
|
|
@@ -6151,22 +6047,6 @@ Status VersionSet::WriteCurrentStateToManifest(
|
|
|
6151
6047
|
}
|
|
6152
6048
|
}
|
|
6153
6049
|
|
|
6154
|
-
// New manifest should rollover the WAL deletion record from previous
|
|
6155
|
-
// manifest. Otherwise, when an addition record of a deleted WAL gets added to
|
|
6156
|
-
// this new manifest later (which can happens in e.g, SyncWAL()), this new
|
|
6157
|
-
// manifest creates an illusion that such WAL hasn't been deleted.
|
|
6158
|
-
VersionEdit wal_deletions;
|
|
6159
|
-
wal_deletions.DeleteWalsBefore(min_log_number_to_keep());
|
|
6160
|
-
std::string wal_deletions_record;
|
|
6161
|
-
if (!wal_deletions.EncodeTo(&wal_deletions_record)) {
|
|
6162
|
-
return Status::Corruption("Unable to Encode VersionEdit: " +
|
|
6163
|
-
wal_deletions.DebugString(true));
|
|
6164
|
-
}
|
|
6165
|
-
io_s = log->AddRecord(wal_deletions_record);
|
|
6166
|
-
if (!io_s.ok()) {
|
|
6167
|
-
return io_s;
|
|
6168
|
-
}
|
|
6169
|
-
|
|
6170
6050
|
for (auto cfd : *column_family_set_) {
|
|
6171
6051
|
assert(cfd);
|
|
6172
6052
|
|
|
@@ -6218,9 +6098,8 @@ Status VersionSet::WriteCurrentStateToManifest(
|
|
|
6218
6098
|
f->fd.smallest_seqno, f->fd.largest_seqno,
|
|
6219
6099
|
f->marked_for_compaction, f->temperature,
|
|
6220
6100
|
f->oldest_blob_file_number, f->oldest_ancester_time,
|
|
6221
|
-
f->file_creation_time, f->
|
|
6222
|
-
f->file_checksum_func_name, f->unique_id
|
|
6223
|
-
f->compensated_range_deletion_size);
|
|
6101
|
+
f->file_creation_time, f->file_checksum,
|
|
6102
|
+
f->file_checksum_func_name, f->unique_id);
|
|
6224
6103
|
}
|
|
6225
6104
|
}
|
|
6226
6105
|
|
|
@@ -6298,9 +6177,8 @@ uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options,
|
|
|
6298
6177
|
const int num_non_empty_levels = vstorage->num_non_empty_levels();
|
|
6299
6178
|
end_level = (end_level == -1) ? num_non_empty_levels
|
|
6300
6179
|
: std::min(end_level, num_non_empty_levels);
|
|
6301
|
-
|
|
6302
|
-
|
|
6303
|
-
}
|
|
6180
|
+
|
|
6181
|
+
assert(start_level <= end_level);
|
|
6304
6182
|
|
|
6305
6183
|
// Outline of the optimization that uses options.files_size_error_margin.
|
|
6306
6184
|
// When approximating the files total size that is used to store a keys range,
|
|
@@ -6592,14 +6470,6 @@ InternalIterator* VersionSet::MakeInputIterator(
|
|
|
6592
6470
|
c->num_input_levels() - 1
|
|
6593
6471
|
: c->num_input_levels());
|
|
6594
6472
|
InternalIterator** list = new InternalIterator*[space];
|
|
6595
|
-
// First item in the pair is a pointer to range tombstones.
|
|
6596
|
-
// Second item is a pointer to a member of a LevelIterator,
|
|
6597
|
-
// that will be initialized to where CompactionMergingIterator stores
|
|
6598
|
-
// pointer to its range tombstones. This is used by LevelIterator
|
|
6599
|
-
// to update pointer to range tombstones as it traverse different SST files.
|
|
6600
|
-
std::vector<
|
|
6601
|
-
std::pair<TruncatedRangeDelIterator*, TruncatedRangeDelIterator***>>
|
|
6602
|
-
range_tombstones;
|
|
6603
6473
|
size_t num = 0;
|
|
6604
6474
|
for (size_t which = 0; which < c->num_input_levels(); which++) {
|
|
6605
6475
|
if (c->input_levels(which)->num_files != 0) {
|
|
@@ -6620,7 +6490,7 @@ InternalIterator* VersionSet::MakeInputIterator(
|
|
|
6620
6490
|
end.value(), fmd.smallest.user_key()) < 0) {
|
|
6621
6491
|
continue;
|
|
6622
6492
|
}
|
|
6623
|
-
|
|
6493
|
+
|
|
6624
6494
|
list[num++] = cfd->table_cache()->NewIterator(
|
|
6625
6495
|
read_options, file_options_compactions,
|
|
6626
6496
|
cfd->internal_comparator(), fmd, range_del_agg,
|
|
@@ -6633,13 +6503,10 @@ InternalIterator* VersionSet::MakeInputIterator(
|
|
|
6633
6503
|
MaxFileSizeForL0MetaPin(*c->mutable_cf_options()),
|
|
6634
6504
|
/*smallest_compaction_key=*/nullptr,
|
|
6635
6505
|
/*largest_compaction_key=*/nullptr,
|
|
6636
|
-
/*allow_unprepared_value=*/false
|
|
6637
|
-
/*range_del_iter=*/&range_tombstone_iter);
|
|
6638
|
-
range_tombstones.emplace_back(range_tombstone_iter, nullptr);
|
|
6506
|
+
/*allow_unprepared_value=*/false);
|
|
6639
6507
|
}
|
|
6640
6508
|
} else {
|
|
6641
6509
|
// Create concatenating iterator for the files from this level
|
|
6642
|
-
TruncatedRangeDelIterator*** tombstone_iter_ptr = nullptr;
|
|
6643
6510
|
list[num++] = new LevelIterator(
|
|
6644
6511
|
cfd->table_cache(), read_options, file_options_compactions,
|
|
6645
6512
|
cfd->internal_comparator(), c->input_levels(which),
|
|
@@ -6648,15 +6515,14 @@ InternalIterator* VersionSet::MakeInputIterator(
|
|
|
6648
6515
|
/*no per level latency histogram=*/nullptr,
|
|
6649
6516
|
TableReaderCaller::kCompaction, /*skip_filters=*/false,
|
|
6650
6517
|
/*level=*/static_cast<int>(c->level(which)), range_del_agg,
|
|
6651
|
-
c->boundaries(which)
|
|
6652
|
-
range_tombstones.emplace_back(nullptr, tombstone_iter_ptr);
|
|
6518
|
+
c->boundaries(which));
|
|
6653
6519
|
}
|
|
6654
6520
|
}
|
|
6655
6521
|
}
|
|
6656
6522
|
assert(num <= space);
|
|
6657
|
-
InternalIterator* result =
|
|
6658
|
-
&c->column_family_data()->internal_comparator(), list,
|
|
6659
|
-
|
|
6523
|
+
InternalIterator* result =
|
|
6524
|
+
NewMergingIterator(&c->column_family_data()->internal_comparator(), list,
|
|
6525
|
+
static_cast<int>(num));
|
|
6660
6526
|
delete[] list;
|
|
6661
6527
|
return result;
|
|
6662
6528
|
}
|
|
@@ -6723,7 +6589,6 @@ void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
|
|
|
6723
6589
|
filemetadata.temperature = file->temperature;
|
|
6724
6590
|
filemetadata.oldest_ancester_time = file->TryGetOldestAncesterTime();
|
|
6725
6591
|
filemetadata.file_creation_time = file->TryGetFileCreationTime();
|
|
6726
|
-
filemetadata.epoch_number = file->epoch_number;
|
|
6727
6592
|
metadata->push_back(filemetadata);
|
|
6728
6593
|
}
|
|
6729
6594
|
}
|
|
@@ -6926,17 +6791,12 @@ Status ReactiveVersionSet::Recover(
|
|
|
6926
6791
|
log::Reader* reader = manifest_reader->get();
|
|
6927
6792
|
assert(reader);
|
|
6928
6793
|
|
|
6929
|
-
manifest_tailer_.reset(
|
|
6930
|
-
|
|
6931
|
-
io_tracer_, EpochNumberRequirement::kMightMissing));
|
|
6794
|
+
manifest_tailer_.reset(new ManifestTailer(
|
|
6795
|
+
column_families, const_cast<ReactiveVersionSet*>(this), io_tracer_));
|
|
6932
6796
|
|
|
6933
6797
|
manifest_tailer_->Iterate(*reader, manifest_reader_status->get());
|
|
6934
6798
|
|
|
6935
|
-
|
|
6936
|
-
if (s.ok()) {
|
|
6937
|
-
RecoverEpochNumbers();
|
|
6938
|
-
}
|
|
6939
|
-
return s;
|
|
6799
|
+
return manifest_tailer_->status();
|
|
6940
6800
|
}
|
|
6941
6801
|
|
|
6942
6802
|
Status ReactiveVersionSet::ReadAndApply(
|
|
@@ -116,10 +116,6 @@ extern bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
|
|
116
116
|
extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
|
|
117
117
|
const std::vector<FileMetaData*>& files,
|
|
118
118
|
Arena* arena);
|
|
119
|
-
enum EpochNumberRequirement {
|
|
120
|
-
kMightMissing,
|
|
121
|
-
kMustPresent,
|
|
122
|
-
};
|
|
123
119
|
|
|
124
120
|
// Information of the storage associated with each Version, including number of
|
|
125
121
|
// levels of LSM tree, files information at each level, files marked for
|
|
@@ -130,9 +126,7 @@ class VersionStorageInfo {
|
|
|
130
126
|
const Comparator* user_comparator, int num_levels,
|
|
131
127
|
CompactionStyle compaction_style,
|
|
132
128
|
VersionStorageInfo* src_vstorage,
|
|
133
|
-
bool _force_consistency_checks
|
|
134
|
-
EpochNumberRequirement epoch_number_requirement =
|
|
135
|
-
EpochNumberRequirement::kMustPresent);
|
|
129
|
+
bool _force_consistency_checks);
|
|
136
130
|
// No copying allowed
|
|
137
131
|
VersionStorageInfo(const VersionStorageInfo&) = delete;
|
|
138
132
|
void operator=(const VersionStorageInfo&) = delete;
|
|
@@ -325,17 +319,6 @@ class VersionStorageInfo {
|
|
|
325
319
|
return files_[level];
|
|
326
320
|
}
|
|
327
321
|
|
|
328
|
-
bool HasMissingEpochNumber() const;
|
|
329
|
-
uint64_t GetMaxEpochNumberOfFiles() const;
|
|
330
|
-
EpochNumberRequirement GetEpochNumberRequirement() const {
|
|
331
|
-
return epoch_number_requirement_;
|
|
332
|
-
}
|
|
333
|
-
void SetEpochNumberRequirement(
|
|
334
|
-
EpochNumberRequirement epoch_number_requirement) {
|
|
335
|
-
epoch_number_requirement_ = epoch_number_requirement;
|
|
336
|
-
}
|
|
337
|
-
void RecoverEpochNumbers(ColumnFamilyData* cfd);
|
|
338
|
-
|
|
339
322
|
class FileLocation {
|
|
340
323
|
public:
|
|
341
324
|
FileLocation() = default;
|
|
@@ -457,11 +440,6 @@ class VersionStorageInfo {
|
|
|
457
440
|
return files_marked_for_compaction_;
|
|
458
441
|
}
|
|
459
442
|
|
|
460
|
-
void TEST_AddFileMarkedForCompaction(int level, FileMetaData* f) {
|
|
461
|
-
f->marked_for_compaction = true;
|
|
462
|
-
files_marked_for_compaction_.emplace_back(level, f);
|
|
463
|
-
}
|
|
464
|
-
|
|
465
443
|
// REQUIRES: ComputeCompactionScore has been called
|
|
466
444
|
// REQUIRES: DB mutex held during access
|
|
467
445
|
const autovector<std::pair<int, FileMetaData*>>& ExpiredTtlFiles() const {
|
|
@@ -745,8 +723,6 @@ class VersionStorageInfo {
|
|
|
745
723
|
// is compiled in release mode
|
|
746
724
|
bool force_consistency_checks_;
|
|
747
725
|
|
|
748
|
-
EpochNumberRequirement epoch_number_requirement_;
|
|
749
|
-
|
|
750
726
|
friend class Version;
|
|
751
727
|
friend class VersionSet;
|
|
752
728
|
};
|
|
@@ -1071,9 +1047,7 @@ class Version {
|
|
|
1071
1047
|
Version(ColumnFamilyData* cfd, VersionSet* vset, const FileOptions& file_opt,
|
|
1072
1048
|
MutableCFOptions mutable_cf_options,
|
|
1073
1049
|
const std::shared_ptr<IOTracer>& io_tracer,
|
|
1074
|
-
uint64_t version_number = 0
|
|
1075
|
-
EpochNumberRequirement epoch_number_requirement =
|
|
1076
|
-
EpochNumberRequirement::kMustPresent);
|
|
1050
|
+
uint64_t version_number = 0);
|
|
1077
1051
|
|
|
1078
1052
|
~Version();
|
|
1079
1053
|
|
|
@@ -1214,10 +1188,6 @@ class VersionSet {
|
|
|
1214
1188
|
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
1215
1189
|
bool read_only, std::string* db_id, bool* has_missing_table_file);
|
|
1216
1190
|
|
|
1217
|
-
// Recover the next epoch number of each CFs and epoch number
|
|
1218
|
-
// of their files (if missing)
|
|
1219
|
-
void RecoverEpochNumbers();
|
|
1220
|
-
|
|
1221
1191
|
// Reads a manifest file and returns a list of column families in
|
|
1222
1192
|
// column_families.
|
|
1223
1193
|
static Status ListColumnFamilies(std::vector<std::string>* column_families,
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
#include "db/db_impl/db_impl.h"
|
|
15
15
|
#include "db/db_test_util.h"
|
|
16
16
|
#include "db/log_writer.h"
|
|
17
|
-
#include "db/version_edit.h"
|
|
18
17
|
#include "rocksdb/advanced_options.h"
|
|
19
18
|
#include "rocksdb/convenience.h"
|
|
20
19
|
#include "rocksdb/file_system.h"
|
|
@@ -50,8 +49,8 @@ class GenerateLevelFilesBriefTest : public testing::Test {
|
|
|
50
49
|
InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
|
|
51
50
|
largest_seq, /* marked_for_compact */ false, Temperature::kUnknown,
|
|
52
51
|
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
|
|
53
|
-
kUnknownFileCreationTime,
|
|
54
|
-
kUnknownFileChecksumFuncName, kNullUniqueId64x2
|
|
52
|
+
kUnknownFileCreationTime, kUnknownFileChecksum,
|
|
53
|
+
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
|
|
55
54
|
files_.push_back(f);
|
|
56
55
|
}
|
|
57
56
|
|
|
@@ -143,27 +142,24 @@ class VersionStorageInfoTestBase : public testing::Test {
|
|
|
143
142
|
|
|
144
143
|
void Add(int level, uint32_t file_number, const char* smallest,
|
|
145
144
|
const char* largest, uint64_t file_size = 0,
|
|
146
|
-
uint64_t oldest_blob_file_number = kInvalidBlobFileNumber
|
|
147
|
-
uint64_t compensated_range_deletion_size = 0) {
|
|
145
|
+
uint64_t oldest_blob_file_number = kInvalidBlobFileNumber) {
|
|
148
146
|
constexpr SequenceNumber dummy_seq = 0;
|
|
149
147
|
|
|
150
148
|
Add(level, file_number, GetInternalKey(smallest, dummy_seq),
|
|
151
|
-
GetInternalKey(largest, dummy_seq), file_size, oldest_blob_file_number
|
|
152
|
-
compensated_range_deletion_size);
|
|
149
|
+
GetInternalKey(largest, dummy_seq), file_size, oldest_blob_file_number);
|
|
153
150
|
}
|
|
154
151
|
|
|
155
152
|
void Add(int level, uint32_t file_number, const InternalKey& smallest,
|
|
156
153
|
const InternalKey& largest, uint64_t file_size = 0,
|
|
157
|
-
uint64_t oldest_blob_file_number = kInvalidBlobFileNumber
|
|
158
|
-
uint64_t compensated_range_deletion_size = 0) {
|
|
154
|
+
uint64_t oldest_blob_file_number = kInvalidBlobFileNumber) {
|
|
159
155
|
assert(level < vstorage_.num_levels());
|
|
160
156
|
FileMetaData* f = new FileMetaData(
|
|
161
157
|
file_number, 0, file_size, smallest, largest, /* smallest_seq */ 0,
|
|
162
158
|
/* largest_seq */ 0, /* marked_for_compact */ false,
|
|
163
159
|
Temperature::kUnknown, oldest_blob_file_number,
|
|
164
160
|
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
|
|
165
|
-
|
|
166
|
-
|
|
161
|
+
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
|
|
162
|
+
f->compensated_file_size = file_size;
|
|
167
163
|
vstorage_.AddFile(level, f);
|
|
168
164
|
}
|
|
169
165
|
|
|
@@ -2138,17 +2134,6 @@ TEST_F(VersionSetTest, AtomicGroupWithWalEdits) {
|
|
|
2138
2134
|
}
|
|
2139
2135
|
}
|
|
2140
2136
|
|
|
2141
|
-
TEST_F(VersionStorageInfoTest, AddRangeDeletionCompensatedFileSize) {
|
|
2142
|
-
// Tests that compensated range deletion size is added to compensated file
|
|
2143
|
-
// size.
|
|
2144
|
-
Add(4, 100U, "1", "2", 100U, kInvalidBlobFileNumber, 1000U);
|
|
2145
|
-
|
|
2146
|
-
UpdateVersionStorageInfo();
|
|
2147
|
-
|
|
2148
|
-
auto meta = vstorage_.GetFileMetaDataByNumber(100U);
|
|
2149
|
-
ASSERT_EQ(meta->compensated_file_size, 100U + 1000U);
|
|
2150
|
-
}
|
|
2151
|
-
|
|
2152
2137
|
class VersionSetWithTimestampTest : public VersionSetTest {
|
|
2153
2138
|
public:
|
|
2154
2139
|
static const std::string kNewCfName;
|
|
@@ -3206,19 +3191,15 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
|
|
|
3206
3191
|
std::string column_family;
|
|
3207
3192
|
std::string key; // the only key
|
|
3208
3193
|
int level = 0;
|
|
3209
|
-
uint64_t epoch_number;
|
|
3210
3194
|
SstInfo(uint64_t file_num, const std::string& cf_name,
|
|
3211
|
-
const std::string& _key
|
|
3212
|
-
|
|
3213
|
-
: SstInfo(file_num, cf_name, _key, 0, _epoch_number) {}
|
|
3195
|
+
const std::string& _key)
|
|
3196
|
+
: SstInfo(file_num, cf_name, _key, 0) {}
|
|
3214
3197
|
SstInfo(uint64_t file_num, const std::string& cf_name,
|
|
3215
|
-
const std::string& _key, int lvl
|
|
3216
|
-
uint64_t _epoch_number = kUnknownEpochNumber)
|
|
3198
|
+
const std::string& _key, int lvl)
|
|
3217
3199
|
: file_number(file_num),
|
|
3218
3200
|
column_family(cf_name),
|
|
3219
3201
|
key(_key),
|
|
3220
|
-
level(lvl)
|
|
3221
|
-
epoch_number(_epoch_number) {}
|
|
3202
|
+
level(lvl) {}
|
|
3222
3203
|
};
|
|
3223
3204
|
|
|
3224
3205
|
// Create dummy sst, return their metadata. Note that only file name and size
|
|
@@ -3254,9 +3235,8 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
|
|
|
3254
3235
|
ASSERT_NE(0, file_size);
|
|
3255
3236
|
file_metas->emplace_back(file_num, /*file_path_id=*/0, file_size, ikey,
|
|
3256
3237
|
ikey, 0, 0, false, Temperature::kUnknown, 0, 0,
|
|
3257
|
-
0,
|
|
3258
|
-
kUnknownFileChecksumFuncName, kNullUniqueId64x2
|
|
3259
|
-
0);
|
|
3238
|
+
0, kUnknownFileChecksum,
|
|
3239
|
+
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
|
|
3260
3240
|
}
|
|
3261
3241
|
}
|
|
3262
3242
|
|
|
@@ -3293,11 +3273,11 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
|
|
|
3293
3273
|
|
|
3294
3274
|
TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
|
|
3295
3275
|
std::vector<SstInfo> existing_files = {
|
|
3296
|
-
SstInfo(100, kDefaultColumnFamilyName, "a"
|
|
3297
|
-
SstInfo(102, kDefaultColumnFamilyName, "b"
|
|
3298
|
-
SstInfo(103, kDefaultColumnFamilyName, "c"
|
|
3299
|
-
SstInfo(107, kDefaultColumnFamilyName, "d"
|
|
3300
|
-
SstInfo(110, kDefaultColumnFamilyName, "e"
|
|
3276
|
+
SstInfo(100, kDefaultColumnFamilyName, "a"),
|
|
3277
|
+
SstInfo(102, kDefaultColumnFamilyName, "b"),
|
|
3278
|
+
SstInfo(103, kDefaultColumnFamilyName, "c"),
|
|
3279
|
+
SstInfo(107, kDefaultColumnFamilyName, "d"),
|
|
3280
|
+
SstInfo(110, kDefaultColumnFamilyName, "e")};
|
|
3301
3281
|
std::vector<FileMetaData> file_metas;
|
|
3302
3282
|
CreateDummyTableFiles(existing_files, &file_metas);
|
|
3303
3283
|
|
|
@@ -3308,12 +3288,10 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
|
|
|
3308
3288
|
std::string largest_ukey = "b";
|
|
3309
3289
|
InternalKey smallest_ikey(smallest_ukey, 1, ValueType::kTypeValue);
|
|
3310
3290
|
InternalKey largest_ikey(largest_ukey, 1, ValueType::kTypeValue);
|
|
3311
|
-
|
|
3312
3291
|
FileMetaData meta = FileMetaData(
|
|
3313
3292
|
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
|
|
3314
3293
|
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
|
|
3315
|
-
|
|
3316
|
-
kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
|
|
3294
|
+
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
|
|
3317
3295
|
added_files.emplace_back(0, meta);
|
|
3318
3296
|
}
|
|
3319
3297
|
WriteFileAdditionAndDeletionToManifest(
|
|
@@ -3343,16 +3321,11 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
|
|
|
3343
3321
|
|
|
3344
3322
|
TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
|
|
3345
3323
|
std::vector<SstInfo> existing_files = {
|
|
3346
|
-
SstInfo(100, kDefaultColumnFamilyName, "a",
|
|
3347
|
-
|
|
3348
|
-
SstInfo(
|
|
3349
|
-
|
|
3350
|
-
SstInfo(
|
|
3351
|
-
103 /* epoch_number */),
|
|
3352
|
-
SstInfo(107, kDefaultColumnFamilyName, "d", 0 /* level */,
|
|
3353
|
-
107 /* epoch_number */),
|
|
3354
|
-
SstInfo(110, kDefaultColumnFamilyName, "e", 0 /* level */,
|
|
3355
|
-
110 /* epoch_number */)};
|
|
3324
|
+
SstInfo(100, kDefaultColumnFamilyName, "a"),
|
|
3325
|
+
SstInfo(102, kDefaultColumnFamilyName, "b"),
|
|
3326
|
+
SstInfo(103, kDefaultColumnFamilyName, "c"),
|
|
3327
|
+
SstInfo(107, kDefaultColumnFamilyName, "d"),
|
|
3328
|
+
SstInfo(110, kDefaultColumnFamilyName, "e")};
|
|
3356
3329
|
std::vector<FileMetaData> file_metas;
|
|
3357
3330
|
CreateDummyTableFiles(existing_files, &file_metas);
|
|
3358
3331
|
|
|
@@ -3373,8 +3346,7 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
|
|
|
3373
3346
|
FileMetaData meta = FileMetaData(
|
|
3374
3347
|
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
|
|
3375
3348
|
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
|
|
3376
|
-
|
|
3377
|
-
kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
|
|
3349
|
+
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
|
|
3378
3350
|
added_files.emplace_back(0, meta);
|
|
3379
3351
|
}
|
|
3380
3352
|
WriteFileAdditionAndDeletionToManifest(
|
|
@@ -3409,16 +3381,11 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
|
|
|
3409
3381
|
|
|
3410
3382
|
TEST_F(VersionSetTestMissingFiles, NoFileMissing) {
|
|
3411
3383
|
std::vector<SstInfo> existing_files = {
|
|
3412
|
-
SstInfo(100, kDefaultColumnFamilyName, "a",
|
|
3413
|
-
|
|
3414
|
-
SstInfo(
|
|
3415
|
-
|
|
3416
|
-
SstInfo(
|
|
3417
|
-
103 /* epoch_number */),
|
|
3418
|
-
SstInfo(107, kDefaultColumnFamilyName, "d", 0 /* level */,
|
|
3419
|
-
107 /* epoch_number */),
|
|
3420
|
-
SstInfo(110, kDefaultColumnFamilyName, "e", 0 /* level */,
|
|
3421
|
-
110 /* epoch_number */)};
|
|
3384
|
+
SstInfo(100, kDefaultColumnFamilyName, "a"),
|
|
3385
|
+
SstInfo(102, kDefaultColumnFamilyName, "b"),
|
|
3386
|
+
SstInfo(103, kDefaultColumnFamilyName, "c"),
|
|
3387
|
+
SstInfo(107, kDefaultColumnFamilyName, "d"),
|
|
3388
|
+
SstInfo(110, kDefaultColumnFamilyName, "e")};
|
|
3422
3389
|
std::vector<FileMetaData> file_metas;
|
|
3423
3390
|
CreateDummyTableFiles(existing_files, &file_metas);
|
|
3424
3391
|
|
|
@@ -3468,8 +3435,7 @@ TEST_F(VersionSetTestMissingFiles, MinLogNumberToKeep2PC) {
|
|
|
3468
3435
|
db_options_.allow_2pc = true;
|
|
3469
3436
|
NewDB();
|
|
3470
3437
|
|
|
3471
|
-
SstInfo sst(100, kDefaultColumnFamilyName, "a"
|
|
3472
|
-
100 /* epoch_number */);
|
|
3438
|
+
SstInfo sst(100, kDefaultColumnFamilyName, "a");
|
|
3473
3439
|
std::vector<FileMetaData> file_metas;
|
|
3474
3440
|
CreateDummyTableFiles({sst}, &file_metas);
|
|
3475
3441
|
|
|
@@ -360,11 +360,8 @@ void WriteThread::EndWriteStall() {
|
|
|
360
360
|
// Unlink write_stall_dummy_ from the write queue. This will unblock
|
|
361
361
|
// pending write threads to enqueue themselves
|
|
362
362
|
assert(newest_writer_.load(std::memory_order_relaxed) == &write_stall_dummy_);
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
if (write_stall_dummy_.link_older) {
|
|
366
|
-
write_stall_dummy_.link_older->link_newer = write_stall_dummy_.link_newer;
|
|
367
|
-
}
|
|
363
|
+
assert(write_stall_dummy_.link_older != nullptr);
|
|
364
|
+
write_stall_dummy_.link_older->link_newer = write_stall_dummy_.link_newer;
|
|
368
365
|
newest_writer_.exchange(write_stall_dummy_.link_older);
|
|
369
366
|
|
|
370
367
|
// Wake up writers
|