@nxtedition/rocksdb 7.1.32 → 7.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +402 -345
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +121 -64
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +28 -18
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +1 -0
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +2 -0
- package/deps/rocksdb/rocksdb/db/builder.cc +2 -1
- package/deps/rocksdb/rocksdb/db/c.cc +563 -673
- package/deps/rocksdb/rocksdb/db/c_test.c +168 -169
- package/deps/rocksdb/rocksdb/db/column_family.cc +16 -15
- package/deps/rocksdb/rocksdb/db/column_family.h +7 -7
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +17 -28
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +4 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +8 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +114 -0
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/convenience.cc +3 -5
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +10 -14
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +9 -13
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +52 -72
- package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +12 -12
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -2
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +1 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +3 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +26 -0
- package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter.cc +12 -6
- package/deps/rocksdb/rocksdb/db/db_iter.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +6 -7
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +10 -8
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +15 -13
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +2 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +7 -4
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +8 -6
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +18 -23
- package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/db_test.cc +10 -5
- package/deps/rocksdb/rocksdb/db/db_test2.cc +172 -169
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +68 -66
- package/deps/rocksdb/rocksdb/db/db_test_util.h +1 -3
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +31 -39
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +182 -2
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +43 -40
- package/deps/rocksdb/rocksdb/db/dbformat.h +15 -0
- package/deps/rocksdb/rocksdb/db/dbformat_test.cc +35 -34
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +10 -11
- package/deps/rocksdb/rocksdb/db/error_handler.cc +6 -6
- package/deps/rocksdb/rocksdb/db/error_handler.h +93 -94
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +1 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.h +3 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +16 -17
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +1 -2
- package/deps/rocksdb/rocksdb/db/file_indexer.cc +2 -0
- package/deps/rocksdb/rocksdb/db/file_indexer.h +2 -1
- package/deps/rocksdb/rocksdb/db/file_indexer_test.cc +4 -2
- package/deps/rocksdb/rocksdb/db/filename_test.cc +27 -29
- package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -13
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +15 -21
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +7 -6
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +2 -2
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +59 -14
- package/deps/rocksdb/rocksdb/db/internal_stats.h +27 -11
- package/deps/rocksdb/rocksdb/db/job_context.h +5 -6
- package/deps/rocksdb/rocksdb/db/listener_test.cc +21 -23
- package/deps/rocksdb/rocksdb/db/log_reader.cc +7 -11
- package/deps/rocksdb/rocksdb/db/log_reader.h +4 -6
- package/deps/rocksdb/rocksdb/db/log_test.cc +6 -12
- package/deps/rocksdb/rocksdb/db/log_writer.h +1 -1
- package/deps/rocksdb/rocksdb/db/logs_with_prep_tracker.h +0 -1
- package/deps/rocksdb/rocksdb/db/lookup_key.h +4 -1
- package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -1
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/memtable.cc +34 -22
- package/deps/rocksdb/rocksdb/db/memtable.h +4 -6
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +37 -13
- package/deps/rocksdb/rocksdb/db/merge_context.h +1 -0
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +128 -14
- package/deps/rocksdb/rocksdb/db/merge_helper.h +15 -7
- package/deps/rocksdb/rocksdb/db/merge_helper_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/merge_operator.cc +5 -6
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +55 -43
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +288 -299
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +22 -27
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +7 -8
- package/deps/rocksdb/rocksdb/db/repair_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/snapshot_impl.cc +4 -5
- package/deps/rocksdb/rocksdb/db/snapshot_impl.h +10 -4
- package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -4
- package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +6 -7
- package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +22 -22
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +12 -12
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +6 -8
- package/deps/rocksdb/rocksdb/db/trim_history_scheduler.h +2 -0
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/version_edit.cc +2 -5
- package/deps/rocksdb/rocksdb/db/version_edit.h +8 -12
- package/deps/rocksdb/rocksdb/db/version_set.cc +74 -102
- package/deps/rocksdb/rocksdb/db/version_set.h +8 -10
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +0 -5
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +47 -45
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +6 -5
- package/deps/rocksdb/rocksdb/db/wal_manager.h +2 -2
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +144 -61
- package/deps/rocksdb/rocksdb/db/write_batch.cc +41 -24
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +2 -7
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +105 -104
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +5 -4
- package/deps/rocksdb/rocksdb/db/write_controller.h +1 -0
- package/deps/rocksdb/rocksdb/db/write_controller_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/write_thread.cc +8 -6
- package/deps/rocksdb/rocksdb/env/io_posix.h +6 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +134 -65
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +29 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +14 -4
- package/deps/rocksdb/rocksdb/table/get_context.cc +52 -7
- package/deps/rocksdb/rocksdb/table/get_context.h +1 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +13 -0
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +36 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +23 -28
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +11 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +19 -17
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +10 -7
- package/index.js +14 -16
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
|
@@ -170,7 +170,8 @@ TEST_P(DBWriteTest, WriteStallRemoveNoSlowdownWrite) {
|
|
|
170
170
|
|
|
171
171
|
TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
|
172
172
|
Options options = GetOptions();
|
|
173
|
-
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger =
|
|
173
|
+
options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger =
|
|
174
|
+
4;
|
|
174
175
|
std::vector<port::Thread> threads;
|
|
175
176
|
std::atomic<int> thread_num(0);
|
|
176
177
|
port::Mutex mutex;
|
|
@@ -195,7 +196,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
|
|
195
196
|
Status s = dbfull()->Put(wo, key, "bar");
|
|
196
197
|
ASSERT_TRUE(s.ok() || s.IsIncomplete());
|
|
197
198
|
};
|
|
198
|
-
std::function<void(void
|
|
199
|
+
std::function<void(void*)> unblock_main_thread_func = [&](void*) {
|
|
199
200
|
mutex.Lock();
|
|
200
201
|
++writers;
|
|
201
202
|
cv.SignalAll();
|
|
@@ -254,8 +255,9 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
|
|
|
254
255
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(nullptr));
|
|
255
256
|
// This would have triggered a write stall. Unblock the write group leader
|
|
256
257
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
|
|
257
|
-
// The leader is going to create missing newer links. When the leader
|
|
258
|
-
// the next leader is going to delay writes and fail writers with
|
|
258
|
+
// The leader is going to create missing newer links. When the leader
|
|
259
|
+
// finishes, the next leader is going to delay writes and fail writers with
|
|
260
|
+
// no_slowdown
|
|
259
261
|
|
|
260
262
|
TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
|
|
261
263
|
for (auto& t : threads) {
|
|
@@ -623,42 +625,43 @@ TEST_P(DBWriteTest, LockWalInEffect) {
|
|
|
623
625
|
}
|
|
624
626
|
|
|
625
627
|
TEST_P(DBWriteTest, ConcurrentlyDisabledWAL) {
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
628
|
+
Options options = GetOptions();
|
|
629
|
+
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
|
|
630
|
+
options.statistics->set_stats_level(StatsLevel::kAll);
|
|
631
|
+
Reopen(options);
|
|
632
|
+
std::string wal_key_prefix = "WAL_KEY_";
|
|
633
|
+
std::string no_wal_key_prefix = "K_";
|
|
634
|
+
// 100 KB value each for NO-WAL operation
|
|
635
|
+
std::string no_wal_value(1024 * 100, 'X');
|
|
636
|
+
// 1B value each for WAL operation
|
|
637
|
+
std::string wal_value = "0";
|
|
638
|
+
std::thread threads[10];
|
|
639
|
+
for (int t = 0; t < 10; t++) {
|
|
640
|
+
threads[t] = std::thread([t, wal_key_prefix, wal_value, no_wal_key_prefix,
|
|
641
|
+
no_wal_value, this] {
|
|
642
|
+
for (int i = 0; i < 10; i++) {
|
|
643
|
+
ROCKSDB_NAMESPACE::WriteOptions write_option_disable;
|
|
644
|
+
write_option_disable.disableWAL = true;
|
|
645
|
+
ROCKSDB_NAMESPACE::WriteOptions write_option_default;
|
|
646
|
+
std::string no_wal_key =
|
|
647
|
+
no_wal_key_prefix + std::to_string(t) + "_" + std::to_string(i);
|
|
648
|
+
ASSERT_OK(this->Put(no_wal_key, no_wal_value, write_option_disable));
|
|
649
|
+
std::string wal_key =
|
|
650
|
+
wal_key_prefix + std::to_string(i) + "_" + std::to_string(i);
|
|
651
|
+
ASSERT_OK(this->Put(wal_key, wal_value, write_option_default));
|
|
652
|
+
ASSERT_OK(dbfull()->SyncWAL());
|
|
653
|
+
}
|
|
654
|
+
return;
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
for (auto& t : threads) {
|
|
658
|
+
t.join();
|
|
659
|
+
}
|
|
660
|
+
uint64_t bytes_num = options.statistics->getTickerCount(
|
|
661
|
+
ROCKSDB_NAMESPACE::Tickers::WAL_FILE_BYTES);
|
|
662
|
+
// written WAL size should less than 100KB (even included HEADER & FOOTER
|
|
663
|
+
// overhead)
|
|
664
|
+
ASSERT_LE(bytes_num, 1024 * 100);
|
|
662
665
|
}
|
|
663
666
|
|
|
664
667
|
INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
|
|
@@ -129,6 +129,12 @@ struct ParsedInternalKey {
|
|
|
129
129
|
const char* addr = user_key.data() + user_key.size() - ts.size();
|
|
130
130
|
memcpy(const_cast<char*>(addr), ts.data(), ts.size());
|
|
131
131
|
}
|
|
132
|
+
|
|
133
|
+
Slice GetTimestamp(size_t ts_sz) {
|
|
134
|
+
assert(ts_sz <= user_key.size());
|
|
135
|
+
const char* addr = user_key.data() + user_key.size() - ts_sz;
|
|
136
|
+
return Slice(const_cast<char*>(addr), ts_sz);
|
|
137
|
+
}
|
|
132
138
|
};
|
|
133
139
|
|
|
134
140
|
// Return the length of the encoding of "key".
|
|
@@ -439,6 +445,8 @@ class IterKey {
|
|
|
439
445
|
void SetIsUserKey(bool is_user_key) { is_user_key_ = is_user_key; }
|
|
440
446
|
|
|
441
447
|
// Returns the key in whichever format that was provided to KeyIter
|
|
448
|
+
// If user-defined timestamp is enabled, then timestamp is included in the
|
|
449
|
+
// return result.
|
|
442
450
|
Slice GetKey() const { return Slice(key_, key_size_); }
|
|
443
451
|
|
|
444
452
|
Slice GetInternalKey() const {
|
|
@@ -446,6 +454,8 @@ class IterKey {
|
|
|
446
454
|
return Slice(key_, key_size_);
|
|
447
455
|
}
|
|
448
456
|
|
|
457
|
+
// If user-defined timestamp is enabled, then timestamp is included in the
|
|
458
|
+
// return result of GetUserKey();
|
|
449
459
|
Slice GetUserKey() const {
|
|
450
460
|
if (IsUserKey()) {
|
|
451
461
|
return Slice(key_, key_size_);
|
|
@@ -495,6 +505,9 @@ class IterKey {
|
|
|
495
505
|
return SetKeyImpl(key, copy);
|
|
496
506
|
}
|
|
497
507
|
|
|
508
|
+
// If user-defined timestamp is enabled, then `key` includes timestamp.
|
|
509
|
+
// TODO(yanqin) this is also used to set prefix, which do not include
|
|
510
|
+
// timestamp. Should be handled.
|
|
498
511
|
Slice SetUserKey(const Slice& key, bool copy = true) {
|
|
499
512
|
is_user_key_ = true;
|
|
500
513
|
return SetKeyImpl(key, copy);
|
|
@@ -689,6 +702,8 @@ extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
|
|
|
689
702
|
// slice they point to.
|
|
690
703
|
// Tag is defined as ValueType.
|
|
691
704
|
// input will be advanced to after the record.
|
|
705
|
+
// If user-defined timestamp is enabled for a column family, then the `key`
|
|
706
|
+
// resulting from this call will include timestamp.
|
|
692
707
|
extern Status ReadRecordFromWriteBatch(Slice* input, char* tag,
|
|
693
708
|
uint32_t* column_family, Slice* key,
|
|
694
709
|
Slice* value, Slice* blob, Slice* xid);
|
|
@@ -15,8 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
namespace ROCKSDB_NAMESPACE {
|
|
17
17
|
|
|
18
|
-
static std::string IKey(const std::string& user_key,
|
|
19
|
-
uint64_t seq,
|
|
18
|
+
static std::string IKey(const std::string& user_key, uint64_t seq,
|
|
20
19
|
ValueType vt) {
|
|
21
20
|
std::string encoded;
|
|
22
21
|
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
|
@@ -37,9 +36,7 @@ static std::string ShortSuccessor(const std::string& s) {
|
|
|
37
36
|
return result;
|
|
38
37
|
}
|
|
39
38
|
|
|
40
|
-
static void TestKey(const std::string& key,
|
|
41
|
-
uint64_t seq,
|
|
42
|
-
ValueType vt) {
|
|
39
|
+
static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
|
|
43
40
|
std::string encoded = IKey(key, seq, vt);
|
|
44
41
|
|
|
45
42
|
Slice in(encoded);
|
|
@@ -56,13 +53,19 @@ static void TestKey(const std::string& key,
|
|
|
56
53
|
class FormatTest : public testing::Test {};
|
|
57
54
|
|
|
58
55
|
TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
|
59
|
-
const char* keys[] = {
|
|
60
|
-
const uint64_t seq[] = {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
56
|
+
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
|
|
57
|
+
const uint64_t seq[] = {1,
|
|
58
|
+
2,
|
|
59
|
+
3,
|
|
60
|
+
(1ull << 8) - 1,
|
|
61
|
+
1ull << 8,
|
|
62
|
+
(1ull << 8) + 1,
|
|
63
|
+
(1ull << 16) - 1,
|
|
64
|
+
1ull << 16,
|
|
65
|
+
(1ull << 16) + 1,
|
|
66
|
+
(1ull << 32) - 1,
|
|
67
|
+
1ull << 32,
|
|
68
|
+
(1ull << 32) + 1};
|
|
66
69
|
for (unsigned int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
|
67
70
|
for (unsigned int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
|
68
71
|
TestKey(keys[k], seq[s], kTypeValue);
|
|
@@ -74,27 +77,25 @@ TEST_F(FormatTest, InternalKey_EncodeDecode) {
|
|
|
74
77
|
TEST_F(FormatTest, InternalKeyShortSeparator) {
|
|
75
78
|
// When user keys are same
|
|
76
79
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
|
77
|
-
Shorten(IKey("foo", 100, kTypeValue),
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
IKey("foo", 100, kTypeDeletion)));
|
|
80
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
|
|
81
|
+
ASSERT_EQ(
|
|
82
|
+
IKey("foo", 100, kTypeValue),
|
|
83
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
|
|
84
|
+
ASSERT_EQ(
|
|
85
|
+
IKey("foo", 100, kTypeValue),
|
|
86
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
|
|
87
|
+
ASSERT_EQ(
|
|
88
|
+
IKey("foo", 100, kTypeValue),
|
|
89
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
|
|
88
90
|
|
|
89
91
|
// When user keys are misordered
|
|
90
92
|
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
|
91
|
-
Shorten(IKey("foo", 100, kTypeValue),
|
|
92
|
-
IKey("bar", 99, kTypeValue)));
|
|
93
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
|
|
93
94
|
|
|
94
95
|
// When user keys are different, but correctly ordered
|
|
95
|
-
ASSERT_EQ(
|
|
96
|
-
|
|
97
|
-
|
|
96
|
+
ASSERT_EQ(
|
|
97
|
+
IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
|
98
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
|
|
98
99
|
|
|
99
100
|
ASSERT_EQ(IKey("ABC2", kMaxSequenceNumber, kValueTypeForSeek),
|
|
100
101
|
Shorten(IKey("ABC1AAAAA", 100, kTypeValue),
|
|
@@ -121,14 +122,14 @@ TEST_F(FormatTest, InternalKeyShortSeparator) {
|
|
|
121
122
|
Shorten(IKey("AAA1", 100, kTypeValue), IKey("AAA2", 200, kTypeValue)));
|
|
122
123
|
|
|
123
124
|
// When start user key is prefix of limit user key
|
|
124
|
-
ASSERT_EQ(
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
ASSERT_EQ(
|
|
126
|
+
IKey("foo", 100, kTypeValue),
|
|
127
|
+
Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
|
|
127
128
|
|
|
128
129
|
// When limit user key is prefix of start user key
|
|
129
|
-
ASSERT_EQ(
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
ASSERT_EQ(
|
|
131
|
+
IKey("foobar", 100, kTypeValue),
|
|
132
|
+
Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
|
|
132
133
|
}
|
|
133
134
|
|
|
134
135
|
TEST_F(FormatTest, InternalKeyShortestSuccessor) {
|
|
@@ -10,9 +10,11 @@
|
|
|
10
10
|
#ifndef ROCKSDB_LITE
|
|
11
11
|
|
|
12
12
|
#include <stdlib.h>
|
|
13
|
+
|
|
13
14
|
#include <map>
|
|
14
15
|
#include <string>
|
|
15
16
|
#include <vector>
|
|
17
|
+
|
|
16
18
|
#include "db/db_impl/db_impl.h"
|
|
17
19
|
#include "db/db_test_util.h"
|
|
18
20
|
#include "db/version_set.h"
|
|
@@ -55,7 +57,7 @@ class DeleteFileTest : public DBTestBase {
|
|
|
55
57
|
WriteOptions options;
|
|
56
58
|
options.sync = false;
|
|
57
59
|
ReadOptions roptions;
|
|
58
|
-
for (int i = startkey; i < (numkeys + startkey)
|
|
60
|
+
for (int i = startkey; i < (numkeys + startkey); i++) {
|
|
59
61
|
std::string temp = std::to_string(i);
|
|
60
62
|
Slice key(temp);
|
|
61
63
|
Slice value(temp);
|
|
@@ -63,10 +65,8 @@ class DeleteFileTest : public DBTestBase {
|
|
|
63
65
|
}
|
|
64
66
|
}
|
|
65
67
|
|
|
66
|
-
int numKeysInLevels(
|
|
67
|
-
|
|
68
|
-
std::vector<int> *keysperlevel = nullptr) {
|
|
69
|
-
|
|
68
|
+
int numKeysInLevels(std::vector<LiveFileMetaData>& metadata,
|
|
69
|
+
std::vector<int>* keysperlevel = nullptr) {
|
|
70
70
|
if (keysperlevel != nullptr) {
|
|
71
71
|
keysperlevel->resize(numlevels_);
|
|
72
72
|
}
|
|
@@ -82,8 +82,7 @@ class DeleteFileTest : public DBTestBase {
|
|
|
82
82
|
}
|
|
83
83
|
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
|
84
84
|
metadata[i].level, metadata[i].name.c_str(),
|
|
85
|
-
metadata[i].smallestkey.c_str(),
|
|
86
|
-
metadata[i].largestkey.c_str());
|
|
85
|
+
metadata[i].smallestkey.c_str(), metadata[i].largestkey.c_str());
|
|
87
86
|
}
|
|
88
87
|
return numKeys;
|
|
89
88
|
}
|
|
@@ -214,7 +213,7 @@ TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
|
|
|
214
213
|
|
|
215
214
|
// this time, we keep an iterator alive
|
|
216
215
|
Reopen(options);
|
|
217
|
-
Iterator
|
|
216
|
+
Iterator* itr = nullptr;
|
|
218
217
|
CreateTwoLevels();
|
|
219
218
|
itr = db_->NewIterator(ReadOptions());
|
|
220
219
|
ASSERT_OK(itr->status());
|
|
@@ -481,12 +480,12 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
|
|
|
481
480
|
}
|
|
482
481
|
|
|
483
482
|
Status status = db_->DeleteFile(level2file);
|
|
484
|
-
fprintf(stdout, "Deletion status %s: %s\n",
|
|
485
|
-
|
|
483
|
+
fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
|
|
484
|
+
status.ToString().c_str());
|
|
486
485
|
ASSERT_OK(status);
|
|
487
486
|
it->SeekToFirst();
|
|
488
487
|
int numKeysIterated = 0;
|
|
489
|
-
while(it->Valid()) {
|
|
488
|
+
while (it->Valid()) {
|
|
490
489
|
numKeysIterated++;
|
|
491
490
|
it->Next();
|
|
492
491
|
}
|
|
@@ -234,8 +234,8 @@ void ErrorHandler::CancelErrorRecovery() {
|
|
|
234
234
|
// We'll release the lock before calling sfm, so make sure no new
|
|
235
235
|
// recovery gets scheduled at that point
|
|
236
236
|
auto_recovery_ = false;
|
|
237
|
-
SstFileManagerImpl* sfm =
|
|
238
|
-
db_options_.sst_file_manager.get());
|
|
237
|
+
SstFileManagerImpl* sfm =
|
|
238
|
+
reinterpret_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
|
|
239
239
|
if (sfm) {
|
|
240
240
|
// This may or may not cancel a pending recovery
|
|
241
241
|
db_mutex_->Unlock();
|
|
@@ -292,8 +292,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
|
|
292
292
|
bool found = false;
|
|
293
293
|
|
|
294
294
|
{
|
|
295
|
-
auto entry = ErrorSeverityMap.find(
|
|
296
|
-
|
|
295
|
+
auto entry = ErrorSeverityMap.find(
|
|
296
|
+
std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid));
|
|
297
297
|
if (entry != ErrorSeverityMap.end()) {
|
|
298
298
|
sev = entry->second;
|
|
299
299
|
found = true;
|
|
@@ -301,8 +301,8 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
|
|
|
301
301
|
}
|
|
302
302
|
|
|
303
303
|
if (!found) {
|
|
304
|
-
auto entry = DefaultErrorSeverityMap.find(
|
|
305
|
-
|
|
304
|
+
auto entry = DefaultErrorSeverityMap.find(
|
|
305
|
+
std::make_tuple(reason, bg_err.code(), paranoid));
|
|
306
306
|
if (entry != DefaultErrorSeverityMap.end()) {
|
|
307
307
|
sev = entry->second;
|
|
308
308
|
found = true;
|
|
@@ -26,100 +26,99 @@ struct DBRecoverContext {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
class ErrorHandler {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
void CheckAndSetRecoveryAndBGError(const Status& bg_err);
|
|
29
|
+
public:
|
|
30
|
+
ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options,
|
|
31
|
+
InstrumentedMutex* db_mutex)
|
|
32
|
+
: db_(db),
|
|
33
|
+
db_options_(db_options),
|
|
34
|
+
cv_(db_mutex),
|
|
35
|
+
end_recovery_(false),
|
|
36
|
+
recovery_thread_(nullptr),
|
|
37
|
+
db_mutex_(db_mutex),
|
|
38
|
+
auto_recovery_(false),
|
|
39
|
+
recovery_in_prog_(false),
|
|
40
|
+
soft_error_no_bg_work_(false),
|
|
41
|
+
is_db_stopped_(false),
|
|
42
|
+
bg_error_stats_(db_options.statistics) {
|
|
43
|
+
// Clear the checked flag for uninitialized errors
|
|
44
|
+
bg_error_.PermitUncheckedError();
|
|
45
|
+
recovery_error_.PermitUncheckedError();
|
|
46
|
+
recovery_io_error_.PermitUncheckedError();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
void EnableAutoRecovery() { auto_recovery_ = true; }
|
|
50
|
+
|
|
51
|
+
Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
|
|
52
|
+
Status::Code code, Status::SubCode subcode);
|
|
53
|
+
|
|
54
|
+
const Status& SetBGError(const Status& bg_err, BackgroundErrorReason reason);
|
|
55
|
+
|
|
56
|
+
Status GetBGError() const { return bg_error_; }
|
|
57
|
+
|
|
58
|
+
Status GetRecoveryError() const { return recovery_error_; }
|
|
59
|
+
|
|
60
|
+
Status ClearBGError();
|
|
61
|
+
|
|
62
|
+
bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
|
|
63
|
+
|
|
64
|
+
bool IsBGWorkStopped() {
|
|
65
|
+
assert(db_mutex_);
|
|
66
|
+
db_mutex_->AssertHeld();
|
|
67
|
+
return !bg_error_.ok() &&
|
|
68
|
+
(bg_error_.severity() >= Status::Severity::kHardError ||
|
|
69
|
+
!auto_recovery_ || soft_error_no_bg_work_);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
|
|
73
|
+
|
|
74
|
+
bool IsRecoveryInProgress() { return recovery_in_prog_; }
|
|
75
|
+
|
|
76
|
+
Status RecoverFromBGError(bool is_manual = false);
|
|
77
|
+
void CancelErrorRecovery();
|
|
78
|
+
|
|
79
|
+
void EndAutoRecovery();
|
|
80
|
+
|
|
81
|
+
private:
|
|
82
|
+
DBImpl* db_;
|
|
83
|
+
const ImmutableDBOptions& db_options_;
|
|
84
|
+
Status bg_error_;
|
|
85
|
+
// A separate Status variable used to record any errors during the
|
|
86
|
+
// recovery process from hard errors
|
|
87
|
+
Status recovery_error_;
|
|
88
|
+
// A separate IO Status variable used to record any IO errors during
|
|
89
|
+
// the recovery process. At the same time, recovery_error_ is also set.
|
|
90
|
+
IOStatus recovery_io_error_;
|
|
91
|
+
// The condition variable used with db_mutex during auto resume for time
|
|
92
|
+
// wait.
|
|
93
|
+
InstrumentedCondVar cv_;
|
|
94
|
+
bool end_recovery_;
|
|
95
|
+
std::unique_ptr<port::Thread> recovery_thread_;
|
|
96
|
+
|
|
97
|
+
InstrumentedMutex* db_mutex_;
|
|
98
|
+
// A flag indicating whether automatic recovery from errors is enabled
|
|
99
|
+
bool auto_recovery_;
|
|
100
|
+
bool recovery_in_prog_;
|
|
101
|
+
// A flag to indicate that for the soft error, we should not allow any
|
|
102
|
+
// background work except the work is from recovery.
|
|
103
|
+
bool soft_error_no_bg_work_;
|
|
104
|
+
|
|
105
|
+
// Used to store the context for recover, such as flush reason.
|
|
106
|
+
DBRecoverContext recover_context_;
|
|
107
|
+
std::atomic<bool> is_db_stopped_;
|
|
108
|
+
|
|
109
|
+
// The pointer of DB statistics.
|
|
110
|
+
std::shared_ptr<Statistics> bg_error_stats_;
|
|
111
|
+
|
|
112
|
+
const Status& HandleKnownErrors(const Status& bg_err,
|
|
113
|
+
BackgroundErrorReason reason);
|
|
114
|
+
Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery);
|
|
115
|
+
void RecoverFromNoSpace();
|
|
116
|
+
const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
|
|
117
|
+
void RecoverFromRetryableBGIOError();
|
|
118
|
+
// First, if it is in recovery and the recovery_error is ok. Set the
|
|
119
|
+
// recovery_error_ to bg_err. Second, if the severity is higher than the
|
|
120
|
+
// current bg_error_, overwrite it.
|
|
121
|
+
void CheckAndSetRecoveryAndBGError(const Status& bg_err);
|
|
123
122
|
};
|
|
124
123
|
|
|
125
124
|
} // namespace ROCKSDB_NAMESPACE
|
|
@@ -39,9 +39,9 @@ class EventHelpers {
|
|
|
39
39
|
const std::string& file_checksum,
|
|
40
40
|
const std::string& file_checksum_func_name);
|
|
41
41
|
static void LogAndNotifyTableFileDeletion(
|
|
42
|
-
EventLogger* event_logger, int job_id,
|
|
43
|
-
|
|
44
|
-
const
|
|
42
|
+
EventLogger* event_logger, int job_id, uint64_t file_number,
|
|
43
|
+
const std::string& file_path, const Status& status,
|
|
44
|
+
const std::string& db_name,
|
|
45
45
|
const std::vector<std::shared_ptr<EventListener>>& listeners);
|
|
46
46
|
static void NotifyOnErrorRecoveryEnd(
|
|
47
47
|
const std::vector<std::shared_ptr<EventListener>>& listeners,
|