@nxtedition/rocksdb 7.1.32 → 7.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +402 -345
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +121 -64
- package/deps/rocksdb/rocksdb/cache/lru_cache_test.cc +28 -18
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.cc +1 -0
- package/deps/rocksdb/rocksdb/db/arena_wrapped_db_iter.h +2 -0
- package/deps/rocksdb/rocksdb/db/builder.cc +2 -1
- package/deps/rocksdb/rocksdb/db/c.cc +563 -673
- package/deps/rocksdb/rocksdb/db/c_test.c +168 -169
- package/deps/rocksdb/rocksdb/db/column_family.cc +16 -15
- package/deps/rocksdb/rocksdb/db/column_family.h +7 -7
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +17 -28
- package/deps/rocksdb/rocksdb/db/compact_files_test.cc +4 -9
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator.cc +8 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_iterator_test.cc +114 -0
- package/deps/rocksdb/rocksdb/db/comparator_db_test.cc +2 -3
- package/deps/rocksdb/rocksdb/db/convenience.cc +3 -5
- package/deps/rocksdb/rocksdb/db/corruption_test.cc +10 -14
- package/deps/rocksdb/rocksdb/db/cuckoo_table_db_test.cc +9 -13
- package/deps/rocksdb/rocksdb/db/db_basic_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_block_cache_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_bloom_filter_test.cc +14 -16
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +52 -72
- package/deps/rocksdb/rocksdb/db/db_dynamic_level_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/db_encryption_test.cc +12 -12
- package/deps/rocksdb/rocksdb/db/db_filesnapshot.cc +1 -2
- package/deps/rocksdb/rocksdb/db/db_flush_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +1 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +3 -0
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +26 -0
- package/deps/rocksdb/rocksdb/db/db_info_dumper.cc +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter.cc +12 -6
- package/deps/rocksdb/rocksdb/db/db_iter.h +1 -0
- package/deps/rocksdb/rocksdb/db/db_iter_stress_test.cc +6 -7
- package/deps/rocksdb/rocksdb/db/db_iter_test.cc +10 -8
- package/deps/rocksdb/rocksdb/db/db_iterator_test.cc +15 -13
- package/deps/rocksdb/rocksdb/db/db_log_iter_test.cc +7 -9
- package/deps/rocksdb/rocksdb/db/db_logical_block_size_cache_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_merge_operand_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_merge_operator_test.cc +2 -4
- package/deps/rocksdb/rocksdb/db/db_options_test.cc +4 -4
- package/deps/rocksdb/rocksdb/db/db_properties_test.cc +7 -4
- package/deps/rocksdb/rocksdb/db/db_range_del_test.cc +7 -5
- package/deps/rocksdb/rocksdb/db/db_secondary_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_statistics_test.cc +8 -6
- package/deps/rocksdb/rocksdb/db/db_table_properties_test.cc +18 -23
- package/deps/rocksdb/rocksdb/db/db_tailing_iter_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/db_test.cc +10 -5
- package/deps/rocksdb/rocksdb/db/db_test2.cc +172 -169
- package/deps/rocksdb/rocksdb/db/db_test_util.cc +68 -66
- package/deps/rocksdb/rocksdb/db/db_test_util.h +1 -3
- package/deps/rocksdb/rocksdb/db/db_universal_compaction_test.cc +31 -39
- package/deps/rocksdb/rocksdb/db/db_with_timestamp_basic_test.cc +182 -2
- package/deps/rocksdb/rocksdb/db/db_write_test.cc +43 -40
- package/deps/rocksdb/rocksdb/db/dbformat.h +15 -0
- package/deps/rocksdb/rocksdb/db/dbformat_test.cc +35 -34
- package/deps/rocksdb/rocksdb/db/deletefile_test.cc +10 -11
- package/deps/rocksdb/rocksdb/db/error_handler.cc +6 -6
- package/deps/rocksdb/rocksdb/db/error_handler.h +93 -94
- package/deps/rocksdb/rocksdb/db/event_helpers.cc +1 -1
- package/deps/rocksdb/rocksdb/db/event_helpers.h +3 -3
- package/deps/rocksdb/rocksdb/db/external_sst_file_ingestion_job.cc +16 -17
- package/deps/rocksdb/rocksdb/db/external_sst_file_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/fault_injection_test.cc +1 -2
- package/deps/rocksdb/rocksdb/db/file_indexer.cc +2 -0
- package/deps/rocksdb/rocksdb/db/file_indexer.h +2 -1
- package/deps/rocksdb/rocksdb/db/file_indexer_test.cc +4 -2
- package/deps/rocksdb/rocksdb/db/filename_test.cc +27 -29
- package/deps/rocksdb/rocksdb/db/flush_job.cc +7 -13
- package/deps/rocksdb/rocksdb/db/flush_job_test.cc +2 -2
- package/deps/rocksdb/rocksdb/db/forward_iterator.cc +15 -21
- package/deps/rocksdb/rocksdb/db/forward_iterator.h +7 -6
- package/deps/rocksdb/rocksdb/db/forward_iterator_bench.cc +4 -2
- package/deps/rocksdb/rocksdb/db/import_column_family_job.cc +2 -2
- package/deps/rocksdb/rocksdb/db/internal_stats.cc +59 -14
- package/deps/rocksdb/rocksdb/db/internal_stats.h +27 -11
- package/deps/rocksdb/rocksdb/db/job_context.h +5 -6
- package/deps/rocksdb/rocksdb/db/listener_test.cc +21 -23
- package/deps/rocksdb/rocksdb/db/log_reader.cc +7 -11
- package/deps/rocksdb/rocksdb/db/log_reader.h +4 -6
- package/deps/rocksdb/rocksdb/db/log_test.cc +6 -12
- package/deps/rocksdb/rocksdb/db/log_writer.h +1 -1
- package/deps/rocksdb/rocksdb/db/logs_with_prep_tracker.h +0 -1
- package/deps/rocksdb/rocksdb/db/lookup_key.h +4 -1
- package/deps/rocksdb/rocksdb/db/malloc_stats.cc +2 -1
- package/deps/rocksdb/rocksdb/db/manual_compaction_test.cc +3 -5
- package/deps/rocksdb/rocksdb/db/memtable.cc +34 -22
- package/deps/rocksdb/rocksdb/db/memtable.h +4 -6
- package/deps/rocksdb/rocksdb/db/memtable_list.cc +7 -0
- package/deps/rocksdb/rocksdb/db/memtable_list_test.cc +37 -13
- package/deps/rocksdb/rocksdb/db/merge_context.h +1 -0
- package/deps/rocksdb/rocksdb/db/merge_helper.cc +128 -14
- package/deps/rocksdb/rocksdb/db/merge_helper.h +15 -7
- package/deps/rocksdb/rocksdb/db/merge_helper_test.cc +2 -1
- package/deps/rocksdb/rocksdb/db/merge_operator.cc +5 -6
- package/deps/rocksdb/rocksdb/db/obsolete_files_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/options_file_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +55 -43
- package/deps/rocksdb/rocksdb/db/plain_table_db_test.cc +288 -299
- package/deps/rocksdb/rocksdb/db/prefix_test.cc +22 -27
- package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc +1 -1
- package/deps/rocksdb/rocksdb/db/range_del_aggregator_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/repair.cc +7 -8
- package/deps/rocksdb/rocksdb/db/repair_test.cc +3 -4
- package/deps/rocksdb/rocksdb/db/snapshot_impl.cc +4 -5
- package/deps/rocksdb/rocksdb/db/snapshot_impl.h +10 -4
- package/deps/rocksdb/rocksdb/db/table_cache.cc +3 -4
- package/deps/rocksdb/rocksdb/db/table_properties_collector.cc +6 -7
- package/deps/rocksdb/rocksdb/db/table_properties_collector_test.cc +22 -22
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.cc +12 -12
- package/deps/rocksdb/rocksdb/db/transaction_log_impl.h +6 -8
- package/deps/rocksdb/rocksdb/db/trim_history_scheduler.h +2 -0
- package/deps/rocksdb/rocksdb/db/version_builder_test.cc +3 -3
- package/deps/rocksdb/rocksdb/db/version_edit.cc +2 -5
- package/deps/rocksdb/rocksdb/db/version_edit.h +8 -12
- package/deps/rocksdb/rocksdb/db/version_set.cc +74 -102
- package/deps/rocksdb/rocksdb/db/version_set.h +8 -10
- package/deps/rocksdb/rocksdb/db/version_set_sync_and_async.h +0 -5
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +47 -45
- package/deps/rocksdb/rocksdb/db/wal_manager.cc +6 -5
- package/deps/rocksdb/rocksdb/db/wal_manager.h +2 -2
- package/deps/rocksdb/rocksdb/db/wal_manager_test.cc +4 -3
- package/deps/rocksdb/rocksdb/db/wide/db_wide_basic_test.cc +144 -61
- package/deps/rocksdb/rocksdb/db/write_batch.cc +41 -24
- package/deps/rocksdb/rocksdb/db/write_batch_internal.h +2 -7
- package/deps/rocksdb/rocksdb/db/write_batch_test.cc +105 -104
- package/deps/rocksdb/rocksdb/db/write_callback_test.cc +5 -4
- package/deps/rocksdb/rocksdb/db/write_controller.h +1 -0
- package/deps/rocksdb/rocksdb/db/write_controller_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/write_thread.cc +8 -6
- package/deps/rocksdb/rocksdb/env/io_posix.h +6 -0
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.cc +134 -65
- package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h +29 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/compaction_filter.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/db.h +1 -4
- package/deps/rocksdb/rocksdb/include/rocksdb/merge_operator.h +1 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/utilities/stackable_db.h +4 -0
- package/deps/rocksdb/rocksdb/include/rocksdb/write_batch.h +14 -4
- package/deps/rocksdb/rocksdb/table/get_context.cc +52 -7
- package/deps/rocksdb/rocksdb/table/get_context.h +1 -2
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +13 -0
- package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc +36 -4
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +6 -6
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.h +23 -28
- package/deps/rocksdb/rocksdb/utilities/transactions/write_committed_transaction_ts_test.cc +11 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +19 -17
- package/deps/rocksdb/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +10 -7
- package/index.js +14 -16
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
|
@@ -29,7 +29,7 @@ int64_t MaybeCurrentTime(Env* env) {
|
|
|
29
29
|
env->GetCurrentTime(&time).PermitUncheckedError();
|
|
30
30
|
return time;
|
|
31
31
|
}
|
|
32
|
-
} // namespace
|
|
32
|
+
} // anonymous namespace
|
|
33
33
|
|
|
34
34
|
// Special Env used to delay background operations
|
|
35
35
|
|
|
@@ -125,54 +125,54 @@ DBTestBase::~DBTestBase() {
|
|
|
125
125
|
|
|
126
126
|
bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
|
|
127
127
|
#ifdef ROCKSDB_LITE
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
128
|
+
// These options are not supported in ROCKSDB_LITE
|
|
129
|
+
if (option_config == kHashSkipList ||
|
|
130
|
+
option_config == kPlainTableFirstBytePrefix ||
|
|
131
|
+
option_config == kPlainTableCappedPrefix ||
|
|
132
|
+
option_config == kPlainTableCappedPrefixNonMmap ||
|
|
133
|
+
option_config == kPlainTableAllBytesPrefix ||
|
|
134
|
+
option_config == kVectorRep || option_config == kHashLinkList ||
|
|
135
|
+
option_config == kUniversalCompaction ||
|
|
136
|
+
option_config == kUniversalCompactionMultiLevel ||
|
|
137
|
+
option_config == kUniversalSubcompactions ||
|
|
138
|
+
option_config == kFIFOCompaction ||
|
|
139
|
+
option_config == kConcurrentSkipList) {
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
142
|
#endif
|
|
143
143
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
144
|
+
if ((skip_mask & kSkipUniversalCompaction) &&
|
|
145
|
+
(option_config == kUniversalCompaction ||
|
|
146
|
+
option_config == kUniversalCompactionMultiLevel ||
|
|
147
|
+
option_config == kUniversalSubcompactions)) {
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
if ((skip_mask & kSkipNoSeekToLast) &&
|
|
154
|
+
(option_config == kHashLinkList || option_config == kHashSkipList)) {
|
|
155
|
+
return true;
|
|
156
|
+
}
|
|
157
|
+
if ((skip_mask & kSkipPlainTable) &&
|
|
158
|
+
(option_config == kPlainTableAllBytesPrefix ||
|
|
159
|
+
option_config == kPlainTableFirstBytePrefix ||
|
|
160
|
+
option_config == kPlainTableCappedPrefix ||
|
|
161
|
+
option_config == kPlainTableCappedPrefixNonMmap)) {
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
if ((skip_mask & kSkipHashIndex) &&
|
|
165
|
+
(option_config == kBlockBasedTableWithPrefixHashIndex ||
|
|
166
|
+
option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
|
|
170
|
+
return true;
|
|
171
|
+
}
|
|
172
|
+
if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
return false;
|
|
176
176
|
}
|
|
177
177
|
|
|
178
178
|
// Switch to a fresh database with the next option configuration to
|
|
@@ -424,13 +424,13 @@ Options DBTestBase::GetOptions(
|
|
|
424
424
|
options.allow_concurrent_memtable_write = false;
|
|
425
425
|
options.unordered_write = false;
|
|
426
426
|
break;
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
427
|
+
case kDirectIO: {
|
|
428
|
+
options.use_direct_reads = true;
|
|
429
|
+
options.use_direct_io_for_flush_and_compaction = true;
|
|
430
|
+
options.compaction_readahead_size = 2 * 1024 * 1024;
|
|
431
|
+
SetupSyncPointsToMockDirectIO();
|
|
432
|
+
break;
|
|
433
|
+
}
|
|
434
434
|
#endif // ROCKSDB_LITE
|
|
435
435
|
case kMergePut:
|
|
436
436
|
options.merge_operator = MergeOperators::CreatePutOperator();
|
|
@@ -1308,12 +1308,14 @@ void DBTestBase::GetSstFiles(Env* env, std::string path,
|
|
|
1308
1308
|
std::vector<std::string>* files) {
|
|
1309
1309
|
EXPECT_OK(env->GetChildren(path, files));
|
|
1310
1310
|
|
|
1311
|
-
files->erase(
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1311
|
+
files->erase(std::remove_if(files->begin(), files->end(),
|
|
1312
|
+
[](std::string name) {
|
|
1313
|
+
uint64_t number;
|
|
1314
|
+
FileType type;
|
|
1315
|
+
return !(ParseFileName(name, &number, &type) &&
|
|
1316
|
+
type == kTableFile);
|
|
1317
|
+
}),
|
|
1318
|
+
files->end());
|
|
1317
1319
|
}
|
|
1318
1320
|
|
|
1319
1321
|
int DBTestBase::GetSstFileCount(std::string path) {
|
|
@@ -1583,8 +1585,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
|
|
1583
1585
|
iter_cnt++;
|
|
1584
1586
|
total_reads++;
|
|
1585
1587
|
}
|
|
1586
|
-
ASSERT_EQ(data_iter, true_data.end())
|
|
1587
|
-
|
|
1588
|
+
ASSERT_EQ(data_iter, true_data.end())
|
|
1589
|
+
<< iter_cnt << " / " << true_data.size();
|
|
1588
1590
|
delete iter;
|
|
1589
1591
|
|
|
1590
1592
|
// Verify Iterator::Prev()
|
|
@@ -1606,8 +1608,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
|
|
1606
1608
|
iter_cnt++;
|
|
1607
1609
|
total_reads++;
|
|
1608
1610
|
}
|
|
1609
|
-
ASSERT_EQ(data_rev, true_data.rend())
|
|
1610
|
-
|
|
1611
|
+
ASSERT_EQ(data_rev, true_data.rend())
|
|
1612
|
+
<< iter_cnt << " / " << true_data.size();
|
|
1611
1613
|
|
|
1612
1614
|
// Verify Iterator::Seek()
|
|
1613
1615
|
for (auto kv : true_data) {
|
|
@@ -1637,8 +1639,8 @@ void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
|
|
1637
1639
|
iter_cnt++;
|
|
1638
1640
|
total_reads++;
|
|
1639
1641
|
}
|
|
1640
|
-
ASSERT_EQ(data_iter, true_data.end())
|
|
1641
|
-
|
|
1642
|
+
ASSERT_EQ(data_iter, true_data.end())
|
|
1643
|
+
<< iter_cnt << " / " << true_data.size();
|
|
1642
1644
|
|
|
1643
1645
|
// Verify ForwardIterator::Seek()
|
|
1644
1646
|
for (auto kv : true_data) {
|
|
@@ -220,9 +220,7 @@ class SpecialEnv : public EnvWrapper {
|
|
|
220
220
|
Env::IOPriority GetIOPriority() override {
|
|
221
221
|
return base_->GetIOPriority();
|
|
222
222
|
}
|
|
223
|
-
bool use_direct_io() const override {
|
|
224
|
-
return base_->use_direct_io();
|
|
225
|
-
}
|
|
223
|
+
bool use_direct_io() const override { return base_->use_direct_io(); }
|
|
226
224
|
Status Allocate(uint64_t offset, uint64_t len) override {
|
|
227
225
|
return base_->Allocate(offset, len);
|
|
228
226
|
}
|
|
@@ -39,8 +39,8 @@ class DBTestUniversalCompactionBase
|
|
|
39
39
|
|
|
40
40
|
class DBTestUniversalCompaction : public DBTestUniversalCompactionBase {
|
|
41
41
|
public:
|
|
42
|
-
DBTestUniversalCompaction()
|
|
43
|
-
DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
|
42
|
+
DBTestUniversalCompaction()
|
|
43
|
+
: DBTestUniversalCompactionBase("/db_universal_compaction_test") {}
|
|
44
44
|
};
|
|
45
45
|
|
|
46
46
|
class DBTestUniversalCompaction2 : public DBTestBase {
|
|
@@ -93,7 +93,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
|
|
93
93
|
std::atomic_bool expect_full_compaction_;
|
|
94
94
|
std::atomic_bool expect_manual_compaction_;
|
|
95
95
|
};
|
|
96
|
-
} // namespace
|
|
96
|
+
} // anonymous namespace
|
|
97
97
|
|
|
98
98
|
// Make sure we don't trigger a problem if the trigger condtion is given
|
|
99
99
|
// to be 0, which is invalid.
|
|
@@ -563,8 +563,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
|
|
563
563
|
}
|
|
564
564
|
|
|
565
565
|
if (compaction_input_file_names.size() == 0) {
|
|
566
|
-
compaction_input_file_names.push_back(
|
|
567
|
-
cf_meta.levels[0].files[0].name);
|
|
566
|
+
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
|
568
567
|
}
|
|
569
568
|
|
|
570
569
|
// expect fail since universal compaction only allow L0 output
|
|
@@ -574,28 +573,23 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
|
|
574
573
|
.ok());
|
|
575
574
|
|
|
576
575
|
// expect ok and verify the compacted files no longer exist.
|
|
577
|
-
ASSERT_OK(dbfull()->CompactFiles(
|
|
578
|
-
|
|
579
|
-
compaction_input_file_names, 0));
|
|
576
|
+
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
|
577
|
+
compaction_input_file_names, 0));
|
|
580
578
|
|
|
581
579
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
|
582
580
|
VerifyCompactionResult(
|
|
583
|
-
cf_meta,
|
|
584
|
-
|
|
585
|
-
compaction_input_file_names.end()));
|
|
581
|
+
cf_meta, std::set<std::string>(compaction_input_file_names.begin(),
|
|
582
|
+
compaction_input_file_names.end()));
|
|
586
583
|
|
|
587
584
|
compaction_input_file_names.clear();
|
|
588
585
|
|
|
589
586
|
// Pick the first and the last file, expect everything is
|
|
590
587
|
// compacted into one single file.
|
|
588
|
+
compaction_input_file_names.push_back(cf_meta.levels[0].files[0].name);
|
|
591
589
|
compaction_input_file_names.push_back(
|
|
592
|
-
cf_meta.levels[0].files[0].name);
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
cf_meta.levels[0].files.size() - 1].name);
|
|
596
|
-
ASSERT_OK(dbfull()->CompactFiles(
|
|
597
|
-
CompactionOptions(), handles_[1],
|
|
598
|
-
compaction_input_file_names, 0));
|
|
590
|
+
cf_meta.levels[0].files[cf_meta.levels[0].files.size() - 1].name);
|
|
591
|
+
ASSERT_OK(dbfull()->CompactFiles(CompactionOptions(), handles_[1],
|
|
592
|
+
compaction_input_file_names, 0));
|
|
599
593
|
|
|
600
594
|
dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
|
601
595
|
ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
|
|
@@ -604,7 +598,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
|
|
604
598
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
|
605
599
|
Options options = CurrentOptions();
|
|
606
600
|
options.compaction_style = kCompactionStyleUniversal;
|
|
607
|
-
options.write_buffer_size = 100 << 10;
|
|
601
|
+
options.write_buffer_size = 100 << 10; // 100KB
|
|
608
602
|
options.num_levels = 7;
|
|
609
603
|
options.disable_auto_compactions = true;
|
|
610
604
|
DestroyAndReopen(options);
|
|
@@ -640,9 +634,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
|
|
|
640
634
|
class DBTestUniversalCompactionMultiLevels
|
|
641
635
|
: public DBTestUniversalCompactionBase {
|
|
642
636
|
public:
|
|
643
|
-
DBTestUniversalCompactionMultiLevels()
|
|
644
|
-
DBTestUniversalCompactionBase(
|
|
645
|
-
|
|
637
|
+
DBTestUniversalCompactionMultiLevels()
|
|
638
|
+
: DBTestUniversalCompactionBase(
|
|
639
|
+
"/db_universal_compaction_multi_levels_test") {}
|
|
646
640
|
};
|
|
647
641
|
|
|
648
642
|
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
|
|
@@ -725,12 +719,11 @@ INSTANTIATE_TEST_CASE_P(MultiLevels, DBTestUniversalCompactionMultiLevels,
|
|
|
725
719
|
::testing::Combine(::testing::Values(3, 20),
|
|
726
720
|
::testing::Bool()));
|
|
727
721
|
|
|
728
|
-
class DBTestUniversalCompactionParallel :
|
|
729
|
-
public DBTestUniversalCompactionBase {
|
|
722
|
+
class DBTestUniversalCompactionParallel : public DBTestUniversalCompactionBase {
|
|
730
723
|
public:
|
|
731
|
-
DBTestUniversalCompactionParallel()
|
|
732
|
-
DBTestUniversalCompactionBase(
|
|
733
|
-
|
|
724
|
+
DBTestUniversalCompactionParallel()
|
|
725
|
+
: DBTestUniversalCompactionBase("/db_universal_compaction_prallel_test") {
|
|
726
|
+
}
|
|
734
727
|
};
|
|
735
728
|
|
|
736
729
|
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
|
|
@@ -919,8 +912,8 @@ INSTANTIATE_TEST_CASE_P(Parallel, DBTestUniversalCompactionParallel,
|
|
|
919
912
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
|
920
913
|
Options options = CurrentOptions();
|
|
921
914
|
options.compaction_style = kCompactionStyleUniversal;
|
|
922
|
-
options.write_buffer_size = 105 << 10;
|
|
923
|
-
options.arena_block_size = 4 << 10;
|
|
915
|
+
options.write_buffer_size = 105 << 10; // 105KB
|
|
916
|
+
options.arena_block_size = 4 << 10; // 4KB
|
|
924
917
|
options.target_file_size_base = 32 << 10; // 32KB
|
|
925
918
|
options.level0_file_num_compaction_trigger = 4;
|
|
926
919
|
options.num_levels = num_levels_;
|
|
@@ -951,8 +944,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
|
|
|
951
944
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
|
|
952
945
|
Options options = CurrentOptions();
|
|
953
946
|
options.compaction_style = kCompactionStyleUniversal;
|
|
954
|
-
options.write_buffer_size = 105 << 10;
|
|
955
|
-
options.arena_block_size = 4 << 10;
|
|
947
|
+
options.write_buffer_size = 105 << 10; // 105KB
|
|
948
|
+
options.arena_block_size = 4 << 10; // 4KB
|
|
956
949
|
options.target_file_size_base = 32 << 10; // 32KB
|
|
957
950
|
// trigger compaction if there are >= 4 files
|
|
958
951
|
options.level0_file_num_compaction_trigger = 4;
|
|
@@ -1353,7 +1346,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
|
|
|
1353
1346
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_3", 500 * 1024);
|
|
1354
1347
|
cf_opt1.cf_paths.emplace_back(dbname_ + "cf1_4", 1024 * 1024 * 1024);
|
|
1355
1348
|
option_vector.emplace_back(DBOptions(options), cf_opt1);
|
|
1356
|
-
CreateColumnFamilies({"one"},option_vector[1]);
|
|
1349
|
+
CreateColumnFamilies({"one"}, option_vector[1]);
|
|
1357
1350
|
|
|
1358
1351
|
// Configura CF2 specific paths.
|
|
1359
1352
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2", 300 * 1024);
|
|
@@ -1361,7 +1354,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
|
|
|
1361
1354
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_3", 500 * 1024);
|
|
1362
1355
|
cf_opt2.cf_paths.emplace_back(dbname_ + "cf2_4", 1024 * 1024 * 1024);
|
|
1363
1356
|
option_vector.emplace_back(DBOptions(options), cf_opt2);
|
|
1364
|
-
CreateColumnFamilies({"two"},option_vector[2]);
|
|
1357
|
+
CreateColumnFamilies({"two"}, option_vector[2]);
|
|
1365
1358
|
|
|
1366
1359
|
ReopenWithColumnFamilies({"default", "one", "two"}, option_vector);
|
|
1367
1360
|
|
|
@@ -1567,7 +1560,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
|
|
|
1567
1560
|
verify_func(max_key3);
|
|
1568
1561
|
}
|
|
1569
1562
|
|
|
1570
|
-
|
|
1571
1563
|
TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
|
|
1572
1564
|
if (!Snappy_Supported()) {
|
|
1573
1565
|
return;
|
|
@@ -1829,9 +1821,9 @@ INSTANTIATE_TEST_CASE_P(NumLevels, DBTestUniversalCompaction,
|
|
|
1829
1821
|
class DBTestUniversalManualCompactionOutputPathId
|
|
1830
1822
|
: public DBTestUniversalCompactionBase {
|
|
1831
1823
|
public:
|
|
1832
|
-
DBTestUniversalManualCompactionOutputPathId()
|
|
1833
|
-
DBTestUniversalCompactionBase(
|
|
1834
|
-
|
|
1824
|
+
DBTestUniversalManualCompactionOutputPathId()
|
|
1825
|
+
: DBTestUniversalCompactionBase(
|
|
1826
|
+
"/db_universal_compaction_manual_pid_test") {}
|
|
1835
1827
|
};
|
|
1836
1828
|
|
|
1837
1829
|
TEST_P(DBTestUniversalManualCompactionOutputPathId,
|
|
@@ -2236,8 +2228,8 @@ int main(int argc, char** argv) {
|
|
|
2236
2228
|
::testing::InitGoogleTest(&argc, argv);
|
|
2237
2229
|
return RUN_ALL_TESTS();
|
|
2238
2230
|
#else
|
|
2239
|
-
(void)
|
|
2240
|
-
(void)
|
|
2231
|
+
(void)argc;
|
|
2232
|
+
(void)argv;
|
|
2241
2233
|
return 0;
|
|
2242
2234
|
#endif
|
|
2243
2235
|
}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
#endif
|
|
19
19
|
#include "test_util/testutil.h"
|
|
20
20
|
#include "utilities/fault_injection_env.h"
|
|
21
|
+
#include "utilities/merge_operators/string_append/stringappend2.h"
|
|
21
22
|
|
|
22
23
|
namespace ROCKSDB_NAMESPACE {
|
|
23
24
|
class DBBasicTestWithTimestamp : public DBBasicTestWithTimestampBase {
|
|
@@ -50,7 +51,7 @@ TEST_F(DBBasicTestWithTimestamp, SanityChecks) {
|
|
|
50
51
|
db_->Put(WriteOptions(), "key", dummy_ts, "value").IsInvalidArgument());
|
|
51
52
|
ASSERT_TRUE(db_->Merge(WriteOptions(), db_->DefaultColumnFamily(), "key",
|
|
52
53
|
dummy_ts, "value")
|
|
53
|
-
.
|
|
54
|
+
.IsInvalidArgument());
|
|
54
55
|
ASSERT_TRUE(db_->Delete(WriteOptions(), "key", dummy_ts).IsInvalidArgument());
|
|
55
56
|
ASSERT_TRUE(
|
|
56
57
|
db_->SingleDelete(WriteOptions(), "key", dummy_ts).IsInvalidArgument());
|
|
@@ -96,7 +97,7 @@ TEST_F(DBBasicTestWithTimestamp, SanityChecks) {
|
|
|
96
97
|
ASSERT_TRUE(db_->Put(WriteOptions(), handle, "key", wrong_ts, "value")
|
|
97
98
|
.IsInvalidArgument());
|
|
98
99
|
ASSERT_TRUE(db_->Merge(WriteOptions(), handle, "key", wrong_ts, "value")
|
|
99
|
-
.
|
|
100
|
+
.IsInvalidArgument());
|
|
100
101
|
ASSERT_TRUE(
|
|
101
102
|
db_->Delete(WriteOptions(), handle, "key", wrong_ts).IsInvalidArgument());
|
|
102
103
|
ASSERT_TRUE(db_->SingleDelete(WriteOptions(), handle, "key", wrong_ts)
|
|
@@ -3690,6 +3691,185 @@ TEST_F(DBBasicTestWithTimestamp, DeleteRangeGetIteratorWithSnapshot) {
|
|
|
3690
3691
|
db_->ReleaseSnapshot(after_tombstone);
|
|
3691
3692
|
Close();
|
|
3692
3693
|
}
|
|
3694
|
+
|
|
3695
|
+
TEST_F(DBBasicTestWithTimestamp, MergeBasic) {
|
|
3696
|
+
Options options = GetDefaultOptions();
|
|
3697
|
+
options.create_if_missing = true;
|
|
3698
|
+
const size_t kTimestampSize = Timestamp(0, 0).size();
|
|
3699
|
+
TestComparator test_cmp(kTimestampSize);
|
|
3700
|
+
options.comparator = &test_cmp;
|
|
3701
|
+
options.merge_operator = std::make_shared<StringAppendTESTOperator>('.');
|
|
3702
|
+
DestroyAndReopen(options);
|
|
3703
|
+
|
|
3704
|
+
const std::array<std::string, 3> write_ts_strs = {
|
|
3705
|
+
Timestamp(100, 0), Timestamp(200, 0), Timestamp(300, 0)};
|
|
3706
|
+
constexpr size_t kNumOfUniqKeys = 100;
|
|
3707
|
+
ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily();
|
|
3708
|
+
|
|
3709
|
+
for (size_t i = 0; i < write_ts_strs.size(); ++i) {
|
|
3710
|
+
for (size_t j = 0; j < kNumOfUniqKeys; ++j) {
|
|
3711
|
+
Status s;
|
|
3712
|
+
if (i == 0) {
|
|
3713
|
+
const std::string val = "v" + std::to_string(j) + "_0";
|
|
3714
|
+
s = db_->Put(WriteOptions(), Key1(j), write_ts_strs[i], val);
|
|
3715
|
+
} else {
|
|
3716
|
+
const std::string merge_op = std::to_string(i);
|
|
3717
|
+
s = db_->Merge(WriteOptions(), default_cf, Key1(j), write_ts_strs[i],
|
|
3718
|
+
merge_op);
|
|
3719
|
+
}
|
|
3720
|
+
ASSERT_OK(s);
|
|
3721
|
+
}
|
|
3722
|
+
}
|
|
3723
|
+
|
|
3724
|
+
std::array<std::string, 3> read_ts_strs = {
|
|
3725
|
+
Timestamp(150, 0), Timestamp(250, 0), Timestamp(350, 0)};
|
|
3726
|
+
|
|
3727
|
+
const auto verify_db_with_get = [&]() {
|
|
3728
|
+
for (size_t i = 0; i < kNumOfUniqKeys; ++i) {
|
|
3729
|
+
const std::string base_val = "v" + std::to_string(i) + "_0";
|
|
3730
|
+
const std::array<std::string, 3> expected_values = {
|
|
3731
|
+
base_val, base_val + ".1", base_val + ".1.2"};
|
|
3732
|
+
const std::array<std::string, 3>& expected_ts = write_ts_strs;
|
|
3733
|
+
ReadOptions read_opts;
|
|
3734
|
+
for (size_t j = 0; j < read_ts_strs.size(); ++j) {
|
|
3735
|
+
Slice read_ts = read_ts_strs[j];
|
|
3736
|
+
read_opts.timestamp = &read_ts;
|
|
3737
|
+
std::string value;
|
|
3738
|
+
std::string ts;
|
|
3739
|
+
const Status s = db_->Get(read_opts, Key1(i), &value, &ts);
|
|
3740
|
+
ASSERT_OK(s);
|
|
3741
|
+
ASSERT_EQ(expected_values[j], value);
|
|
3742
|
+
ASSERT_EQ(expected_ts[j], ts);
|
|
3743
|
+
|
|
3744
|
+
// Do Seek/SeekForPrev
|
|
3745
|
+
std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
|
|
3746
|
+
it->Seek(Key1(i));
|
|
3747
|
+
ASSERT_TRUE(it->Valid());
|
|
3748
|
+
ASSERT_EQ(expected_values[j], it->value());
|
|
3749
|
+
ASSERT_EQ(expected_ts[j], it->timestamp());
|
|
3750
|
+
|
|
3751
|
+
it->SeekForPrev(Key1(i));
|
|
3752
|
+
ASSERT_TRUE(it->Valid());
|
|
3753
|
+
ASSERT_EQ(expected_values[j], it->value());
|
|
3754
|
+
ASSERT_EQ(expected_ts[j], it->timestamp());
|
|
3755
|
+
}
|
|
3756
|
+
}
|
|
3757
|
+
};
|
|
3758
|
+
|
|
3759
|
+
const auto verify_db_with_iterator = [&]() {
|
|
3760
|
+
std::string value_suffix;
|
|
3761
|
+
for (size_t i = 0; i < read_ts_strs.size(); ++i) {
|
|
3762
|
+
ReadOptions read_opts;
|
|
3763
|
+
Slice read_ts = read_ts_strs[i];
|
|
3764
|
+
read_opts.timestamp = &read_ts;
|
|
3765
|
+
std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
|
|
3766
|
+
size_t key_int_val = 0;
|
|
3767
|
+
for (it->SeekToFirst(); it->Valid(); it->Next(), ++key_int_val) {
|
|
3768
|
+
const std::string key = Key1(key_int_val);
|
|
3769
|
+
const std::string value =
|
|
3770
|
+
"v" + std::to_string(key_int_val) + "_0" + value_suffix;
|
|
3771
|
+
ASSERT_EQ(key, it->key());
|
|
3772
|
+
ASSERT_EQ(value, it->value());
|
|
3773
|
+
ASSERT_EQ(write_ts_strs[i], it->timestamp());
|
|
3774
|
+
}
|
|
3775
|
+
ASSERT_EQ(kNumOfUniqKeys, key_int_val);
|
|
3776
|
+
|
|
3777
|
+
key_int_val = kNumOfUniqKeys - 1;
|
|
3778
|
+
for (it->SeekToLast(); it->Valid(); it->Prev(), --key_int_val) {
|
|
3779
|
+
const std::string key = Key1(key_int_val);
|
|
3780
|
+
const std::string value =
|
|
3781
|
+
"v" + std::to_string(key_int_val) + "_0" + value_suffix;
|
|
3782
|
+
ASSERT_EQ(key, it->key());
|
|
3783
|
+
ASSERT_EQ(value, it->value());
|
|
3784
|
+
ASSERT_EQ(write_ts_strs[i], it->timestamp());
|
|
3785
|
+
}
|
|
3786
|
+
ASSERT_EQ(std::numeric_limits<size_t>::max(), key_int_val);
|
|
3787
|
+
|
|
3788
|
+
value_suffix = value_suffix + "." + std::to_string(i + 1);
|
|
3789
|
+
}
|
|
3790
|
+
};
|
|
3791
|
+
|
|
3792
|
+
verify_db_with_get();
|
|
3793
|
+
verify_db_with_iterator();
|
|
3794
|
+
|
|
3795
|
+
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
3796
|
+
|
|
3797
|
+
verify_db_with_get();
|
|
3798
|
+
verify_db_with_iterator();
|
|
3799
|
+
|
|
3800
|
+
Close();
|
|
3801
|
+
}
|
|
3802
|
+
|
|
3803
|
+
TEST_F(DBBasicTestWithTimestamp, MergeAfterDeletion) {
|
|
3804
|
+
Options options = GetDefaultOptions();
|
|
3805
|
+
options.create_if_missing = true;
|
|
3806
|
+
const size_t kTimestampSize = Timestamp(0, 0).size();
|
|
3807
|
+
TestComparator test_cmp(kTimestampSize);
|
|
3808
|
+
options.comparator = &test_cmp;
|
|
3809
|
+
options.merge_operator = std::make_shared<StringAppendTESTOperator>('.');
|
|
3810
|
+
DestroyAndReopen(options);
|
|
3811
|
+
|
|
3812
|
+
ColumnFamilyHandle* const column_family = db_->DefaultColumnFamily();
|
|
3813
|
+
|
|
3814
|
+
const size_t num_keys_per_file = 10;
|
|
3815
|
+
const size_t num_merges_per_key = 2;
|
|
3816
|
+
for (size_t i = 0; i < num_keys_per_file; ++i) {
|
|
3817
|
+
std::string ts = Timestamp(i + 10000, 0);
|
|
3818
|
+
Status s = db_->Delete(WriteOptions(), Key1(i), ts);
|
|
3819
|
+
ASSERT_OK(s);
|
|
3820
|
+
for (size_t j = 1; j <= num_merges_per_key; ++j) {
|
|
3821
|
+
ts = Timestamp(i + 10000 + j, 0);
|
|
3822
|
+
s = db_->Merge(WriteOptions(), column_family, Key1(i), ts,
|
|
3823
|
+
std::to_string(j));
|
|
3824
|
+
ASSERT_OK(s);
|
|
3825
|
+
}
|
|
3826
|
+
}
|
|
3827
|
+
|
|
3828
|
+
const auto verify_db = [&]() {
|
|
3829
|
+
ReadOptions read_opts;
|
|
3830
|
+
std::string read_ts_str = Timestamp(20000, 0);
|
|
3831
|
+
Slice ts = read_ts_str;
|
|
3832
|
+
read_opts.timestamp = &ts;
|
|
3833
|
+
std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
|
|
3834
|
+
size_t count = 0;
|
|
3835
|
+
for (it->SeekToFirst(); it->Valid(); it->Next(), ++count) {
|
|
3836
|
+
std::string key = Key1(count);
|
|
3837
|
+
ASSERT_EQ(key, it->key());
|
|
3838
|
+
std::string value;
|
|
3839
|
+
for (size_t j = 1; j <= num_merges_per_key; ++j) {
|
|
3840
|
+
value.append(std::to_string(j));
|
|
3841
|
+
if (j < num_merges_per_key) {
|
|
3842
|
+
value.push_back('.');
|
|
3843
|
+
}
|
|
3844
|
+
}
|
|
3845
|
+
ASSERT_EQ(value, it->value());
|
|
3846
|
+
std::string ts1 = Timestamp(count + 10000 + num_merges_per_key, 0);
|
|
3847
|
+
ASSERT_EQ(ts1, it->timestamp());
|
|
3848
|
+
}
|
|
3849
|
+
ASSERT_OK(it->status());
|
|
3850
|
+
ASSERT_EQ(num_keys_per_file, count);
|
|
3851
|
+
for (it->SeekToLast(); it->Valid(); it->Prev(), --count) {
|
|
3852
|
+
std::string key = Key1(count - 1);
|
|
3853
|
+
ASSERT_EQ(key, it->key());
|
|
3854
|
+
std::string value;
|
|
3855
|
+
for (size_t j = 1; j <= num_merges_per_key; ++j) {
|
|
3856
|
+
value.append(std::to_string(j));
|
|
3857
|
+
if (j < num_merges_per_key) {
|
|
3858
|
+
value.push_back('.');
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
ASSERT_EQ(value, it->value());
|
|
3862
|
+
std::string ts1 = Timestamp(count - 1 + 10000 + num_merges_per_key, 0);
|
|
3863
|
+
ASSERT_EQ(ts1, it->timestamp());
|
|
3864
|
+
}
|
|
3865
|
+
ASSERT_OK(it->status());
|
|
3866
|
+
ASSERT_EQ(0, count);
|
|
3867
|
+
};
|
|
3868
|
+
|
|
3869
|
+
verify_db();
|
|
3870
|
+
|
|
3871
|
+
Close();
|
|
3872
|
+
}
|
|
3693
3873
|
} // namespace ROCKSDB_NAMESPACE
|
|
3694
3874
|
|
|
3695
3875
|
int main(int argc, char** argv) {
|