@nxtedition/rocksdb 7.0.37 → 7.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +17 -26
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +2 -2
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +13 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +273 -134
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +33 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +11 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +133 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +130 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +8 -4
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +11 -9
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +209 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +54 -39
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +102 -19
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +30 -11
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +28 -25
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +0 -14
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +63 -54
- package/deps/rocksdb/rocksdb/db/db_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/error_handler.cc +7 -0
- package/deps/rocksdb/rocksdb/db/error_handler.h +10 -9
- package/deps/rocksdb/rocksdb/db/log_test.cc +13 -6
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/table_cache.cc +21 -0
- package/deps/rocksdb/rocksdb/db/table_cache.h +5 -0
- package/deps/rocksdb/rocksdb/db/version_set.cc +3 -2
- package/deps/rocksdb/rocksdb/db/version_set.h +6 -4
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +8 -6
- package/deps/rocksdb/rocksdb/db/wal_edit.cc +22 -15
- package/deps/rocksdb/rocksdb/db/wal_edit.h +10 -0
- package/deps/rocksdb/rocksdb/db/wal_edit_test.cc +4 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +0 -36
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +1 -12
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +23 -29
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +0 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +7 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +0 -5
- package/deps/rocksdb/rocksdb/env/io_posix.cc +1 -7
- package/deps/rocksdb/rocksdb/options/options_test.cc +16 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +51 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +3 -0
- package/deps/rocksdb/rocksdb/table/table_reader.h +14 -0
- package/deps/rocksdb/rocksdb/table/table_test.cc +52 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +8 -38
- package/deps/rocksdb/rocksdb/util/rate_limiter.cc +27 -21
- package/deps/rocksdb/rocksdb/util/rate_limiter.h +12 -10
- package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +11 -8
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +59 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +12 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +31 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +0 -3
- package/index.js +2 -2
- package/iterator.js +1 -1
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
|
@@ -758,6 +758,9 @@ DEFINE_bool(show_table_properties, false,
|
|
|
758
758
|
|
|
759
759
|
DEFINE_string(db, "", "Use the db with the following name.");
|
|
760
760
|
|
|
761
|
+
DEFINE_bool(progress_reports, true,
|
|
762
|
+
"If true, db_bench will report number of finished operations.");
|
|
763
|
+
|
|
761
764
|
// Read cache flags
|
|
762
765
|
|
|
763
766
|
DEFINE_string(read_cache_path, "",
|
|
@@ -2252,7 +2255,7 @@ class Stats {
|
|
|
2252
2255
|
}
|
|
2253
2256
|
|
|
2254
2257
|
done_ += num_ops;
|
|
2255
|
-
if (done_ >= next_report_) {
|
|
2258
|
+
if (done_ >= next_report_ && FLAGS_progress_reports) {
|
|
2256
2259
|
if (!FLAGS_stats_interval) {
|
|
2257
2260
|
if (next_report_ < 1000) next_report_ += 100;
|
|
2258
2261
|
else if (next_report_ < 5000) next_report_ += 500;
|
|
@@ -3814,6 +3817,10 @@ class Benchmark {
|
|
|
3814
3817
|
perf_context.EnablePerLevelPerfContext();
|
|
3815
3818
|
thread->stats.Start(thread->tid);
|
|
3816
3819
|
(arg->bm->*(arg->method))(thread);
|
|
3820
|
+
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
3821
|
+
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
3822
|
+
get_perf_context()->ToString());
|
|
3823
|
+
}
|
|
3817
3824
|
thread->stats.Stop();
|
|
3818
3825
|
|
|
3819
3826
|
{
|
|
@@ -5743,10 +5750,6 @@ class Benchmark {
|
|
|
5743
5750
|
|
|
5744
5751
|
delete iter;
|
|
5745
5752
|
thread->stats.AddBytes(bytes);
|
|
5746
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
5747
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
5748
|
-
get_perf_context()->ToString());
|
|
5749
|
-
}
|
|
5750
5753
|
}
|
|
5751
5754
|
|
|
5752
5755
|
void ReadToRowCache(ThreadState* thread) {
|
|
@@ -5800,11 +5803,6 @@ class Benchmark {
|
|
|
5800
5803
|
|
|
5801
5804
|
thread->stats.AddBytes(bytes);
|
|
5802
5805
|
thread->stats.AddMessage(msg);
|
|
5803
|
-
|
|
5804
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
5805
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
5806
|
-
get_perf_context()->ToString());
|
|
5807
|
-
}
|
|
5808
5806
|
}
|
|
5809
5807
|
|
|
5810
5808
|
void ReadReverse(ThreadState* thread) {
|
|
@@ -5896,11 +5894,6 @@ class Benchmark {
|
|
|
5896
5894
|
found, read, nonexist);
|
|
5897
5895
|
|
|
5898
5896
|
thread->stats.AddMessage(msg);
|
|
5899
|
-
|
|
5900
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
5901
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
5902
|
-
get_perf_context()->ToString());
|
|
5903
|
-
}
|
|
5904
5897
|
}
|
|
5905
5898
|
|
|
5906
5899
|
int64_t GetRandomKey(Random64* rand) {
|
|
@@ -6036,11 +6029,6 @@ class Benchmark {
|
|
|
6036
6029
|
|
|
6037
6030
|
thread->stats.AddBytes(bytes);
|
|
6038
6031
|
thread->stats.AddMessage(msg);
|
|
6039
|
-
|
|
6040
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
6041
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
6042
|
-
get_perf_context()->ToString());
|
|
6043
|
-
}
|
|
6044
6032
|
}
|
|
6045
6033
|
|
|
6046
6034
|
// Calls MultiGet over a list of keys from a random distribution.
|
|
@@ -6602,11 +6590,6 @@ class Benchmark {
|
|
|
6602
6590
|
|
|
6603
6591
|
thread->stats.AddBytes(bytes);
|
|
6604
6592
|
thread->stats.AddMessage(msg);
|
|
6605
|
-
|
|
6606
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
6607
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
6608
|
-
get_perf_context()->ToString());
|
|
6609
|
-
}
|
|
6610
6593
|
}
|
|
6611
6594
|
|
|
6612
6595
|
void IteratorCreation(ThreadState* thread) {
|
|
@@ -6756,10 +6739,6 @@ class Benchmark {
|
|
|
6756
6739
|
found, read);
|
|
6757
6740
|
thread->stats.AddBytes(bytes);
|
|
6758
6741
|
thread->stats.AddMessage(msg);
|
|
6759
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
6760
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
6761
|
-
get_perf_context()->ToString());
|
|
6762
|
-
}
|
|
6763
6742
|
}
|
|
6764
6743
|
|
|
6765
6744
|
void SeekRandomWhileWriting(ThreadState* thread) {
|
|
@@ -7782,11 +7761,6 @@ class Benchmark {
|
|
|
7782
7761
|
snprintf(msg, sizeof(msg), "( batches:%" PRIu64 " )", transactions_done);
|
|
7783
7762
|
}
|
|
7784
7763
|
thread->stats.AddMessage(msg);
|
|
7785
|
-
|
|
7786
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
7787
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
7788
|
-
get_perf_context()->ToString());
|
|
7789
|
-
}
|
|
7790
7764
|
thread->stats.AddBytes(static_cast<int64_t>(inserter.GetBytesInserted()));
|
|
7791
7765
|
}
|
|
7792
7766
|
|
|
@@ -7965,10 +7939,6 @@ class Benchmark {
|
|
|
7965
7939
|
read);
|
|
7966
7940
|
thread->stats.AddBytes(bytes);
|
|
7967
7941
|
thread->stats.AddMessage(msg);
|
|
7968
|
-
if (FLAGS_perf_level > ROCKSDB_NAMESPACE::PerfLevel::kDisable) {
|
|
7969
|
-
thread->stats.AddMessage(std::string("PERF_CONTEXT:\n") +
|
|
7970
|
-
get_perf_context()->ToString());
|
|
7971
|
-
}
|
|
7972
7942
|
}
|
|
7973
7943
|
|
|
7974
7944
|
void TimeSeriesWrite(ThreadState* thread) {
|
|
@@ -54,20 +54,20 @@ GenericRateLimiter::GenericRateLimiter(
|
|
|
54
54
|
rate_bytes_per_sec_(auto_tuned ? rate_bytes_per_sec / 2
|
|
55
55
|
: rate_bytes_per_sec),
|
|
56
56
|
refill_bytes_per_period_(
|
|
57
|
-
|
|
57
|
+
CalculateRefillBytesPerPeriodLocked(rate_bytes_per_sec_)),
|
|
58
58
|
clock_(clock),
|
|
59
59
|
stop_(false),
|
|
60
60
|
exit_cv_(&request_mutex_),
|
|
61
61
|
requests_to_wait_(0),
|
|
62
62
|
available_bytes_(0),
|
|
63
|
-
next_refill_us_(
|
|
63
|
+
next_refill_us_(NowMicrosMonotonicLocked()),
|
|
64
64
|
fairness_(fairness > 100 ? 100 : fairness),
|
|
65
65
|
rnd_((uint32_t)time(nullptr)),
|
|
66
66
|
wait_until_refill_pending_(false),
|
|
67
67
|
auto_tuned_(auto_tuned),
|
|
68
68
|
num_drains_(0),
|
|
69
69
|
max_bytes_per_sec_(rate_bytes_per_sec),
|
|
70
|
-
tuned_time_(
|
|
70
|
+
tuned_time_(NowMicrosMonotonicLocked()) {
|
|
71
71
|
for (int i = Env::IO_LOW; i < Env::IO_TOTAL; ++i) {
|
|
72
72
|
total_requests_[i] = 0;
|
|
73
73
|
total_bytes_through_[i] = 0;
|
|
@@ -97,10 +97,15 @@ GenericRateLimiter::~GenericRateLimiter() {
|
|
|
97
97
|
|
|
98
98
|
// This API allows user to dynamically change rate limiter's bytes per second.
|
|
99
99
|
void GenericRateLimiter::SetBytesPerSecond(int64_t bytes_per_second) {
|
|
100
|
+
MutexLock g(&request_mutex_);
|
|
101
|
+
SetBytesPerSecondLocked(bytes_per_second);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
void GenericRateLimiter::SetBytesPerSecondLocked(int64_t bytes_per_second) {
|
|
100
105
|
assert(bytes_per_second > 0);
|
|
101
|
-
rate_bytes_per_sec_
|
|
106
|
+
rate_bytes_per_sec_.store(bytes_per_second, std::memory_order_relaxed);
|
|
102
107
|
refill_bytes_per_period_.store(
|
|
103
|
-
|
|
108
|
+
CalculateRefillBytesPerPeriodLocked(bytes_per_second),
|
|
104
109
|
std::memory_order_relaxed);
|
|
105
110
|
}
|
|
106
111
|
|
|
@@ -115,10 +120,10 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
115
120
|
|
|
116
121
|
if (auto_tuned_) {
|
|
117
122
|
static const int kRefillsPerTune = 100;
|
|
118
|
-
std::chrono::microseconds now(
|
|
123
|
+
std::chrono::microseconds now(NowMicrosMonotonicLocked());
|
|
119
124
|
if (now - tuned_time_ >=
|
|
120
125
|
kRefillsPerTune * std::chrono::microseconds(refill_period_us_)) {
|
|
121
|
-
Status s =
|
|
126
|
+
Status s = TuneLocked();
|
|
122
127
|
s.PermitUncheckedError(); //**TODO: What to do on error?
|
|
123
128
|
}
|
|
124
129
|
}
|
|
@@ -152,7 +157,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
152
157
|
// (1) Waiting for the next refill time.
|
|
153
158
|
// (2) Refilling the bytes and granting requests.
|
|
154
159
|
do {
|
|
155
|
-
int64_t time_until_refill_us = next_refill_us_ -
|
|
160
|
+
int64_t time_until_refill_us = next_refill_us_ - NowMicrosMonotonicLocked();
|
|
156
161
|
if (time_until_refill_us > 0) {
|
|
157
162
|
if (wait_until_refill_pending_) {
|
|
158
163
|
// Somebody is performing (1). Trust we'll be woken up when our request
|
|
@@ -173,7 +178,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
173
178
|
} else {
|
|
174
179
|
// Whichever thread reaches here first performs duty (2) as described
|
|
175
180
|
// above.
|
|
176
|
-
|
|
181
|
+
RefillBytesAndGrantRequestsLocked();
|
|
177
182
|
if (r.granted) {
|
|
178
183
|
// If there is any remaining requests, make sure there exists at least
|
|
179
184
|
// one candidate is awake for future duties by signaling a front request
|
|
@@ -215,7 +220,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
215
220
|
}
|
|
216
221
|
|
|
217
222
|
std::vector<Env::IOPriority>
|
|
218
|
-
GenericRateLimiter::
|
|
223
|
+
GenericRateLimiter::GeneratePriorityIterationOrderLocked() {
|
|
219
224
|
std::vector<Env::IOPriority> pri_iteration_order(Env::IO_TOTAL /* 4 */);
|
|
220
225
|
// We make Env::IO_USER a superior priority by always iterating its queue
|
|
221
226
|
// first
|
|
@@ -223,12 +228,12 @@ GenericRateLimiter::GeneratePriorityIterationOrder() {
|
|
|
223
228
|
|
|
224
229
|
bool high_pri_iterated_after_mid_low_pri = rnd_.OneIn(fairness_);
|
|
225
230
|
TEST_SYNC_POINT_CALLBACK(
|
|
226
|
-
"GenericRateLimiter::
|
|
231
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
227
232
|
"PostRandomOneInFairnessForHighPri",
|
|
228
233
|
&high_pri_iterated_after_mid_low_pri);
|
|
229
234
|
bool mid_pri_itereated_after_low_pri = rnd_.OneIn(fairness_);
|
|
230
235
|
TEST_SYNC_POINT_CALLBACK(
|
|
231
|
-
"GenericRateLimiter::
|
|
236
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
232
237
|
"PostRandomOneInFairnessForMidPri",
|
|
233
238
|
&mid_pri_itereated_after_low_pri);
|
|
234
239
|
|
|
@@ -247,15 +252,16 @@ GenericRateLimiter::GeneratePriorityIterationOrder() {
|
|
|
247
252
|
}
|
|
248
253
|
|
|
249
254
|
TEST_SYNC_POINT_CALLBACK(
|
|
250
|
-
"GenericRateLimiter::
|
|
255
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
251
256
|
"PreReturnPriIterationOrder",
|
|
252
257
|
&pri_iteration_order);
|
|
253
258
|
return pri_iteration_order;
|
|
254
259
|
}
|
|
255
260
|
|
|
256
|
-
void GenericRateLimiter::
|
|
257
|
-
|
|
258
|
-
|
|
261
|
+
void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
|
|
262
|
+
TEST_SYNC_POINT_CALLBACK(
|
|
263
|
+
"GenericRateLimiter::RefillBytesAndGrantRequestsLocked", &request_mutex_);
|
|
264
|
+
next_refill_us_ = NowMicrosMonotonicLocked() + refill_period_us_;
|
|
259
265
|
// Carry over the left over quota from the last period
|
|
260
266
|
auto refill_bytes_per_period =
|
|
261
267
|
refill_bytes_per_period_.load(std::memory_order_relaxed);
|
|
@@ -264,7 +270,7 @@ void GenericRateLimiter::RefillBytesAndGrantRequests() {
|
|
|
264
270
|
}
|
|
265
271
|
|
|
266
272
|
std::vector<Env::IOPriority> pri_iteration_order =
|
|
267
|
-
|
|
273
|
+
GeneratePriorityIterationOrderLocked();
|
|
268
274
|
|
|
269
275
|
for (int i = Env::IO_LOW; i < Env::IO_TOTAL; ++i) {
|
|
270
276
|
assert(!pri_iteration_order.empty());
|
|
@@ -293,7 +299,7 @@ void GenericRateLimiter::RefillBytesAndGrantRequests() {
|
|
|
293
299
|
}
|
|
294
300
|
}
|
|
295
301
|
|
|
296
|
-
int64_t GenericRateLimiter::
|
|
302
|
+
int64_t GenericRateLimiter::CalculateRefillBytesPerPeriodLocked(
|
|
297
303
|
int64_t rate_bytes_per_sec) {
|
|
298
304
|
if (std::numeric_limits<int64_t>::max() / rate_bytes_per_sec <
|
|
299
305
|
refill_period_us_) {
|
|
@@ -305,7 +311,7 @@ int64_t GenericRateLimiter::CalculateRefillBytesPerPeriod(
|
|
|
305
311
|
}
|
|
306
312
|
}
|
|
307
313
|
|
|
308
|
-
Status GenericRateLimiter::
|
|
314
|
+
Status GenericRateLimiter::TuneLocked() {
|
|
309
315
|
const int kLowWatermarkPct = 50;
|
|
310
316
|
const int kHighWatermarkPct = 90;
|
|
311
317
|
const int kAdjustFactorPct = 5;
|
|
@@ -314,7 +320,7 @@ Status GenericRateLimiter::Tune() {
|
|
|
314
320
|
const int kAllowedRangeFactor = 20;
|
|
315
321
|
|
|
316
322
|
std::chrono::microseconds prev_tuned_time = tuned_time_;
|
|
317
|
-
tuned_time_ = std::chrono::microseconds(
|
|
323
|
+
tuned_time_ = std::chrono::microseconds(NowMicrosMonotonicLocked());
|
|
318
324
|
|
|
319
325
|
int64_t elapsed_intervals = (tuned_time_ - prev_tuned_time +
|
|
320
326
|
std::chrono::microseconds(refill_period_us_) -
|
|
@@ -349,7 +355,7 @@ Status GenericRateLimiter::Tune() {
|
|
|
349
355
|
new_bytes_per_sec = prev_bytes_per_sec;
|
|
350
356
|
}
|
|
351
357
|
if (new_bytes_per_sec != prev_bytes_per_sec) {
|
|
352
|
-
|
|
358
|
+
SetBytesPerSecondLocked(new_bytes_per_sec);
|
|
353
359
|
}
|
|
354
360
|
num_drains_ = 0;
|
|
355
361
|
return Status::OK();
|
|
@@ -92,30 +92,32 @@ class GenericRateLimiter : public RateLimiter {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
virtual int64_t GetBytesPerSecond() const override {
|
|
95
|
-
return rate_bytes_per_sec_;
|
|
95
|
+
return rate_bytes_per_sec_.load(std::memory_order_relaxed);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
virtual void TEST_SetClock(std::shared_ptr<SystemClock> clock) {
|
|
99
99
|
MutexLock g(&request_mutex_);
|
|
100
100
|
clock_ = std::move(clock);
|
|
101
|
-
next_refill_us_ =
|
|
101
|
+
next_refill_us_ = NowMicrosMonotonicLocked();
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
private:
|
|
105
|
-
void
|
|
106
|
-
std::vector<Env::IOPriority>
|
|
107
|
-
int64_t
|
|
108
|
-
Status
|
|
109
|
-
|
|
110
|
-
|
|
105
|
+
void RefillBytesAndGrantRequestsLocked();
|
|
106
|
+
std::vector<Env::IOPriority> GeneratePriorityIterationOrderLocked();
|
|
107
|
+
int64_t CalculateRefillBytesPerPeriodLocked(int64_t rate_bytes_per_sec);
|
|
108
|
+
Status TuneLocked();
|
|
109
|
+
void SetBytesPerSecondLocked(int64_t bytes_per_second);
|
|
110
|
+
|
|
111
|
+
uint64_t NowMicrosMonotonicLocked() {
|
|
112
|
+
return clock_->NowNanos() / std::milli::den;
|
|
113
|
+
}
|
|
111
114
|
|
|
112
115
|
// This mutex guard all internal states
|
|
113
116
|
mutable port::Mutex request_mutex_;
|
|
114
117
|
|
|
115
118
|
const int64_t refill_period_us_;
|
|
116
119
|
|
|
117
|
-
int64_t rate_bytes_per_sec_;
|
|
118
|
-
// This variable can be changed dynamically.
|
|
120
|
+
std::atomic<int64_t> rate_bytes_per_sec_;
|
|
119
121
|
std::atomic<int64_t> refill_bytes_per_period_;
|
|
120
122
|
std::shared_ptr<SystemClock> clock_;
|
|
121
123
|
|
|
@@ -202,7 +202,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
202
202
|
bool mid_pri_itereated_after_low_pri_set = false;
|
|
203
203
|
bool pri_iteration_order_verified = false;
|
|
204
204
|
SyncPoint::GetInstance()->SetCallBack(
|
|
205
|
-
"GenericRateLimiter::
|
|
205
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
206
206
|
"PostRandomOneInFairnessForHighPri",
|
|
207
207
|
[&](void* arg) {
|
|
208
208
|
bool* high_pri_iterated_after_mid_low_pri = (bool*)arg;
|
|
@@ -212,7 +212,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
212
212
|
});
|
|
213
213
|
|
|
214
214
|
SyncPoint::GetInstance()->SetCallBack(
|
|
215
|
-
"GenericRateLimiter::
|
|
215
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
216
216
|
"PostRandomOneInFairnessForMidPri",
|
|
217
217
|
[&](void* arg) {
|
|
218
218
|
bool* mid_pri_itereated_after_low_pri = (bool*)arg;
|
|
@@ -222,7 +222,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
222
222
|
});
|
|
223
223
|
|
|
224
224
|
SyncPoint::GetInstance()->SetCallBack(
|
|
225
|
-
"GenericRateLimiter::
|
|
225
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
226
226
|
"PreReturnPriIterationOrder",
|
|
227
227
|
[&](void* arg) {
|
|
228
228
|
std::vector<Env::IOPriority>* pri_iteration_order =
|
|
@@ -249,13 +249,13 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
249
249
|
ASSERT_EQ(pri_iteration_order_verified, true);
|
|
250
250
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
251
251
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
252
|
-
"GenericRateLimiter::
|
|
252
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
253
253
|
"PreReturnPriIterationOrder");
|
|
254
254
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
255
|
-
"GenericRateLimiter::
|
|
255
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
256
256
|
"PostRandomOneInFairnessForMidPri");
|
|
257
257
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
258
|
-
"GenericRateLimiter::
|
|
258
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
259
259
|
"PostRandomOneInFairnessForHighPri");
|
|
260
260
|
}
|
|
261
261
|
}
|
|
@@ -346,7 +346,7 @@ TEST_F(RateLimiterTest, Rate) {
|
|
|
346
346
|
|
|
347
347
|
// This can fail in heavily loaded CI environments
|
|
348
348
|
bool skip_minimum_rate_check =
|
|
349
|
-
#if
|
|
349
|
+
#if defined(CIRCLECI) && defined(OS_MACOSX)
|
|
350
350
|
true;
|
|
351
351
|
#else
|
|
352
352
|
getenv("SANDCASTLE");
|
|
@@ -387,11 +387,14 @@ TEST_F(RateLimiterTest, LimitChangeTest) {
|
|
|
387
387
|
std::make_shared<GenericRateLimiter>(
|
|
388
388
|
target, refill_period, 10, RateLimiter::Mode::kWritesOnly,
|
|
389
389
|
SystemClock::Default(), false /* auto_tuned */);
|
|
390
|
+
// After "GenericRateLimiter::Request:1" the mutex is held until the bytes
|
|
391
|
+
// are refilled. This test could be improved to change the limit when lock
|
|
392
|
+
// is released in `TimedWait()`.
|
|
390
393
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
|
391
394
|
{{"GenericRateLimiter::Request",
|
|
392
395
|
"RateLimiterTest::LimitChangeTest:changeLimitStart"},
|
|
393
396
|
{"RateLimiterTest::LimitChangeTest:changeLimitEnd",
|
|
394
|
-
"GenericRateLimiter::
|
|
397
|
+
"GenericRateLimiter::Request:1"}});
|
|
395
398
|
Arg arg(target, Env::IO_HIGH, limiter);
|
|
396
399
|
// The idea behind is to start a request first, then before it refills,
|
|
397
400
|
// update limit to a different value (2X/0.5X). No starvation should
|
|
@@ -1115,7 +1115,8 @@ TEST_P(BackupEngineTestWithParam, OfflineIntegrationTest) {
|
|
|
1115
1115
|
destroy_data = false;
|
|
1116
1116
|
// kAutoFlushOnly to preserve legacy test behavior (consider updating)
|
|
1117
1117
|
FillDB(db_.get(), keys_iteration * i, fill_up_to, kAutoFlushOnly);
|
|
1118
|
-
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0))
|
|
1118
|
+
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0))
|
|
1119
|
+
<< "iter: " << iter << ", idx: " << i;
|
|
1119
1120
|
CloseDBAndBackupEngine();
|
|
1120
1121
|
DestroyDB(dbname_, options_);
|
|
1121
1122
|
|
|
@@ -382,6 +382,51 @@ Status PessimisticTransactionDB::CreateColumnFamily(
|
|
|
382
382
|
return s;
|
|
383
383
|
}
|
|
384
384
|
|
|
385
|
+
Status PessimisticTransactionDB::CreateColumnFamilies(
|
|
386
|
+
const ColumnFamilyOptions& options,
|
|
387
|
+
const std::vector<std::string>& column_family_names,
|
|
388
|
+
std::vector<ColumnFamilyHandle*>* handles) {
|
|
389
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
390
|
+
|
|
391
|
+
Status s = VerifyCFOptions(options);
|
|
392
|
+
if (!s.ok()) {
|
|
393
|
+
return s;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
s = db_->CreateColumnFamilies(options, column_family_names, handles);
|
|
397
|
+
if (s.ok()) {
|
|
398
|
+
for (auto* handle : *handles) {
|
|
399
|
+
lock_manager_->AddColumnFamily(handle);
|
|
400
|
+
UpdateCFComparatorMap(handle);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
return s;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
Status PessimisticTransactionDB::CreateColumnFamilies(
|
|
408
|
+
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
409
|
+
std::vector<ColumnFamilyHandle*>* handles) {
|
|
410
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
411
|
+
|
|
412
|
+
for (auto& cf_desc : column_families) {
|
|
413
|
+
Status s = VerifyCFOptions(cf_desc.options);
|
|
414
|
+
if (!s.ok()) {
|
|
415
|
+
return s;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
Status s = db_->CreateColumnFamilies(column_families, handles);
|
|
420
|
+
if (s.ok()) {
|
|
421
|
+
for (auto* handle : *handles) {
|
|
422
|
+
lock_manager_->AddColumnFamily(handle);
|
|
423
|
+
UpdateCFComparatorMap(handle);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
return s;
|
|
428
|
+
}
|
|
429
|
+
|
|
385
430
|
// Let LockManager know that it can deallocate the LockMap for this
|
|
386
431
|
// column family.
|
|
387
432
|
Status PessimisticTransactionDB::DropColumnFamily(
|
|
@@ -396,6 +441,20 @@ Status PessimisticTransactionDB::DropColumnFamily(
|
|
|
396
441
|
return s;
|
|
397
442
|
}
|
|
398
443
|
|
|
444
|
+
Status PessimisticTransactionDB::DropColumnFamilies(
|
|
445
|
+
const std::vector<ColumnFamilyHandle*>& column_families) {
|
|
446
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
447
|
+
|
|
448
|
+
Status s = db_->DropColumnFamilies(column_families);
|
|
449
|
+
if (s.ok()) {
|
|
450
|
+
for (auto* handle : column_families) {
|
|
451
|
+
lock_manager_->RemoveColumnFamily(handle);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
return s;
|
|
456
|
+
}
|
|
457
|
+
|
|
399
458
|
Status PessimisticTransactionDB::TryLock(PessimisticTransaction* txn,
|
|
400
459
|
uint32_t cfh_id,
|
|
401
460
|
const std::string& key,
|
|
@@ -101,9 +101,21 @@ class PessimisticTransactionDB : public TransactionDB {
|
|
|
101
101
|
const std::string& column_family_name,
|
|
102
102
|
ColumnFamilyHandle** handle) override;
|
|
103
103
|
|
|
104
|
+
Status CreateColumnFamilies(
|
|
105
|
+
const ColumnFamilyOptions& options,
|
|
106
|
+
const std::vector<std::string>& column_family_names,
|
|
107
|
+
std::vector<ColumnFamilyHandle*>* handles) override;
|
|
108
|
+
|
|
109
|
+
Status CreateColumnFamilies(
|
|
110
|
+
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
111
|
+
std::vector<ColumnFamilyHandle*>* handles) override;
|
|
112
|
+
|
|
104
113
|
using StackableDB::DropColumnFamily;
|
|
105
114
|
virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
|
|
106
115
|
|
|
116
|
+
Status DropColumnFamilies(
|
|
117
|
+
const std::vector<ColumnFamilyHandle*>& column_families) override;
|
|
118
|
+
|
|
107
119
|
Status TryLock(PessimisticTransaction* txn, uint32_t cfh_id,
|
|
108
120
|
const std::string& key, bool exclusive);
|
|
109
121
|
Status TryRangeLock(PessimisticTransaction* txn, uint32_t cfh_id,
|
|
@@ -6499,6 +6499,37 @@ TEST_P(TransactionTest, OpenAndEnableU32Timestamp) {
|
|
|
6499
6499
|
}
|
|
6500
6500
|
}
|
|
6501
6501
|
|
|
6502
|
+
TEST_P(TransactionTest, WriteWithBulkCreatedColumnFamilies) {
|
|
6503
|
+
ColumnFamilyOptions cf_options;
|
|
6504
|
+
WriteOptions write_options;
|
|
6505
|
+
|
|
6506
|
+
std::vector<std::string> cf_names;
|
|
6507
|
+
std::vector<ColumnFamilyHandle*> cf_handles;
|
|
6508
|
+
|
|
6509
|
+
cf_names.push_back("test_cf");
|
|
6510
|
+
|
|
6511
|
+
ASSERT_OK(db->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
|
6512
|
+
ASSERT_OK(db->Put(write_options, cf_handles[0], "foo", "bar"));
|
|
6513
|
+
ASSERT_OK(db->DropColumnFamilies(cf_handles));
|
|
6514
|
+
|
|
6515
|
+
for (auto* h : cf_handles) {
|
|
6516
|
+
delete h;
|
|
6517
|
+
}
|
|
6518
|
+
cf_handles.clear();
|
|
6519
|
+
|
|
6520
|
+
std::vector<ColumnFamilyDescriptor> cf_descriptors;
|
|
6521
|
+
|
|
6522
|
+
cf_descriptors.emplace_back("test_cf", ColumnFamilyOptions());
|
|
6523
|
+
|
|
6524
|
+
ASSERT_OK(db->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
|
6525
|
+
ASSERT_OK(db->Put(write_options, cf_handles[0], "foo", "bar"));
|
|
6526
|
+
ASSERT_OK(db->DropColumnFamilies(cf_handles));
|
|
6527
|
+
for (auto* h : cf_handles) {
|
|
6528
|
+
delete h;
|
|
6529
|
+
}
|
|
6530
|
+
cf_handles.clear();
|
|
6531
|
+
}
|
|
6532
|
+
|
|
6502
6533
|
} // namespace ROCKSDB_NAMESPACE
|
|
6503
6534
|
|
|
6504
6535
|
int main(int argc, char** argv) {
|
|
@@ -1166,8 +1166,6 @@ TEST_P(WritePreparedTransactionTest, CheckAgainstSnapshots) {
|
|
|
1166
1166
|
}
|
|
1167
1167
|
}
|
|
1168
1168
|
|
|
1169
|
-
// This test is too slow for travis
|
|
1170
|
-
#ifndef TRAVIS
|
|
1171
1169
|
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
1172
1170
|
// Test that CheckAgainstSnapshots will not miss a live snapshot if it is run in
|
|
1173
1171
|
// parallel with UpdateSnapshots.
|
|
@@ -1249,7 +1247,6 @@ TEST_P(SnapshotConcurrentAccessTest, SnapshotConcurrentAccess) {
|
|
|
1249
1247
|
printf("\n");
|
|
1250
1248
|
}
|
|
1251
1249
|
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
1252
|
-
#endif // TRAVIS
|
|
1253
1250
|
|
|
1254
1251
|
// This test clarifies the contract of AdvanceMaxEvictedSeq method
|
|
1255
1252
|
TEST_P(WritePreparedTransactionTest, AdvanceMaxEvictedSeqBasic) {
|
package/index.js
CHANGED
|
@@ -60,7 +60,7 @@ class RocksLevel extends AbstractLevel {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
get sequence () {
|
|
63
|
-
return
|
|
63
|
+
return binding.db_get_latest_sequence(this[kContext])
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
get location () {
|
|
@@ -216,7 +216,7 @@ class RocksLevel extends AbstractLevel {
|
|
|
216
216
|
} else {
|
|
217
217
|
resolve({
|
|
218
218
|
rows,
|
|
219
|
-
sequence:
|
|
219
|
+
sequence: binding.iterator_get_sequence(context),
|
|
220
220
|
finished
|
|
221
221
|
})
|
|
222
222
|
}
|
package/iterator.js
CHANGED
package/package.json
CHANGED
|
Binary file
|
|
Binary file
|