@nxtedition/rocksdb 7.0.38 → 7.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.cc +62 -33
- package/deps/rocksdb/rocksdb/cache/cache_bench_tool.cc +27 -11
- package/deps/rocksdb/rocksdb/cache/clock_cache.cc +310 -337
- package/deps/rocksdb/rocksdb/cache/clock_cache.h +394 -352
- package/deps/rocksdb/rocksdb/db/blob/blob_file_reader.cc +1 -1
- package/deps/rocksdb/rocksdb/db/column_family.cc +2 -2
- package/deps/rocksdb/rocksdb/db/column_family_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction.cc +13 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.cc +273 -134
- package/deps/rocksdb/rocksdb/db/compaction/compaction_job.h +33 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.cc +11 -3
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker.h +2 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_fifo.cc +2 -2
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_level.cc +133 -5
- package/deps/rocksdb/rocksdb/db/compaction/compaction_picker_test.cc +130 -1
- package/deps/rocksdb/rocksdb/db/compaction/compaction_service_job.cc +8 -4
- package/deps/rocksdb/rocksdb/db/compaction/subcompaction_state.h +11 -9
- package/deps/rocksdb/rocksdb/db/db_compaction_test.cc +209 -12
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.cc +54 -39
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl.h +102 -19
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_compaction_flush.cc +30 -11
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_debug.cc +1 -1
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_files.cc +28 -25
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_open.cc +0 -14
- package/deps/rocksdb/rocksdb/db/db_impl/db_impl_write.cc +63 -54
- package/deps/rocksdb/rocksdb/db/db_test.cc +6 -6
- package/deps/rocksdb/rocksdb/db/error_handler.cc +7 -0
- package/deps/rocksdb/rocksdb/db/error_handler.h +10 -9
- package/deps/rocksdb/rocksdb/db/log_test.cc +13 -6
- package/deps/rocksdb/rocksdb/db/perf_context_test.cc +1 -1
- package/deps/rocksdb/rocksdb/db/table_cache.cc +21 -0
- package/deps/rocksdb/rocksdb/db/table_cache.h +5 -0
- package/deps/rocksdb/rocksdb/db/version_set.cc +3 -2
- package/deps/rocksdb/rocksdb/db/version_set.h +6 -4
- package/deps/rocksdb/rocksdb/db/version_set_test.cc +8 -6
- package/deps/rocksdb/rocksdb/db/wal_edit.cc +22 -15
- package/deps/rocksdb/rocksdb/db/wal_edit.h +10 -0
- package/deps/rocksdb/rocksdb/db/wal_edit_test.cc +4 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_common.cc +0 -36
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_driver.cc +1 -12
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.cc +23 -29
- package/deps/rocksdb/rocksdb/db_stress_tool/db_stress_test_base.h +0 -5
- package/deps/rocksdb/rocksdb/db_stress_tool/multi_ops_txns_stress.cc +7 -0
- package/deps/rocksdb/rocksdb/env/env_test.cc +0 -5
- package/deps/rocksdb/rocksdb/env/io_posix.cc +1 -7
- package/deps/rocksdb/rocksdb/memtable/hash_linklist_rep.cc +100 -78
- package/deps/rocksdb/rocksdb/options/options_test.cc +16 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.cc +51 -0
- package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader.h +3 -0
- package/deps/rocksdb/rocksdb/table/table_reader.h +14 -0
- package/deps/rocksdb/rocksdb/table/table_test.cc +52 -0
- package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc +8 -38
- package/deps/rocksdb/rocksdb/util/rate_limiter.cc +27 -21
- package/deps/rocksdb/rocksdb/util/rate_limiter.h +12 -10
- package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc +11 -8
- package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc +2 -1
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +59 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/pessimistic_transaction_db.h +12 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc +31 -0
- package/deps/rocksdb/rocksdb/utilities/transactions/write_prepared_transaction_test.cc +0 -3
- package/index.js +2 -2
- package/iterator.js +1 -1
- package/max_rev_operator.h +114 -0
- package/package.json +1 -1
- package/prebuilds/darwin-arm64/node.napi.node +0 -0
- package/prebuilds/linux-x64/node.napi.node +0 -0
|
@@ -54,20 +54,20 @@ GenericRateLimiter::GenericRateLimiter(
|
|
|
54
54
|
rate_bytes_per_sec_(auto_tuned ? rate_bytes_per_sec / 2
|
|
55
55
|
: rate_bytes_per_sec),
|
|
56
56
|
refill_bytes_per_period_(
|
|
57
|
-
|
|
57
|
+
CalculateRefillBytesPerPeriodLocked(rate_bytes_per_sec_)),
|
|
58
58
|
clock_(clock),
|
|
59
59
|
stop_(false),
|
|
60
60
|
exit_cv_(&request_mutex_),
|
|
61
61
|
requests_to_wait_(0),
|
|
62
62
|
available_bytes_(0),
|
|
63
|
-
next_refill_us_(
|
|
63
|
+
next_refill_us_(NowMicrosMonotonicLocked()),
|
|
64
64
|
fairness_(fairness > 100 ? 100 : fairness),
|
|
65
65
|
rnd_((uint32_t)time(nullptr)),
|
|
66
66
|
wait_until_refill_pending_(false),
|
|
67
67
|
auto_tuned_(auto_tuned),
|
|
68
68
|
num_drains_(0),
|
|
69
69
|
max_bytes_per_sec_(rate_bytes_per_sec),
|
|
70
|
-
tuned_time_(
|
|
70
|
+
tuned_time_(NowMicrosMonotonicLocked()) {
|
|
71
71
|
for (int i = Env::IO_LOW; i < Env::IO_TOTAL; ++i) {
|
|
72
72
|
total_requests_[i] = 0;
|
|
73
73
|
total_bytes_through_[i] = 0;
|
|
@@ -97,10 +97,15 @@ GenericRateLimiter::~GenericRateLimiter() {
|
|
|
97
97
|
|
|
98
98
|
// This API allows user to dynamically change rate limiter's bytes per second.
|
|
99
99
|
void GenericRateLimiter::SetBytesPerSecond(int64_t bytes_per_second) {
|
|
100
|
+
MutexLock g(&request_mutex_);
|
|
101
|
+
SetBytesPerSecondLocked(bytes_per_second);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
void GenericRateLimiter::SetBytesPerSecondLocked(int64_t bytes_per_second) {
|
|
100
105
|
assert(bytes_per_second > 0);
|
|
101
|
-
rate_bytes_per_sec_
|
|
106
|
+
rate_bytes_per_sec_.store(bytes_per_second, std::memory_order_relaxed);
|
|
102
107
|
refill_bytes_per_period_.store(
|
|
103
|
-
|
|
108
|
+
CalculateRefillBytesPerPeriodLocked(bytes_per_second),
|
|
104
109
|
std::memory_order_relaxed);
|
|
105
110
|
}
|
|
106
111
|
|
|
@@ -115,10 +120,10 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
115
120
|
|
|
116
121
|
if (auto_tuned_) {
|
|
117
122
|
static const int kRefillsPerTune = 100;
|
|
118
|
-
std::chrono::microseconds now(
|
|
123
|
+
std::chrono::microseconds now(NowMicrosMonotonicLocked());
|
|
119
124
|
if (now - tuned_time_ >=
|
|
120
125
|
kRefillsPerTune * std::chrono::microseconds(refill_period_us_)) {
|
|
121
|
-
Status s =
|
|
126
|
+
Status s = TuneLocked();
|
|
122
127
|
s.PermitUncheckedError(); //**TODO: What to do on error?
|
|
123
128
|
}
|
|
124
129
|
}
|
|
@@ -152,7 +157,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
152
157
|
// (1) Waiting for the next refill time.
|
|
153
158
|
// (2) Refilling the bytes and granting requests.
|
|
154
159
|
do {
|
|
155
|
-
int64_t time_until_refill_us = next_refill_us_ -
|
|
160
|
+
int64_t time_until_refill_us = next_refill_us_ - NowMicrosMonotonicLocked();
|
|
156
161
|
if (time_until_refill_us > 0) {
|
|
157
162
|
if (wait_until_refill_pending_) {
|
|
158
163
|
// Somebody is performing (1). Trust we'll be woken up when our request
|
|
@@ -173,7 +178,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
173
178
|
} else {
|
|
174
179
|
// Whichever thread reaches here first performs duty (2) as described
|
|
175
180
|
// above.
|
|
176
|
-
|
|
181
|
+
RefillBytesAndGrantRequestsLocked();
|
|
177
182
|
if (r.granted) {
|
|
178
183
|
// If there is any remaining requests, make sure there exists at least
|
|
179
184
|
// one candidate is awake for future duties by signaling a front request
|
|
@@ -215,7 +220,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
|
|
|
215
220
|
}
|
|
216
221
|
|
|
217
222
|
std::vector<Env::IOPriority>
|
|
218
|
-
GenericRateLimiter::
|
|
223
|
+
GenericRateLimiter::GeneratePriorityIterationOrderLocked() {
|
|
219
224
|
std::vector<Env::IOPriority> pri_iteration_order(Env::IO_TOTAL /* 4 */);
|
|
220
225
|
// We make Env::IO_USER a superior priority by always iterating its queue
|
|
221
226
|
// first
|
|
@@ -223,12 +228,12 @@ GenericRateLimiter::GeneratePriorityIterationOrder() {
|
|
|
223
228
|
|
|
224
229
|
bool high_pri_iterated_after_mid_low_pri = rnd_.OneIn(fairness_);
|
|
225
230
|
TEST_SYNC_POINT_CALLBACK(
|
|
226
|
-
"GenericRateLimiter::
|
|
231
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
227
232
|
"PostRandomOneInFairnessForHighPri",
|
|
228
233
|
&high_pri_iterated_after_mid_low_pri);
|
|
229
234
|
bool mid_pri_itereated_after_low_pri = rnd_.OneIn(fairness_);
|
|
230
235
|
TEST_SYNC_POINT_CALLBACK(
|
|
231
|
-
"GenericRateLimiter::
|
|
236
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
232
237
|
"PostRandomOneInFairnessForMidPri",
|
|
233
238
|
&mid_pri_itereated_after_low_pri);
|
|
234
239
|
|
|
@@ -247,15 +252,16 @@ GenericRateLimiter::GeneratePriorityIterationOrder() {
|
|
|
247
252
|
}
|
|
248
253
|
|
|
249
254
|
TEST_SYNC_POINT_CALLBACK(
|
|
250
|
-
"GenericRateLimiter::
|
|
255
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
251
256
|
"PreReturnPriIterationOrder",
|
|
252
257
|
&pri_iteration_order);
|
|
253
258
|
return pri_iteration_order;
|
|
254
259
|
}
|
|
255
260
|
|
|
256
|
-
void GenericRateLimiter::
|
|
257
|
-
|
|
258
|
-
|
|
261
|
+
void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
|
|
262
|
+
TEST_SYNC_POINT_CALLBACK(
|
|
263
|
+
"GenericRateLimiter::RefillBytesAndGrantRequestsLocked", &request_mutex_);
|
|
264
|
+
next_refill_us_ = NowMicrosMonotonicLocked() + refill_period_us_;
|
|
259
265
|
// Carry over the left over quota from the last period
|
|
260
266
|
auto refill_bytes_per_period =
|
|
261
267
|
refill_bytes_per_period_.load(std::memory_order_relaxed);
|
|
@@ -264,7 +270,7 @@ void GenericRateLimiter::RefillBytesAndGrantRequests() {
|
|
|
264
270
|
}
|
|
265
271
|
|
|
266
272
|
std::vector<Env::IOPriority> pri_iteration_order =
|
|
267
|
-
|
|
273
|
+
GeneratePriorityIterationOrderLocked();
|
|
268
274
|
|
|
269
275
|
for (int i = Env::IO_LOW; i < Env::IO_TOTAL; ++i) {
|
|
270
276
|
assert(!pri_iteration_order.empty());
|
|
@@ -293,7 +299,7 @@ void GenericRateLimiter::RefillBytesAndGrantRequests() {
|
|
|
293
299
|
}
|
|
294
300
|
}
|
|
295
301
|
|
|
296
|
-
int64_t GenericRateLimiter::
|
|
302
|
+
int64_t GenericRateLimiter::CalculateRefillBytesPerPeriodLocked(
|
|
297
303
|
int64_t rate_bytes_per_sec) {
|
|
298
304
|
if (std::numeric_limits<int64_t>::max() / rate_bytes_per_sec <
|
|
299
305
|
refill_period_us_) {
|
|
@@ -305,7 +311,7 @@ int64_t GenericRateLimiter::CalculateRefillBytesPerPeriod(
|
|
|
305
311
|
}
|
|
306
312
|
}
|
|
307
313
|
|
|
308
|
-
Status GenericRateLimiter::
|
|
314
|
+
Status GenericRateLimiter::TuneLocked() {
|
|
309
315
|
const int kLowWatermarkPct = 50;
|
|
310
316
|
const int kHighWatermarkPct = 90;
|
|
311
317
|
const int kAdjustFactorPct = 5;
|
|
@@ -314,7 +320,7 @@ Status GenericRateLimiter::Tune() {
|
|
|
314
320
|
const int kAllowedRangeFactor = 20;
|
|
315
321
|
|
|
316
322
|
std::chrono::microseconds prev_tuned_time = tuned_time_;
|
|
317
|
-
tuned_time_ = std::chrono::microseconds(
|
|
323
|
+
tuned_time_ = std::chrono::microseconds(NowMicrosMonotonicLocked());
|
|
318
324
|
|
|
319
325
|
int64_t elapsed_intervals = (tuned_time_ - prev_tuned_time +
|
|
320
326
|
std::chrono::microseconds(refill_period_us_) -
|
|
@@ -349,7 +355,7 @@ Status GenericRateLimiter::Tune() {
|
|
|
349
355
|
new_bytes_per_sec = prev_bytes_per_sec;
|
|
350
356
|
}
|
|
351
357
|
if (new_bytes_per_sec != prev_bytes_per_sec) {
|
|
352
|
-
|
|
358
|
+
SetBytesPerSecondLocked(new_bytes_per_sec);
|
|
353
359
|
}
|
|
354
360
|
num_drains_ = 0;
|
|
355
361
|
return Status::OK();
|
|
@@ -92,30 +92,32 @@ class GenericRateLimiter : public RateLimiter {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
virtual int64_t GetBytesPerSecond() const override {
|
|
95
|
-
return rate_bytes_per_sec_;
|
|
95
|
+
return rate_bytes_per_sec_.load(std::memory_order_relaxed);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
virtual void TEST_SetClock(std::shared_ptr<SystemClock> clock) {
|
|
99
99
|
MutexLock g(&request_mutex_);
|
|
100
100
|
clock_ = std::move(clock);
|
|
101
|
-
next_refill_us_ =
|
|
101
|
+
next_refill_us_ = NowMicrosMonotonicLocked();
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
private:
|
|
105
|
-
void
|
|
106
|
-
std::vector<Env::IOPriority>
|
|
107
|
-
int64_t
|
|
108
|
-
Status
|
|
109
|
-
|
|
110
|
-
|
|
105
|
+
void RefillBytesAndGrantRequestsLocked();
|
|
106
|
+
std::vector<Env::IOPriority> GeneratePriorityIterationOrderLocked();
|
|
107
|
+
int64_t CalculateRefillBytesPerPeriodLocked(int64_t rate_bytes_per_sec);
|
|
108
|
+
Status TuneLocked();
|
|
109
|
+
void SetBytesPerSecondLocked(int64_t bytes_per_second);
|
|
110
|
+
|
|
111
|
+
uint64_t NowMicrosMonotonicLocked() {
|
|
112
|
+
return clock_->NowNanos() / std::milli::den;
|
|
113
|
+
}
|
|
111
114
|
|
|
112
115
|
// This mutex guard all internal states
|
|
113
116
|
mutable port::Mutex request_mutex_;
|
|
114
117
|
|
|
115
118
|
const int64_t refill_period_us_;
|
|
116
119
|
|
|
117
|
-
int64_t rate_bytes_per_sec_;
|
|
118
|
-
// This variable can be changed dynamically.
|
|
120
|
+
std::atomic<int64_t> rate_bytes_per_sec_;
|
|
119
121
|
std::atomic<int64_t> refill_bytes_per_period_;
|
|
120
122
|
std::shared_ptr<SystemClock> clock_;
|
|
121
123
|
|
|
@@ -202,7 +202,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
202
202
|
bool mid_pri_itereated_after_low_pri_set = false;
|
|
203
203
|
bool pri_iteration_order_verified = false;
|
|
204
204
|
SyncPoint::GetInstance()->SetCallBack(
|
|
205
|
-
"GenericRateLimiter::
|
|
205
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
206
206
|
"PostRandomOneInFairnessForHighPri",
|
|
207
207
|
[&](void* arg) {
|
|
208
208
|
bool* high_pri_iterated_after_mid_low_pri = (bool*)arg;
|
|
@@ -212,7 +212,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
212
212
|
});
|
|
213
213
|
|
|
214
214
|
SyncPoint::GetInstance()->SetCallBack(
|
|
215
|
-
"GenericRateLimiter::
|
|
215
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
216
216
|
"PostRandomOneInFairnessForMidPri",
|
|
217
217
|
[&](void* arg) {
|
|
218
218
|
bool* mid_pri_itereated_after_low_pri = (bool*)arg;
|
|
@@ -222,7 +222,7 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
222
222
|
});
|
|
223
223
|
|
|
224
224
|
SyncPoint::GetInstance()->SetCallBack(
|
|
225
|
-
"GenericRateLimiter::
|
|
225
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
226
226
|
"PreReturnPriIterationOrder",
|
|
227
227
|
[&](void* arg) {
|
|
228
228
|
std::vector<Env::IOPriority>* pri_iteration_order =
|
|
@@ -249,13 +249,13 @@ TEST_F(RateLimiterTest, GeneratePriorityIterationOrder) {
|
|
|
249
249
|
ASSERT_EQ(pri_iteration_order_verified, true);
|
|
250
250
|
SyncPoint::GetInstance()->DisableProcessing();
|
|
251
251
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
252
|
-
"GenericRateLimiter::
|
|
252
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
253
253
|
"PreReturnPriIterationOrder");
|
|
254
254
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
255
|
-
"GenericRateLimiter::
|
|
255
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
256
256
|
"PostRandomOneInFairnessForMidPri");
|
|
257
257
|
SyncPoint::GetInstance()->ClearCallBack(
|
|
258
|
-
"GenericRateLimiter::
|
|
258
|
+
"GenericRateLimiter::GeneratePriorityIterationOrderLocked::"
|
|
259
259
|
"PostRandomOneInFairnessForHighPri");
|
|
260
260
|
}
|
|
261
261
|
}
|
|
@@ -346,7 +346,7 @@ TEST_F(RateLimiterTest, Rate) {
|
|
|
346
346
|
|
|
347
347
|
// This can fail in heavily loaded CI environments
|
|
348
348
|
bool skip_minimum_rate_check =
|
|
349
|
-
#if
|
|
349
|
+
#if defined(CIRCLECI) && defined(OS_MACOSX)
|
|
350
350
|
true;
|
|
351
351
|
#else
|
|
352
352
|
getenv("SANDCASTLE");
|
|
@@ -387,11 +387,14 @@ TEST_F(RateLimiterTest, LimitChangeTest) {
|
|
|
387
387
|
std::make_shared<GenericRateLimiter>(
|
|
388
388
|
target, refill_period, 10, RateLimiter::Mode::kWritesOnly,
|
|
389
389
|
SystemClock::Default(), false /* auto_tuned */);
|
|
390
|
+
// After "GenericRateLimiter::Request:1" the mutex is held until the bytes
|
|
391
|
+
// are refilled. This test could be improved to change the limit when lock
|
|
392
|
+
// is released in `TimedWait()`.
|
|
390
393
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
|
391
394
|
{{"GenericRateLimiter::Request",
|
|
392
395
|
"RateLimiterTest::LimitChangeTest:changeLimitStart"},
|
|
393
396
|
{"RateLimiterTest::LimitChangeTest:changeLimitEnd",
|
|
394
|
-
"GenericRateLimiter::
|
|
397
|
+
"GenericRateLimiter::Request:1"}});
|
|
395
398
|
Arg arg(target, Env::IO_HIGH, limiter);
|
|
396
399
|
// The idea behind is to start a request first, then before it refills,
|
|
397
400
|
// update limit to a different value (2X/0.5X). No starvation should
|
|
@@ -1115,7 +1115,8 @@ TEST_P(BackupEngineTestWithParam, OfflineIntegrationTest) {
|
|
|
1115
1115
|
destroy_data = false;
|
|
1116
1116
|
// kAutoFlushOnly to preserve legacy test behavior (consider updating)
|
|
1117
1117
|
FillDB(db_.get(), keys_iteration * i, fill_up_to, kAutoFlushOnly);
|
|
1118
|
-
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0))
|
|
1118
|
+
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0))
|
|
1119
|
+
<< "iter: " << iter << ", idx: " << i;
|
|
1119
1120
|
CloseDBAndBackupEngine();
|
|
1120
1121
|
DestroyDB(dbname_, options_);
|
|
1121
1122
|
|
|
@@ -382,6 +382,51 @@ Status PessimisticTransactionDB::CreateColumnFamily(
|
|
|
382
382
|
return s;
|
|
383
383
|
}
|
|
384
384
|
|
|
385
|
+
Status PessimisticTransactionDB::CreateColumnFamilies(
|
|
386
|
+
const ColumnFamilyOptions& options,
|
|
387
|
+
const std::vector<std::string>& column_family_names,
|
|
388
|
+
std::vector<ColumnFamilyHandle*>* handles) {
|
|
389
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
390
|
+
|
|
391
|
+
Status s = VerifyCFOptions(options);
|
|
392
|
+
if (!s.ok()) {
|
|
393
|
+
return s;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
s = db_->CreateColumnFamilies(options, column_family_names, handles);
|
|
397
|
+
if (s.ok()) {
|
|
398
|
+
for (auto* handle : *handles) {
|
|
399
|
+
lock_manager_->AddColumnFamily(handle);
|
|
400
|
+
UpdateCFComparatorMap(handle);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
return s;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
Status PessimisticTransactionDB::CreateColumnFamilies(
|
|
408
|
+
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
409
|
+
std::vector<ColumnFamilyHandle*>* handles) {
|
|
410
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
411
|
+
|
|
412
|
+
for (auto& cf_desc : column_families) {
|
|
413
|
+
Status s = VerifyCFOptions(cf_desc.options);
|
|
414
|
+
if (!s.ok()) {
|
|
415
|
+
return s;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
Status s = db_->CreateColumnFamilies(column_families, handles);
|
|
420
|
+
if (s.ok()) {
|
|
421
|
+
for (auto* handle : *handles) {
|
|
422
|
+
lock_manager_->AddColumnFamily(handle);
|
|
423
|
+
UpdateCFComparatorMap(handle);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
return s;
|
|
428
|
+
}
|
|
429
|
+
|
|
385
430
|
// Let LockManager know that it can deallocate the LockMap for this
|
|
386
431
|
// column family.
|
|
387
432
|
Status PessimisticTransactionDB::DropColumnFamily(
|
|
@@ -396,6 +441,20 @@ Status PessimisticTransactionDB::DropColumnFamily(
|
|
|
396
441
|
return s;
|
|
397
442
|
}
|
|
398
443
|
|
|
444
|
+
Status PessimisticTransactionDB::DropColumnFamilies(
|
|
445
|
+
const std::vector<ColumnFamilyHandle*>& column_families) {
|
|
446
|
+
InstrumentedMutexLock l(&column_family_mutex_);
|
|
447
|
+
|
|
448
|
+
Status s = db_->DropColumnFamilies(column_families);
|
|
449
|
+
if (s.ok()) {
|
|
450
|
+
for (auto* handle : column_families) {
|
|
451
|
+
lock_manager_->RemoveColumnFamily(handle);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
return s;
|
|
456
|
+
}
|
|
457
|
+
|
|
399
458
|
Status PessimisticTransactionDB::TryLock(PessimisticTransaction* txn,
|
|
400
459
|
uint32_t cfh_id,
|
|
401
460
|
const std::string& key,
|
|
@@ -101,9 +101,21 @@ class PessimisticTransactionDB : public TransactionDB {
|
|
|
101
101
|
const std::string& column_family_name,
|
|
102
102
|
ColumnFamilyHandle** handle) override;
|
|
103
103
|
|
|
104
|
+
Status CreateColumnFamilies(
|
|
105
|
+
const ColumnFamilyOptions& options,
|
|
106
|
+
const std::vector<std::string>& column_family_names,
|
|
107
|
+
std::vector<ColumnFamilyHandle*>* handles) override;
|
|
108
|
+
|
|
109
|
+
Status CreateColumnFamilies(
|
|
110
|
+
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
111
|
+
std::vector<ColumnFamilyHandle*>* handles) override;
|
|
112
|
+
|
|
104
113
|
using StackableDB::DropColumnFamily;
|
|
105
114
|
virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
|
|
106
115
|
|
|
116
|
+
Status DropColumnFamilies(
|
|
117
|
+
const std::vector<ColumnFamilyHandle*>& column_families) override;
|
|
118
|
+
|
|
107
119
|
Status TryLock(PessimisticTransaction* txn, uint32_t cfh_id,
|
|
108
120
|
const std::string& key, bool exclusive);
|
|
109
121
|
Status TryRangeLock(PessimisticTransaction* txn, uint32_t cfh_id,
|
|
@@ -6499,6 +6499,37 @@ TEST_P(TransactionTest, OpenAndEnableU32Timestamp) {
|
|
|
6499
6499
|
}
|
|
6500
6500
|
}
|
|
6501
6501
|
|
|
6502
|
+
TEST_P(TransactionTest, WriteWithBulkCreatedColumnFamilies) {
|
|
6503
|
+
ColumnFamilyOptions cf_options;
|
|
6504
|
+
WriteOptions write_options;
|
|
6505
|
+
|
|
6506
|
+
std::vector<std::string> cf_names;
|
|
6507
|
+
std::vector<ColumnFamilyHandle*> cf_handles;
|
|
6508
|
+
|
|
6509
|
+
cf_names.push_back("test_cf");
|
|
6510
|
+
|
|
6511
|
+
ASSERT_OK(db->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
|
6512
|
+
ASSERT_OK(db->Put(write_options, cf_handles[0], "foo", "bar"));
|
|
6513
|
+
ASSERT_OK(db->DropColumnFamilies(cf_handles));
|
|
6514
|
+
|
|
6515
|
+
for (auto* h : cf_handles) {
|
|
6516
|
+
delete h;
|
|
6517
|
+
}
|
|
6518
|
+
cf_handles.clear();
|
|
6519
|
+
|
|
6520
|
+
std::vector<ColumnFamilyDescriptor> cf_descriptors;
|
|
6521
|
+
|
|
6522
|
+
cf_descriptors.emplace_back("test_cf", ColumnFamilyOptions());
|
|
6523
|
+
|
|
6524
|
+
ASSERT_OK(db->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
|
6525
|
+
ASSERT_OK(db->Put(write_options, cf_handles[0], "foo", "bar"));
|
|
6526
|
+
ASSERT_OK(db->DropColumnFamilies(cf_handles));
|
|
6527
|
+
for (auto* h : cf_handles) {
|
|
6528
|
+
delete h;
|
|
6529
|
+
}
|
|
6530
|
+
cf_handles.clear();
|
|
6531
|
+
}
|
|
6532
|
+
|
|
6502
6533
|
} // namespace ROCKSDB_NAMESPACE
|
|
6503
6534
|
|
|
6504
6535
|
int main(int argc, char** argv) {
|
|
@@ -1166,8 +1166,6 @@ TEST_P(WritePreparedTransactionTest, CheckAgainstSnapshots) {
|
|
|
1166
1166
|
}
|
|
1167
1167
|
}
|
|
1168
1168
|
|
|
1169
|
-
// This test is too slow for travis
|
|
1170
|
-
#ifndef TRAVIS
|
|
1171
1169
|
#if !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
1172
1170
|
// Test that CheckAgainstSnapshots will not miss a live snapshot if it is run in
|
|
1173
1171
|
// parallel with UpdateSnapshots.
|
|
@@ -1249,7 +1247,6 @@ TEST_P(SnapshotConcurrentAccessTest, SnapshotConcurrentAccess) {
|
|
|
1249
1247
|
printf("\n");
|
|
1250
1248
|
}
|
|
1251
1249
|
#endif // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)
|
|
1252
|
-
#endif // TRAVIS
|
|
1253
1250
|
|
|
1254
1251
|
// This test clarifies the contract of AdvanceMaxEvictedSeq method
|
|
1255
1252
|
TEST_P(WritePreparedTransactionTest, AdvanceMaxEvictedSeqBasic) {
|
package/index.js
CHANGED
|
@@ -60,7 +60,7 @@ class RocksLevel extends AbstractLevel {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
get sequence () {
|
|
63
|
-
return
|
|
63
|
+
return binding.db_get_latest_sequence(this[kContext])
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
get location () {
|
|
@@ -216,7 +216,7 @@ class RocksLevel extends AbstractLevel {
|
|
|
216
216
|
} else {
|
|
217
217
|
resolve({
|
|
218
218
|
rows,
|
|
219
|
-
sequence:
|
|
219
|
+
sequence: binding.iterator_get_sequence(context),
|
|
220
220
|
finished
|
|
221
221
|
})
|
|
222
222
|
}
|
package/iterator.js
CHANGED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <memory.h>
|
|
4
|
+
|
|
5
|
+
int compareRev(const rocksdb::Slice& a, const rocksdb::Slice& b) {
|
|
6
|
+
auto indexA = 0;
|
|
7
|
+
const auto endA = a.size();
|
|
8
|
+
auto lenA = endA;
|
|
9
|
+
|
|
10
|
+
auto indexB = 0;
|
|
11
|
+
const auto endB = b.size();
|
|
12
|
+
auto lenB = endB;
|
|
13
|
+
|
|
14
|
+
// Skip leading zeroes
|
|
15
|
+
while (a[indexA] == '0') {
|
|
16
|
+
++indexA;
|
|
17
|
+
--lenA;
|
|
18
|
+
}
|
|
19
|
+
while (b[indexB] == '0') {
|
|
20
|
+
++indexB;
|
|
21
|
+
--lenB;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Compare the revision number
|
|
25
|
+
auto result = 0;
|
|
26
|
+
const auto end = std::min(endA, endB);
|
|
27
|
+
while (indexA < end) {
|
|
28
|
+
const auto ac = a[indexA++];
|
|
29
|
+
const auto bc = b[indexB++];
|
|
30
|
+
|
|
31
|
+
if (ac == '-') {
|
|
32
|
+
if (bc == '-') {
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
return -1;
|
|
36
|
+
} else if (bc == '-') {
|
|
37
|
+
return 1;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (!result) {
|
|
41
|
+
result = ac == bc ? 0 : ac < bc ? -1 : 1;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (result) {
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Compare the rest
|
|
50
|
+
while (indexA < end) {
|
|
51
|
+
const auto ac = a[indexA++];
|
|
52
|
+
const auto bc = b[indexB++];
|
|
53
|
+
if (ac != bc) {
|
|
54
|
+
return ac < bc ? -1 : 1;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return lenA - lenB;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Merge operator that picks the maximum operand, Comparison is based on
|
|
62
|
+
// Slice::compare
|
|
63
|
+
class MaxRevOperator : public rocksdb::MergeOperator {
|
|
64
|
+
public:
|
|
65
|
+
bool FullMergeV2(const MergeOperationInput& merge_in,
|
|
66
|
+
MergeOperationOutput* merge_out) const override {
|
|
67
|
+
rocksdb::Slice& max = merge_out->existing_operand;
|
|
68
|
+
if (merge_in.existing_value) {
|
|
69
|
+
max = rocksdb::Slice(merge_in.existing_value->data(),
|
|
70
|
+
merge_in.existing_value->size());
|
|
71
|
+
} else if (max.data() == nullptr) {
|
|
72
|
+
max = rocksdb::Slice();
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
for (const auto& op : merge_in.operand_list) {
|
|
76
|
+
if (compareRev(max, op) < 0) {
|
|
77
|
+
max = op;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left_operand,
|
|
85
|
+
const rocksdb::Slice& right_operand, std::string* new_value,
|
|
86
|
+
rocksdb::Logger* /*logger*/) const override {
|
|
87
|
+
if (compareRev(left_operand, right_operand) >= 0) {
|
|
88
|
+
new_value->assign(left_operand.data(), left_operand.size());
|
|
89
|
+
} else {
|
|
90
|
+
new_value->assign(right_operand.data(), right_operand.size());
|
|
91
|
+
}
|
|
92
|
+
return true;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
bool PartialMergeMulti(const rocksdb::Slice& /*key*/,
|
|
96
|
+
const std::deque<rocksdb::Slice>& operand_list,
|
|
97
|
+
std::string* new_value,
|
|
98
|
+
rocksdb::Logger* /*logger*/) const override {
|
|
99
|
+
rocksdb::Slice max;
|
|
100
|
+
for (const auto& operand : operand_list) {
|
|
101
|
+
if (compareRev(max, operand) < 0) {
|
|
102
|
+
max = operand;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
new_value->assign(max.data(), max.size());
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static const char* kClassName() { return "MaxRevOperator"; }
|
|
111
|
+
static const char* kNickName() { return "maxRev"; }
|
|
112
|
+
const char* Name() const override { return kClassName(); }
|
|
113
|
+
const char* NickName() const override { return kNickName(); }
|
|
114
|
+
};
|
package/package.json
CHANGED
|
Binary file
|
|
Binary file
|